mirror of
https://github.com/openai/codex.git
synced 2026-10-24 18:18:41 +00:00
feat: support local development using Buck2
This commit is contained in:
Michael Bolin
@@ -0,0 +1,636 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate BUCK files for first-party Rust crates in the codex-rs workspace.
|
||||
|
||||
Reindeer generates third-party targets (in `codex-rs/third-party/BUCK`). This
|
||||
script generates Buck targets for workspace members (local crates) that depend
|
||||
on those third-party targets.
|
||||
|
||||
The generated BUCK files are intentionally gitignored for now.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
CODEX_RS_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = CODEX_RS_ROOT.parent
|
||||
BUILDIFIER = REPO_ROOT / "scripts" / "buildifier"
|
||||
|
||||
# All labels are rooted at the repo root Buck project.
|
||||
CODEX_RS_LABEL_PREFIX = "//codex-rs"
|
||||
|
||||
# Some integration tests rely on Cargo's behavior of allowing `include_str!`
|
||||
# paths that traverse outside of a package directory. Buck does not allow `..`
|
||||
# paths in `srcs`, so we skip those tests for now.
|
||||
SKIP_INTEGRATION_TESTS_BY_PACKAGE: dict[str, set[str]] = {
|
||||
"codex-core": {"all"},
|
||||
}
|
||||
|
||||
|
||||
def cargo_metadata() -> dict[str, Any]:
|
||||
out = subprocess.check_output(
|
||||
["cargo", "metadata", "--format-version=1", "--locked"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuckDep:
|
||||
label: str
|
||||
local_name: str
|
||||
crate_name: str
|
||||
cfg: Optional[str]
|
||||
|
||||
|
||||
def parse_dep_cfg(cfg: Optional[str]) -> list[str]:
|
||||
"""
|
||||
Map a subset of Cargo cfg(...) strings to Buck2 constraint keys.
|
||||
|
||||
Returns a list of constraint keys that should include the dependency.
|
||||
An empty list means "unconditional".
|
||||
"""
|
||||
if cfg is None:
|
||||
return []
|
||||
|
||||
cfg = cfg.strip()
|
||||
m = re.fullmatch(r'cfg\(target_os\s*=\s*"([^"]+)"\)', cfg)
|
||||
if m:
|
||||
os_name = m.group(1)
|
||||
if os_name in ("linux", "macos", "windows"):
|
||||
return [f"prelude//os:{os_name}"]
|
||||
|
||||
if cfg == "cfg(windows)":
|
||||
return ["prelude//os:windows"]
|
||||
|
||||
if cfg == "cfg(unix)":
|
||||
return ["prelude//os:linux", "prelude//os:macos"]
|
||||
|
||||
# If we don't recognize the expression, treat it as unconditional to avoid
|
||||
# silently dropping deps. This may cause platform build issues, in which
|
||||
# case we can extend this mapping.
|
||||
return []
|
||||
|
||||
|
||||
def buckify_features(features: list[str]) -> list[str]:
|
||||
# Cargo metadata includes lots of feature names, including "default".
|
||||
# Keep stable output for diffs.
|
||||
return sorted(features)
|
||||
|
||||
|
||||
def package_lib_target(pkg: dict[str, Any]) -> Optional[dict[str, Any]]:
|
||||
for t in pkg.get("targets") or []:
|
||||
kind = t.get("kind") or []
|
||||
if "proc-macro" in kind or "lib" in kind:
|
||||
return t
|
||||
return None
|
||||
|
||||
|
||||
def package_bin_targets(pkg: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
return [t for t in (pkg.get("targets") or []) if "bin" in (t.get("kind") or [])]
|
||||
|
||||
def package_test_targets(pkg: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
# These correspond to Cargo integration tests under `tests/`.
|
||||
return [t for t in (pkg.get("targets") or []) if "test" in (t.get("kind") or [])]
|
||||
|
||||
|
||||
def relpath_from_codex_rs(path: str) -> str:
|
||||
return str(pathlib.Path(path).resolve().relative_to(CODEX_RS_ROOT))
|
||||
|
||||
|
||||
def write_buck_file(crate_dir: pathlib.Path, content: str) -> None:
|
||||
buck_path = crate_dir / "BUCK"
|
||||
buck_path.write_text(content, encoding="utf-8")
|
||||
|
||||
def buildifier(paths: list[pathlib.Path]) -> None:
|
||||
if not paths:
|
||||
return
|
||||
if not BUILDIFIER.exists():
|
||||
# Generated BUCK files are local-only; keep generation working even if
|
||||
# buildifier isn't available.
|
||||
return
|
||||
|
||||
# Run buildifier from the repo root so it can infer workspace-relative paths.
|
||||
proc = subprocess.run(
|
||||
[str(BUILDIFIER), "-lint=off", "-mode=fix", *[str(p) for p in paths]],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(f"warning: buildifier failed (exit {proc.returncode}); leaving BUCK files unformatted", file=sys.stderr)
|
||||
|
||||
|
||||
def starlark_list(items: list[str], indent: str = " ") -> str:
|
||||
if not items:
|
||||
return "[]"
|
||||
lines = ["["]
|
||||
for it in items:
|
||||
lines.append(f'{indent}"{it}",')
|
||||
lines.append("]")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def starlark_dict(d: dict[str, str], indent: str = " ") -> str:
|
||||
if not d:
|
||||
return "{}"
|
||||
lines = ["{"] # insertion order is stable for our dict construction
|
||||
for k, v in d.items():
|
||||
lines.append(f'{indent}"{k}": "{v}",')
|
||||
lines.append("}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def starlark_deps_expr(base: list[str], conditional: dict[str, list[str]]) -> str:
|
||||
"""
|
||||
Render deps as a list plus (optional) `select()` fragments.
|
||||
"""
|
||||
expr = starlark_list(base)
|
||||
for constraint, deps in sorted(conditional.items()):
|
||||
expr = (
|
||||
f"{expr} + select({{\n"
|
||||
f' "{constraint}": {starlark_list(deps, indent=" ")},\n'
|
||||
f' "DEFAULT": [],\n'
|
||||
"})"
|
||||
)
|
||||
return expr
|
||||
|
||||
|
||||
def starlark_str(s: str) -> str:
|
||||
# Conservative escaping for readability (most strings here are already safe).
|
||||
return s.replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def parse_semver(version: str) -> tuple[str, str, str, str]:
|
||||
m = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-(.*))?$", version)
|
||||
if not m:
|
||||
return ("0", "0", "0", "")
|
||||
return (m.group(1), m.group(2), m.group(3), m.group(4) or "")
|
||||
|
||||
def group_deps(deps: list[BuckDep]) -> tuple[list[str], dict[str, list[str]], dict[str, str]]:
|
||||
"""
|
||||
Group deps into (base_deps, conditional_deps, named_deps) for emitting in BUCK.
|
||||
"""
|
||||
base_deps: list[str] = []
|
||||
conditional_deps: dict[str, list[str]] = {}
|
||||
named_deps: dict[str, str] = {}
|
||||
for d in deps:
|
||||
dep_constraints = parse_dep_cfg(d.cfg)
|
||||
if dep_constraints:
|
||||
for c in dep_constraints:
|
||||
conditional_deps.setdefault(c, []).append(d.label)
|
||||
else:
|
||||
base_deps.append(d.label)
|
||||
|
||||
# Handle renamed deps (Cargo `package = ...` style).
|
||||
if d.local_name != d.crate_name:
|
||||
named_deps[d.local_name] = d.label
|
||||
|
||||
base_deps = sorted(set(base_deps))
|
||||
for k in list(conditional_deps.keys()):
|
||||
conditional_deps[k] = sorted(set(conditional_deps[k]))
|
||||
|
||||
return (base_deps, conditional_deps, named_deps)
|
||||
|
||||
|
||||
def label_for_workspace_pkg(crate_rel_dir: str, rule_name: str) -> str:
|
||||
# crate_rel_dir is relative to codex-rs/, but buck labels are rooted at the
|
||||
# repo root, so we prefix with //codex-rs.
|
||||
return f"{CODEX_RS_LABEL_PREFIX}/{crate_rel_dir}:{rule_name}"
|
||||
|
||||
|
||||
def label_for_third_party(pkg_name: str, version: str) -> str:
|
||||
# Reindeer generates versioned third-party targets and we disable the
|
||||
# unversioned alias layer.
|
||||
return f"{CODEX_RS_LABEL_PREFIX}/third-party:{pkg_name}-{version}"
|
||||
|
||||
def collect_snap_resources(crate_dir: pathlib.Path) -> dict[str, str]:
|
||||
"""
|
||||
Collect `.snap` files for insta snapshot tests.
|
||||
|
||||
Buck's rust_test uses an external runner that executes tests from the
|
||||
project root with project-relative paths. Many snapshot tests expect to find
|
||||
their `.snap` files under `codex-rs/<crate>/...` from that root, which is
|
||||
exactly where the rust rules place `resources`.
|
||||
"""
|
||||
resources: dict[str, str] = {}
|
||||
for p in sorted(crate_dir.rglob("*.snap")):
|
||||
rel = str(p.relative_to(crate_dir))
|
||||
resources[rel] = rel
|
||||
return resources
|
||||
|
||||
def buck_bin_rule_name(pkg_name: str, bin_name: str) -> str:
|
||||
# Avoid collisions with the package's rust_library target, which we name
|
||||
# after the package (pkg_name).
|
||||
if bin_name == pkg_name:
|
||||
return f"{bin_name}-bin"
|
||||
return bin_name
|
||||
|
||||
|
||||
def main() -> int:
|
||||
meta = cargo_metadata()
|
||||
packages = meta.get("packages", [])
|
||||
|
||||
# Package ID -> package json
|
||||
by_id: dict[str, Any] = {p["id"]: p for p in packages}
|
||||
|
||||
# Workspace member IDs -> crate directory relative to codex-rs/
|
||||
workspace_members: list[str] = list(meta.get("workspace_members") or [])
|
||||
workspace_dirs: dict[str, str] = {}
|
||||
for pkg_id in workspace_members:
|
||||
pkg = by_id[pkg_id]
|
||||
manifest_path = pkg["manifest_path"]
|
||||
crate_rel = relpath_from_codex_rs(manifest_path)
|
||||
crate_dir = str(pathlib.Path(crate_rel).parent)
|
||||
workspace_dirs[pkg_id] = crate_dir
|
||||
|
||||
# Resolve graph nodes for dependency edges (including cfg/platform edges).
|
||||
resolve = meta.get("resolve") or {}
|
||||
nodes = resolve.get("nodes") or []
|
||||
node_by_id: dict[str, Any] = {n["id"]: n for n in nodes}
|
||||
|
||||
# Build a workspace-wide mapping of Cargo binary name -> Buck label for the
|
||||
# corresponding rust_binary target. Many integration tests use
|
||||
# assert_cmd/escargot helpers that look for CARGO_BIN_EXE_* env vars (set by
|
||||
# Cargo) to find binaries; we emulate that under Buck.
|
||||
cargo_bin_to_label: dict[str, str] = {}
|
||||
for pkg_id in workspace_members:
|
||||
pkg = by_id[pkg_id]
|
||||
crate_dir_rel = workspace_dirs[pkg_id]
|
||||
for bin_t in package_bin_targets(pkg):
|
||||
bin_name = bin_t["name"]
|
||||
cargo_bin_to_label[bin_name] = label_for_workspace_pkg(
|
||||
crate_dir_rel,
|
||||
buck_bin_rule_name(pkg["name"], bin_name),
|
||||
)
|
||||
|
||||
generated_buck_files: list[pathlib.Path] = []
|
||||
|
||||
for pkg_id in workspace_members:
|
||||
pkg = by_id[pkg_id]
|
||||
crate_dir_rel = workspace_dirs[pkg_id]
|
||||
crate_dir = CODEX_RS_ROOT / crate_dir_rel
|
||||
|
||||
lib_t = package_lib_target(pkg)
|
||||
bin_ts = package_bin_targets(pkg)
|
||||
test_ts = package_test_targets(pkg)
|
||||
if lib_t is None and not bin_ts:
|
||||
continue
|
||||
|
||||
node = node_by_id.get(pkg_id) or {}
|
||||
|
||||
normal_deps: list[BuckDep] = []
|
||||
dev_deps: list[BuckDep] = []
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
if dep_id == pkg_id:
|
||||
# Cargo sometimes models "self dependencies" (commonly as a
|
||||
# dev-dependency to enable features for tests). Buck should not
|
||||
# treat this as a real crate edge, and using it creates cycles.
|
||||
continue
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
|
||||
local_name = dep.get("name") or dep_pkg["name"]
|
||||
crate_name = dep_pkg.get("targets", [{}])[0].get("name") or local_name
|
||||
|
||||
if dep_id in workspace_dirs:
|
||||
dep_label = label_for_workspace_pkg(workspace_dirs[dep_id], dep_pkg["name"])
|
||||
else:
|
||||
dep_label = label_for_third_party(dep_pkg["name"], dep_pkg["version"])
|
||||
|
||||
# A single dependency may appear multiple times with different Cargo
|
||||
# cfg(...) selectors (e.g. target.'cfg(target_os = "...")'). Cargo
|
||||
# metadata encodes these as multiple dep_kind entries, so we emit a
|
||||
# Buck dep entry for each one.
|
||||
if not dep_kinds:
|
||||
dep_kinds = [{"kind": None, "target": None}]
|
||||
|
||||
for k in dep_kinds:
|
||||
kind = k.get("kind")
|
||||
if kind not in (None, "normal", "dev"):
|
||||
continue
|
||||
|
||||
dep_entry = BuckDep(
|
||||
label=dep_label,
|
||||
local_name=local_name,
|
||||
crate_name=crate_name,
|
||||
cfg=k.get("target"),
|
||||
)
|
||||
if kind is None or kind == "normal":
|
||||
normal_deps.append(dep_entry)
|
||||
elif kind == "dev":
|
||||
dev_deps.append(dep_entry)
|
||||
|
||||
# Deduplicate while preserving stable output.
|
||||
normal_deps = list({(d.label, d.local_name, d.crate_name, d.cfg): d for d in normal_deps}.values())
|
||||
dev_deps = list({(d.label, d.local_name, d.crate_name, d.cfg): d for d in dev_deps}.values())
|
||||
|
||||
base_deps, conditional_deps, named_deps = group_deps(normal_deps)
|
||||
test_base_deps, test_conditional_deps, test_named_deps = group_deps(normal_deps + dev_deps)
|
||||
|
||||
# Enabled features for this package (as resolved by Cargo).
|
||||
features = buckify_features(list(node.get("features") or []))
|
||||
|
||||
edition = pkg.get("edition") or "2021"
|
||||
ver_major, ver_minor, ver_patch, ver_pre = parse_semver(pkg.get("version") or "0.0.0")
|
||||
|
||||
# Emulate Cargo's CARGO_BIN_EXE_* env vars for all known workspace
|
||||
# binaries. This avoids test helpers trying to discover binaries via a
|
||||
# Cargo target directory layout.
|
||||
#
|
||||
# These are runtime-only and should be provided via `run_env` so we
|
||||
# don't accidentally make compile actions depend on executable outputs.
|
||||
cargo_bin_env = {f"CARGO_BIN_EXE_{k}": f"$(location {v})" for k, v in sorted(cargo_bin_to_label.items())}
|
||||
cargo_run_env = cargo_bin_env
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("# @generated by scripts/gen_buck_first_party.py")
|
||||
lines.append("# Regenerate with: (cd codex-rs && python3 scripts/gen_buck_first_party.py)")
|
||||
lines.append("")
|
||||
|
||||
# NOTE: We use a broad src glob to support include_str!/include_bytes!
|
||||
# and other build-time file reads without crate-specific buckification.
|
||||
src_glob = 'glob(["**"], exclude = ["BUCK", "target/**"])'
|
||||
snap_resources = collect_snap_resources(crate_dir)
|
||||
|
||||
deps_expr = starlark_deps_expr(base_deps, conditional_deps)
|
||||
|
||||
if lib_t is not None:
|
||||
crate_root = relpath_from_codex_rs(lib_t["src_path"])
|
||||
crate_root_rel = str(pathlib.Path(crate_root).relative_to(crate_dir.relative_to(CODEX_RS_ROOT)))
|
||||
|
||||
lines.append("rust_library(")
|
||||
lines.append(f' name = "{pkg["name"]}",')
|
||||
lines.append(f' crate = "{lib_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": lib_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
# Insta snapshots use a compile-time workspace root
|
||||
# (option_env!("INSTA_WORKSPACE_ROOT")) and otherwise
|
||||
# fall back to cargo metadata, which doesn't work under
|
||||
# Buck's test sandbox. Pin this to repo root.
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {deps_expr},")
|
||||
if named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
# Unit tests for the library (i.e., `#[cfg(test)]` within src/).
|
||||
test_deps_expr = starlark_deps_expr(test_base_deps, test_conditional_deps)
|
||||
lines.append("")
|
||||
lines.append("rust_test(")
|
||||
lines.append(f' name = "{pkg["name"]}-unit-tests",')
|
||||
lines.append(f' crate = "{lib_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": lib_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if cargo_run_env:
|
||||
lines.append(" run_env = " + starlark_dict(cargo_run_env, indent=" ") + ",")
|
||||
if snap_resources:
|
||||
lines.append(" resources = " + starlark_dict(snap_resources, indent=" ") + ",")
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {test_deps_expr},")
|
||||
if test_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(test_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
for bin_t in bin_ts:
|
||||
bin_rule_name = buck_bin_rule_name(pkg["name"], bin_t["name"])
|
||||
crate_root = relpath_from_codex_rs(bin_t["src_path"])
|
||||
crate_root_rel = str(pathlib.Path(crate_root).relative_to(crate_dir.relative_to(CODEX_RS_ROOT)))
|
||||
|
||||
# Cargo makes the package's library available to binaries in the
|
||||
# same package, and package dependencies are also visible.
|
||||
bin_base_deps = base_deps
|
||||
if lib_t is not None:
|
||||
bin_base_deps = [f":{pkg['name']}"] + base_deps
|
||||
bin_deps_expr = starlark_deps_expr(bin_base_deps, conditional_deps)
|
||||
|
||||
lines.append("")
|
||||
lines.append("rust_binary(")
|
||||
lines.append(f' name = "{bin_rule_name}",')
|
||||
lines.append(f' crate = "{bin_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": bin_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {bin_deps_expr},")
|
||||
if named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
# Unit tests for the binary (i.e., `#[cfg(test)]` within src/main.rs).
|
||||
bin_test_base_deps = list(test_base_deps)
|
||||
if lib_t is not None:
|
||||
# Cargo makes the package's library available to binaries; the
|
||||
# unit test harness for `src/main.rs` needs that dependency too.
|
||||
bin_test_base_deps = [f":{pkg['name']}"] + bin_test_base_deps
|
||||
test_deps_expr = starlark_deps_expr(bin_test_base_deps, test_conditional_deps)
|
||||
lines.append("")
|
||||
lines.append("rust_test(")
|
||||
lines.append(f' name = "{bin_rule_name}-unit-tests",')
|
||||
lines.append(f' crate = "{bin_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": bin_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if cargo_run_env:
|
||||
lines.append(" run_env = " + starlark_dict(cargo_run_env, indent=" ") + ",")
|
||||
if snap_resources:
|
||||
lines.append(" resources = " + starlark_dict(snap_resources, indent=" ") + ",")
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {test_deps_expr},")
|
||||
if test_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(test_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
# Integration tests under `tests/` (Cargo `kind = ["test"]`).
|
||||
#
|
||||
# Cargo gives these crates access to both normal + dev dependencies, and
|
||||
# also makes the package's library available if it exists.
|
||||
if test_ts:
|
||||
test_deps_expr = starlark_deps_expr(test_base_deps, test_conditional_deps)
|
||||
for test_t in test_ts:
|
||||
test_name = test_t["name"]
|
||||
if test_name in SKIP_INTEGRATION_TESTS_BY_PACKAGE.get(pkg["name"], set()):
|
||||
continue
|
||||
crate_root = relpath_from_codex_rs(test_t["src_path"])
|
||||
crate_root_rel = str(pathlib.Path(crate_root).relative_to(crate_dir.relative_to(CODEX_RS_ROOT)))
|
||||
|
||||
integration_base_deps = list(test_base_deps)
|
||||
if lib_t is not None:
|
||||
integration_base_deps = [f":{pkg['name']}"] + integration_base_deps
|
||||
integration_deps_expr = starlark_deps_expr(integration_base_deps, test_conditional_deps)
|
||||
|
||||
lines.append("")
|
||||
lines.append("rust_test(")
|
||||
lines.append(f' name = "{test_name}-integration-test",')
|
||||
lines.append(f' crate = "{test_name}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": test_name,
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if cargo_run_env:
|
||||
lines.append(" run_env = " + starlark_dict(cargo_run_env, indent=" ") + ",")
|
||||
if snap_resources:
|
||||
lines.append(" resources = " + starlark_dict(snap_resources, indent=" ") + ",")
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {integration_deps_expr},")
|
||||
if test_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(test_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
write_buck_file(crate_dir, "\n".join(lines) + "\n")
|
||||
generated_buck_files.append(crate_dir / "BUCK")
|
||||
|
||||
buildifier(generated_buck_files)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate Reindeer fixups for third-party crates.
|
||||
|
||||
Reindeer requires an explicit decision for each crate with a build script:
|
||||
either run it or ignore it. For most third-party crates we run build scripts.
|
||||
|
||||
We intentionally do not generate `extra_srcs` fixups here:
|
||||
- Reindeer validates fixup globs against its chosen crate source directory,
|
||||
and with vendoring enabled this can be a filtered view of the package that
|
||||
does not include top-level docs/fixtures (README, tests data, etc).
|
||||
- Instead, we include common non-Rust sources via the `codex_rust_*` wrapper
|
||||
macros in `codex-rs/buck2/reindeer_macros.bzl` using Buck `glob(...)`.
|
||||
|
||||
This script is checked in; its outputs are not.
|
||||
|
||||
The generated fixups live under `codex-rs/third-party/fixups/` and are
|
||||
intentionally gitignored for now (see the repo root `.gitignore`, which ignores
|
||||
`codex-rs/third-party/`). This script is designed to be re-run and will
|
||||
overwrite any existing generated fixups.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
CODEX_RS_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
THIRD_PARTY_DIR = CODEX_RS_ROOT / "third-party"
|
||||
VENDOR_DIR = THIRD_PARTY_DIR / "vendor"
|
||||
FIXUPS_DIR = THIRD_PARTY_DIR / "fixups"
|
||||
|
||||
|
||||
def cargo_metadata() -> dict[str, Any]:
|
||||
out = subprocess.check_output(
|
||||
["cargo", "metadata", "--format-version=1", "--locked"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def reindeer_reachable_package_ids(meta: dict[str, Any]) -> set[str]:
|
||||
"""
|
||||
Approximate the set of packages Reindeer cares about for `buckify`.
|
||||
|
||||
Reindeer buckifies dependencies of workspace members, but it does not need
|
||||
deps that are only reachable through `dev` edges (tests/benches/examples).
|
||||
"""
|
||||
|
||||
resolve = meta.get("resolve") or {}
|
||||
nodes = resolve.get("nodes") or []
|
||||
by_id: dict[str, Any] = {n["id"]: n for n in nodes}
|
||||
|
||||
roots = list(meta.get("workspace_members") or [])
|
||||
reachable: set[str] = set()
|
||||
stack = list(roots)
|
||||
while stack:
|
||||
pkg_id = stack.pop()
|
||||
if pkg_id in reachable:
|
||||
continue
|
||||
reachable.add(pkg_id)
|
||||
|
||||
node = by_id.get(pkg_id)
|
||||
if not node:
|
||||
continue
|
||||
|
||||
for dep in node.get("deps") or []:
|
||||
kinds = dep.get("dep_kinds") or []
|
||||
if kinds and all(k.get("kind") == "dev" for k in kinds):
|
||||
continue
|
||||
stack.append(dep["pkg"])
|
||||
|
||||
return reachable
|
||||
|
||||
|
||||
def vendored_links_value(name: str, version: str) -> str | None:
|
||||
"""
|
||||
If the vendored crate declares `links = "..."` in its Cargo.toml, return it.
|
||||
|
||||
Cargo provides this to build scripts via the CARGO_MANIFEST_LINKS env var.
|
||||
Some build scripts rely on it (e.g. ring).
|
||||
"""
|
||||
|
||||
cargo_toml = VENDOR_DIR / f"{name}-{version}" / "Cargo.toml"
|
||||
if not cargo_toml.exists():
|
||||
return None
|
||||
|
||||
text = cargo_toml.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
# Very small parser: search within the [package] section first, then fall back.
|
||||
pkg_idx = text.find("[package]")
|
||||
if pkg_idx != -1:
|
||||
rest = text[pkg_idx:]
|
||||
next_table = rest.find("\n[", 1)
|
||||
pkg_block = rest if next_table == -1 else rest[:next_table]
|
||||
m = re.search(r'(?m)^\s*links\s*=\s*"([^"]+)"\s*$', pkg_block)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
m = re.search(r'(?m)^\s*links\s*=\s*"([^"]+)"\s*$', text)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def openssl_dep_version_number_hex() -> str | None:
|
||||
"""
|
||||
Derive a DEP_OPENSSL_VERSION_NUMBER value (hex string, no 0x prefix).
|
||||
|
||||
Under Cargo, openssl-sys emits a `cargo:version_number=...` metadata line
|
||||
and Cargo converts it into DEP_OPENSSL_VERSION_NUMBER for dependents.
|
||||
Buck's buildscript runner does not currently propagate those DEP_* env vars,
|
||||
so we approximate the value here for Buck builds.
|
||||
"""
|
||||
|
||||
def pkg_config_modversion() -> str | None:
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
["pkg-config", "--modversion", "openssl"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return None
|
||||
v = out.strip()
|
||||
return v if v else None
|
||||
|
||||
def parse_openssl_version(
|
||||
v: str,
|
||||
) -> tuple[int, int, int, str | None] | None:
|
||||
"""
|
||||
Parse versions like:
|
||||
- 3.0.13
|
||||
- 1.1.1w
|
||||
- 1.1.0h
|
||||
"""
|
||||
|
||||
m = re.match(r"^(\d+)\.(\d+)\.(\d+)([a-z])?$", v)
|
||||
if not m:
|
||||
return None
|
||||
major, minor, patch = (int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
suffix = m.group(4)
|
||||
return (major, minor, patch, suffix)
|
||||
|
||||
v = pkg_config_modversion()
|
||||
parsed = parse_openssl_version(v) if v else None
|
||||
|
||||
# In CI we install libssl-dev, so pkg-config should generally be available.
|
||||
# If it isn't, guess OpenSSL 3.x on modern Linux distros.
|
||||
if parsed is None:
|
||||
if os.name == "posix":
|
||||
# 3.0.0: (3<<28)|(0<<20)|(0<<4)
|
||||
return f"{(3 << 28):x}"
|
||||
return None
|
||||
|
||||
major, minor, patch, suffix = parsed
|
||||
|
||||
# OpenSSL 3 uses (major<<28)|(minor<<20)|(patch<<4).
|
||||
if major >= 3:
|
||||
version = (major << 28) | (minor << 20) | (patch << 4)
|
||||
return f"{version:x}"
|
||||
|
||||
# OpenSSL 1.x uses 0xMNNFFPPS where PP is the patch number and S is the
|
||||
# patch status nibble. For lettered patch releases, 'a' => 1, 'b' => 2, etc.
|
||||
if major == 1:
|
||||
patch_num = 0
|
||||
if suffix:
|
||||
patch_num = ord(suffix) - ord("a") + 1
|
||||
# Release status (0xf) matches Cargo/rust-openssl default behavior.
|
||||
version = (major << 28) | (minor << 20) | (patch << 12) | (patch_num << 4) | 0xF
|
||||
return f"{version:x}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def openssl_dep_conf() -> str | None:
|
||||
"""
|
||||
Derive a DEP_OPENSSL_CONF value (comma-separated macro names).
|
||||
|
||||
Under Cargo, openssl-sys emits `cargo:conf=...` and Cargo converts it into
|
||||
DEP_OPENSSL_CONF for dependents. The `openssl` crate's build script reads
|
||||
DEP_OPENSSL_CONF and re-emits per-macro `osslconf="..."` cfgs so it can
|
||||
conditionalize APIs that are disabled in the system OpenSSL build (e.g.
|
||||
OPENSSL_NO_IDEA).
|
||||
"""
|
||||
|
||||
def pkg_config_cflags() -> list[str] | None:
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
["pkg-config", "--cflags", "openssl"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return None
|
||||
return [w for w in out.split() if w]
|
||||
|
||||
def preprocess_opensslconf(cflags: list[str]) -> str | None:
|
||||
snippet = "#include <openssl/opensslconf.h>\n"
|
||||
for cc in ("cc", "clang"):
|
||||
try:
|
||||
return subprocess.check_output(
|
||||
[cc, "-dM", "-E", "-xc", "-", *cflags],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
input=snippet,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
continue
|
||||
return None
|
||||
|
||||
cflags = pkg_config_cflags()
|
||||
if not cflags:
|
||||
return None
|
||||
|
||||
out = preprocess_opensslconf(cflags)
|
||||
if not out:
|
||||
return None
|
||||
|
||||
# Match the behavior expected by openssl/build.rs: it splits DEP_OPENSSL_CONF
|
||||
# by ',' and emits cfgs `osslconf="..."` for each token.
|
||||
conf: set[str] = set()
|
||||
for line in out.splitlines():
|
||||
# Lines look like: "#define OPENSSL_NO_IDEA 1"
|
||||
m = re.match(r"^#define\s+(OPENSSL_[A-Z0-9_]+)\b", line)
|
||||
if not m:
|
||||
continue
|
||||
macro = m.group(1)
|
||||
if macro.startswith("OPENSSL_NO_"):
|
||||
conf.add(macro)
|
||||
return ",".join(sorted(conf)) if conf else ""
|
||||
|
||||
|
||||
def toml_inline_table(entries: dict[str, str]) -> str:
|
||||
# Emit stable TOML with double-quoted strings.
|
||||
parts = [f"{k} = {json.dumps(v)}" for k, v in sorted(entries.items())]
|
||||
return "{ " + ", ".join(parts) + " }"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
meta = cargo_metadata()
|
||||
packages = meta.get("packages", [])
|
||||
reachable_ids = reindeer_reachable_package_ids(meta)
|
||||
|
||||
# First-party packages whose build.rs we intentionally ignore under Buck.
|
||||
buildscript_run_overrides: dict[str, bool] = {
|
||||
# build.rs only adds rerun-if-changed
|
||||
"codex-execpolicy-legacy": False,
|
||||
# Windows-only resource compilation
|
||||
"codex-windows-sandbox": False,
|
||||
}
|
||||
|
||||
# Packages where Cargo metadata reports a build script, but Reindeer does not
|
||||
# require/accept a buildscript fixup in this workspace.
|
||||
skip_packages: set[str] = {
|
||||
"indexmap",
|
||||
"quinn",
|
||||
}
|
||||
|
||||
reachable_pkgs = [p for p in packages if p["id"] in reachable_ids]
|
||||
|
||||
# name -> version -> has_buildscript
|
||||
by_name: dict[str, dict[str, bool]] = {}
|
||||
for pkg in reachable_pkgs:
|
||||
name = pkg["name"]
|
||||
version = pkg["version"]
|
||||
has_buildscript = any(
|
||||
"custom-build" in (t.get("kind") or []) for t in (pkg.get("targets") or [])
|
||||
)
|
||||
by_name.setdefault(name, {})[version] = has_buildscript
|
||||
|
||||
# Nuke and regenerate: fixups are gitignored and generated.
|
||||
if FIXUPS_DIR.exists():
|
||||
shutil.rmtree(FIXUPS_DIR)
|
||||
FIXUPS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
wrote = 0
|
||||
for name, versions in sorted(by_name.items()):
|
||||
if name in skip_packages:
|
||||
continue
|
||||
|
||||
buildscript_versions = sorted([v for v, has in versions.items() if has])
|
||||
if not buildscript_versions:
|
||||
continue
|
||||
|
||||
run = buildscript_run_overrides.get(name, True)
|
||||
|
||||
fixup_path = FIXUPS_DIR / name / "fixups.toml"
|
||||
fixup_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
stanzas: list[str] = []
|
||||
for v in buildscript_versions:
|
||||
if run:
|
||||
links = vendored_links_value(name, v)
|
||||
stanzas.append(f"['cfg(version = \"={v}\")'.buildscript.run]")
|
||||
stanzas.append("rustc_link_lib = true")
|
||||
stanzas.append("rustc_link_search = true")
|
||||
env_vars: dict[str, str] = {}
|
||||
# Some crates rely on this being set when `links = "..."`
|
||||
# exists in Cargo.toml (Cargo passes it to build scripts).
|
||||
if links:
|
||||
env_vars["CARGO_MANIFEST_LINKS"] = links
|
||||
# The `openssl` crate's build script derives cfgs from
|
||||
# DEP_OPENSSL_VERSION_NUMBER (emitted by openssl-sys). Buck's
|
||||
# buildscript runner does not currently propagate those DEP_*
|
||||
# env vars, so approximate the value for Buck builds.
|
||||
if name == "openssl" and v == "0.10.73":
|
||||
dep_ver = openssl_dep_version_number_hex()
|
||||
if dep_ver:
|
||||
env_vars["DEP_OPENSSL_VERSION_NUMBER"] = dep_ver
|
||||
dep_conf = openssl_dep_conf()
|
||||
if dep_conf is not None:
|
||||
# Empty string is meaningful: "no conf macros enabled".
|
||||
env_vars["DEP_OPENSSL_CONF"] = dep_conf
|
||||
elif sys.platform.startswith("linux"):
|
||||
# If we can't determine the conf macros, be conservative
|
||||
# on Linux (Ubuntu CI commonly disables IDEA).
|
||||
env_vars["DEP_OPENSSL_CONF"] = "OPENSSL_NO_IDEA"
|
||||
if env_vars:
|
||||
stanzas.append(f"env = {toml_inline_table(env_vars)}")
|
||||
stanzas.append("")
|
||||
else:
|
||||
stanzas.append(f"['cfg(version = \"={v}\")']")
|
||||
stanzas.append("buildscript.run = false")
|
||||
stanzas.append("")
|
||||
|
||||
content = "\n".join(stanzas).rstrip() + "\n"
|
||||
fixup_path.write_text(content, encoding="utf-8")
|
||||
wrote += 1
|
||||
|
||||
if wrote:
|
||||
print(
|
||||
f"Wrote buildscript fixups.toml for {wrote} crates under "
|
||||
f"{os.path.relpath(FIXUPS_DIR, CODEX_RS_ROOT)}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,510 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Patch the Reindeer-generated `codex-rs/third-party/BUCK` to add a small overlay
|
||||
of third-party crates that are only used as *dev-dependencies* in the Cargo
|
||||
workspace.
|
||||
|
||||
Reindeer currently generates Buck targets for Cargo "normal" deps (plus build
|
||||
deps for build scripts), but it does not emit targets for dev-dependencies
|
||||
because it does not generate rules for Cargo test/example/bench targets.
|
||||
|
||||
For Buck2-driven `rust_test()` targets in first-party crates, we still need a
|
||||
handful of dev-only third-party crates to exist as targets in
|
||||
`//codex-rs/third-party`.
|
||||
|
||||
This script is run as part of `scripts/setup_buck2_local.sh` and is intended to
|
||||
be safe to run repeatedly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
CODEX_RS_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = CODEX_RS_ROOT.parent
|
||||
BUILDIFIER = REPO_ROOT / "scripts" / "buildifier"
|
||||
THIRD_PARTY_BUCK = CODEX_RS_ROOT / "third-party" / "BUCK"
|
||||
VENDOR_DIR = CODEX_RS_ROOT / "third-party" / "vendor"
|
||||
|
||||
BEGIN_MARKER = "# BEGIN CODEX LOCAL DEV TEST DEPS (generated)\n"
|
||||
END_MARKER = "# END CODEX LOCAL DEV TEST DEPS (generated)\n"
|
||||
|
||||
|
||||
def cargo_metadata() -> dict[str, Any]:
|
||||
out = subprocess.check_output(
|
||||
["cargo", "metadata", "--format-version=1", "--locked"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def parse_semver(version: str) -> tuple[str, str, str, str]:
|
||||
m = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-(.*))?$", version)
|
||||
if not m:
|
||||
return ("0", "0", "0", "")
|
||||
return (m.group(1), m.group(2), m.group(3), m.group(4) or "")
|
||||
|
||||
|
||||
def parse_dep_cfg(cfg: Optional[str]) -> list[str]:
|
||||
"""
|
||||
Best-effort mapping for cfg(...) strings to Buck2 constraint keys.
|
||||
This mirrors the mapping in scripts/gen_buck_first_party.py.
|
||||
"""
|
||||
if cfg is None:
|
||||
return []
|
||||
|
||||
cfg = cfg.strip()
|
||||
m = re.fullmatch(r'cfg\(target_os\s*=\s*"([^"]+)"\)', cfg)
|
||||
if m:
|
||||
os_name = m.group(1)
|
||||
if os_name in ("linux", "macos", "windows"):
|
||||
return [f"prelude//os:{os_name}"]
|
||||
|
||||
if cfg == "cfg(windows)":
|
||||
return ["prelude//os:windows"]
|
||||
|
||||
if cfg == "cfg(unix)":
|
||||
return ["prelude//os:linux", "prelude//os:macos"]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def starlark_list(items: list[str], indent: str = " ") -> str:
|
||||
if not items:
|
||||
return "[]"
|
||||
lines = ["["]
|
||||
for it in items:
|
||||
lines.append(f'{indent}"{it}",')
|
||||
lines.append("]")
|
||||
return "\n".join(lines)
|
||||
|
||||
def starlark_str(s: str) -> str:
|
||||
# Conservative escaping suitable for Starlark string literals.
|
||||
return (
|
||||
s.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\r", "\\r")
|
||||
.replace("\n", "\\n")
|
||||
)
|
||||
|
||||
|
||||
def starlark_dict(d: dict[str, str], indent: str = " ") -> str:
|
||||
if not d:
|
||||
return "{}"
|
||||
lines = ["{"]
|
||||
for k, v in d.items():
|
||||
lines.append(f'{indent}"{starlark_str(k)}": "{starlark_str(v)}",')
|
||||
lines.append("}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def starlark_deps_expr(base: list[str], conditional: dict[str, list[str]]) -> str:
|
||||
expr = starlark_list(base)
|
||||
for constraint, deps in sorted(conditional.items()):
|
||||
expr = (
|
||||
f"{expr} + select({{\n"
|
||||
f' "{constraint}": {starlark_list(deps, indent=" ")},\n'
|
||||
' "DEFAULT": [],\n'
|
||||
"})"
|
||||
)
|
||||
return expr
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuckDep:
|
||||
label: str
|
||||
local_name: str
|
||||
crate_name: str
|
||||
cfg: Optional[str]
|
||||
|
||||
|
||||
def group_deps(deps: list[BuckDep]) -> tuple[list[str], dict[str, list[str]], dict[str, str]]:
|
||||
base_deps: list[str] = []
|
||||
conditional_deps: dict[str, list[str]] = {}
|
||||
named_deps: dict[str, str] = {}
|
||||
for d in deps:
|
||||
dep_constraints = parse_dep_cfg(d.cfg)
|
||||
if dep_constraints:
|
||||
for c in dep_constraints:
|
||||
conditional_deps.setdefault(c, []).append(d.label)
|
||||
else:
|
||||
base_deps.append(d.label)
|
||||
|
||||
if d.local_name != d.crate_name:
|
||||
named_deps[d.local_name] = d.label
|
||||
|
||||
base_deps = sorted(set(base_deps))
|
||||
for k in list(conditional_deps.keys()):
|
||||
conditional_deps[k] = sorted(set(conditional_deps[k]))
|
||||
|
||||
return (base_deps, conditional_deps, named_deps)
|
||||
|
||||
|
||||
def strip_existing_overlay(text: str) -> str:
|
||||
start = text.find(BEGIN_MARKER)
|
||||
if start == -1:
|
||||
return text
|
||||
end = text.find(END_MARKER, start)
|
||||
if end == -1:
|
||||
# If the file is corrupted, be conservative and do not try to patch.
|
||||
raise SystemExit(f"Found BEGIN marker but not END marker in {THIRD_PARTY_BUCK}")
|
||||
return text[:start] + text[end + len(END_MARKER) :]
|
||||
|
||||
|
||||
def patch_reindeer_output_for_test_features(text: str) -> str:
|
||||
"""
|
||||
Best-effort patching of Reindeer output to make certain dev-only crates build.
|
||||
|
||||
Today, our locally-generated targets for dev-dependencies (e.g. `insta`) may
|
||||
require feature combinations on their transitive deps that are not enabled
|
||||
by the normal (non-test) workspace graph that Reindeer buckifies.
|
||||
|
||||
We keep this narrowly-scoped and idempotent.
|
||||
"""
|
||||
# `insta` requires `similar` with the `inline` feature.
|
||||
similar_rule_name = "similar-2.7.0"
|
||||
lines = text.splitlines(keepends=True)
|
||||
for i, line in enumerate(lines):
|
||||
if "name" in line and similar_rule_name in line and "name" in line:
|
||||
# Walk until the end of this rule.
|
||||
j = i
|
||||
while j < len(lines) and lines[j].strip() != ")":
|
||||
j += 1
|
||||
if j >= len(lines):
|
||||
break
|
||||
|
||||
block = lines[i : j + 1]
|
||||
# Only treat it as patched if the *feature* is present. The crate
|
||||
# itself contains an `inline.rs` source file, so substring checks
|
||||
# would be a false positive.
|
||||
if any(l.strip() == '"inline",' for l in block):
|
||||
break
|
||||
|
||||
for k, bl in enumerate(block):
|
||||
if bl.lstrip().startswith("features = ["):
|
||||
block.insert(k + 1, ' "inline",\n')
|
||||
lines[i : j + 1] = block
|
||||
return "".join(lines)
|
||||
|
||||
# No existing features list; add one just before `visibility`.
|
||||
for k, bl in enumerate(block):
|
||||
if bl.lstrip().startswith("visibility ="):
|
||||
block.insert(k, ' features = ["inline"],\n')
|
||||
lines[i : j + 1] = block
|
||||
return "".join(lines)
|
||||
break
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def present_target_names(text: str) -> set[str]:
|
||||
# This is intentionally simple: it's used only to decide whether a target
|
||||
# already exists so we don't duplicate it.
|
||||
# Reindeer emits string literals with escaped quotes (e.g. `name = \"foo\"`)
|
||||
# so we accept an optional backslash before both quote characters.
|
||||
return set(re.findall(r'(?m)^\s*name\s*=\s*\\?"([^"]+)\\?"', text))
|
||||
|
||||
|
||||
def vendor_crate_root(pkg_name: str, version: str) -> pathlib.Path:
|
||||
crate_dir = VENDOR_DIR / f"{pkg_name}-{version}"
|
||||
# Prefer src/lib.rs, but fall back to whichever crate root Cargo reports.
|
||||
lib_rs = crate_dir / "src" / "lib.rs"
|
||||
if lib_rs.exists():
|
||||
# Paths in `codex-rs/third-party/BUCK` are relative to the `third-party`
|
||||
# package directory, so use `vendor/...` rather than an absolute path or
|
||||
# a `third-party/vendor/...` path.
|
||||
return pathlib.Path(f"vendor/{pkg_name}-{version}/src/lib.rs")
|
||||
raise SystemExit(f"Could not find {crate_dir}/src/lib.rs (needed for {pkg_name} {version})")
|
||||
|
||||
|
||||
def append_overlay(meta: dict[str, Any], existing_buck: str) -> str:
|
||||
by_id: dict[str, Any] = {p["id"]: p for p in (meta.get("packages") or [])}
|
||||
workspace_members: set[str] = set(meta.get("workspace_members") or [])
|
||||
node_by_id: dict[str, Any] = {n["id"]: n for n in (meta.get("resolve") or {}).get("nodes") or []}
|
||||
|
||||
existing_targets = present_target_names(existing_buck)
|
||||
|
||||
def target_name_for_pkg_id(pid: str) -> str:
|
||||
p = by_id[pid]
|
||||
return f"{p['name']}-{p['version']}"
|
||||
|
||||
# Find third-party packages that are reachable via any dev-dependency edge
|
||||
# from any workspace member.
|
||||
dev_needed_pkg_ids: set[str] = set()
|
||||
for pkg_id in workspace_members:
|
||||
node = node_by_id.get(pkg_id) or {}
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
if dep_id in workspace_members:
|
||||
continue
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
if not any(k.get("kind") == "dev" for k in dep_kinds):
|
||||
continue
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
# Only patch registry crates. Workspace members have source=None.
|
||||
if dep_pkg.get("source") is None:
|
||||
continue
|
||||
dev_needed_pkg_ids.add(dep_id)
|
||||
|
||||
# Reindeer buckify does not emit targets for dev-deps. We patch in any dev
|
||||
# deps that are missing *and* any of their transitive normal deps that are
|
||||
# also missing (e.g., proc-macro helper crates).
|
||||
to_add: set[str] = {pid for pid in dev_needed_pkg_ids if target_name_for_pkg_id(pid) not in existing_targets}
|
||||
queue = list(to_add)
|
||||
|
||||
while queue:
|
||||
pid = queue.pop()
|
||||
node = node_by_id.get(pid) or {}
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
if dep_pkg.get("source") is None:
|
||||
continue
|
||||
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
if not any(k.get("kind") is None or k.get("kind") == "normal" for k in dep_kinds):
|
||||
continue
|
||||
|
||||
tgt = target_name_for_pkg_id(dep_id)
|
||||
if tgt in existing_targets or dep_id in to_add:
|
||||
continue
|
||||
|
||||
to_add.add(dep_id)
|
||||
queue.append(dep_id)
|
||||
|
||||
missing_pkg_ids = sorted(to_add, key=lambda pid: (by_id[pid]["name"], by_id[pid]["version"]))
|
||||
if os.environ.get("CODEX_PATCH_THIRD_PARTY_DEBUG") == "1":
|
||||
missing_targets = [target_name_for_pkg_id(pid) for pid in missing_pkg_ids]
|
||||
preview = existing_buck[:120].replace("\n", "\\n")
|
||||
print(f"patch_third_party_buck_for_tests: buck_preview={preview!r}")
|
||||
print(f"patch_third_party_buck_for_tests: existing_targets={len(existing_targets)}")
|
||||
print(f"patch_third_party_buck_for_tests: dev_needed={len(dev_needed_pkg_ids)} missing={len(missing_targets)}")
|
||||
for t in sorted(missing_targets)[:50]:
|
||||
print(f" missing: {t}")
|
||||
if not missing_pkg_ids:
|
||||
return existing_buck
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("\n" + BEGIN_MARKER.rstrip("\n"))
|
||||
lines.append("# This section is appended by codex-rs/scripts/patch_third_party_buck_for_tests.py")
|
||||
lines.append("# to make Buck `rust_test()` targets usable for first-party dev-deps.")
|
||||
lines.append("")
|
||||
|
||||
for pkg_id in missing_pkg_ids:
|
||||
pkg = by_id[pkg_id]
|
||||
name = pkg["name"]
|
||||
version = pkg["version"]
|
||||
rule_name = f"{name}-{version}"
|
||||
if rule_name in existing_targets:
|
||||
continue
|
||||
|
||||
# Find the lib target to determine the Rust crate name and edition.
|
||||
lib_targets = [
|
||||
t
|
||||
for t in (pkg.get("targets") or [])
|
||||
if "lib" in (t.get("kind") or []) or "proc-macro" in (t.get("kind") or [])
|
||||
]
|
||||
if not lib_targets:
|
||||
# These dev deps should all be libraries.
|
||||
continue
|
||||
lib_t = lib_targets[0]
|
||||
crate_name = lib_t["name"]
|
||||
edition = pkg.get("edition") or "2021"
|
||||
ver_major, ver_minor, ver_patch, ver_pre = parse_semver(version)
|
||||
proc_macro = "proc-macro" in (lib_t.get("kind") or [])
|
||||
|
||||
# Compute deps for the library (normal deps only).
|
||||
node = node_by_id.get(pkg_id) or {}
|
||||
deps: list[BuckDep] = []
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
local_name = dep.get("name") or dep_pkg["name"]
|
||||
dep_crate_name = dep_pkg.get("targets", [{}])[0].get("name") or local_name
|
||||
dep_label = f":{dep_pkg['name']}-{dep_pkg['version']}"
|
||||
|
||||
# One dependency may have multiple cfg(...) selectors; include them all.
|
||||
for k in dep_kinds:
|
||||
kind = k.get("kind")
|
||||
if kind is None or kind == "normal":
|
||||
deps.append(
|
||||
BuckDep(
|
||||
label=dep_label,
|
||||
local_name=local_name,
|
||||
crate_name=dep_crate_name,
|
||||
cfg=k.get("target"),
|
||||
)
|
||||
)
|
||||
|
||||
base_deps, conditional_deps, named_deps = group_deps(deps)
|
||||
deps_expr = starlark_deps_expr(base_deps, conditional_deps)
|
||||
|
||||
crate_root_rel = vendor_crate_root(name, version)
|
||||
|
||||
env = {
|
||||
"CARGO_CRATE_NAME": crate_name,
|
||||
"CARGO_MANIFEST_DIR": f"vendor/{name}-{version}",
|
||||
"CARGO_PKG_AUTHORS": ":".join(pkg.get("authors") or []),
|
||||
"CARGO_PKG_DESCRIPTION": pkg.get("description") or "",
|
||||
"CARGO_PKG_NAME": name,
|
||||
"CARGO_PKG_REPOSITORY": pkg.get("repository") or "",
|
||||
"CARGO_PKG_VERSION": version,
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
}
|
||||
|
||||
has_build_rs = (VENDOR_DIR / f"{name}-{version}" / "build.rs").exists()
|
||||
if has_build_rs:
|
||||
build_bin = f"{rule_name}-build-script-build"
|
||||
build_run = f"{rule_name}-build-script-run"
|
||||
build_rs_rel = pathlib.Path(f"vendor/{name}-{version}/build.rs")
|
||||
|
||||
# Build deps for compiling the build script itself.
|
||||
build_deps: list[BuckDep] = []
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
local_name = dep.get("name") or dep_pkg["name"]
|
||||
dep_crate_name = dep_pkg.get("targets", [{}])[0].get("name") or local_name
|
||||
dep_label = f":{dep_pkg['name']}-{dep_pkg['version']}"
|
||||
for k in dep_kinds:
|
||||
if k.get("kind") == "build":
|
||||
build_deps.append(
|
||||
BuckDep(
|
||||
label=dep_label,
|
||||
local_name=local_name,
|
||||
crate_name=dep_crate_name,
|
||||
cfg=k.get("target"),
|
||||
)
|
||||
)
|
||||
build_base_deps, build_conditional_deps, build_named_deps = group_deps(build_deps)
|
||||
build_deps_expr = starlark_deps_expr(build_base_deps, build_conditional_deps)
|
||||
|
||||
lines.append("codex_rust_binary(")
|
||||
lines.append(f' name = "{build_bin}",')
|
||||
lines.append(f' srcs = ["{build_rs_rel}"],')
|
||||
lines.append(' crate = "build_script_build",')
|
||||
lines.append(f' crate_root = "{build_rs_rel}",')
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(" env = " + starlark_dict({**env, "CARGO_CRATE_NAME": "build_script_build"}, indent=" ") + ",")
|
||||
lines.append(' visibility = [],')
|
||||
if build_base_deps or build_conditional_deps:
|
||||
lines.append(f" deps = {build_deps_expr},")
|
||||
if build_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(build_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(")")
|
||||
lines.append("")
|
||||
|
||||
lines.append("codex_buildscript_run(")
|
||||
lines.append(f' name = "{build_run}",')
|
||||
lines.append(f' package_name = "{name}",')
|
||||
lines.append(f' buildscript_rule = ":{build_bin}",')
|
||||
# Keep env small; codex_buildscript_run fills in common profile vars.
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_PKG_AUTHORS": env["CARGO_PKG_AUTHORS"],
|
||||
"CARGO_PKG_DESCRIPTION": env["CARGO_PKG_DESCRIPTION"],
|
||||
"CARGO_PKG_REPOSITORY": env["CARGO_PKG_REPOSITORY"],
|
||||
"CARGO_PKG_VERSION_MAJOR": env["CARGO_PKG_VERSION_MAJOR"],
|
||||
"CARGO_PKG_VERSION_MINOR": env["CARGO_PKG_VERSION_MINOR"],
|
||||
"CARGO_PKG_VERSION_PATCH": env["CARGO_PKG_VERSION_PATCH"],
|
||||
"CARGO_PKG_VERSION_PRE": env["CARGO_PKG_VERSION_PRE"],
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
lines.append(" rustc_link_lib = True,")
|
||||
lines.append(" rustc_link_search = True,")
|
||||
lines.append(f' version = "{version}",')
|
||||
lines.append(")")
|
||||
lines.append("")
|
||||
|
||||
# Build-script outputs for crates that use `OUT_DIR`.
|
||||
env = {**env, "OUT_DIR": f"$(location :{build_run}[out_dir])"}
|
||||
|
||||
lines.append("codex_rust_library(")
|
||||
lines.append(f' name = "{rule_name}",')
|
||||
lines.append(f' srcs = ["{crate_root_rel}"],')
|
||||
lines.append(f' crate = "{crate_name}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(" env = " + starlark_dict(env, indent=" ") + ",")
|
||||
features = sorted([f for f in (node.get("features") or []) if not f.startswith("dep:")])
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
if proc_macro:
|
||||
lines.append(" proc_macro = True,")
|
||||
if has_build_rs:
|
||||
lines.append(f' rustc_flags = ["@$(location :{rule_name}-build-script-run[rustc_flags])"],')
|
||||
lines.append(" visibility = [],")
|
||||
if base_deps or conditional_deps:
|
||||
lines.append(f" deps = {deps_expr},")
|
||||
if named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(")")
|
||||
lines.append("")
|
||||
|
||||
lines.append(END_MARKER.rstrip("\n") + "\n")
|
||||
return existing_buck + "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not THIRD_PARTY_BUCK.exists():
|
||||
raise SystemExit(f"Missing {THIRD_PARTY_BUCK}; run `reindeer buckify` first.")
|
||||
meta = cargo_metadata()
|
||||
existing = THIRD_PARTY_BUCK.read_text(encoding="utf-8")
|
||||
existing = strip_existing_overlay(existing)
|
||||
existing = patch_reindeer_output_for_test_features(existing)
|
||||
patched = append_overlay(meta, existing)
|
||||
if patched != existing:
|
||||
THIRD_PARTY_BUCK.write_text(patched, encoding="utf-8")
|
||||
if BUILDIFIER.exists():
|
||||
# Keep the local-only generated file readable.
|
||||
proc = subprocess.run(
|
||||
[str(BUILDIFIER), "-lint=off", "-mode=fix", str(THIRD_PARTY_BUCK)],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(
|
||||
f"warning: buildifier failed (exit {proc.returncode}); leaving {THIRD_PARTY_BUCK} unformatted",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user