mirror of
https://github.com/openai/codex.git
synced 2026-09-11 20:36:49 +00:00
aa4a870e06
## Why Windows libffi builds invoke tools by name, and Cygwin provides a different `link.exe` from MSVC. Inherited search paths can select unintended tools or SDK inputs. ## What changed Add optional `--windows-build-inputs <json>` to `third_party/voice/build_native.py` to select installed tools and SDK directories for Windows MSVC targets. Validate the target, tool paths, architecture-specific assembler, SDK directories, and agreement with CLI tool arguments. Build the search path from the selection with MSVC ahead of Cygwin, reject shadowed tools, replace inherited `INCLUDE` and `LIB`, and remove `LIBPATH`. Retain the selection in build receipts. Builds without the option continue using the normal Visual Studio environment. ## Testing Add tests for linker precedence and subprocess selection, invalid or mismatched inputs, shadowed SDK tools, environment replacement, and receipt recording. GitOrigin-RevId: c80a6d9eb29556157ba1316d62c9b5083f4038b9
574 lines
21 KiB
Python
574 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Build the candidate native voice prefix with upstream build systems."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
|
|
# Match the package builder: import this script's sibling under PYTHONSAFEPATH,
|
|
# without adding the caller's working directory to the module search path.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from prepare_sources import MANIFEST, load_sources, prepare_sources
|
|
from windows_build_inputs import build_environment
|
|
|
|
TARGET_SYSTEMS = {
|
|
"apple-darwin": "Darwin",
|
|
"unknown-linux-gnu": "Linux",
|
|
"pc-windows-msvc": "Windows",
|
|
}
|
|
|
|
|
|
def validate_target(target, system, machine, libc, deployment_target):
|
|
architecture, separator, suffix = target.partition("-")
|
|
host_architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(
|
|
machine.lower(), machine.lower()
|
|
)
|
|
if (
|
|
not separator
|
|
or architecture not in ("aarch64", "x86_64")
|
|
or suffix not in TARGET_SYSTEMS
|
|
):
|
|
raise ValueError(f"Unsupported native voice target: {target}")
|
|
if TARGET_SYSTEMS[suffix] != system or architecture != host_architecture:
|
|
raise ValueError("Use a native build host matching the requested target")
|
|
if system == "Linux" and libc != "glibc":
|
|
raise ValueError("Native Linux voice builds require glibc")
|
|
if system == "Darwin" and not re.fullmatch(
|
|
r"[0-9]+\.[0-9]+(?:\.[0-9]+)?", deployment_target or ""
|
|
):
|
|
raise ValueError(
|
|
"Declare the macOS deployment target; do not inherit the host default"
|
|
)
|
|
|
|
|
|
class NativeBuild:
|
|
def __init__(self, args, inherited_environment):
|
|
validate_target(
|
|
args.target,
|
|
platform.system(),
|
|
platform.machine(),
|
|
platform.libc_ver()[0],
|
|
args.deployment_target,
|
|
)
|
|
self.args = args
|
|
self.output = args.output.absolute()
|
|
self.prefix = self.output / "prefix"
|
|
self.tools = self.output / "tools"
|
|
self.windows = args.target.endswith("windows-msvc")
|
|
# Compiler drivers such as clang++ select link behavior from argv[0].
|
|
self.toolchain = {
|
|
name: getattr(args, name).absolute()
|
|
for name in ("cc", "cxx", "cmake", "make", "pkg_config", "shell")
|
|
}
|
|
self.toolchain.update(
|
|
(name, path.absolute())
|
|
for name in ("ar", "ranlib")
|
|
if (path := getattr(args, name, None)) is not None
|
|
)
|
|
self.quote = subprocess.list2cmdline if self.windows else shlex.join
|
|
self.flags = {
|
|
variable: list(getattr(args, argument, []))
|
|
for variable, argument in (
|
|
("CFLAGS", "c_flag"),
|
|
("CXXFLAGS", "cxx_flag"),
|
|
("LDFLAGS", "link_flag"),
|
|
)
|
|
}
|
|
if self.windows and (
|
|
any(self.flags.values())
|
|
or any(name in self.toolchain for name in ("ar", "ranlib"))
|
|
):
|
|
raise ValueError("Explicit toolchain overrides require a Unix build host")
|
|
self.bootstrap_make = (args.bootstrap_make or args.make).absolute()
|
|
for tool in (*self.toolchain.values(), self.bootstrap_make):
|
|
if not tool.is_file():
|
|
raise ValueError(f"Missing build tool: {tool}")
|
|
windows_inputs = getattr(args, "windows_build_inputs", None)
|
|
input_record = None
|
|
if windows_inputs is not None:
|
|
if not self.windows:
|
|
raise ValueError("Windows build inputs require a Windows MSVC target")
|
|
explicit, input_record = build_environment(
|
|
windows_inputs,
|
|
args.target,
|
|
{
|
|
**self.toolchain,
|
|
"bootstrap_make": self.bootstrap_make,
|
|
"python": Path(sys.executable),
|
|
},
|
|
)
|
|
inherited_environment = {**inherited_environment, **explicit}
|
|
if self.windows and not all(
|
|
inherited_environment.get(key) for key in ("INCLUDE", "LIB")
|
|
):
|
|
raise ValueError(
|
|
"Initialize the standard Visual Studio build environment first"
|
|
)
|
|
self.environment = {
|
|
key: value
|
|
for key, value in inherited_environment.items()
|
|
if key
|
|
in (
|
|
"HOME",
|
|
"TMPDIR",
|
|
"TMP",
|
|
"TEMP",
|
|
"SYSTEMROOT",
|
|
"SystemRoot",
|
|
"WINDIR",
|
|
"COMSPEC",
|
|
"INCLUDE",
|
|
"LIB",
|
|
"LIBPATH",
|
|
"HTTPS_PROXY",
|
|
"HTTP_PROXY",
|
|
"NO_PROXY",
|
|
)
|
|
or (self.windows and key == "USERPROFILE")
|
|
}
|
|
if windows_inputs is not None:
|
|
self.environment.pop("LIBPATH", None)
|
|
paths = [
|
|
str(self.tools / "bin"),
|
|
*(
|
|
[]
|
|
if windows_inputs is not None
|
|
else [str(p.parent) for p in self.toolchain.values()]
|
|
),
|
|
]
|
|
paths += (
|
|
inherited_environment.get("PATH", "").split(os.pathsep)
|
|
if self.windows
|
|
else ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
|
|
)
|
|
self.environment.update(
|
|
{
|
|
"PATH": os.pathsep.join(dict.fromkeys(paths)),
|
|
"LANG": "C",
|
|
"LC_ALL": "C",
|
|
"PYTHONDONTWRITEBYTECODE": "1",
|
|
"CC": str(self.toolchain["cc"]),
|
|
"CXX": str(self.toolchain["cxx"]),
|
|
"PKG_CONFIG": str(self.toolchain["pkg_config"]),
|
|
"PKG_CONFIG_PATH": "",
|
|
"PKG_CONFIG_LIBDIR": os.pathsep.join(
|
|
str(self.prefix / p) for p in ("lib/pkgconfig", "share/pkgconfig")
|
|
),
|
|
"CMAKE_PREFIX_PATH": str(self.prefix),
|
|
"NINJA": str(
|
|
self.tools / "bin" / ("ninja.exe" if self.windows else "ninja")
|
|
),
|
|
}
|
|
)
|
|
self.cmake_platform = []
|
|
self.environment.update(
|
|
(name, self.quote(flags)) for name, flags in self.flags.items() if flags
|
|
)
|
|
self.environment.update(
|
|
(name.upper(), self.quote([str(self.toolchain[name])]))
|
|
for name in ("ar", "ranlib")
|
|
if name in self.toolchain
|
|
)
|
|
if args.target.endswith("apple-darwin"):
|
|
self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target
|
|
self.cmake_platform = [
|
|
f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}",
|
|
"-DCMAKE_INSTALL_NAME_DIR=@rpath",
|
|
"-DCMAKE_INSTALL_RPATH=@loader_path",
|
|
]
|
|
elif not self.windows:
|
|
self.cmake_platform = [
|
|
"-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON",
|
|
"-DCMAKE_INSTALL_RPATH=$ORIGIN",
|
|
]
|
|
self.record = {
|
|
"target": args.target,
|
|
"deployment_target": args.deployment_target,
|
|
"flags": self.flags,
|
|
"steps": [],
|
|
}
|
|
if input_record is not None:
|
|
self.record["windows_build_inputs"] = input_record
|
|
|
|
def run(self, name, command, cwd=None, environment=None):
|
|
command = [str(part) for part in command]
|
|
step = {"name": name, "command": command}
|
|
self.record["steps"].append(step)
|
|
print(f"Building {name}", flush=True)
|
|
with (self.output / f"{name}.log").open("w", encoding="utf-8") as log:
|
|
result = subprocess.run(
|
|
command,
|
|
cwd=cwd or self.output,
|
|
env=environment or self.environment,
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
check=False,
|
|
)
|
|
step["exit_code"] = result.returncode
|
|
(self.output / "build-state.json").write_text(
|
|
json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
result.check_returncode()
|
|
|
|
def posix_path(self, path):
|
|
if not self.windows:
|
|
return path.as_posix()
|
|
return subprocess.check_output(
|
|
[self.toolchain["shell"].with_name("cygpath.exe"), "-u", str(path)],
|
|
env=self.environment,
|
|
text=True,
|
|
).strip()
|
|
|
|
def cmake(self, name, options, *, bootstrap=False):
|
|
directory = self.output / "build" / name
|
|
prefix = self.tools if bootstrap else self.prefix
|
|
generator = (
|
|
("NMake Makefiles" if self.windows else "Unix Makefiles")
|
|
if bootstrap
|
|
else "Ninja"
|
|
)
|
|
make = self.bootstrap_make if bootstrap else self.environment["NINJA"]
|
|
self.run(
|
|
name + "-configure",
|
|
[
|
|
self.toolchain["cmake"],
|
|
"-S",
|
|
self.sources[name],
|
|
"-B",
|
|
directory,
|
|
"-G",
|
|
generator,
|
|
f"-DCMAKE_MAKE_PROGRAM={make}",
|
|
"-DCMAKE_BUILD_TYPE=Release",
|
|
f"-DCMAKE_INSTALL_PREFIX={prefix}",
|
|
"-DCMAKE_INSTALL_LIBDIR=lib",
|
|
f"-DCMAKE_C_COMPILER={self.toolchain['cc']}",
|
|
f"-DCMAKE_CXX_COMPILER={self.toolchain['cxx']}",
|
|
*(
|
|
f"-DCMAKE_{name.upper()}={self.toolchain[name]}"
|
|
for name in ("ar", "ranlib")
|
|
if name in self.toolchain
|
|
),
|
|
f"-DCMAKE_PREFIX_PATH={self.prefix}",
|
|
"-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF",
|
|
"-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF",
|
|
"-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF",
|
|
"-DFETCHCONTENT_FULLY_DISCONNECTED=ON",
|
|
*self.cmake_platform,
|
|
*options,
|
|
],
|
|
)
|
|
self.run(
|
|
name + "-build",
|
|
[
|
|
self.toolchain["cmake"],
|
|
"--build",
|
|
directory,
|
|
"--parallel",
|
|
self.args.jobs,
|
|
],
|
|
)
|
|
self.run(name + "-install", [self.toolchain["cmake"], "--install", directory])
|
|
|
|
def meson(self, name, options):
|
|
directory = self.output / "build" / name
|
|
meson = [sys.executable, self.sources["meson"] / "meson.py"]
|
|
include = f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"
|
|
link = (
|
|
[f"/LIBPATH:{self.prefix / 'lib'}"]
|
|
if self.windows
|
|
else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"]
|
|
)
|
|
if self.args.target.endswith("unknown-linux-gnu"):
|
|
link[-1] = "-Wl,-rpath,$ORIGIN:$ORIGIN/.."
|
|
self.environment.update(
|
|
{
|
|
name: self.quote([*flags, *(link if name == "LDFLAGS" else [include])])
|
|
for name, flags in self.flags.items()
|
|
}
|
|
)
|
|
if self.args.target.endswith("apple-darwin"):
|
|
self.environment["OBJC"] = str(self.toolchain["cc"])
|
|
self.environment["OBJCFLAGS"] = self.environment["CFLAGS"]
|
|
self.run(
|
|
name + "-configure",
|
|
[
|
|
*meson,
|
|
"setup",
|
|
directory,
|
|
self.sources[name],
|
|
f"--prefix={self.prefix}",
|
|
"--libdir=lib",
|
|
"--buildtype=release",
|
|
"--wrap-mode=nofallback",
|
|
"-Dauto_features=disabled",
|
|
"-Ddefault_library=shared",
|
|
"-Dpkgconfig.relocatable=true",
|
|
*options,
|
|
],
|
|
)
|
|
self.run(
|
|
name + "-build", [*meson, "compile", "-C", directory, "-j", self.args.jobs]
|
|
)
|
|
self.run(
|
|
name + "-install", [*meson, "install", "-C", directory, "--no-rebuild"]
|
|
)
|
|
|
|
def build(self):
|
|
self.output.mkdir()
|
|
manifest = MANIFEST.read_bytes()
|
|
prepare_sources(self.args.archives, self.output / "sources", manifest)
|
|
self.sources = {
|
|
s.name: self.output / "sources" / s.root for s in load_sources(manifest)
|
|
}
|
|
self.record["manifest_sha256"] = hashlib.sha256(manifest).hexdigest()
|
|
self.record["tools"] = {
|
|
name: str(path) for name, path in self.toolchain.items()
|
|
}
|
|
self.record["python"] = sys.version
|
|
self.run("cmake-version", [self.toolchain["cmake"], "--version"])
|
|
self.run("pkg-config-version", [self.toolchain["pkg_config"], "--version"])
|
|
self.cmake("ninja", ["-DBUILD_TESTING=OFF"], bootstrap=True)
|
|
self.cmake(
|
|
"zlib",
|
|
[
|
|
"-DZLIB_BUILD_TESTING=OFF",
|
|
"-DZLIB_BUILD_SHARED=ON",
|
|
"-DZLIB_BUILD_STATIC=OFF",
|
|
],
|
|
)
|
|
self.cmake(
|
|
"pcre2",
|
|
[
|
|
"-DBUILD_SHARED_LIBS=ON",
|
|
"-DBUILD_STATIC_LIBS=OFF",
|
|
"-DPCRE2_BUILD_TESTS=OFF",
|
|
"-DPCRE2_BUILD_PCRE2GREP=OFF",
|
|
"-DPCRE2_SUPPORT_LIBZ=OFF",
|
|
"-DPCRE2_SUPPORT_LIBBZ2=OFF",
|
|
"-DPCRE2_SUPPORT_LIBREADLINE=OFF",
|
|
"-DPCRE2_SUPPORT_LIBEDIT=OFF",
|
|
],
|
|
)
|
|
ffi_build = self.output / "build/libffi"
|
|
ffi_build.mkdir()
|
|
environment = self.environment.copy()
|
|
configure_options = []
|
|
if self.windows:
|
|
wrapper = shlex.quote(self.posix_path(self.sources["libffi"] / "msvcc.sh"))
|
|
architecture = (
|
|
"-m64" if self.args.target.startswith("x86_64-") else "-marm64"
|
|
)
|
|
# Match upstream's MSVC recipe, including native ARM64 outputs
|
|
# from x64-emulated Cygwin tools. Never infer the target from uname.
|
|
host = self.args.target.partition("-")[0] + "-w64-mingw32"
|
|
configure_options = [f"--build={host}", f"--host={host}"]
|
|
automake = subprocess.check_output(
|
|
[
|
|
self.toolchain["shell"],
|
|
"--noprofile",
|
|
"--norc",
|
|
"-c",
|
|
"automake-1.18 --print-libdir",
|
|
],
|
|
env=environment,
|
|
text=True,
|
|
).strip()
|
|
environment.update(
|
|
{
|
|
"CC": f"{wrapper} {architecture}",
|
|
"CXX": f"{wrapper} {architecture}",
|
|
"AR": f"{shlex.quote(automake + '/ar-lib')} lib",
|
|
"RANLIB": ":",
|
|
"LD": "link",
|
|
"NM": "dumpbin -symbols",
|
|
"STRIP": ":",
|
|
"LDFLAGS": "-no-undefined",
|
|
# Libffi clears MAKEOVERRIDES. Use its recursion hook to
|
|
# name the import library expected by libtool's installer.
|
|
"AM_MAKEFLAGS": shlex.quote(
|
|
"LTLDFLAGS=-no-undefined -Wc,-link,/IMPLIB:.libs/libffi.lib"
|
|
),
|
|
"CPP": "cl -nologo -EP",
|
|
"CXXCPP": "cl -nologo -EP",
|
|
"CPPFLAGS": "-DFFI_BUILDING_DLL",
|
|
"CONFIG_SHELL": self.posix_path(self.toolchain["shell"]),
|
|
}
|
|
)
|
|
else:
|
|
if self.args.target.endswith("apple-darwin"):
|
|
# Libtool's partial links need -r, which ld64.lld does not support.
|
|
environment["CC"] += " -fuse-ld=/usr/bin/ld"
|
|
for name in ("ar", "ranlib"):
|
|
if name in self.toolchain:
|
|
path = str(self.toolchain[name])
|
|
if shlex.quote(path) != path:
|
|
raise ValueError(
|
|
f"libffi {name} path cannot require shell quoting"
|
|
)
|
|
# Response files preserve compiler arguments through Autoconf's word
|
|
# splitting and Make/libtool's shell expansion without losing flags
|
|
# added by configure (such as -fexceptions).
|
|
for name, flags in self.flags.items():
|
|
if any(
|
|
any(character.isspace() for character in flag) for flag in flags
|
|
):
|
|
raise ValueError("libffi flags cannot contain whitespace")
|
|
if flags:
|
|
response = ffi_build / f"{name.lower()}.rsp"
|
|
if shlex.quote(str(response)) != str(response):
|
|
raise ValueError(
|
|
"libffi response-file path cannot require shell quoting"
|
|
)
|
|
response.write_text(
|
|
"\n".join(
|
|
'"' + flag.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
for flag in flags
|
|
)
|
|
+ "\n"
|
|
)
|
|
environment[name] = f"@{response}"
|
|
if self.args.target.endswith("unknown-linux-gnu"):
|
|
# Libtool otherwise drops Clang's declared CRT/runtime options.
|
|
environment["AM_LTLDFLAGS"] = shlex.join(
|
|
argument
|
|
for flag in self.flags["LDFLAGS"]
|
|
for argument in ("-Xcompiler", flag)
|
|
)
|
|
self.run(
|
|
"libffi-configure",
|
|
[
|
|
self.toolchain["shell"],
|
|
self.posix_path(self.sources["libffi"] / "configure"),
|
|
f"--prefix={self.posix_path(self.prefix)}",
|
|
"--enable-shared",
|
|
"--disable-static",
|
|
"--disable-docs",
|
|
*configure_options,
|
|
],
|
|
cwd=ffi_build,
|
|
environment=environment,
|
|
)
|
|
self.run(
|
|
"libffi-build",
|
|
[self.toolchain["make"], f"-j{self.args.jobs}"],
|
|
cwd=ffi_build,
|
|
environment=environment,
|
|
)
|
|
self.run(
|
|
"libffi-install",
|
|
[self.toolchain["make"], "install"],
|
|
cwd=ffi_build,
|
|
environment=environment,
|
|
)
|
|
if self.windows:
|
|
self.run(
|
|
"libffi-pkg-config",
|
|
[self.toolchain["pkg_config"], "--cflags", "--libs", "libffi"],
|
|
)
|
|
if "/cygdrive/" in (self.output / "libffi-pkg-config.log").read_text():
|
|
raise ValueError("pkg-config did not relocate libffi to native paths")
|
|
self.cmake(
|
|
"opus",
|
|
[
|
|
"-DOPUS_BUILD_SHARED_LIBRARY=ON",
|
|
"-DOPUS_BUILD_TESTING=OFF",
|
|
"-DOPUS_BUILD_PROGRAMS=OFF",
|
|
# Windows ARM64 guarantees NEON; upstream misses its ARM64 spelling.
|
|
*(
|
|
["-DOPUS_PRESUME_NEON=ON"]
|
|
if self.args.target == "aarch64-pc-windows-msvc"
|
|
else []
|
|
),
|
|
],
|
|
)
|
|
self.meson("proxy-libintl", [])
|
|
self.meson(
|
|
"glib",
|
|
[
|
|
"-Dtests=false",
|
|
"-Dinstalled_tests=false",
|
|
"-Dnls=disabled",
|
|
"-Ddocumentation=false",
|
|
"-Dintrospection=disabled",
|
|
"-Dlibmount=disabled",
|
|
"-Dselinux=disabled",
|
|
"-Dxattr=false",
|
|
],
|
|
)
|
|
self.meson(
|
|
"gstreamer",
|
|
[
|
|
"-Dregistry=false",
|
|
"-Doption-parsing=false",
|
|
"-Dtracer_hooks=false",
|
|
"-Dgst_parse=false",
|
|
"-Dtools=disabled",
|
|
"-Dptp-helper=disabled",
|
|
],
|
|
)
|
|
self.meson(
|
|
"gst-plugins-base",
|
|
[
|
|
"-Dapp=enabled",
|
|
"-Daudioconvert=enabled",
|
|
"-Daudioresample=enabled",
|
|
"-Dopus=enabled",
|
|
"-Dgl=disabled",
|
|
],
|
|
)
|
|
self.meson("gst-plugins-good", ["-Drtp=enabled", "-Drtpmanager=enabled"])
|
|
(self.output / "built.json").write_text(
|
|
json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
for name in (
|
|
"archives",
|
|
"output",
|
|
"cc",
|
|
"cxx",
|
|
"cmake",
|
|
"make",
|
|
"pkg-config",
|
|
"shell",
|
|
):
|
|
parser.add_argument(f"--{name}", type=Path, required=True)
|
|
parser.add_argument(
|
|
"--bootstrap-make",
|
|
type=Path,
|
|
help="NMake on Windows; defaults to --make elsewhere",
|
|
)
|
|
parser.add_argument(
|
|
"--windows-build-inputs",
|
|
type=Path,
|
|
help="Explicit Windows tool and SDK selection; no unrelated PATH fallback",
|
|
)
|
|
for name in ("ar", "ranlib"):
|
|
parser.add_argument(f"--{name}", type=Path)
|
|
for name in ("c-flag", "cxx-flag", "link-flag"):
|
|
parser.add_argument(f"--{name}", action="append", default=[])
|
|
parser.add_argument("--target", required=True)
|
|
parser.add_argument(
|
|
"--deployment-target",
|
|
help="Required on macOS; use the existing supported release minimum",
|
|
)
|
|
parser.add_argument("--jobs", type=int, default=8)
|
|
args = parser.parse_args()
|
|
if sys.version_info < (3, 12) or args.jobs < 1:
|
|
parser.error("Python 3.12+ and a positive --jobs value are required")
|
|
NativeBuild(args, os.environ).build()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|