Files
codex/third_party/voice/build_native.py
T
zbarsky-openai 1099fe0694 Use hermetic LLVM for Windows MSVC Bazel builds (#60507)
## Why

Windows MSVC builds relied on installed C++ toolchains, and V8 snapshot generation required native Windows execution. Cross-compiling V8 also needs a Windows snapshot generator to preserve Windows unwind metadata.

## What changed

- Use the registered hermetic LLVM toolchain and declared Microsoft SDK for Windows MSVC builds, with static CRT linkage and the toolchain's declared `lld-link` for Rust.
- Run Windows V8 snapshot generators natively or through `rules_wine` on Linux, preserving target CPU, OS, and sandbox settings. Keep separate archive and binding pairs for the workspace's ISO stdio ABI and the distributed MSVC stdio ABI.
- Update build dependencies and bindgen, scope V8's Chromium libc++ settings to its own targets, and adapt native voice dependencies for clang-cl and Windows ARM64.
- Build SQLite directly with Bazel and give `openssl-sys` an explicit build using Bazel's OpenSSL.
- Remove build-worker source-directory dependencies from Bazel resource lookup, schema output discovery, and snapshot paths; embed assets in debug builds as well.

GitOrigin-RevId: fd224d9d28d73a67baf90f6d73c6079d60f20351
2026-11-10 03:02:25 +00:00

585 lines
22 KiB
Python

#!/usr/bin/env python3
"""Build the candidate native voice prefix with upstream build systems."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import shlex
import subprocess
import sys
# Match the package builder: import this script's sibling under PYTHONSAFEPATH,
# without adding the caller's working directory to the module search path.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from prepare_sources import MANIFEST, load_sources, prepare_sources
from windows_build_inputs import build_environment
TARGET_SYSTEMS = {
"apple-darwin": "Darwin",
"unknown-linux-gnu": "Linux",
"pc-windows-msvc": "Windows",
}
def validate_target(target, system, machine, libc, deployment_target):
architecture, separator, suffix = target.partition("-")
host_architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(
machine.lower(), machine.lower()
)
if (
not separator
or architecture not in ("aarch64", "x86_64")
or suffix not in TARGET_SYSTEMS
):
raise ValueError(f"Unsupported native voice target: {target}")
if TARGET_SYSTEMS[suffix] != system or architecture != host_architecture:
raise ValueError("Use a native build host matching the requested target")
if system == "Linux" and libc != "glibc":
raise ValueError("Native Linux voice builds require glibc")
if system == "Darwin" and not re.fullmatch(
r"[0-9]+\.[0-9]+(?:\.[0-9]+)?", deployment_target or ""
):
raise ValueError(
"Declare the macOS deployment target; do not inherit the host default"
)
class NativeBuild:
def __init__(self, args, inherited_environment):
validate_target(
args.target,
platform.system(),
platform.machine(),
platform.libc_ver()[0],
args.deployment_target,
)
self.args = args
self.output = args.output.absolute()
self.prefix = self.output / "prefix"
self.tools = self.output / "tools"
self.windows = args.target.endswith("windows-msvc")
# Compiler drivers such as clang++ select link behavior from argv[0].
self.toolchain = {
name: getattr(args, name).absolute()
for name in ("cc", "cxx", "cmake", "make", "pkg_config", "shell")
}
self.toolchain.update(
(name, path.absolute())
for name in ("ar", "ranlib")
if (path := getattr(args, name, None)) is not None
)
self.quote = subprocess.list2cmdline if self.windows else shlex.join
self.flags = {
variable: list(getattr(args, argument, []))
for variable, argument in (
("CFLAGS", "c_flag"),
("CXXFLAGS", "cxx_flag"),
("LDFLAGS", "link_flag"),
)
}
if self.windows and (
any(self.flags.values())
or any(name in self.toolchain for name in ("ar", "ranlib"))
):
raise ValueError("Explicit toolchain overrides require a Unix build host")
self.bootstrap_make = (args.bootstrap_make or args.make).absolute()
for tool in (*self.toolchain.values(), self.bootstrap_make):
if not tool.is_file():
raise ValueError(f"Missing build tool: {tool}")
windows_inputs = getattr(args, "windows_build_inputs", None)
input_record = None
if windows_inputs is not None:
if not self.windows:
raise ValueError("Windows build inputs require a Windows MSVC target")
explicit, input_record = build_environment(
windows_inputs,
args.target,
{
**self.toolchain,
"bootstrap_make": self.bootstrap_make,
"python": Path(sys.executable),
},
)
inherited_environment = {**inherited_environment, **explicit}
if self.windows and not all(
inherited_environment.get(key) for key in ("INCLUDE", "LIB")
):
raise ValueError(
"Initialize the standard Visual Studio build environment first"
)
self.environment = {
key: value
for key, value in inherited_environment.items()
if key
in (
"HOME",
"TMPDIR",
"TMP",
"TEMP",
"SYSTEMROOT",
"SystemRoot",
"WINDIR",
"COMSPEC",
"INCLUDE",
"LIB",
"LIBPATH",
"HTTPS_PROXY",
"HTTP_PROXY",
"NO_PROXY",
)
or (self.windows and key == "USERPROFILE")
}
if windows_inputs is not None:
self.environment.pop("LIBPATH", None)
paths = [
str(self.tools / "bin"),
*(
[]
if windows_inputs is not None
else [str(p.parent) for p in self.toolchain.values()]
),
]
paths += (
inherited_environment.get("PATH", "").split(os.pathsep)
if self.windows
else ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
)
self.environment.update(
{
"PATH": os.pathsep.join(dict.fromkeys(paths)),
"LANG": "C",
"LC_ALL": "C",
"PYTHONDONTWRITEBYTECODE": "1",
"CC": str(self.toolchain["cc"]),
"CXX": str(self.toolchain["cxx"]),
"PKG_CONFIG": str(self.toolchain["pkg_config"]),
"PKG_CONFIG_PATH": "",
"PKG_CONFIG_LIBDIR": os.pathsep.join(
str(self.prefix / p) for p in ("lib/pkgconfig", "share/pkgconfig")
),
"CMAKE_PREFIX_PATH": str(self.prefix),
"NINJA": str(
self.tools / "bin" / ("ninja.exe" if self.windows else "ninja")
),
}
)
self.cmake_platform = []
self.environment.update(
(name, self.quote(flags)) for name, flags in self.flags.items() if flags
)
self.environment.update(
(name.upper(), self.quote([str(self.toolchain[name])]))
for name in ("ar", "ranlib")
if name in self.toolchain
)
if args.target.endswith("apple-darwin"):
self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target
self.cmake_platform = [
f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}",
"-DCMAKE_INSTALL_NAME_DIR=@rpath",
"-DCMAKE_INSTALL_RPATH=@loader_path",
]
elif not self.windows:
self.cmake_platform = [
"-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON",
"-DCMAKE_INSTALL_RPATH=$ORIGIN",
]
self.record = {
"target": args.target,
"deployment_target": args.deployment_target,
"flags": self.flags,
"steps": [],
}
if input_record is not None:
self.record["windows_build_inputs"] = input_record
def run(self, name, command, cwd=None, environment=None):
command = [str(part) for part in command]
step = {"name": name, "command": command}
self.record["steps"].append(step)
print(f"Building {name}", flush=True)
with (self.output / f"{name}.log").open("w", encoding="utf-8") as log:
result = subprocess.run(
command,
cwd=cwd or self.output,
env=environment or self.environment,
stdout=log,
stderr=subprocess.STDOUT,
check=False,
)
step["exit_code"] = result.returncode
(self.output / "build-state.json").write_text(
json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
)
result.check_returncode()
def posix_path(self, path):
if not self.windows:
return path.as_posix()
return subprocess.check_output(
[self.toolchain["shell"].with_name("cygpath.exe"), "-u", str(path)],
env=self.environment,
text=True,
).strip()
def cmake(self, name, options, *, bootstrap=False):
directory = self.output / "build" / name
prefix = self.tools if bootstrap else self.prefix
generator = (
("NMake Makefiles" if self.windows else "Unix Makefiles")
if bootstrap
else "Ninja"
)
make = self.bootstrap_make if bootstrap else self.environment["NINJA"]
self.run(
name + "-configure",
[
self.toolchain["cmake"],
"-S",
self.sources[name],
"-B",
directory,
"-G",
generator,
f"-DCMAKE_MAKE_PROGRAM={make}",
"-DCMAKE_BUILD_TYPE=Release",
f"-DCMAKE_INSTALL_PREFIX={prefix}",
"-DCMAKE_INSTALL_LIBDIR=lib",
f"-DCMAKE_C_COMPILER={self.toolchain['cc']}",
f"-DCMAKE_CXX_COMPILER={self.toolchain['cxx']}",
*(
f"-DCMAKE_{name.upper()}={self.toolchain[name]}"
for name in ("ar", "ranlib")
if name in self.toolchain
),
f"-DCMAKE_PREFIX_PATH={self.prefix}",
"-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF",
"-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF",
"-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF",
"-DFETCHCONTENT_FULLY_DISCONNECTED=ON",
*self.cmake_platform,
*options,
],
)
self.run(
name + "-build",
[
self.toolchain["cmake"],
"--build",
directory,
"--parallel",
self.args.jobs,
],
)
self.run(name + "-install", [self.toolchain["cmake"], "--install", directory])
def meson(self, name, options):
directory = self.output / "build" / name
meson = [sys.executable, self.sources["meson"] / "meson.py"]
include = f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"
link = (
[f"/LIBPATH:{self.prefix / 'lib'}"]
if self.windows
else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"]
)
if self.args.target.endswith("unknown-linux-gnu"):
link[-1] = "-Wl,-rpath,$ORIGIN:$ORIGIN/.."
self.environment.update(
{
name: self.quote([*flags, *(link if name == "LDFLAGS" else [include])])
for name, flags in self.flags.items()
}
)
languages = {"c": "CFLAGS", "cpp": "CXXFLAGS"}
if self.args.target.endswith("apple-darwin"):
self.environment["OBJC"] = str(self.toolchain["cc"])
self.environment["OBJCFLAGS"] = self.environment["CFLAGS"]
languages["objc"] = "CFLAGS"
# Environment CFLAGS are also linked for Autotools compatibility. Use
# Meson's phase-specific options to preserve Bazel's toolchain contract.
meson_flags = []
for language, variable in languages.items():
meson_flags += [
f"-D{language}_args={json.dumps([*self.flags[variable], include])}",
f"-D{language}_link_args={json.dumps([*self.flags['LDFLAGS'], *link])}",
]
self.run(
name + "-configure",
[
*meson,
"setup",
directory,
self.sources[name],
f"--prefix={self.prefix}",
"--libdir=lib",
"--buildtype=release",
"--wrap-mode=nofallback",
"-Dauto_features=disabled",
"-Ddefault_library=shared",
"-Dpkgconfig.relocatable=true",
*meson_flags,
*options,
],
)
self.run(
name + "-build", [*meson, "compile", "-C", directory, "-j", self.args.jobs]
)
self.run(
name + "-install", [*meson, "install", "-C", directory, "--no-rebuild"]
)
def build(self):
self.output.mkdir()
manifest = MANIFEST.read_bytes()
prepare_sources(self.args.archives, self.output / "sources", manifest)
self.sources = {
s.name: self.output / "sources" / s.root for s in load_sources(manifest)
}
self.record["manifest_sha256"] = hashlib.sha256(manifest).hexdigest()
self.record["tools"] = {
name: str(path) for name, path in self.toolchain.items()
}
self.record["python"] = sys.version
self.run("cmake-version", [self.toolchain["cmake"], "--version"])
self.run("pkg-config-version", [self.toolchain["pkg_config"], "--version"])
self.cmake("ninja", ["-DBUILD_TESTING=OFF"], bootstrap=True)
self.cmake(
"zlib",
[
"-DZLIB_BUILD_TESTING=OFF",
"-DZLIB_BUILD_SHARED=ON",
"-DZLIB_BUILD_STATIC=OFF",
],
)
self.cmake(
"pcre2",
[
"-DBUILD_SHARED_LIBS=ON",
"-DBUILD_STATIC_LIBS=OFF",
"-DPCRE2_BUILD_TESTS=OFF",
"-DPCRE2_BUILD_PCRE2GREP=OFF",
"-DPCRE2_SUPPORT_LIBZ=OFF",
"-DPCRE2_SUPPORT_LIBBZ2=OFF",
"-DPCRE2_SUPPORT_LIBREADLINE=OFF",
"-DPCRE2_SUPPORT_LIBEDIT=OFF",
],
)
ffi_build = self.output / "build/libffi"
ffi_build.mkdir()
environment = self.environment.copy()
configure_options = []
if self.windows:
wrapper = shlex.quote(self.posix_path(self.sources["libffi"] / "msvcc.sh"))
architecture = (
"-m64" if self.args.target.startswith("x86_64-") else "-marm64"
)
# Match upstream's MSVC recipe, including native ARM64 outputs
# from x64-emulated Cygwin tools. Never infer the target from uname.
host = self.args.target.partition("-")[0] + "-w64-mingw32"
configure_options = [f"--build={host}", f"--host={host}"]
automake = subprocess.check_output(
[
self.toolchain["shell"],
"--noprofile",
"--norc",
"-c",
"automake-1.18 --print-libdir",
],
env=environment,
text=True,
).strip()
environment.update(
{
"CC": f"{wrapper} {architecture}",
"CXX": f"{wrapper} {architecture}",
"AR": f"{shlex.quote(automake + '/ar-lib')} lib",
"RANLIB": ":",
"LD": "link",
"NM": "dumpbin -symbols",
"STRIP": ":",
"LDFLAGS": "-no-undefined",
# Libffi clears MAKEOVERRIDES. Use its recursion hook to
# name the import library expected by libtool's installer.
"AM_MAKEFLAGS": shlex.quote(
"LTLDFLAGS=-no-undefined -Wc,-link,/IMPLIB:.libs/libffi.lib"
),
"CPP": "cl -nologo -EP",
"CXXCPP": "cl -nologo -EP",
"CPPFLAGS": "-DFFI_BUILDING_DLL",
"CONFIG_SHELL": self.posix_path(self.toolchain["shell"]),
}
)
else:
if self.args.target.endswith("apple-darwin"):
# Libtool's partial links need -r, which ld64.lld does not support.
environment["CC"] += " -fuse-ld=/usr/bin/ld"
for name in ("ar", "ranlib"):
if name in self.toolchain:
path = str(self.toolchain[name])
if shlex.quote(path) != path:
raise ValueError(
f"libffi {name} path cannot require shell quoting"
)
# Response files preserve compiler arguments through Autoconf's word
# splitting and Make/libtool's shell expansion without losing flags
# added by configure (such as -fexceptions).
for name, flags in self.flags.items():
if any(
any(character.isspace() for character in flag) for flag in flags
):
raise ValueError("libffi flags cannot contain whitespace")
if flags:
response = ffi_build / f"{name.lower()}.rsp"
if shlex.quote(str(response)) != str(response):
raise ValueError(
"libffi response-file path cannot require shell quoting"
)
response.write_text(
"\n".join(
'"' + flag.replace("\\", "\\\\").replace('"', '\\"') + '"'
for flag in flags
)
+ "\n"
)
environment[name] = f"@{response}"
if self.args.target.endswith("unknown-linux-gnu"):
# Libtool otherwise drops Clang's declared CRT/runtime options.
environment["AM_LTLDFLAGS"] = shlex.join(
argument
for flag in self.flags["LDFLAGS"]
for argument in ("-Xcompiler", flag)
)
self.run(
"libffi-configure",
[
self.toolchain["shell"],
self.posix_path(self.sources["libffi"] / "configure"),
f"--prefix={self.posix_path(self.prefix)}",
"--enable-shared",
"--disable-static",
"--disable-docs",
*configure_options,
],
cwd=ffi_build,
environment=environment,
)
self.run(
"libffi-build",
[self.toolchain["make"], f"-j{self.args.jobs}"],
cwd=ffi_build,
environment=environment,
)
self.run(
"libffi-install",
[self.toolchain["make"], "install"],
cwd=ffi_build,
environment=environment,
)
if self.windows:
self.run(
"libffi-pkg-config",
[self.toolchain["pkg_config"], "--cflags", "--libs", "libffi"],
)
if "/cygdrive/" in (self.output / "libffi-pkg-config.log").read_text():
raise ValueError("pkg-config did not relocate libffi to native paths")
self.cmake(
"opus",
[
"-DOPUS_BUILD_SHARED_LIBRARY=ON",
"-DOPUS_BUILD_TESTING=OFF",
"-DOPUS_BUILD_PROGRAMS=OFF",
# Windows ARM64 guarantees NEON; upstream misses its ARM64 spelling.
*(
["-DOPUS_PRESUME_NEON=ON"]
if self.args.target == "aarch64-pc-windows-msvc"
else []
),
],
)
self.meson("proxy-libintl", [])
self.meson(
"glib",
[
"-Dtests=false",
"-Dinstalled_tests=false",
"-Dnls=disabled",
"-Ddocumentation=false",
"-Dintrospection=disabled",
"-Dlibmount=disabled",
"-Dselinux=disabled",
"-Dxattr=false",
],
)
self.meson(
"gstreamer",
[
"-Dregistry=false",
"-Doption-parsing=false",
"-Dtracer_hooks=false",
"-Dgst_parse=false",
"-Dtools=disabled",
"-Dptp-helper=disabled",
],
)
self.meson(
"gst-plugins-base",
[
"-Dapp=enabled",
"-Daudioconvert=enabled",
"-Daudioresample=enabled",
"-Dopus=enabled",
"-Dgl=disabled",
],
)
self.meson("gst-plugins-good", ["-Drtp=enabled", "-Drtpmanager=enabled"])
(self.output / "built.json").write_text(
json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
)
def main():
parser = argparse.ArgumentParser(description=__doc__)
for name in (
"archives",
"output",
"cc",
"cxx",
"cmake",
"make",
"pkg-config",
"shell",
):
parser.add_argument(f"--{name}", type=Path, required=True)
parser.add_argument(
"--bootstrap-make",
type=Path,
help="NMake on Windows; defaults to --make elsewhere",
)
parser.add_argument(
"--windows-build-inputs",
type=Path,
help="Explicit Windows tool and SDK selection; no unrelated PATH fallback",
)
for name in ("ar", "ranlib"):
parser.add_argument(f"--{name}", type=Path)
for name in ("c-flag", "cxx-flag", "link-flag"):
parser.add_argument(f"--{name}", action="append", default=[])
parser.add_argument("--target", required=True)
parser.add_argument(
"--deployment-target",
help="Required on macOS; use the existing supported release minimum",
)
parser.add_argument("--jobs", type=int, default=8)
args = parser.parse_args()
if sys.version_info < (3, 12) or args.jobs < 1:
parser.error("Python 3.12+ and a positive --jobs value are required")
NativeBuild(args, os.environ).build()
if __name__ == "__main__":
main()