From fc7d34ad67d0c398b8d2745e329273ee9ecebbaa Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Mon, 31 Aug 2026 17:53:01 +0000 Subject: [PATCH] Add native voice dependency build recipe (#41890) ## Why The pinned voice source inputs can be prepared but do not yet provide compiled native libraries for downstream integration. ## What changed - Add `build_native.py` to build a shared-library prefix for the supported voice dependencies on native x64 and ARM64 GNU/Linux, macOS, and Windows MSVC hosts. - Require explicit toolchain inputs, isolate dependency discovery to the output prefix, and record command logs and build state for provenance and failures. - Expose the recipe and source inputs through `//third_party/voice:build_inputs` and document its prerequisites, outputs, and integration boundaries. ## Testing - `PYTHONSAFEPATH=1 python3 -m unittest discover -s third_party/voice -p 'test_build_native.py'` GitOrigin-RevId: 132d93561a8e1bb178518f1248cf87d031060aae --- third_party/voice/BUILD.bazel | 9 + third_party/voice/README.md | 29 ++ third_party/voice/build_native.py | 401 +++++++++++++++++++++++++ third_party/voice/test_build_native.py | 182 +++++++++++ 4 files changed, 621 insertions(+) create mode 100644 third_party/voice/build_native.py create mode 100644 third_party/voice/test_build_native.py diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 817a9d67f8..efcc1950fd 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -27,3 +27,12 @@ filegroup( tags = ["manual"], visibility = ["//visibility:public"], ) + +filegroup( + name = "build_inputs", + srcs = [ + "build_native.py", + ":source_inputs", + ], + visibility = ["//visibility:public"], +) diff --git a/third_party/voice/README.md b/third_party/voice/README.md index 90a853529c..fe3235fff8 100644 --- a/third_party/voice/README.md +++ b/third_party/voice/README.md @@ -46,3 +46,32 @@ Checksums establish input identity, not security or license approval. Native compilation, final Cargo/Bazel linking, installed packages, minimum OS support and duplex audio validation remain separate stages. These inputs do not establish a shared Opus build with Rust consumers or a reduced dependency count. + +Rust `opus` 0.4.0 is available through Socket. Adding Rust transport dependencies +and establishing a shared Opus build remain separate integration work. + +## Native build recipe + +`build_native.py` runs the unmodified upstream build systems in a new output +directory, using the same archives. Specify the target and existing compiler, +CMake, make, pkg-config and shell paths explicitly. It requires a matching +native host: GNU Linux, macOS, or Windows MSVC, on x64 or ARM64. + +On macOS, specify the existing release deployment target with +`--deployment-target`; the host OS version is not an acceptable default. +Windows requires the normal Visual Studio SDK environment, GNU make and a +POSIX shell for upstream libffi, and `--bootstrap-make` pointing to NMake. +The recipe does not install these build prerequisites or patch upstream sources. + +Outputs are under `prefix/`, build tools under `tools/`, and logs beside them. +`build-state.json` records completed commands and failures; `built.json` exists +only when every build/install command succeeds. Failed builds retain their logs +and must use a new output directory on retry. CMake compiler-identification logs +and the recorded tool/configuration inputs remain part of the build provenance. + +The recipe disables optional plugins and Meson fallback dependency resolution, +with pkg-config restricted to this prefix. Only system ABI libraries/frameworks +may remain external; runtime closure inspection must verify that independently. +`//third_party/voice:build_inputs` exposes the recipe and source inputs to Bazel. +Neither this filegroup nor a successful prefix build proves final Cargo/Bazel +linkage, safe private runtime loading, or an installed voice-capable Codex package. diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.py new file mode 100644 index 0000000000..33e16e03f8 --- /dev/null +++ b/third_party/voice/build_native.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +"""Build the candidate native voice prefix with upstream build systems.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shlex +import subprocess +import sys + +# Match the package builder: import this script's sibling under PYTHONSAFEPATH, +# without adding the caller's working directory to the module search path. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from prepare_sources import MANIFEST, load_sources, prepare_sources + +TARGET_SYSTEMS = { + "apple-darwin": "Darwin", + "unknown-linux-gnu": "Linux", + "pc-windows-msvc": "Windows", +} + + +def validate_target(target, system, machine, libc, deployment_target): + architecture, separator, suffix = target.partition("-") + host_architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get( + machine.lower(), machine.lower() + ) + if ( + not separator + or architecture not in ("aarch64", "x86_64") + or suffix not in TARGET_SYSTEMS + ): + raise ValueError(f"Unsupported native voice target: {target}") + if TARGET_SYSTEMS[suffix] != system or architecture != host_architecture: + raise ValueError("Use a native build host matching the requested target") + if system == "Linux" and libc != "glibc": + raise ValueError("Native Linux voice builds require glibc") + if system == "Darwin" and not re.fullmatch( + r"[0-9]+\.[0-9]+(?:\.[0-9]+)?", deployment_target or "" + ): + raise ValueError( + "Declare the macOS deployment target; do not inherit the host default" + ) + + +class NativeBuild: + def __init__(self, args, inherited_environment): + validate_target( + args.target, + platform.system(), + platform.machine(), + platform.libc_ver()[0], + args.deployment_target, + ) + self.args = args + self.output = args.output.absolute() + self.prefix = self.output / "prefix" + self.tools = self.output / "tools" + self.windows = args.target.endswith("windows-msvc") + # Compiler drivers such as clang++ select link behavior from argv[0]. + self.toolchain = { + name: getattr(args, name).absolute() + for name in ("cc", "cxx", "cmake", "make", "pkg_config", "shell") + } + self.bootstrap_make = (args.bootstrap_make or args.make).absolute() + for tool in (*self.toolchain.values(), self.bootstrap_make): + if not tool.is_file(): + raise ValueError(f"Missing build tool: {tool}") + if self.windows and not all( + inherited_environment.get(key) for key in ("INCLUDE", "LIB") + ): + raise ValueError( + "Initialize the standard Visual Studio build environment first" + ) + self.environment = { + key: value + for key, value in inherited_environment.items() + if key + in ( + "HOME", + "TMPDIR", + "TMP", + "TEMP", + "SYSTEMROOT", + "SystemRoot", + "WINDIR", + "COMSPEC", + "INCLUDE", + "LIB", + "LIBPATH", + "HTTPS_PROXY", + "HTTP_PROXY", + "NO_PROXY", + ) + } + paths = [ + str(self.tools / "bin"), + *(str(p.parent) for p in self.toolchain.values()), + ] + paths += ( + inherited_environment.get("PATH", "").split(os.pathsep) + if self.windows + else ["/usr/bin", "/bin", "/usr/sbin", "/sbin"] + ) + self.environment.update( + { + "PATH": os.pathsep.join(dict.fromkeys(paths)), + "LANG": "C", + "LC_ALL": "C", + "PYTHONDONTWRITEBYTECODE": "1", + "CC": str(self.toolchain["cc"]), + "CXX": str(self.toolchain["cxx"]), + "PKG_CONFIG": str(self.toolchain["pkg_config"]), + "PKG_CONFIG_PATH": "", + "PKG_CONFIG_LIBDIR": os.pathsep.join( + str(self.prefix / p) for p in ("lib/pkgconfig", "share/pkgconfig") + ), + "CMAKE_PREFIX_PATH": str(self.prefix), + "NINJA": str( + self.tools / "bin" / ("ninja.exe" if self.windows else "ninja") + ), + } + ) + self.cmake_platform = [] + if args.target.endswith("apple-darwin"): + self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target + self.cmake_platform = [ + f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}" + ] + self.record = { + "target": args.target, + "deployment_target": args.deployment_target, + "steps": [], + } + + def run(self, name, command, cwd=None, environment=None): + command = [str(part) for part in command] + step = {"name": name, "command": command} + self.record["steps"].append(step) + print(f"Building {name}", flush=True) + with (self.output / f"{name}.log").open("w", encoding="utf-8") as log: + result = subprocess.run( + command, + cwd=cwd or self.output, + env=environment or self.environment, + stdout=log, + stderr=subprocess.STDOUT, + check=False, + ) + step["exit_code"] = result.returncode + (self.output / "build-state.json").write_text( + json.dumps(self.record, indent=2) + "\n", encoding="utf-8" + ) + result.check_returncode() + + def cmake(self, name, options, *, bootstrap=False): + directory = self.output / "build" / name + prefix = self.tools if bootstrap else self.prefix + generator = ( + ("NMake Makefiles" if self.windows else "Unix Makefiles") + if bootstrap + else "Ninja" + ) + make = self.bootstrap_make if bootstrap else self.environment["NINJA"] + self.run( + name + "-configure", + [ + self.toolchain["cmake"], + "-S", + self.sources[name], + "-B", + directory, + "-G", + generator, + f"-DCMAKE_MAKE_PROGRAM={make}", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_INSTALL_PREFIX={prefix}", + "-DCMAKE_INSTALL_LIBDIR=lib", + f"-DCMAKE_C_COMPILER={self.toolchain['cc']}", + f"-DCMAKE_CXX_COMPILER={self.toolchain['cxx']}", + f"-DCMAKE_PREFIX_PATH={self.prefix}", + "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF", + "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF", + "-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF", + "-DFETCHCONTENT_FULLY_DISCONNECTED=ON", + *self.cmake_platform, + *options, + ], + ) + self.run( + name + "-build", + [ + self.toolchain["cmake"], + "--build", + directory, + "--parallel", + self.args.jobs, + ], + ) + self.run(name + "-install", [self.toolchain["cmake"], "--install", directory]) + + def meson(self, name, options): + directory = self.output / "build" / name + meson = [sys.executable, self.sources["meson"] / "meson.py"] + quote = subprocess.list2cmdline if self.windows else shlex.join + include = quote([f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"]) + link = ( + [f"/LIBPATH:{self.prefix / 'lib'}"] + if self.windows + else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"] + ) + self.environment.update( + {"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)} + ) + self.run( + name + "-configure", + [ + *meson, + "setup", + directory, + self.sources[name], + f"--prefix={self.prefix}", + "--libdir=lib", + "--buildtype=release", + "--wrap-mode=nofallback", + "-Dauto_features=disabled", + "-Ddefault_library=shared", + *options, + ], + ) + self.run( + name + "-build", [*meson, "compile", "-C", directory, "-j", self.args.jobs] + ) + self.run( + name + "-install", [*meson, "install", "-C", directory, "--no-rebuild"] + ) + + def build(self): + self.output.mkdir() + manifest = MANIFEST.read_bytes() + prepare_sources(self.args.archives, self.output / "sources", manifest) + self.sources = { + s.name: self.output / "sources" / s.root for s in load_sources(manifest) + } + self.record["manifest_sha256"] = hashlib.sha256(manifest).hexdigest() + self.record["tools"] = { + name: str(path) for name, path in self.toolchain.items() + } + self.record["python"] = sys.version + self.run("cmake-version", [self.toolchain["cmake"], "--version"]) + self.run("pkg-config-version", [self.toolchain["pkg_config"], "--version"]) + self.cmake("ninja", ["-DBUILD_TESTING=OFF"], bootstrap=True) + self.cmake( + "zlib", + [ + "-DZLIB_BUILD_TESTING=OFF", + "-DZLIB_BUILD_SHARED=ON", + "-DZLIB_BUILD_STATIC=OFF", + ], + ) + self.cmake( + "pcre2", + [ + "-DBUILD_SHARED_LIBS=ON", + "-DBUILD_STATIC_LIBS=OFF", + "-DPCRE2_BUILD_TESTS=OFF", + "-DPCRE2_BUILD_PCRE2GREP=OFF", + "-DPCRE2_SUPPORT_LIBZ=OFF", + "-DPCRE2_SUPPORT_LIBBZ2=OFF", + "-DPCRE2_SUPPORT_LIBREADLINE=OFF", + "-DPCRE2_SUPPORT_LIBEDIT=OFF", + ], + ) + ffi_build = self.output / "build/libffi" + ffi_build.mkdir() + environment = self.environment.copy() + if self.windows: + wrapper = shlex.quote((self.sources["libffi"] / "msvcc.sh").as_posix()) + architecture = ( + "-m64" if self.args.target.startswith("x86_64-") else "-marm64" + ) + environment.update( + { + "CC": f"{wrapper} {architecture}", + "CXX": f"{wrapper} {architecture}", + "LD": "link", + "CPP": "cl -nologo -EP", + "CXXCPP": "cl -nologo -EP", + "CPPFLAGS": "-DFFI_BUILDING_DLL", + } + ) + self.run( + "libffi-configure", + [ + self.toolchain["shell"], + (self.sources["libffi"] / "configure").as_posix(), + f"--prefix={self.prefix.as_posix()}", + "--enable-shared", + "--disable-static", + "--disable-docs", + ], + cwd=ffi_build, + environment=environment, + ) + self.run( + "libffi-build", + [self.toolchain["make"], f"-j{self.args.jobs}"], + cwd=ffi_build, + environment=environment, + ) + self.run( + "libffi-install", + [self.toolchain["make"], "install"], + cwd=ffi_build, + environment=environment, + ) + self.cmake( + "opus", + [ + "-DOPUS_BUILD_SHARED_LIBRARY=ON", + "-DOPUS_BUILD_TESTING=OFF", + "-DOPUS_BUILD_PROGRAMS=OFF", + ], + ) + self.meson("proxy-libintl", []) + self.meson( + "glib", + [ + "-Dtests=false", + "-Dinstalled_tests=false", + "-Dnls=disabled", + "-Ddocumentation=false", + "-Dintrospection=disabled", + "-Dlibmount=disabled", + "-Dselinux=disabled", + "-Dxattr=false", + ], + ) + self.meson( + "gstreamer", + [ + "-Dregistry=false", + "-Doption-parsing=false", + "-Dtracer_hooks=false", + "-Dgst_parse=false", + "-Dtools=disabled", + "-Dptp-helper=disabled", + ], + ) + self.meson( + "gst-plugins-base", + [ + "-Dapp=enabled", + "-Daudioconvert=enabled", + "-Daudioresample=enabled", + "-Dopus=enabled", + "-Dgl=disabled", + ], + ) + self.meson("gst-plugins-good", ["-Drtp=enabled", "-Drtpmanager=enabled"]) + (self.output / "built.json").write_text( + json.dumps(self.record, indent=2) + "\n", encoding="utf-8" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ( + "archives", + "output", + "cc", + "cxx", + "cmake", + "make", + "pkg-config", + "shell", + ): + parser.add_argument(f"--{name}", type=Path, required=True) + parser.add_argument( + "--bootstrap-make", + type=Path, + help="NMake on Windows; defaults to --make elsewhere", + ) + parser.add_argument("--target", required=True) + parser.add_argument( + "--deployment-target", + help="Required on macOS; use the existing supported release minimum", + ) + parser.add_argument("--jobs", type=int, default=8) + args = parser.parse_args() + if sys.version_info < (3, 12) or args.jobs < 1: + parser.error("Python 3.12+ and a positive --jobs value are required") + NativeBuild(args, os.environ).build() + + +if __name__ == "__main__": + main() diff --git a/third_party/voice/test_build_native.py b/third_party/voice/test_build_native.py new file mode 100644 index 0000000000..bf04af8a2c --- /dev/null +++ b/third_party/voice/test_build_native.py @@ -0,0 +1,182 @@ +"""Check native build preconditions and subprocess failure propagation.""" + +import json +import os +from pathlib import Path +import platform +import shlex +import shutil +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from build_native import NativeBuild, validate_target + + +class NativeBuildTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + machine = platform.machine().lower() + architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(machine, machine) + suffix = { + "Darwin": "apple-darwin", + "Linux": "unknown-linux-gnu", + "Windows": "pc-windows-msvc", + }[platform.system()] + self.args = SimpleNamespace( + target=f"{architecture}-{suffix}", + deployment_target="11.0", + output=self.root / "build output", + archives=self.root / "archives", + cc=Path(sys.executable), + cxx=Path(sys.executable), + cmake=Path(sys.executable), + make=Path(sys.executable), + pkg_config=Path(sys.executable), + shell=Path(sys.executable), + bootstrap_make=None, + jobs=2, + ) + self.environment = { + **os.environ, + "INCLUDE": "fixture include", + "LIB": "fixture lib", + } + + def test_cli_entrypoint_imports_its_sibling_with_safe_path_enabled(self): + result = subprocess.run( + [ + sys.executable, + "-P", + str(Path(__file__).with_name("build_native.py")), + "--help", + ], + cwd=self.root, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("--archives", result.stdout) + + def test_rejects_cross_host_and_musl_builds(self): + for target, system, machine, libc in [ + ("x86_64-pc-windows-msvc", "Darwin", "x86_64", ""), + ("aarch64-apple-darwin", "Darwin", "x86_64", ""), + ("x86_64-unknown-linux-gnu", "Linux", "x86_64", "musl"), + ("x86_64-unknown-linux-musl", "Linux", "x86_64", "musl"), + ]: + with self.subTest(target=target, system=system): + with self.assertRaises(ValueError): + validate_target(target, system, machine, libc, "11.0") + + def test_requires_explicit_macos_deployment_target(self): + with self.assertRaisesRegex(ValueError, "Declare the macOS deployment target"): + validate_target("aarch64-apple-darwin", "Darwin", "arm64", "", None) + + def test_missing_tool_does_not_create_output(self): + self.args.cc = self.root / "missing compiler" + with self.assertRaisesRegex(ValueError, "Missing build tool"): + NativeBuild(self.args, self.environment) + self.assertFalse(self.args.output.exists()) + + @unittest.skipIf(os.name == "nt", "Exercises the Unix clang++ driver symlink") + def test_bootstrap_links_cpp_runtime_through_compiler_symlink(self): + tools = {name: shutil.which(name) for name in ("clang", "cmake", "make")} + if not all(tools.values()): + self.skipTest("Requires clang, CMake and make") + compiler = self.root / "clang++" + compiler.symlink_to(Path(tools["clang"]).resolve()) + self.args.cc = Path(tools["clang"]) + self.args.cxx = compiler + self.args.cmake = Path(tools["cmake"]) + self.args.make = Path(tools["make"]) + source = self.root / "cpp-source" + source.mkdir() + (source / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.15)\n" + "project(cpp_driver LANGUAGES CXX)\n" + "add_executable(cpp_driver main.cpp)\n" + "install(TARGETS cpp_driver DESTINATION bin)\n" + ) + (source / "main.cpp").write_text( + '#include \nint main() { std::cout << "linked"; }\n' + ) + build = NativeBuild(self.args, self.environment) + build.output.mkdir() + build.sources = {"cpp-driver": source} + build.cmake("cpp-driver", [], bootstrap=True) + result = subprocess.run( + [build.tools / "bin" / "cpp_driver"], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(result.stdout, "linked") + + def test_build_refuses_existing_output_before_reading_sources(self): + self.args.output.mkdir() + (self.args.output / "keep").write_text("untouched") + with self.assertRaises(FileExistsError): + NativeBuild(self.args, self.environment).build() + self.assertEqual( + {p.name: p.read_text() for p in self.args.output.iterdir()}, + {"keep": "untouched"}, + ) + + def test_failure_retains_log_and_state_without_completion_marker(self): + build = NativeBuild(self.args, self.environment) + build.output.mkdir() + command = [ + sys.executable, + "-c", + "print('synthetic failure'); raise SystemExit(23)", + ] + with self.assertRaises(subprocess.CalledProcessError) as error: + build.run("failure", command) + self.assertEqual(error.exception.returncode, 23) + self.assertEqual( + (build.output / "failure.log").read_text().strip(), "synthetic failure" + ) + self.assertEqual( + json.loads((build.output / "build-state.json").read_text()), + { + "target": self.args.target, + "deployment_target": "11.0", + "steps": [{"name": "failure", "command": command, "exit_code": 23}], + }, + ) + self.assertFalse((build.output / "built.json").exists()) + + def test_ambient_native_discovery_variables_are_not_inherited(self): + inherited = { + **self.environment, + "PKG_CONFIG_PATH": "/ambient", + "CMAKE_PREFIX_PATH": "/ambient", + "CPATH": "/ambient", + "CFLAGS": "-I/ambient", + "LDFLAGS": "-L/ambient", + } + environment = NativeBuild(self.args, inherited).environment + self.assertFalse(any("/ambient" in value for value in environment.values())) + self.assertEqual(environment["PKG_CONFIG_PATH"], "") + + def test_meson_receives_private_link_inputs_with_spaces(self): + build = NativeBuild(self.args, self.environment) + build.sources = {"meson": self.root / "meson", "glib": self.root / "glib"} + with patch.object(build, "run"): + build.meson("glib", []) + if build.windows: + self.assertEqual( + build.environment["LDFLAGS"], f'"/LIBPATH:{build.prefix / "lib"}"' + ) + else: + self.assertEqual( + shlex.split(build.environment["LDFLAGS"]), + [f"-L{build.prefix / 'lib'}", f"-Wl,-rpath,{build.prefix / 'lib'}"], + )