Add native voice dependency build recipe (#41890)

## Why

The pinned voice source inputs can be prepared but do not yet provide compiled
native libraries for downstream integration.

## What changed

- Add `build_native.py` to build a shared-library prefix for the supported voice
  dependencies on native x64 and ARM64 GNU/Linux, macOS, and Windows MSVC hosts.
- Require explicit toolchain inputs, isolate dependency discovery to the output
  prefix, and record command logs and build state for provenance and failures.
- Expose the recipe and source inputs through `//third_party/voice:build_inputs`
  and document its prerequisites, outputs, and integration boundaries.

## Testing

- `PYTHONSAFEPATH=1 python3 -m unittest discover -s third_party/voice -p 'test_build_native.py'`

GitOrigin-RevId: 132d93561a8e1bb178518f1248cf87d031060aae
This commit is contained in:
Benjamin Carlsson
2026-08-31 17:53:01 +00:00
committed by copyberry
parent 379d50be35
commit fc7d34ad67
4 changed files with 621 additions and 0 deletions

View File

@@ -27,3 +27,12 @@ filegroup(
tags = ["manual"],
visibility = ["//visibility:public"],
)
filegroup(
name = "build_inputs",
srcs = [
"build_native.py",
":source_inputs",
],
visibility = ["//visibility:public"],
)

View File

@@ -46,3 +46,32 @@ Checksums establish input identity, not security or license approval. Native
compilation, final Cargo/Bazel linking, installed packages, minimum OS support
and duplex audio validation remain separate stages. These inputs do not establish
a shared Opus build with Rust consumers or a reduced dependency count.
Rust `opus` 0.4.0 is available through Socket. Adding Rust transport dependencies
and establishing a shared Opus build remain separate integration work.
## Native build recipe
`build_native.py` runs the unmodified upstream build systems in a new output
directory, using the same archives. Specify the target and existing compiler,
CMake, make, pkg-config and shell paths explicitly. It requires a matching
native host: GNU Linux, macOS, or Windows MSVC, on x64 or ARM64.
On macOS, specify the existing release deployment target with
`--deployment-target`; the host OS version is not an acceptable default.
Windows requires the normal Visual Studio SDK environment, GNU make and a
POSIX shell for upstream libffi, and `--bootstrap-make` pointing to NMake.
The recipe does not install these build prerequisites or patch upstream sources.
Outputs are under `prefix/`, build tools under `tools/`, and logs beside them.
`build-state.json` records completed commands and failures; `built.json` exists
only when every build/install command succeeds. Failed builds retain their logs
and must use a new output directory on retry. CMake compiler-identification logs
and the recorded tool/configuration inputs remain part of the build provenance.
The recipe disables optional plugins and Meson fallback dependency resolution,
with pkg-config restricted to this prefix. Only system ABI libraries/frameworks
may remain external; runtime closure inspection must verify that independently.
`//third_party/voice:build_inputs` exposes the recipe and source inputs to Bazel.
Neither this filegroup nor a successful prefix build proves final Cargo/Bazel
linkage, safe private runtime loading, or an installed voice-capable Codex package.

401
third_party/voice/build_native.py vendored Normal file
View File

@@ -0,0 +1,401 @@
#!/usr/bin/env python3
"""Build the candidate native voice prefix with upstream build systems."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import re
import shlex
import subprocess
import sys
# Match the package builder: import this script's sibling under PYTHONSAFEPATH,
# without adding the caller's working directory to the module search path.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from prepare_sources import MANIFEST, load_sources, prepare_sources
TARGET_SYSTEMS = {
"apple-darwin": "Darwin",
"unknown-linux-gnu": "Linux",
"pc-windows-msvc": "Windows",
}
def validate_target(target, system, machine, libc, deployment_target):
architecture, separator, suffix = target.partition("-")
host_architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(
machine.lower(), machine.lower()
)
if (
not separator
or architecture not in ("aarch64", "x86_64")
or suffix not in TARGET_SYSTEMS
):
raise ValueError(f"Unsupported native voice target: {target}")
if TARGET_SYSTEMS[suffix] != system or architecture != host_architecture:
raise ValueError("Use a native build host matching the requested target")
if system == "Linux" and libc != "glibc":
raise ValueError("Native Linux voice builds require glibc")
if system == "Darwin" and not re.fullmatch(
r"[0-9]+\.[0-9]+(?:\.[0-9]+)?", deployment_target or ""
):
raise ValueError(
"Declare the macOS deployment target; do not inherit the host default"
)
class NativeBuild:
def __init__(self, args, inherited_environment):
validate_target(
args.target,
platform.system(),
platform.machine(),
platform.libc_ver()[0],
args.deployment_target,
)
self.args = args
self.output = args.output.absolute()
self.prefix = self.output / "prefix"
self.tools = self.output / "tools"
self.windows = args.target.endswith("windows-msvc")
# Compiler drivers such as clang++ select link behavior from argv[0].
self.toolchain = {
name: getattr(args, name).absolute()
for name in ("cc", "cxx", "cmake", "make", "pkg_config", "shell")
}
self.bootstrap_make = (args.bootstrap_make or args.make).absolute()
for tool in (*self.toolchain.values(), self.bootstrap_make):
if not tool.is_file():
raise ValueError(f"Missing build tool: {tool}")
if self.windows and not all(
inherited_environment.get(key) for key in ("INCLUDE", "LIB")
):
raise ValueError(
"Initialize the standard Visual Studio build environment first"
)
self.environment = {
key: value
for key, value in inherited_environment.items()
if key
in (
"HOME",
"TMPDIR",
"TMP",
"TEMP",
"SYSTEMROOT",
"SystemRoot",
"WINDIR",
"COMSPEC",
"INCLUDE",
"LIB",
"LIBPATH",
"HTTPS_PROXY",
"HTTP_PROXY",
"NO_PROXY",
)
}
paths = [
str(self.tools / "bin"),
*(str(p.parent) for p in self.toolchain.values()),
]
paths += (
inherited_environment.get("PATH", "").split(os.pathsep)
if self.windows
else ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
)
self.environment.update(
{
"PATH": os.pathsep.join(dict.fromkeys(paths)),
"LANG": "C",
"LC_ALL": "C",
"PYTHONDONTWRITEBYTECODE": "1",
"CC": str(self.toolchain["cc"]),
"CXX": str(self.toolchain["cxx"]),
"PKG_CONFIG": str(self.toolchain["pkg_config"]),
"PKG_CONFIG_PATH": "",
"PKG_CONFIG_LIBDIR": os.pathsep.join(
str(self.prefix / p) for p in ("lib/pkgconfig", "share/pkgconfig")
),
"CMAKE_PREFIX_PATH": str(self.prefix),
"NINJA": str(
self.tools / "bin" / ("ninja.exe" if self.windows else "ninja")
),
}
)
self.cmake_platform = []
if args.target.endswith("apple-darwin"):
self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target
self.cmake_platform = [
f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}"
]
self.record = {
"target": args.target,
"deployment_target": args.deployment_target,
"steps": [],
}
def run(self, name, command, cwd=None, environment=None):
command = [str(part) for part in command]
step = {"name": name, "command": command}
self.record["steps"].append(step)
print(f"Building {name}", flush=True)
with (self.output / f"{name}.log").open("w", encoding="utf-8") as log:
result = subprocess.run(
command,
cwd=cwd or self.output,
env=environment or self.environment,
stdout=log,
stderr=subprocess.STDOUT,
check=False,
)
step["exit_code"] = result.returncode
(self.output / "build-state.json").write_text(
json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
)
result.check_returncode()
def cmake(self, name, options, *, bootstrap=False):
directory = self.output / "build" / name
prefix = self.tools if bootstrap else self.prefix
generator = (
("NMake Makefiles" if self.windows else "Unix Makefiles")
if bootstrap
else "Ninja"
)
make = self.bootstrap_make if bootstrap else self.environment["NINJA"]
self.run(
name + "-configure",
[
self.toolchain["cmake"],
"-S",
self.sources[name],
"-B",
directory,
"-G",
generator,
f"-DCMAKE_MAKE_PROGRAM={make}",
"-DCMAKE_BUILD_TYPE=Release",
f"-DCMAKE_INSTALL_PREFIX={prefix}",
"-DCMAKE_INSTALL_LIBDIR=lib",
f"-DCMAKE_C_COMPILER={self.toolchain['cc']}",
f"-DCMAKE_CXX_COMPILER={self.toolchain['cxx']}",
f"-DCMAKE_PREFIX_PATH={self.prefix}",
"-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF",
"-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF",
"-DCMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH=OFF",
"-DFETCHCONTENT_FULLY_DISCONNECTED=ON",
*self.cmake_platform,
*options,
],
)
self.run(
name + "-build",
[
self.toolchain["cmake"],
"--build",
directory,
"--parallel",
self.args.jobs,
],
)
self.run(name + "-install", [self.toolchain["cmake"], "--install", directory])
def meson(self, name, options):
directory = self.output / "build" / name
meson = [sys.executable, self.sources["meson"] / "meson.py"]
quote = subprocess.list2cmdline if self.windows else shlex.join
include = quote([f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"])
link = (
[f"/LIBPATH:{self.prefix / 'lib'}"]
if self.windows
else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"]
)
self.environment.update(
{"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)}
)
self.run(
name + "-configure",
[
*meson,
"setup",
directory,
self.sources[name],
f"--prefix={self.prefix}",
"--libdir=lib",
"--buildtype=release",
"--wrap-mode=nofallback",
"-Dauto_features=disabled",
"-Ddefault_library=shared",
*options,
],
)
self.run(
name + "-build", [*meson, "compile", "-C", directory, "-j", self.args.jobs]
)
self.run(
name + "-install", [*meson, "install", "-C", directory, "--no-rebuild"]
)
def build(self):
self.output.mkdir()
manifest = MANIFEST.read_bytes()
prepare_sources(self.args.archives, self.output / "sources", manifest)
self.sources = {
s.name: self.output / "sources" / s.root for s in load_sources(manifest)
}
self.record["manifest_sha256"] = hashlib.sha256(manifest).hexdigest()
self.record["tools"] = {
name: str(path) for name, path in self.toolchain.items()
}
self.record["python"] = sys.version
self.run("cmake-version", [self.toolchain["cmake"], "--version"])
self.run("pkg-config-version", [self.toolchain["pkg_config"], "--version"])
self.cmake("ninja", ["-DBUILD_TESTING=OFF"], bootstrap=True)
self.cmake(
"zlib",
[
"-DZLIB_BUILD_TESTING=OFF",
"-DZLIB_BUILD_SHARED=ON",
"-DZLIB_BUILD_STATIC=OFF",
],
)
self.cmake(
"pcre2",
[
"-DBUILD_SHARED_LIBS=ON",
"-DBUILD_STATIC_LIBS=OFF",
"-DPCRE2_BUILD_TESTS=OFF",
"-DPCRE2_BUILD_PCRE2GREP=OFF",
"-DPCRE2_SUPPORT_LIBZ=OFF",
"-DPCRE2_SUPPORT_LIBBZ2=OFF",
"-DPCRE2_SUPPORT_LIBREADLINE=OFF",
"-DPCRE2_SUPPORT_LIBEDIT=OFF",
],
)
ffi_build = self.output / "build/libffi"
ffi_build.mkdir()
environment = self.environment.copy()
if self.windows:
wrapper = shlex.quote((self.sources["libffi"] / "msvcc.sh").as_posix())
architecture = (
"-m64" if self.args.target.startswith("x86_64-") else "-marm64"
)
environment.update(
{
"CC": f"{wrapper} {architecture}",
"CXX": f"{wrapper} {architecture}",
"LD": "link",
"CPP": "cl -nologo -EP",
"CXXCPP": "cl -nologo -EP",
"CPPFLAGS": "-DFFI_BUILDING_DLL",
}
)
self.run(
"libffi-configure",
[
self.toolchain["shell"],
(self.sources["libffi"] / "configure").as_posix(),
f"--prefix={self.prefix.as_posix()}",
"--enable-shared",
"--disable-static",
"--disable-docs",
],
cwd=ffi_build,
environment=environment,
)
self.run(
"libffi-build",
[self.toolchain["make"], f"-j{self.args.jobs}"],
cwd=ffi_build,
environment=environment,
)
self.run(
"libffi-install",
[self.toolchain["make"], "install"],
cwd=ffi_build,
environment=environment,
)
self.cmake(
"opus",
[
"-DOPUS_BUILD_SHARED_LIBRARY=ON",
"-DOPUS_BUILD_TESTING=OFF",
"-DOPUS_BUILD_PROGRAMS=OFF",
],
)
self.meson("proxy-libintl", [])
self.meson(
"glib",
[
"-Dtests=false",
"-Dinstalled_tests=false",
"-Dnls=disabled",
"-Ddocumentation=false",
"-Dintrospection=disabled",
"-Dlibmount=disabled",
"-Dselinux=disabled",
"-Dxattr=false",
],
)
self.meson(
"gstreamer",
[
"-Dregistry=false",
"-Doption-parsing=false",
"-Dtracer_hooks=false",
"-Dgst_parse=false",
"-Dtools=disabled",
"-Dptp-helper=disabled",
],
)
self.meson(
"gst-plugins-base",
[
"-Dapp=enabled",
"-Daudioconvert=enabled",
"-Daudioresample=enabled",
"-Dopus=enabled",
"-Dgl=disabled",
],
)
self.meson("gst-plugins-good", ["-Drtp=enabled", "-Drtpmanager=enabled"])
(self.output / "built.json").write_text(
json.dumps(self.record, indent=2) + "\n", encoding="utf-8"
)
def main():
parser = argparse.ArgumentParser(description=__doc__)
for name in (
"archives",
"output",
"cc",
"cxx",
"cmake",
"make",
"pkg-config",
"shell",
):
parser.add_argument(f"--{name}", type=Path, required=True)
parser.add_argument(
"--bootstrap-make",
type=Path,
help="NMake on Windows; defaults to --make elsewhere",
)
parser.add_argument("--target", required=True)
parser.add_argument(
"--deployment-target",
help="Required on macOS; use the existing supported release minimum",
)
parser.add_argument("--jobs", type=int, default=8)
args = parser.parse_args()
if sys.version_info < (3, 12) or args.jobs < 1:
parser.error("Python 3.12+ and a positive --jobs value are required")
NativeBuild(args, os.environ).build()
if __name__ == "__main__":
main()

182
third_party/voice/test_build_native.py vendored Normal file
View File

@@ -0,0 +1,182 @@
"""Check native build preconditions and subprocess failure propagation."""
import json
import os
from pathlib import Path
import platform
import shlex
import shutil
import subprocess
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from build_native import NativeBuild, validate_target
class NativeBuildTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
machine = platform.machine().lower()
architecture = {"arm64": "aarch64", "amd64": "x86_64"}.get(machine, machine)
suffix = {
"Darwin": "apple-darwin",
"Linux": "unknown-linux-gnu",
"Windows": "pc-windows-msvc",
}[platform.system()]
self.args = SimpleNamespace(
target=f"{architecture}-{suffix}",
deployment_target="11.0",
output=self.root / "build output",
archives=self.root / "archives",
cc=Path(sys.executable),
cxx=Path(sys.executable),
cmake=Path(sys.executable),
make=Path(sys.executable),
pkg_config=Path(sys.executable),
shell=Path(sys.executable),
bootstrap_make=None,
jobs=2,
)
self.environment = {
**os.environ,
"INCLUDE": "fixture include",
"LIB": "fixture lib",
}
def test_cli_entrypoint_imports_its_sibling_with_safe_path_enabled(self):
result = subprocess.run(
[
sys.executable,
"-P",
str(Path(__file__).with_name("build_native.py")),
"--help",
],
cwd=self.root,
capture_output=True,
text=True,
check=False,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("--archives", result.stdout)
def test_rejects_cross_host_and_musl_builds(self):
for target, system, machine, libc in [
("x86_64-pc-windows-msvc", "Darwin", "x86_64", ""),
("aarch64-apple-darwin", "Darwin", "x86_64", ""),
("x86_64-unknown-linux-gnu", "Linux", "x86_64", "musl"),
("x86_64-unknown-linux-musl", "Linux", "x86_64", "musl"),
]:
with self.subTest(target=target, system=system):
with self.assertRaises(ValueError):
validate_target(target, system, machine, libc, "11.0")
def test_requires_explicit_macos_deployment_target(self):
with self.assertRaisesRegex(ValueError, "Declare the macOS deployment target"):
validate_target("aarch64-apple-darwin", "Darwin", "arm64", "", None)
def test_missing_tool_does_not_create_output(self):
self.args.cc = self.root / "missing compiler"
with self.assertRaisesRegex(ValueError, "Missing build tool"):
NativeBuild(self.args, self.environment)
self.assertFalse(self.args.output.exists())
@unittest.skipIf(os.name == "nt", "Exercises the Unix clang++ driver symlink")
def test_bootstrap_links_cpp_runtime_through_compiler_symlink(self):
tools = {name: shutil.which(name) for name in ("clang", "cmake", "make")}
if not all(tools.values()):
self.skipTest("Requires clang, CMake and make")
compiler = self.root / "clang++"
compiler.symlink_to(Path(tools["clang"]).resolve())
self.args.cc = Path(tools["clang"])
self.args.cxx = compiler
self.args.cmake = Path(tools["cmake"])
self.args.make = Path(tools["make"])
source = self.root / "cpp-source"
source.mkdir()
(source / "CMakeLists.txt").write_text(
"cmake_minimum_required(VERSION 3.15)\n"
"project(cpp_driver LANGUAGES CXX)\n"
"add_executable(cpp_driver main.cpp)\n"
"install(TARGETS cpp_driver DESTINATION bin)\n"
)
(source / "main.cpp").write_text(
'#include <iostream>\nint main() { std::cout << "linked"; }\n'
)
build = NativeBuild(self.args, self.environment)
build.output.mkdir()
build.sources = {"cpp-driver": source}
build.cmake("cpp-driver", [], bootstrap=True)
result = subprocess.run(
[build.tools / "bin" / "cpp_driver"],
check=True,
capture_output=True,
text=True,
)
self.assertEqual(result.stdout, "linked")
def test_build_refuses_existing_output_before_reading_sources(self):
self.args.output.mkdir()
(self.args.output / "keep").write_text("untouched")
with self.assertRaises(FileExistsError):
NativeBuild(self.args, self.environment).build()
self.assertEqual(
{p.name: p.read_text() for p in self.args.output.iterdir()},
{"keep": "untouched"},
)
def test_failure_retains_log_and_state_without_completion_marker(self):
build = NativeBuild(self.args, self.environment)
build.output.mkdir()
command = [
sys.executable,
"-c",
"print('synthetic failure'); raise SystemExit(23)",
]
with self.assertRaises(subprocess.CalledProcessError) as error:
build.run("failure", command)
self.assertEqual(error.exception.returncode, 23)
self.assertEqual(
(build.output / "failure.log").read_text().strip(), "synthetic failure"
)
self.assertEqual(
json.loads((build.output / "build-state.json").read_text()),
{
"target": self.args.target,
"deployment_target": "11.0",
"steps": [{"name": "failure", "command": command, "exit_code": 23}],
},
)
self.assertFalse((build.output / "built.json").exists())
def test_ambient_native_discovery_variables_are_not_inherited(self):
inherited = {
**self.environment,
"PKG_CONFIG_PATH": "/ambient",
"CMAKE_PREFIX_PATH": "/ambient",
"CPATH": "/ambient",
"CFLAGS": "-I/ambient",
"LDFLAGS": "-L/ambient",
}
environment = NativeBuild(self.args, inherited).environment
self.assertFalse(any("/ambient" in value for value in environment.values()))
self.assertEqual(environment["PKG_CONFIG_PATH"], "")
def test_meson_receives_private_link_inputs_with_spaces(self):
build = NativeBuild(self.args, self.environment)
build.sources = {"meson": self.root / "meson", "glib": self.root / "glib"}
with patch.object(build, "run"):
build.meson("glib", [])
if build.windows:
self.assertEqual(
build.environment["LDFLAGS"], f'"/LIBPATH:{build.prefix / "lib"}"'
)
else:
self.assertEqual(
shlex.split(build.environment["LDFLAGS"]),
[f"-L{build.prefix / 'lib'}", f"-Wl,-rpath,{build.prefix / 'lib'}"],
)