Add explicit toolchain inputs for native voice builds (#43109)

## Why

Native voice builds ignore ambient compiler flags, but need a way to accept declared compiler, linker, and archive-tool inputs.

## What changed

- Add repeated `--c-flag`, `--cxx-flag`, and `--link-flag` options plus optional `--ar` and `--ranlib` paths for Unix builds. Record flags and pass them through CMake, Meson, and libffi, including Objective-C flags on macOS.
- Preserve literal libffi compiler arguments with response files. Reject flags containing whitespace, libffi paths requiring shell quoting, and Unix toolchain overrides on Windows.
- Add a pinned Bazel `pkg-config` tool built with LLVM archive tools and native macOS configure checks. Register the pinned Make toolchain and patch its bootstrap to pass preprocessor flags and enable cross-compilation configuration.

## Testing

Extend native-build tests to verify compiler and linker flags, explicit archive tools, Meson and Objective-C inputs, literal definitions through shell and recursive Make expansion, and rejection of unsupported overrides.

GitOrigin-RevId: e5ea541ec76abfa170f6b0411b0f14cd698c07e7
This commit is contained in:
Benjamin Carlsson
2026-09-05 21:47:17 +00:00
committed by copyberry
parent e4ce83419b
commit e67a8ae6c9
11 changed files with 402 additions and 15 deletions
+58
View File
@@ -1,3 +1,6 @@
load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain")
load(":pkg_config.bzl", "pkg_config")
# All native build consumers must receive the same pinned source manifest.
exports_files(["opus-toolchain.cmake"])
@@ -45,3 +48,58 @@ filegroup(
],
visibility = ["//visibility:public"],
)
alias(
name = "pkg_config_linker",
actual = select({
"@platforms//os:macos": "@llvm//tools:ld64.lld",
"//conditions:default": "@llvm//tools:ld.lld",
}),
)
# Keep the pinned pkg-config sources, but explicitly use LLVM archive tools.
# The upstream bootstrap otherwise selects host ar/ranlib on macOS.
pkg_config(
name = "pkg_config",
build_data = [
":pkg_config_linker",
"@llvm//tools:llvm-ar",
"@llvm//tools:llvm-ranlib",
],
# foreign_cc normalizes copied source timestamps, preserving the release
# configure files when remote execution materializes inputs in a new order.
configure_in_place = True,
configure_options = [
"--with-internal-glib",
"--disable-shared",
],
copts = ["-Wno-int-conversion"],
env = {
"AR": "$(execpath @llvm//tools:llvm-ar)",
"LD": "$(execpath :pkg_config_linker)",
"PKG_CONFIG": "/bin/false",
"RANLIB": "$(execpath @llvm//tools:llvm-ranlib)",
},
lib_source = "@pkgconfig_src//:all_srcs",
# foreign_cc rewrites *-config files as text. Keep the executable out of
# that transformation, which otherwise corrupts its native code signature.
out_binaries = ["pkg-config-tool"],
out_include_dir = "",
out_static_libs = [],
postfix_script = 'mv "$$INSTALLDIR/bin/pkg-config" "$$INSTALLDIR/bin/pkg-config-tool"',
tags = ["manual"],
target_compatible_with = select({
"@platforms//os:windows": ["@platforms//:incompatible"],
"//conditions:default": [],
}),
# Internal GLib needs no pkg-config. Avoid bootstrapping another copy first;
# the explicit failing command above forbids falling back to a host copy.
toolchain = "@rules_foreign_cc//toolchains:preinstalled_pkgconfig_toolchain",
)
native_tool_toolchain(
name = "pkg_config_tool",
path = "$(execpath :pkg_config)",
tags = ["manual"],
target = ":pkg_config",
)
+9 -1
View File
@@ -77,7 +77,15 @@ transitive imports must satisfy the same private-library and system-import check
## Native build recipe
`build_native.py` runs the unmodified upstream build systems in a new output
directory, using the same archives. Specify the target and existing compiler,
directory, using the same archives. Unix builds accept repeated `--c-flag=...`,
`--cxx-flag=...`, `--link-flag=...` and optional `--ar`/`--ranlib` inputs.
These overrides are rejected on Windows; ambient flags remain ignored.
Libffi uses compiler response files to preserve literal definitions through
configure, recursive Make and libtool. Response-file paths must not require shell
quoting. Its
Autoconf recipe cannot preserve flags containing whitespace; those are rejected.
Declared archiver and ranlib paths must not require shell quoting for libffi.
Specify the target and existing compiler,
CMake, make, pkg-config and shell paths explicitly. It requires a matching
native host: GNU Linux, macOS, or Windows MSVC, on x64 or ARM64.
+75 -3
View File
@@ -66,6 +66,25 @@ class NativeBuild:
name: getattr(args, name).absolute()
for name in ("cc", "cxx", "cmake", "make", "pkg_config", "shell")
}
self.toolchain.update(
(name, path.absolute())
for name in ("ar", "ranlib")
if (path := getattr(args, name, None)) is not None
)
self.quote = subprocess.list2cmdline if self.windows else shlex.join
self.flags = {
variable: list(getattr(args, argument, []))
for variable, argument in (
("CFLAGS", "c_flag"),
("CXXFLAGS", "cxx_flag"),
("LDFLAGS", "link_flag"),
)
}
if self.windows and (
any(self.flags.values())
or any(name in self.toolchain for name in ("ar", "ranlib"))
):
raise ValueError("Explicit toolchain overrides require a Unix build host")
self.bootstrap_make = (args.bootstrap_make or args.make).absolute()
for tool in (*self.toolchain.values(), self.bootstrap_make):
if not tool.is_file():
@@ -127,6 +146,14 @@ class NativeBuild:
}
)
self.cmake_platform = []
self.environment.update(
(name, self.quote(flags)) for name, flags in self.flags.items() if flags
)
self.environment.update(
(name.upper(), self.quote([str(self.toolchain[name])]))
for name in ("ar", "ranlib")
if name in self.toolchain
)
if args.target.endswith("apple-darwin"):
self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target
self.cmake_platform = [
@@ -142,6 +169,7 @@ class NativeBuild:
self.record = {
"target": args.target,
"deployment_target": args.deployment_target,
"flags": self.flags,
"steps": [],
}
@@ -199,6 +227,11 @@ class NativeBuild:
"-DCMAKE_INSTALL_LIBDIR=lib",
f"-DCMAKE_C_COMPILER={self.toolchain['cc']}",
f"-DCMAKE_CXX_COMPILER={self.toolchain['cxx']}",
*(
f"-DCMAKE_{name.upper()}={self.toolchain[name]}"
for name in ("ar", "ranlib")
if name in self.toolchain
),
f"-DCMAKE_PREFIX_PATH={self.prefix}",
"-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF",
"-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF",
@@ -223,8 +256,7 @@ class NativeBuild:
def meson(self, name, options):
directory = self.output / "build" / name
meson = [sys.executable, self.sources["meson"] / "meson.py"]
quote = subprocess.list2cmdline if self.windows else shlex.join
include = quote([f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"])
include = f"{'/I' if self.windows else '-I'}{self.prefix / 'include'}"
link = (
[f"/LIBPATH:{self.prefix / 'lib'}"]
if self.windows
@@ -233,8 +265,14 @@ class NativeBuild:
if self.args.target.endswith("unknown-linux-gnu"):
link[-1] = "-Wl,-rpath,$ORIGIN:$ORIGIN/.."
self.environment.update(
{"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)}
{
name: self.quote([*flags, *(link if name == "LDFLAGS" else [include])])
for name, flags in self.flags.items()
}
)
if self.args.target.endswith("apple-darwin"):
self.environment["OBJC"] = str(self.toolchain["cc"])
self.environment["OBJCFLAGS"] = self.environment["CFLAGS"]
self.run(
name + "-configure",
[
@@ -340,6 +378,36 @@ class NativeBuild:
"CONFIG_SHELL": self.posix_path(self.toolchain["shell"]),
}
)
else:
for name in ("ar", "ranlib"):
if name in self.toolchain:
path = str(self.toolchain[name])
if shlex.quote(path) != path:
raise ValueError(
f"libffi {name} path cannot require shell quoting"
)
# Response files preserve compiler arguments through Autoconf's word
# splitting and Make/libtool's shell expansion without losing flags
# added by configure (such as -fexceptions).
for name, flags in self.flags.items():
if any(
any(character.isspace() for character in flag) for flag in flags
):
raise ValueError("libffi flags cannot contain whitespace")
if flags:
response = ffi_build / f"{name.lower()}.rsp"
if shlex.quote(str(response)) != str(response):
raise ValueError(
"libffi response-file path cannot require shell quoting"
)
response.write_text(
"\n".join(
'"' + flag.replace("\\", "\\\\").replace('"', '\\"') + '"'
for flag in flags
)
+ "\n"
)
environment[name] = f"@{response}"
self.run(
"libffi-configure",
[
@@ -446,6 +514,10 @@ def main():
type=Path,
help="NMake on Windows; defaults to --make elsewhere",
)
for name in ("ar", "ranlib"):
parser.add_argument(f"--{name}", type=Path)
for name in ("c-flag", "cxx-flag", "link-flag"):
parser.add_argument(f"--{name}", action="append", default=[])
parser.add_argument("--target", required=True)
parser.add_argument(
"--deployment-target",
+33
View File
@@ -0,0 +1,33 @@
"""Build the pinned pkg-config with native Mac configure checks."""
load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make_variant")
def pkg_config(name, **kwargs):
"""Keep Linux remote builds and select native Mac producers by OS and CPU."""
tags = kwargs.pop("tags", [])
configure_make_variant(
name = name + "_default",
tags = tags,
**kwargs
)
for arch in ["aarch64", "x86_64"]:
native.config_setting(
name = name + "_macos_" + arch,
constraint_values = ["@platforms//os:macos", "@platforms//cpu:" + arch],
)
configure_make_variant(
name = name + "_native_macos_" + arch,
exec_compatible_with = ["@platforms//os:macos", "@platforms//cpu:" + arch],
tags = tags + ["no-remote-exec"],
**kwargs
)
native.alias(
name = name,
tags = tags,
actual = select({
":" + name + "_macos_aarch64": ":" + name + "_native_macos_aarch64",
":" + name + "_macos_x86_64": ":" + name + "_native_macos_x86_64",
"//conditions:default": ":" + name + "_default",
}),
visibility = kwargs.get("visibility"),
)
+182 -9
View File
@@ -96,16 +96,39 @@ class NativeBuildTests(unittest.TestCase):
self.args.cxx = compiler
self.args.cmake = Path(tools["cmake"])
self.args.make = Path(tools["make"])
headers = self.root / "declared headers"
headers.mkdir()
(headers / "build_flag.h").write_text(
'#define BUILD_FLAG "linked with flags"\n'
)
self.args.cxx_flag = [f"-I{headers}"]
for name in ("ar", "ranlib"):
tool = shutil.which(name)
if tool is None:
self.skipTest(f"Requires {name}")
wrapper = self.root / f"declared {name}"
wrapper.write_text(
"#!/bin/sh\n"
f"touch {shlex.quote(str(self.root / (name + '.used')))}\n"
f'exec {shlex.quote(tool)} "$@"\n'
)
wrapper.chmod(0o755)
setattr(self.args, name, wrapper)
source = self.root / "cpp-source"
source.mkdir()
(source / "CMakeLists.txt").write_text(
"cmake_minimum_required(VERSION 3.15)\n"
"project(cpp_driver LANGUAGES CXX)\n"
"add_library(message STATIC message.cpp)\n"
"add_executable(cpp_driver main.cpp)\n"
"target_link_libraries(cpp_driver PRIVATE message)\n"
"install(TARGETS cpp_driver DESTINATION bin)\n"
)
(source / "main.cpp").write_text(
'#include <iostream>\nint main() { std::cout << "linked"; }\n'
"#include <iostream>\nconst char* message();\nint main() { std::cout << message(); }\n"
)
(source / "message.cpp").write_text(
'#include "build_flag.h"\nconst char* message() { return BUILD_FLAG; }\n'
)
build = NativeBuild(self.args, self.environment)
build.output.mkdir()
@@ -117,7 +140,98 @@ class NativeBuildTests(unittest.TestCase):
capture_output=True,
text=True,
)
self.assertEqual(result.stdout, "linked")
self.assertEqual(result.stdout, "linked with flags")
self.assertTrue(
all((self.root / (name + ".used")).is_file() for name in ("ar", "ranlib"))
)
@unittest.skipIf(os.name == "nt", "Exercises Autoconf's Unix tool expansion")
def test_libffi_rejects_quoted_archive_tool_paths(self):
for name in ("ar", "ranlib"):
for index, path in enumerate(("declared tool", "declared'tool")):
with self.subTest(name=name, path=path):
(self.root / path).touch()
setattr(self.args, name, self.root / path)
self.args.output = self.root / f"{name}-{index}"
build = NativeBuild(self.args, self.environment)
with (
patch(
"build_native.prepare_sources",
side_effect=lambda *args: (build.output / "build").mkdir(),
),
patch.object(build, "cmake"),
patch.object(build, "meson"),
patch.object(build, "run"),
self.assertRaisesRegex(ValueError, "libffi .* path"),
):
build.build()
setattr(self.args, name, None)
@unittest.skipIf(os.name == "nt", "Exercises Autoconf's Unix flag expansion")
def test_libffi_compiler_receives_literal_defines_without_shell_quotes(self):
compiler = shutil.which("cc")
if compiler is None:
self.skipTest("Requires a C compiler")
self.args.cc = Path(compiler)
make = shutil.which("make")
if make is None:
self.skipTest("Requires Make")
for index, (flag, expected) in enumerate(
[
('-DBUILD_FLAG="redacted"', "redacted\n"),
(r'-DBUILD_FLAG="C:\\voice"', "C:\\voice\n"),
('-DBUILD_FLAG="two words"', None),
]
):
with self.subTest(flag=flag):
self.args.c_flag = [flag]
self.args.output = self.root / str(index)
build = NativeBuild(self.args, self.environment)
with (
patch(
"build_native.prepare_sources",
side_effect=lambda *args: (build.output / "build").mkdir(),
),
patch.object(build, "cmake"),
patch.object(build, "meson"),
patch.object(build, "run") as run,
):
if " " in flag:
with self.assertRaisesRegex(ValueError, "libffi flags"):
build.build()
continue
build.build()
environment = next(
call.kwargs["environment"]
for call in run.call_args_list
if call.args[0] == "libffi-configure"
)
(build.output / "probe.c").write_text(
"#include <stdio.h>\nint main(void) { puts(BUILD_FLAG); }\n"
)
subprocess.run(
["/bin/sh", "-c", "$CC $CFLAGS probe.c -o probe"],
cwd=build.output,
env=environment,
check=True,
)
result = subprocess.run(
[build.output / "probe"], check=True, capture_output=True, text=True
)
self.assertEqual(result.stdout, expected)
(build.output / "Makefile").write_text(
f"CC={compiler}\nCFLAGS={environment['CFLAGS']} -fexceptions\n"
"MAKEOVERRIDES=\nall:\n\t$(MAKE) nested\n"
"nested:\n\t$(CC) $(CFLAGS) probe.c -o probe\n\t./probe\n"
)
result = subprocess.run(
[make, "--silent"],
cwd=build.output,
check=True,
capture_output=True,
text=True,
)
self.assertEqual(result.stdout, expected)
@unittest.skipIf(os.name == "nt", "Unix install paths; Windows layout is unchanged")
def test_cmake_installs_relative_library_paths(self):
@@ -130,11 +244,13 @@ class NativeBuildTests(unittest.TestCase):
setattr(self.args, argument, Path(shutil.which(tool)))
source = self.root / "library-source"
source.mkdir()
self.args.c_flag = ["-DBUILD_FLAG=42"]
self.args.link_flag = ["-Wl,-rpath,declared-relative-path"]
(source / "CMakeLists.txt").write_text(
"cmake_minimum_required(VERSION 3.15)\nproject(relative_paths LANGUAGES C)\n"
"add_library(fixture SHARED fixture.c)\ninstall(TARGETS fixture DESTINATION lib)\n"
)
(source / "fixture.c").write_text("int fixture(void) { return 42; }\n")
(source / "fixture.c").write_text("int fixture(void) { return BUILD_FLAG; }\n")
build = NativeBuild(self.args, self.environment)
build.output.mkdir()
build.sources = {"fixture": source}
@@ -145,14 +261,15 @@ class NativeBuildTests(unittest.TestCase):
metadata = inspect(build.tools / "lib/libfixture.dylib", self.args.target)
self.assertEqual(
(metadata.identity, metadata.rpaths),
("@rpath/libfixture.dylib", ("@loader_path",)),
("@rpath/libfixture.dylib", ("declared-relative-path", "@loader_path")),
)
else:
from linux_runtime import inspect
metadata = inspect(build.tools / "lib/libfixture.so", self.args.target)
self.assertEqual(
(metadata.identity, metadata.rpaths), ("libfixture.so", ("29=$ORIGIN",))
(metadata.identity, metadata.rpaths),
("libfixture.so", ("29=declared-relative-path:$ORIGIN",)),
)
def test_build_refuses_existing_output_before_reading_sources(self):
@@ -184,6 +301,7 @@ class NativeBuildTests(unittest.TestCase):
{
"target": self.args.target,
"deployment_target": "11.0",
"flags": {"CFLAGS": [], "CXXFLAGS": [], "LDFLAGS": []},
"steps": [{"name": "failure", "command": command, "exit_code": 23}],
},
)
@@ -203,22 +321,64 @@ class NativeBuildTests(unittest.TestCase):
self.assertEqual(environment["PKG_CONFIG_PATH"], "")
def test_meson_receives_private_link_inputs_with_spaces(self):
if platform.system() == "Darwin":
self.args.cc = Path(shutil.which("cc"))
self.args.c_flag = [] if os.name == "nt" else ["-DDECLARED_C=1"]
self.args.cxx_flag = [] if os.name == "nt" else ["-DDECLARED_CXX=1"]
self.args.link_flag = (
[] if os.name == "nt" else ["-Ldeclared input with spaces"]
)
build = NativeBuild(self.args, self.environment)
self.assertEqual(
build.record["flags"],
{
"CFLAGS": self.args.c_flag,
"CXXFLAGS": self.args.cxx_flag,
"LDFLAGS": self.args.link_flag,
},
)
build.sources = {"meson": self.root / "meson", "glib": self.root / "glib"}
with patch.object(build, "run"):
build.meson("glib", [])
if platform.system() == "Darwin":
subprocess.run(
[
build.environment["OBJC"],
*shlex.split(build.environment["OBJCFLAGS"]),
"-x",
"objective-c",
"-fsyntax-only",
"-",
],
input="#if DECLARED_C != 1\n#error missing declared flags\n#endif\n",
text=True,
check=True,
)
quote = subprocess.list2cmdline if build.windows else shlex.join
include = f"{'/I' if build.windows else '-I'}{build.prefix / 'include'}"
self.assertEqual(
{name: build.environment[name] for name in ("CFLAGS", "CXXFLAGS")},
{
"CFLAGS": quote([*self.args.c_flag, include]),
"CXXFLAGS": quote([*self.args.cxx_flag, include]),
},
)
if build.windows:
self.assertEqual(
build.environment["LDFLAGS"], f'"/LIBPATH:{build.prefix / "lib"}"'
build.environment["LDFLAGS"],
quote([*self.args.link_flag, f"/LIBPATH:{build.prefix / 'lib'}"]),
)
else:
self.assertEqual(
shlex.split(build.environment["LDFLAGS"]),
[
*self.args.link_flag,
f"-L{build.prefix / 'lib'}",
"-Wl,-rpath,$ORIGIN:$ORIGIN/.."
if build.args.target.endswith("unknown-linux-gnu")
else f"-Wl,-rpath,{build.prefix / 'lib'}",
(
"-Wl,-rpath,$ORIGIN:$ORIGIN/.."
if build.args.target.endswith("unknown-linux-gnu")
else f"-Wl,-rpath,{build.prefix / 'lib'}"
),
],
)
@@ -242,6 +402,19 @@ class NativeBuildTests(unittest.TestCase):
with self.assertRaises(subprocess.CalledProcessError):
build.posix_path(path)
def test_windows_rejects_unix_overrides_before_creating_output(self):
self.args.target = "x86_64-pc-windows-msvc"
for name, value in (
("c_flag", [r"/IC:\SDK\include"]),
("ranlib", Path(sys.executable)),
):
with patch("build_native.validate_target"):
setattr(self.args, name, value)
with self.assertRaisesRegex(ValueError, "Unix build host"):
NativeBuild(self.args, self.environment)
delattr(self.args, name)
self.assertFalse(self.args.output.exists())
def test_windows_recipes_use_explicit_targets_and_posix_paths(self):
self.environment["USERPROFILE"] = str(self.root / "user profile")
for architecture, flag in (("x86_64", "-m64"), ("aarch64", "-marm64")):