From f5a71ff40a713eff0bb3feeb22e2ca0e1c208a08 Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Sat, 5 Sep 2026 22:30:56 +0000 Subject: [PATCH] Link Unix Bazel bindings against the prepared voice runtime (#43117) ## What changed - Export `native_sdk` from the same inspected build as `native_runtime`. - Configure GLib and GStreamer Rust build scripts to use the declared `pkg-config` executable and SDK, preserving upstream version checks while directing linking to prepared libraries. - Add `native_link` to expose canonical libraries, development aliases, transitive dependencies, plugins, and the runtime manifest through `CcInfo` and runfiles. Replace GStreamer SDK absolute rpaths with relative Bazel runpaths. ## Testing Extend the macOS archive preparation test to cover absent and empty SDK output directories, verify SDK file hashes, and check that SDK and runtime manifests share the same source commit and target. GitOrigin-RevId: 3fc33af06225c69e4c293437f1985290f228dc0a --- MODULE.bazel | 88 +++++++++++++++ third_party/voice/BUILD.bazel | 35 ++++++ third_party/voice/README.md | 13 +++ third_party/voice/native_link.bzl | 104 ++++++++++++++++++ third_party/voice/prepare_built_runtime.py | 24 +++- third_party/voice/runtime.bzl | 10 +- .../voice/test_prepare_built_runtime.py | 28 +++++ 7 files changed, 296 insertions(+), 6 deletions(-) create mode 100644 third_party/voice/native_link.bzl diff --git a/MODULE.bazel b/MODULE.bazel index ff5bbf21af..e656eab604 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -647,6 +647,94 @@ crate.annotation( inject_repo(crate, "alsa_lib") +# Keep upstream pkg-config version probes, but link only the prepared runtime. +# system-deps applies these path/flag overrides after a successful version probe. +[ + crate.annotation( + build_script_data = [ + "//third_party/voice:native_link", + "//third_party/voice:native_sdk", + ], + build_script_env = { + "PKG_CONFIG": "$(execpath @@//third_party/voice:pkg_config)", + "PKG_CONFIG_LIBDIR": "$(execpath @@//third_party/voice:native_sdk)/lib/pkgconfig", + "PKG_CONFIG_PATH": "", + } | { + "SYSTEM_DEPS_" + key + "_SEARCH_NATIVE": "$(execpath @@//third_party/voice:native_link)/.." + for key in keys + } | { + # GStreamer's .pc files add an absolute rpath; CcInfo supplies the + # relative Bazel runpath instead. Library/version checks still run. + "SYSTEM_DEPS_" + key + "_LDFLAGS": "" + for key in keys + if key.startswith("GSTREAMER_") + }, + build_script_tools = ["//third_party/voice:pkg_config"], + crate = name, + gen_build_script = "on", + version = version, + deps = ["//third_party/voice:native_link"], + ) + for name, version, keys in [ + ( + "glib-sys", + "0.22.8", + [ + "GLIB_2_0", + "GOBJECT_2_0", + ], + ), + ( + "gobject-sys", + "0.22.6", + ["GOBJECT_2_0"], + ), + ( + "gio-sys", + "0.22.8", + ["GIO_2_0"], + ), + ( + "gstreamer-sys", + "0.25.2", + ["GSTREAMER_1_0"], + ), + ( + "gstreamer-base-sys", + "0.25.3", + ["GSTREAMER_BASE_1_0"], + ), + ( + "gstreamer-app-sys", + "0.25.0", + ["GSTREAMER_APP_1_0"], + ), + ( + "gstreamer-audio-sys", + "0.25.3", + ["GSTREAMER_AUDIO_1_0"], + ), + ( + "glib-sys", + "0.22.9", + [ + "GLIB_2_0", + "GOBJECT_2_0", + ], + ), + ( + "gobject-sys", + "0.22.9", + ["GOBJECT_2_0"], + ), + ( + "gio-sys", + "0.22.9", + ["GIO_2_0"], + ), + ] +] + bazel_dep(name = "v8", version = "15.0.245.2") archive_override( module_name = "v8", diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 6c807508a0..7a98b0ec58 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain") load(":native.bzl", "native_prefix") +load(":native_link.bzl", "native_link") load(":pkg_config.bzl", "pkg_config") load(":runtime.bzl", "native_runtime") @@ -82,6 +83,7 @@ pkg_config( configure_options = [ "--with-internal-glib", "--disable-shared", + "--enable-define-prefix", ], copts = ["-Wno-int-conversion"], env = { @@ -105,6 +107,7 @@ pkg_config( # Internal GLib needs no pkg-config. Avoid bootstrapping another copy first; # the explicit failing command above forbids falling back to a host copy. toolchain = "@rules_foreign_cc//toolchains:preinstalled_pkgconfig_toolchain", + visibility = ["//visibility:public"], ) native_tool_toolchain( @@ -206,3 +209,35 @@ alias( tags = ["manual"], visibility = ["//visibility:public"], ) + +filegroup( + name = "native_sdk", + srcs = [":native_runtime"], + output_group = "sdk", + tags = ["manual"], + visibility = ["//visibility:public"], +) + +[ + native_link( + name = "native_link_" + os + "_" + cpu, + runtime = ":native_runtime_" + os + "_" + cpu, + tags = ["manual"], + target = cpu + "-" + suffix, + target_compatible_with = select({ + ":" + os + "_" + cpu: [], + "//conditions:default": ["@platforms//:incompatible"], + }), + ) + for os, cpu, suffix in _NATIVE_PLATFORMS +] + +alias( + name = "native_link", + actual = select({ + ":" + os + "_" + cpu: ":native_link_" + os + "_" + cpu + for os, cpu, _ in _NATIVE_PLATFORMS + }), + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/third_party/voice/README.md b/third_party/voice/README.md index 7488eef434..d04f070633 100644 --- a/third_party/voice/README.md +++ b/third_party/voice/README.md @@ -195,6 +195,19 @@ This manual target requires the same host inspection/signing tools as the standa platform preparer. It does not link Rust, change Windows builds, assemble a CLI package, or enable voice. The prepared runtime is the input to those later steps. +The `native_sdk` output exports the existing development SDK from that same +inspected build. Unix Bazel Rust bindings keep their upstream pkg-config version +checks, restricted to this SDK and the declared pkg-config executable. The +supported `system-deps` search-path override directs linking to `native_link`'s +prepared libraries; its GStreamer linker-flag override removes the SDK's absolute +rpath. Standard CcInfo supplies relative Bazel runpaths and explicit runfiles. +Canonical ABI names and development aliases stay together, including transitive +native dependencies. No host library fallback or version-probe bypass is used. +Plugins and their manifest are exported beside those same canonical libraries; +bindings and plugin imports must resolve to one physical copy of each library. +Cargo still consumes an explicitly supplied SDK; Windows MSVC and final installed +helper loader paths remain separate packaging steps. This does not enable voice. + ## Private Windows runtime preparation `windows_runtime.py` takes the same arguments for x64/ARM64 MSVC build prefixes. diff --git a/third_party/voice/native_link.bzl b/third_party/voice/native_link.bzl new file mode 100644 index 0000000000..c3931bd2d7 --- /dev/null +++ b/third_party/voice/native_link.bzl @@ -0,0 +1,104 @@ +"""Expose prepared native libraries to standard CcInfo linking and runfiles.""" + +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") + +# ABI filenames from the pinned native sources. Missing outputs fail the build. +_ABI_VERSIONS = { + "ffi": "8", + "gio-2.0": "0", + "glib-2.0": "0", + "gmodule-2.0": "0", + "gobject-2.0": "0", + "gstapp-1.0": "0", + "gstaudio-1.0": "0", + "gstbase-1.0": "0", + "gstnet-1.0": "0", + "gstpbutils-1.0": "0", + "gstreamer-1.0": "0", + "gstrtp-1.0": "0", + "gsttag-1.0": "0", + "gstvideo-1.0": "0", + "intl": "8", + "opus": "0", + "pcre2-8": "0", + "z": "1", +} + +def _native_link_impl(ctx): + runtime = ctx.file.runtime + versions = dict(_ABI_VERSIONS) + macos = ctx.attr.target.endswith("apple-darwin") + if not macos: + versions["gstallocators-1.0"] = "0" + locator = ctx.actions.declare_directory(ctx.label.name + "/lib/search-path") + originals, aliases, libraries = [], [], [] + arguments = [locator.path] + cc = find_cpp_toolchain(ctx) + features = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + for name, version in versions.items(): + filename = "lib" + name + ("." + version + ".dylib" if macos else ".so." + version) + alias = "lib" + name + (".dylib" if macos else ".so") + library = ctx.actions.declare_file(ctx.label.name + "/lib/" + filename) + development = ctx.actions.declare_file(ctx.label.name + "/lib/" + alias) + originals.append(library) + aliases.append(development) + arguments.extend([runtime.path + "/lib/" + filename, library.path]) + arguments.extend([runtime.path + "/lib/" + filename, development.path]) + libraries.append(cc_common.create_library_to_link( + actions = ctx.actions, + feature_configuration = features, + cc_toolchain = cc, + dynamic_library = library, + # @loader_path/$ORIGIN dependencies must stay beside one another. + dynamic_library_symlink_path = "voice/" + ctx.label.name + "/" + filename, + )) + payloads = [] + paths = ["runtime.json"] + [ + ("plugins/libgst" + plugin + ".dylib" if macos else "lib/gstreamer-1.0/libgst" + plugin + ".so") + for plugin in "app audioconvert audioresample coreelements opus rtp rtpmanager".split(" ") + ] + for path in paths: + payload = ctx.actions.declare_file(ctx.label.name + "/" + path) + payloads.append(payload) + arguments.extend([runtime.path + "/" + path, payload.path]) + ctx.actions.run_shell( + inputs = [runtime], + outputs = [locator] + originals + aliases + payloads, + arguments = arguments, + command = 'set -eu; /bin/mkdir -p "$1"; shift; while [ "$#" -gt 0 ]; do /bin/mkdir -p "${2%/*}"; /bin/cp "$1" "$2"; shift 2; done', + mnemonic = "VoiceNativeLinkInputs", + ) + linker = cc_common.create_linker_input( + owner = ctx.label, + user_link_flags = depset([ + "-Wl,-rpath," + ("@loader_path/../lib" if macos else "$ORIGIN/../lib"), + ]), + libraries = depset(libraries), + additional_inputs = depset([locator] + aliases), + ) + return [ + CcInfo(linking_context = cc_common.create_linking_context(linker_inputs = depset([linker]))), + DefaultInfo( + # A real directory permits unambiguous $(execpath :native_link)/.. + # in build-script settings; a marker file followed by /.. would fail. + files = depset([locator]), + runfiles = ctx.runfiles(files = [locator] + originals + aliases + payloads + [lib.dynamic_library for lib in libraries]), + ), + ] + +native_link = rule( + implementation = _native_link_impl, + attrs = { + "runtime": attr.label(mandatory = True, allow_single_file = True), + "target": attr.string(mandatory = True), + }, + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) diff --git a/third_party/voice/prepare_built_runtime.py b/third_party/voice/prepare_built_runtime.py index 9a070e479b..c773ffbfb0 100644 --- a/third_party/voice/prepare_built_runtime.py +++ b/third_party/voice/prepare_built_runtime.py @@ -15,9 +15,10 @@ import tempfile sys.path.insert(0, str(Path(__file__).resolve().parent)) from runtime import digest +from sdk import export_sdk -def prepare_built(prefix, build_receipt, status, target, output): +def prepare_built(prefix, build_receipt, status, target, output, *, sdk_output=None): prefix = prefix.resolve(strict=True) if build_receipt.stat().st_size > 1024 * 1024 or status.stat().st_size > 65536: raise ValueError("native build metadata exceeds limits") @@ -86,15 +87,19 @@ def prepare_built(prefix, build_receipt, status, target, output): ) ) platform.project(prefix, receipts, target, output) + if sdk_output is not None: + export_sdk(prefix, receipts, target, sdk_output) -def prepare_archive(archive, build_receipt, status, target, output): +def prepare_archive(archive, build_receipt, status, target, output, *, sdk_output=None): # Archives preserve native aliases through Bazel's cache and sandbox links. with tempfile.TemporaryDirectory(prefix="voice-prefix-") as temporary: prefix = Path(temporary) with tarfile.open(archive) as source: source.extractall(prefix, filter="data") - prepare_built(prefix, build_receipt, status, target, output) + prepare_built( + prefix, build_receipt, status, target, output, sdk_output=sdk_output + ) if __name__ == "__main__": @@ -102,12 +107,23 @@ if __name__ == "__main__": for name in ("prefix", "build-receipt", "status", "output"): parser.add_argument("--" + name, type=Path, required=True) parser.add_argument("--target", required=True) + parser.add_argument("--sdk-output", type=Path) args = parser.parse_args() # Executors may leave the TreeArtifact absent or create an empty directory. try: args.output.rmdir() except FileNotFoundError: pass + if args.sdk_output is not None: + try: + args.sdk_output.rmdir() + except FileNotFoundError: + pass prepare_archive( - args.prefix, args.build_receipt, args.status, args.target, args.output + args.prefix, + args.build_receipt, + args.status, + args.target, + args.output, + sdk_output=args.sdk_output, ) diff --git a/third_party/voice/runtime.bzl b/third_party/voice/runtime.bzl index 3334e3ffd7..4c52fc5a6a 100644 --- a/third_party/voice/runtime.bzl +++ b/third_party/voice/runtime.bzl @@ -7,6 +7,7 @@ def _native_runtime_impl(ctx): prefix = ctx.attr.prefix[DefaultInfo].files.to_list()[0] receipt = ctx.attr.prefix[OutputGroupInfo].receipt.to_list()[0] output = ctx.actions.declare_directory(ctx.label.name) + sdk = ctx.actions.declare_directory(ctx.label.name + "_sdk") ctx.actions.run( executable = python.interpreter, arguments = [ @@ -21,18 +22,23 @@ def _native_runtime_impl(ctx): ctx.attr.target, "--output", output.path, + "--sdk-output", + sdk.path, ], inputs = depset( [prefix, receipt, ctx.info_file, ctx.file._driver, python.interpreter] + ctx.files._preparers, transitive = [python.files], ), - outputs = [output], + outputs = [output, sdk], env = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, execution_requirements = {"no-remote-exec": "1", "no-remote-cache": "1"} if ctx.attr.target.endswith("apple-darwin") else {}, mnemonic = "VoiceNativeRuntime", progress_message = "Preparing private voice runtime for " + ctx.attr.target, ) - return [DefaultInfo(files = depset([output]))] + return [ + DefaultInfo(files = depset([output])), + OutputGroupInfo(sdk = depset([sdk])), + ] native_runtime = rule( implementation = _native_runtime_impl, diff --git a/third_party/voice/test_prepare_built_runtime.py b/third_party/voice/test_prepare_built_runtime.py index 6aa51d0884..e9bc654cdd 100644 --- a/third_party/voice/test_prepare_built_runtime.py +++ b/third_party/voice/test_prepare_built_runtime.py @@ -4,6 +4,7 @@ import json from pathlib import Path import sys import subprocess +import shutil import tarfile import tempfile import unittest @@ -13,6 +14,7 @@ import macos_runtime from prepare_built_runtime import prepare_archive, prepare_built from runtime import digest import test_macos_runtime +import test_sdk class BuiltRuntimeTests(unittest.TestCase): @@ -124,6 +126,11 @@ class BuiltRuntimeTests(unittest.TestCase): fixture = test_macos_runtime.RuntimeTests("runTest") self.addCleanup(fixture.doCleanups) fixture.setUp() + sdk_fixture = test_sdk.SdkTests("runTest") + self.addCleanup(sdk_fixture.doCleanups) + sdk_fixture.setUp() + for relative in ("include", "lib/glib-2.0", "lib/pkgconfig"): + shutil.copytree(sdk_fixture.prefix / relative, fixture.prefix / relative) self.receipt.write_text(json.dumps({**self.build, "target": fixture.target})) library = next((fixture.prefix / "lib").glob("*.dylib")) (library.parent / "development-alias.dylib").symlink_to(library.name) @@ -137,6 +144,9 @@ class BuiltRuntimeTests(unittest.TestCase): output = self.root / f"runtime-{state}" if state == "empty": output.mkdir() + sdk_output = self.root / f"sdk-{state}" + if state == "empty": + sdk_output.mkdir() subprocess.run( [ sys.executable, @@ -151,10 +161,28 @@ class BuiltRuntimeTests(unittest.TestCase): fixture.target, "--output", str(output), + "--sdk-output", + str(sdk_output), ], check=True, ) + sdk_manifest = json.loads((sdk_output / "sdk.json").read_text()) + self.assertEqual( + { + record["path"]: record["sha256"] + for record in sdk_manifest["files"] + }, + { + path.relative_to(sdk_output).as_posix(): digest(path) + for path in sdk_output.rglob("*") + if path.is_file() and path.name != "sdk.json" + }, + ) manifest = json.loads((output / "runtime.json").read_text()) + self.assertEqual( + (sdk_manifest["sourceCommit"], sdk_manifest["target"]), + (manifest["sourceCommit"], manifest["target"]), + ) self.assertEqual(manifest["sourceCommit"], "a" * 40) self.assertEqual(manifest["target"], fixture.target) self.assertTrue(