diff --git a/MODULE.bazel b/MODULE.bazel index ff5bbf21af..e656eab604 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -647,6 +647,94 @@ crate.annotation( inject_repo(crate, "alsa_lib") +# Keep upstream pkg-config version probes, but link only the prepared runtime. +# system-deps applies these path/flag overrides after a successful version probe. +[ + crate.annotation( + build_script_data = [ + "//third_party/voice:native_link", + "//third_party/voice:native_sdk", + ], + build_script_env = { + "PKG_CONFIG": "$(execpath @@//third_party/voice:pkg_config)", + "PKG_CONFIG_LIBDIR": "$(execpath @@//third_party/voice:native_sdk)/lib/pkgconfig", + "PKG_CONFIG_PATH": "", + } | { + "SYSTEM_DEPS_" + key + "_SEARCH_NATIVE": "$(execpath @@//third_party/voice:native_link)/.." + for key in keys + } | { + # GStreamer's .pc files add an absolute rpath; CcInfo supplies the + # relative Bazel runpath instead. Library/version checks still run. + "SYSTEM_DEPS_" + key + "_LDFLAGS": "" + for key in keys + if key.startswith("GSTREAMER_") + }, + build_script_tools = ["//third_party/voice:pkg_config"], + crate = name, + gen_build_script = "on", + version = version, + deps = ["//third_party/voice:native_link"], + ) + for name, version, keys in [ + ( + "glib-sys", + "0.22.8", + [ + "GLIB_2_0", + "GOBJECT_2_0", + ], + ), + ( + "gobject-sys", + "0.22.6", + ["GOBJECT_2_0"], + ), + ( + "gio-sys", + "0.22.8", + ["GIO_2_0"], + ), + ( + "gstreamer-sys", + "0.25.2", + ["GSTREAMER_1_0"], + ), + ( + "gstreamer-base-sys", + "0.25.3", + ["GSTREAMER_BASE_1_0"], + ), + ( + "gstreamer-app-sys", + "0.25.0", + ["GSTREAMER_APP_1_0"], + ), + ( + "gstreamer-audio-sys", + "0.25.3", + ["GSTREAMER_AUDIO_1_0"], + ), + ( + "glib-sys", + "0.22.9", + [ + "GLIB_2_0", + "GOBJECT_2_0", + ], + ), + ( + "gobject-sys", + "0.22.9", + ["GOBJECT_2_0"], + ), + ( + "gio-sys", + "0.22.9", + ["GIO_2_0"], + ), + ] +] + bazel_dep(name = "v8", version = "15.0.245.2") archive_override( module_name = "v8", diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 6c807508a0..7a98b0ec58 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain") load(":native.bzl", "native_prefix") +load(":native_link.bzl", "native_link") load(":pkg_config.bzl", "pkg_config") load(":runtime.bzl", "native_runtime") @@ -82,6 +83,7 @@ pkg_config( configure_options = [ "--with-internal-glib", "--disable-shared", + "--enable-define-prefix", ], copts = ["-Wno-int-conversion"], env = { @@ -105,6 +107,7 @@ pkg_config( # Internal GLib needs no pkg-config. Avoid bootstrapping another copy first; # the explicit failing command above forbids falling back to a host copy. toolchain = "@rules_foreign_cc//toolchains:preinstalled_pkgconfig_toolchain", + visibility = ["//visibility:public"], ) native_tool_toolchain( @@ -206,3 +209,35 @@ alias( tags = ["manual"], visibility = ["//visibility:public"], ) + +filegroup( + name = "native_sdk", + srcs = [":native_runtime"], + output_group = "sdk", + tags = ["manual"], + visibility = ["//visibility:public"], +) + +[ + native_link( + name = "native_link_" + os + "_" + cpu, + runtime = ":native_runtime_" + os + "_" + cpu, + tags = ["manual"], + target = cpu + "-" + suffix, + target_compatible_with = select({ + ":" + os + "_" + cpu: [], + "//conditions:default": ["@platforms//:incompatible"], + }), + ) + for os, cpu, suffix in _NATIVE_PLATFORMS +] + +alias( + name = "native_link", + actual = select({ + ":" + os + "_" + cpu: ":native_link_" + os + "_" + cpu + for os, cpu, _ in _NATIVE_PLATFORMS + }), + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/third_party/voice/README.md b/third_party/voice/README.md index 7488eef434..d04f070633 100644 --- a/third_party/voice/README.md +++ b/third_party/voice/README.md @@ -195,6 +195,19 @@ This manual target requires the same host inspection/signing tools as the standa platform preparer. It does not link Rust, change Windows builds, assemble a CLI package, or enable voice. The prepared runtime is the input to those later steps. +The `native_sdk` output exports the existing development SDK from that same +inspected build. Unix Bazel Rust bindings keep their upstream pkg-config version +checks, restricted to this SDK and the declared pkg-config executable. The +supported `system-deps` search-path override directs linking to `native_link`'s +prepared libraries; its GStreamer linker-flag override removes the SDK's absolute +rpath. Standard CcInfo supplies relative Bazel runpaths and explicit runfiles. +Canonical ABI names and development aliases stay together, including transitive +native dependencies. No host library fallback or version-probe bypass is used. +Plugins and their manifest are exported beside those same canonical libraries; +bindings and plugin imports must resolve to one physical copy of each library. +Cargo still consumes an explicitly supplied SDK; Windows MSVC and final installed +helper loader paths remain separate packaging steps. This does not enable voice. + ## Private Windows runtime preparation `windows_runtime.py` takes the same arguments for x64/ARM64 MSVC build prefixes. diff --git a/third_party/voice/native_link.bzl b/third_party/voice/native_link.bzl new file mode 100644 index 0000000000..c3931bd2d7 --- /dev/null +++ b/third_party/voice/native_link.bzl @@ -0,0 +1,104 @@ +"""Expose prepared native libraries to standard CcInfo linking and runfiles.""" + +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") + +# ABI filenames from the pinned native sources. Missing outputs fail the build. +_ABI_VERSIONS = { + "ffi": "8", + "gio-2.0": "0", + "glib-2.0": "0", + "gmodule-2.0": "0", + "gobject-2.0": "0", + "gstapp-1.0": "0", + "gstaudio-1.0": "0", + "gstbase-1.0": "0", + "gstnet-1.0": "0", + "gstpbutils-1.0": "0", + "gstreamer-1.0": "0", + "gstrtp-1.0": "0", + "gsttag-1.0": "0", + "gstvideo-1.0": "0", + "intl": "8", + "opus": "0", + "pcre2-8": "0", + "z": "1", +} + +def _native_link_impl(ctx): + runtime = ctx.file.runtime + versions = dict(_ABI_VERSIONS) + macos = ctx.attr.target.endswith("apple-darwin") + if not macos: + versions["gstallocators-1.0"] = "0" + locator = ctx.actions.declare_directory(ctx.label.name + "/lib/search-path") + originals, aliases, libraries = [], [], [] + arguments = [locator.path] + cc = find_cpp_toolchain(ctx) + features = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + for name, version in versions.items(): + filename = "lib" + name + ("." + version + ".dylib" if macos else ".so." + version) + alias = "lib" + name + (".dylib" if macos else ".so") + library = ctx.actions.declare_file(ctx.label.name + "/lib/" + filename) + development = ctx.actions.declare_file(ctx.label.name + "/lib/" + alias) + originals.append(library) + aliases.append(development) + arguments.extend([runtime.path + "/lib/" + filename, library.path]) + arguments.extend([runtime.path + "/lib/" + filename, development.path]) + libraries.append(cc_common.create_library_to_link( + actions = ctx.actions, + feature_configuration = features, + cc_toolchain = cc, + dynamic_library = library, + # @loader_path/$ORIGIN dependencies must stay beside one another. + dynamic_library_symlink_path = "voice/" + ctx.label.name + "/" + filename, + )) + payloads = [] + paths = ["runtime.json"] + [ + ("plugins/libgst" + plugin + ".dylib" if macos else "lib/gstreamer-1.0/libgst" + plugin + ".so") + for plugin in "app audioconvert audioresample coreelements opus rtp rtpmanager".split(" ") + ] + for path in paths: + payload = ctx.actions.declare_file(ctx.label.name + "/" + path) + payloads.append(payload) + arguments.extend([runtime.path + "/" + path, payload.path]) + ctx.actions.run_shell( + inputs = [runtime], + outputs = [locator] + originals + aliases + payloads, + arguments = arguments, + command = 'set -eu; /bin/mkdir -p "$1"; shift; while [ "$#" -gt 0 ]; do /bin/mkdir -p "${2%/*}"; /bin/cp "$1" "$2"; shift 2; done', + mnemonic = "VoiceNativeLinkInputs", + ) + linker = cc_common.create_linker_input( + owner = ctx.label, + user_link_flags = depset([ + "-Wl,-rpath," + ("@loader_path/../lib" if macos else "$ORIGIN/../lib"), + ]), + libraries = depset(libraries), + additional_inputs = depset([locator] + aliases), + ) + return [ + CcInfo(linking_context = cc_common.create_linking_context(linker_inputs = depset([linker]))), + DefaultInfo( + # A real directory permits unambiguous $(execpath :native_link)/.. + # in build-script settings; a marker file followed by /.. would fail. + files = depset([locator]), + runfiles = ctx.runfiles(files = [locator] + originals + aliases + payloads + [lib.dynamic_library for lib in libraries]), + ), + ] + +native_link = rule( + implementation = _native_link_impl, + attrs = { + "runtime": attr.label(mandatory = True, allow_single_file = True), + "target": attr.string(mandatory = True), + }, + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) diff --git a/third_party/voice/prepare_built_runtime.py b/third_party/voice/prepare_built_runtime.py index 9a070e479b..c773ffbfb0 100644 --- a/third_party/voice/prepare_built_runtime.py +++ b/third_party/voice/prepare_built_runtime.py @@ -15,9 +15,10 @@ import tempfile sys.path.insert(0, str(Path(__file__).resolve().parent)) from runtime import digest +from sdk import export_sdk -def prepare_built(prefix, build_receipt, status, target, output): +def prepare_built(prefix, build_receipt, status, target, output, *, sdk_output=None): prefix = prefix.resolve(strict=True) if build_receipt.stat().st_size > 1024 * 1024 or status.stat().st_size > 65536: raise ValueError("native build metadata exceeds limits") @@ -86,15 +87,19 @@ def prepare_built(prefix, build_receipt, status, target, output): ) ) platform.project(prefix, receipts, target, output) + if sdk_output is not None: + export_sdk(prefix, receipts, target, sdk_output) -def prepare_archive(archive, build_receipt, status, target, output): +def prepare_archive(archive, build_receipt, status, target, output, *, sdk_output=None): # Archives preserve native aliases through Bazel's cache and sandbox links. with tempfile.TemporaryDirectory(prefix="voice-prefix-") as temporary: prefix = Path(temporary) with tarfile.open(archive) as source: source.extractall(prefix, filter="data") - prepare_built(prefix, build_receipt, status, target, output) + prepare_built( + prefix, build_receipt, status, target, output, sdk_output=sdk_output + ) if __name__ == "__main__": @@ -102,12 +107,23 @@ if __name__ == "__main__": for name in ("prefix", "build-receipt", "status", "output"): parser.add_argument("--" + name, type=Path, required=True) parser.add_argument("--target", required=True) + parser.add_argument("--sdk-output", type=Path) args = parser.parse_args() # Executors may leave the TreeArtifact absent or create an empty directory. try: args.output.rmdir() except FileNotFoundError: pass + if args.sdk_output is not None: + try: + args.sdk_output.rmdir() + except FileNotFoundError: + pass prepare_archive( - args.prefix, args.build_receipt, args.status, args.target, args.output + args.prefix, + args.build_receipt, + args.status, + args.target, + args.output, + sdk_output=args.sdk_output, ) diff --git a/third_party/voice/runtime.bzl b/third_party/voice/runtime.bzl index 3334e3ffd7..4c52fc5a6a 100644 --- a/third_party/voice/runtime.bzl +++ b/third_party/voice/runtime.bzl @@ -7,6 +7,7 @@ def _native_runtime_impl(ctx): prefix = ctx.attr.prefix[DefaultInfo].files.to_list()[0] receipt = ctx.attr.prefix[OutputGroupInfo].receipt.to_list()[0] output = ctx.actions.declare_directory(ctx.label.name) + sdk = ctx.actions.declare_directory(ctx.label.name + "_sdk") ctx.actions.run( executable = python.interpreter, arguments = [ @@ -21,18 +22,23 @@ def _native_runtime_impl(ctx): ctx.attr.target, "--output", output.path, + "--sdk-output", + sdk.path, ], inputs = depset( [prefix, receipt, ctx.info_file, ctx.file._driver, python.interpreter] + ctx.files._preparers, transitive = [python.files], ), - outputs = [output], + outputs = [output, sdk], env = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, execution_requirements = {"no-remote-exec": "1", "no-remote-cache": "1"} if ctx.attr.target.endswith("apple-darwin") else {}, mnemonic = "VoiceNativeRuntime", progress_message = "Preparing private voice runtime for " + ctx.attr.target, ) - return [DefaultInfo(files = depset([output]))] + return [ + DefaultInfo(files = depset([output])), + OutputGroupInfo(sdk = depset([sdk])), + ] native_runtime = rule( implementation = _native_runtime_impl, diff --git a/third_party/voice/test_prepare_built_runtime.py b/third_party/voice/test_prepare_built_runtime.py index 6aa51d0884..e9bc654cdd 100644 --- a/third_party/voice/test_prepare_built_runtime.py +++ b/third_party/voice/test_prepare_built_runtime.py @@ -4,6 +4,7 @@ import json from pathlib import Path import sys import subprocess +import shutil import tarfile import tempfile import unittest @@ -13,6 +14,7 @@ import macos_runtime from prepare_built_runtime import prepare_archive, prepare_built from runtime import digest import test_macos_runtime +import test_sdk class BuiltRuntimeTests(unittest.TestCase): @@ -124,6 +126,11 @@ class BuiltRuntimeTests(unittest.TestCase): fixture = test_macos_runtime.RuntimeTests("runTest") self.addCleanup(fixture.doCleanups) fixture.setUp() + sdk_fixture = test_sdk.SdkTests("runTest") + self.addCleanup(sdk_fixture.doCleanups) + sdk_fixture.setUp() + for relative in ("include", "lib/glib-2.0", "lib/pkgconfig"): + shutil.copytree(sdk_fixture.prefix / relative, fixture.prefix / relative) self.receipt.write_text(json.dumps({**self.build, "target": fixture.target})) library = next((fixture.prefix / "lib").glob("*.dylib")) (library.parent / "development-alias.dylib").symlink_to(library.name) @@ -137,6 +144,9 @@ class BuiltRuntimeTests(unittest.TestCase): output = self.root / f"runtime-{state}" if state == "empty": output.mkdir() + sdk_output = self.root / f"sdk-{state}" + if state == "empty": + sdk_output.mkdir() subprocess.run( [ sys.executable, @@ -151,10 +161,28 @@ class BuiltRuntimeTests(unittest.TestCase): fixture.target, "--output", str(output), + "--sdk-output", + str(sdk_output), ], check=True, ) + sdk_manifest = json.loads((sdk_output / "sdk.json").read_text()) + self.assertEqual( + { + record["path"]: record["sha256"] + for record in sdk_manifest["files"] + }, + { + path.relative_to(sdk_output).as_posix(): digest(path) + for path in sdk_output.rglob("*") + if path.is_file() and path.name != "sdk.json" + }, + ) manifest = json.loads((output / "runtime.json").read_text()) + self.assertEqual( + (sdk_manifest["sourceCommit"], sdk_manifest["target"]), + (manifest["sourceCommit"], manifest["target"]), + ) self.assertEqual(manifest["sourceCommit"], "a" * 40) self.assertEqual(manifest["target"], fixture.target) self.assertTrue(