diff --git a/.bazelrc b/.bazelrc index 0aec7b8953..ea06d44080 100644 --- a/.bazelrc +++ b/.bazelrc @@ -11,6 +11,9 @@ startup --experimental_remote_repo_contents_cache common --experimental_platform_in_output_dir +build --workspace_status_command=./scripts/workspace-status.sh +build:windows --workspace_status_command=./scripts/workspace-status.cmd + # Runfiles strategy rationale: codex-rs/utils/cargo-bin/README.md common --noenable_runfiles @@ -85,6 +88,7 @@ common --jobs=30 common:remote --strategy=remote # Native Mac voice actions need the local Apple tools; Linux remains remote. common:macos --strategy=VoiceNativePrefix=remote,sandboxed,local +common:macos --strategy=VoiceNativeRuntime=remote,sandboxed,local common:remote --extra_execution_platforms=//:rbe common:remote --jobs=800 # TODO(team): Evaluate if this actually helps, zbarsky is not sure, everything seems bottlenecked on `core` either way. diff --git a/scripts/workspace-status.cmd b/scripts/workspace-status.cmd new file mode 100644 index 0000000000..6a63292c78 --- /dev/null +++ b/scripts/workspace-status.cmd @@ -0,0 +1,20 @@ +@echo off +setlocal + +if defined STABLE_GIT_COMMIT ( + echo STABLE_GIT_COMMIT %STABLE_GIT_COMMIT% + exit /b 0 +) + +for /f "delims=" %%I in ('git rev-parse --verify HEAD 2^>nul') do set "BUILD_COMMIT=%%I" +if defined BUILD_COMMIT ( + echo STABLE_GIT_COMMIT %BUILD_COMMIT% + exit /b 0 +) + +if defined GITHUB_SHA ( + echo STABLE_GIT_COMMIT %GITHUB_SHA% + exit /b 0 +) + +echo STABLE_GIT_COMMIT unknown diff --git a/scripts/workspace-status.sh b/scripts/workspace-status.sh new file mode 100755 index 0000000000..ba4cd3869e --- /dev/null +++ b/scripts/workspace-status.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ -n "${STABLE_GIT_COMMIT:-}" ]]; then + build_commit="${STABLE_GIT_COMMIT}" +elif build_commit="$(git rev-parse --verify HEAD 2>/dev/null)"; then + : +elif [[ -n "${GITHUB_SHA:-}" ]]; then + build_commit="${GITHUB_SHA}" +else + build_commit="unknown" +fi + +printf 'STABLE_GIT_COMMIT %s\n' "${build_commit}" diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 87ea515b98..6c807508a0 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -1,6 +1,7 @@ load("@rules_foreign_cc//toolchains/native_tools:native_tools_toolchain.bzl", "native_tool_toolchain") load(":native.bzl", "native_prefix") load(":pkg_config.bzl", "pkg_config") +load(":runtime.bzl", "native_runtime") # All native build consumers must receive the same pinned source manifest. exports_files(["opus-toolchain.cmake"]) @@ -177,3 +178,31 @@ alias( tags = ["manual"], visibility = ["//visibility:public"], ) + +[ + native_runtime( + name = "native_runtime_" + os + "_" + cpu, + exec_compatible_with = [ + "@platforms//os:" + os, + "@platforms//cpu:" + cpu, + ], + prefix = ":native_prefix_" + os + "_" + cpu, + tags = ["manual"], + target = cpu + "-" + suffix, + target_compatible_with = select({ + ":" + os + "_" + cpu: [], + "//conditions:default": ["@platforms//:incompatible"], + }), + ) + for os, cpu, suffix in _NATIVE_PLATFORMS +] + +alias( + name = "native_runtime", + actual = select({ + ":" + os + "_" + cpu: ":native_runtime_" + os + "_" + cpu + for os, cpu, _ in _NATIVE_PLATFORMS + }), + tags = ["manual"], + visibility = ["//visibility:public"], +) diff --git a/third_party/voice/README.md b/third_party/voice/README.md index a4b7499e58..7488eef434 100644 --- a/third_party/voice/README.md +++ b/third_party/voice/README.md @@ -183,6 +183,18 @@ preparation or shipped in the runtime. The output is development-only, uses the host glibc, and does not establish musl or minimum-glibc support, dynamic-only dependency closure, helper loading policy or working voice. +## Prepare Bazel's native build output + +`bazel build //third_party/voice:native_runtime` prepares the selected Mac or GNU +Linux prefix archive using the existing platform inspection and preparation code. +It checks the completed build receipt, records the current workspace build commit, +inspects physical libraries, and prepares verified copies. These receipts describe +build inputs and inspection; they are not signatures or release approval. + +This manual target requires the same host inspection/signing tools as the standalone +platform preparer. It does not link Rust, change Windows builds, assemble a CLI +package, or enable voice. The prepared runtime is the input to those later steps. + ## Private Windows runtime preparation `windows_runtime.py` takes the same arguments for x64/ARM64 MSVC build prefixes. diff --git a/third_party/voice/macos_runtime.py b/third_party/voice/macos_runtime.py index 23b674207e..38fdf009df 100644 --- a/third_party/voice/macos_runtime.py +++ b/third_party/voice/macos_runtime.py @@ -15,6 +15,7 @@ from runtime import PLUGINS, RuntimeFormat, digest, prepare, required_library_pa SYSTEM_IMPORTS = frozenset( { "/usr/lib/libSystem.B.dylib", + "/usr/lib/libc++.1.dylib", "/usr/lib/libobjc.A.dylib", "/usr/lib/libiconv.2.dylib", "/usr/lib/libresolv.9.dylib", diff --git a/third_party/voice/prepare_built_runtime.py b/third_party/voice/prepare_built_runtime.py new file mode 100644 index 0000000000..9a070e479b --- /dev/null +++ b/third_party/voice/prepare_built_runtime.py @@ -0,0 +1,113 @@ +"""Prepare an inspected build output using the existing platform runtime policy. + +Receipts describe the declared build inputs and inspection, not authenticity or +approval. The current build commit comes from Bazel's workspace status file. +""" + +import argparse +import importlib +import json +from pathlib import Path +import re +import sys +import tarfile +import tempfile + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from runtime import digest + + +def prepare_built(prefix, build_receipt, status, target, output): + prefix = prefix.resolve(strict=True) + if build_receipt.stat().st_size > 1024 * 1024 or status.stat().st_size > 65536: + raise ValueError("native build metadata exceeds limits") + build = json.loads(build_receipt.read_text()) + commits = [ + line.removeprefix("STABLE_GIT_COMMIT ") + for line in status.read_text().splitlines() + if line.startswith("STABLE_GIT_COMMIT ") + ] + manifest_hash = digest(Path(__file__).with_name("sources.json")) + steps = build.get("steps", []) + if ( + len(commits) != 1 + or not re.fullmatch(r"[0-9a-f]{40}", commits[0]) + or build.get("target") != target + or build.get("manifest_sha256") != manifest_hash + or not 1 <= len(steps) <= 128 + or any(step.get("exit_code") != 0 for step in steps) + or steps[-1].get("name") != "gst-plugins-good-install" + ): + raise ValueError("native build receipt is incomplete or mismatched") + suffix = target.partition("-")[2] + modules = { + "apple-darwin": "macos_runtime", + "unknown-linux-gnu": "linux_runtime", + "pc-windows-msvc": "windows_runtime", + } + if suffix not in modules or target.partition("-")[0] not in ("aarch64", "x86_64"): + raise ValueError("unsupported native runtime target") + platform = importlib.import_module(modules[suffix]) + records = [] + for path in sorted(prefix.rglob("*")): + if not re.fullmatch(r".+\.(?:dylib|dll|so(?:\.[0-9]+)*)", path.name): + continue + if path.is_symlink(): + if not path.resolve(strict=True).is_relative_to(prefix): + raise ValueError("native library link escapes its prefix") + continue + if not path.is_file(): + continue + if len(records) >= 128: + raise ValueError("native inventory exceeds limits") + platform.inspect(path, target) + records.append( + { + "path": path.relative_to(prefix).as_posix(), + "target": target, + "sha256": digest(path), + } + ) + if not records: + raise ValueError("native prefix contains no libraries") + with tempfile.TemporaryDirectory(prefix="voice-receipts-") as temporary: + receipts = Path(temporary) + (receipts / "inspection").mkdir() + (receipts / "inspection/binaries.json").write_text(json.dumps(records)) + (receipts / "ci.json").write_text( + json.dumps( + { + "commit": commits[0], + "target": target, + "manifest_sha256": manifest_hash, + "build_complete": True, + "inspection_complete": True, + } + ) + ) + platform.project(prefix, receipts, target, output) + + +def prepare_archive(archive, build_receipt, status, target, output): + # Archives preserve native aliases through Bazel's cache and sandbox links. + with tempfile.TemporaryDirectory(prefix="voice-prefix-") as temporary: + prefix = Path(temporary) + with tarfile.open(archive) as source: + source.extractall(prefix, filter="data") + prepare_built(prefix, build_receipt, status, target, output) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + for name in ("prefix", "build-receipt", "status", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--target", required=True) + args = parser.parse_args() + # Executors may leave the TreeArtifact absent or create an empty directory. + try: + args.output.rmdir() + except FileNotFoundError: + pass + prepare_archive( + args.prefix, args.build_receipt, args.status, args.target, args.output + ) diff --git a/third_party/voice/runtime.bzl b/third_party/voice/runtime.bzl new file mode 100644 index 0000000000..3334e3ffd7 --- /dev/null +++ b/third_party/voice/runtime.bzl @@ -0,0 +1,46 @@ +"""Prepare build-produced native libraries with the existing platform policy.""" + +load("@rules_python//python:py_runtime_info.bzl", "PyRuntimeInfo") + +def _native_runtime_impl(ctx): + python = ctx.attr._python[PyRuntimeInfo] + prefix = ctx.attr.prefix[DefaultInfo].files.to_list()[0] + receipt = ctx.attr.prefix[OutputGroupInfo].receipt.to_list()[0] + output = ctx.actions.declare_directory(ctx.label.name) + ctx.actions.run( + executable = python.interpreter, + arguments = [ + ctx.file._driver.path, + "--prefix", + prefix.path, + "--build-receipt", + receipt.path, + "--status", + ctx.info_file.path, + "--target", + ctx.attr.target, + "--output", + output.path, + ], + inputs = depset( + [prefix, receipt, ctx.info_file, ctx.file._driver, python.interpreter] + ctx.files._preparers, + transitive = [python.files], + ), + outputs = [output], + env = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, + execution_requirements = {"no-remote-exec": "1", "no-remote-cache": "1"} if ctx.attr.target.endswith("apple-darwin") else {}, + mnemonic = "VoiceNativeRuntime", + progress_message = "Preparing private voice runtime for " + ctx.attr.target, + ) + return [DefaultInfo(files = depset([output]))] + +native_runtime = rule( + implementation = _native_runtime_impl, + attrs = { + "prefix": attr.label(mandatory = True, allow_single_file = True), + "target": attr.string(mandatory = True), + "_driver": attr.label(default = "//third_party/voice:prepare_built_runtime.py", allow_single_file = True), + "_preparers": attr.label(default = "//third_party/voice:build_inputs"), + "_python": attr.label(default = "@python_3_12//:py3_runtime", cfg = "exec"), + }, +) diff --git a/third_party/voice/test_prepare_built_runtime.py b/third_party/voice/test_prepare_built_runtime.py new file mode 100644 index 0000000000..6aa51d0884 --- /dev/null +++ b/third_party/voice/test_prepare_built_runtime.py @@ -0,0 +1,181 @@ +"""Exercise the build receipt boundary and real runtime preparation adapter.""" + +import json +from pathlib import Path +import sys +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +import macos_runtime +from prepare_built_runtime import prepare_archive, prepare_built +from runtime import digest +import test_macos_runtime + + +class BuiltRuntimeTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.prefix = self.root / "prefix" + self.prefix.mkdir() + self.library = self.prefix / "libfixture.dylib" + self.library.write_bytes(b"synthetic inventory fixture") + self.target = "aarch64-apple-darwin" + self.status = self.root / "status" + self.status.write_text("STABLE_GIT_COMMIT " + "a" * 40 + "\n") + self.receipt = self.root / "built.json" + self.build = { + "target": self.target, + "manifest_sha256": digest(Path(__file__).with_name("sources.json")), + "steps": [{"name": "gst-plugins-good-install", "exit_code": 0}], + } + self.receipt.write_text(json.dumps(self.build)) + + def test_inspection_precedes_truthful_receipts_and_projection(self): + def project(prefix, receipts, target, output): + inspect.assert_called_once_with(self.library.resolve(), target) + self.assertEqual( + json.loads((receipts / "ci.json").read_text()), + { + "commit": "a" * 40, + "target": self.target, + "manifest_sha256": self.build["manifest_sha256"], + "build_complete": True, + "inspection_complete": True, + }, + ) + self.assertEqual( + json.loads((receipts / "inspection/binaries.json").read_text()), + [ + { + "path": self.library.name, + "target": self.target, + "sha256": digest(self.library), + } + ], + ) + + with ( + patch.object(macos_runtime, "inspect") as inspect, + patch.object(macos_runtime, "project", side_effect=project), + ): + prepare_built( + self.prefix, self.receipt, self.status, self.target, self.root / "out" + ) + + def test_rejects_failed_incomplete_and_mismatched_builds_before_inspection(self): + for change in ( + {"target": "x86_64-apple-darwin"}, + {"manifest_sha256": "0" * 64}, + {"steps": []}, + {"steps": [{"name": "glib-install", "exit_code": 0}]}, + {"steps": [{"name": "gst-plugins-good-install", "exit_code": 1}]}, + ): + with ( + self.subTest(change=change), + patch.object(macos_runtime, "inspect") as inspect, + ): + self.receipt.write_text(json.dumps({**self.build, **change})) + with self.assertRaises(ValueError): + prepare_built( + self.prefix, + self.receipt, + self.status, + self.target, + self.root / "out", + ) + inspect.assert_not_called() + + def test_rejects_unknown_or_ambiguous_build_commit(self): + for text in ("STABLE_GIT_COMMIT unknown\n", self.status.read_text() * 2): + self.status.write_text(text) + with self.assertRaises(ValueError): + prepare_built( + self.prefix, + self.receipt, + self.status, + self.target, + self.root / "out", + ) + + def test_inspection_failure_never_reaches_projection(self): + with ( + patch.object( + macos_runtime, "inspect", side_effect=ValueError("bad library") + ), + patch.object(macos_runtime, "project") as project, + ): + with self.assertRaisesRegex(ValueError, "bad library"): + prepare_built( + self.prefix, + self.receipt, + self.status, + self.target, + self.root / "out", + ) + project.assert_not_called() + + @unittest.skipUnless(sys.platform == "darwin", "Uses real Mach-O fixture libraries") + def test_projects_archive_through_sandbox_link_with_native_aliases(self): + fixture = test_macos_runtime.RuntimeTests("runTest") + self.addCleanup(fixture.doCleanups) + fixture.setUp() + self.receipt.write_text(json.dumps({**self.build, "target": fixture.target})) + library = next((fixture.prefix / "lib").glob("*.dylib")) + (library.parent / "development-alias.dylib").symlink_to(library.name) + archive = self.root / "prefix.tar" + with tarfile.open(archive, "w") as source: + source.add(fixture.prefix, arcname=".") + sandbox_input = self.root / "sandbox-input.tar" + sandbox_input.symlink_to(archive) + for state in ("absent", "empty"): + with self.subTest(output_state=state): + output = self.root / f"runtime-{state}" + if state == "empty": + output.mkdir() + subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name("prepare_built_runtime.py")), + "--prefix", + str(sandbox_input), + "--build-receipt", + str(self.receipt), + "--status", + str(self.status), + "--target", + fixture.target, + "--output", + str(output), + ], + check=True, + ) + manifest = json.loads((output / "runtime.json").read_text()) + self.assertEqual(manifest["sourceCommit"], "a" * 40) + self.assertEqual(manifest["target"], fixture.target) + self.assertTrue( + all( + not macos_runtime.inspect( + output / record["path"], fixture.target + ).rpaths + for record in manifest["libraries"] + ) + ) + + def test_archive_rejects_escaping_native_alias_before_inspection(self): + archive = self.root / "prefix.tar" + with tarfile.open(archive, "w") as source: + link = tarfile.TarInfo("lib/escape.dylib") + link.type = tarfile.SYMTYPE + link.linkname = "../../outside.dylib" + source.addfile(link) + with patch.object(macos_runtime, "inspect") as inspect: + with self.assertRaises(tarfile.LinkOutsideDestinationError): + prepare_archive( + archive, self.receipt, self.status, self.target, self.root / "out" + ) + inspect.assert_not_called()