diff --git a/third_party/voice/BUILD.bazel b/third_party/voice/BUILD.bazel index 33fac3e46a..4e6f3cb62c 100644 --- a/third_party/voice/BUILD.bazel +++ b/third_party/voice/BUILD.bazel @@ -33,6 +33,7 @@ filegroup( srcs = [ "assemble_package.py", "build_native.py", + "linux_runtime.py", "macos_runtime.py", "runtime.py", ":source_inputs", diff --git a/third_party/voice/README.md b/third_party/voice/README.md index b60234138a..efc73f2ddb 100644 --- a/third_party/voice/README.md +++ b/third_party/voice/README.md @@ -74,6 +74,12 @@ must still match the real native target. Native pkgconf relocates libffi's POSIX prefix metadata; CI rejects residual Cygwin paths. These are build prerequisites, not shipped runtime components or evidence of working voice. +CMake libraries use relative install runpaths (`$ORIGIN` on Linux and +`@loader_path` on Mac), with `@rpath` install names on Mac. Linux Meson links +use `$ORIGIN:$ORIGIN/..`, matching libraries in `lib/` and plugins in +`lib/gstreamer-1.0/`. Mac Meson and libffi still need packaging-time fixups; +these options do not make every Mac library relocatable at installation. + Outputs are under `prefix/`, build tools under `tools/`, and logs beside them. `build-state.json` records completed commands and failures; `built.json` exists only when every build/install command succeeds. Failed builds retain their logs @@ -110,3 +116,18 @@ audio behavior. Dynamic-only dependencies, native helper linkage, LGPL notices, production signing/notarization, Windows/Linux loading and security approval remain separate requirements. No microphone, device, plugin scanner or backend is started by projection. Run its native relocation tests on macOS with Python 3.12 or newer. + +## Private GNU Linux runtime preparation + +`linux_runtime.py` takes the same prefix, receipts, target and output arguments. +It reads bounded ELF64 headers, segments and dynamic tables directly and accepts +x64/ARM64 GNU Linux libraries. The shared Python coordinator selects the seven +plugins and their declared dependencies without changing their bytes. The native +build must have emitted package-relative runpaths; older absolute paths are +rejected with a rebuild instruction. Output preserves `lib/gstreamer-1.0/` so +those relative paths remain valid. Loader audit/filter dependencies and +path-bearing imports are rejected. Native tests require Python 3.12, a C compiler +and `patchelf`; the latter constructs malformed inputs and is not needed during +preparation or shipped in the runtime. The output is development-only, uses the +host glibc, and does not establish musl or minimum-glibc support, dynamic-only +dependency closure, helper loading policy or working voice. diff --git a/third_party/voice/build_native.py b/third_party/voice/build_native.py index d75cc1fc81..adb75b6e4a 100644 --- a/third_party/voice/build_native.py +++ b/third_party/voice/build_native.py @@ -130,7 +130,14 @@ class NativeBuild: if args.target.endswith("apple-darwin"): self.environment["MACOSX_DEPLOYMENT_TARGET"] = args.deployment_target self.cmake_platform = [ - f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}" + f"-DCMAKE_OSX_DEPLOYMENT_TARGET={args.deployment_target}", + "-DCMAKE_INSTALL_NAME_DIR=@rpath", + "-DCMAKE_INSTALL_RPATH=@loader_path", + ] + elif not self.windows: + self.cmake_platform = [ + "-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON", + "-DCMAKE_INSTALL_RPATH=$ORIGIN", ] self.record = { "target": args.target, @@ -223,6 +230,8 @@ class NativeBuild: if self.windows else [f"-L{self.prefix / 'lib'}", f"-Wl,-rpath,{self.prefix / 'lib'}"] ) + if self.args.target.endswith("unknown-linux-gnu"): + link[-1] = "-Wl,-rpath,$ORIGIN:$ORIGIN/.." self.environment.update( {"CFLAGS": include, "CXXFLAGS": include, "LDFLAGS": quote(link)} ) diff --git a/third_party/voice/linux_runtime.py b/third_party/voice/linux_runtime.py new file mode 100644 index 0000000000..124e5d40e1 --- /dev/null +++ b/third_party/voice/linux_runtime.py @@ -0,0 +1,152 @@ +"""Prepare verified GNU Linux audio libraries with package-relative loader paths.""" + +import argparse +import os +from pathlib import Path +import re +import struct +import sys + +# Import only this script's siblings, including under PYTHONSAFEPATH. +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from runtime import Binary, PLUGINS, RuntimeFormat, prepare + +SYSTEM_IMPORTS = frozenset( + { + "libc.so.6", + "libm.so.6", + "libdl.so.2", + "libpthread.so.0", + "librt.so.1", + "libresolv.so.2", + "ld-linux-x86-64.so.2", + "ld-linux-aarch64.so.1", + } +) + + +def inspect(path, target): + machine = {"x86_64-unknown-linux-gnu": 62, "aarch64-unknown-linux-gnu": 183}[target] + with path.open("rb") as source: + data = source.read(64 * 1024 * 1024 + 1) + if not 64 <= len(data) <= 64 * 1024 * 1024: + raise ValueError("invalid ELF file size") + header = struct.unpack_from("<16sHHIQQQIHHHHHH", data) + if header[0][:7] != b"\x7fELF\x02\x01\x01" or header[0][7] not in (0, 3): + raise ValueError("expected a little-endian ELF64 library") + if header[1:4] != (3, machine, 1) or header[8:10] != (64, 56): + raise ValueError(f"expected a {target} shared library") + phoff, count = header[5], header[10] + if not 1 <= count <= 128 or phoff < 64 or phoff + count * 56 > len(data): + raise ValueError("invalid ELF program headers") + loads, dynamic = [], [] + for index in range(count): + kind, _, offset, address, _, size, memory_size, _ = struct.unpack_from( + " memory_size or offset + size > len(data): + raise ValueError("invalid ELF segment bounds") + if kind == 1: + loads.append((address, offset, size)) + elif kind == 2: + dynamic.append((offset, size, address)) + elif kind == 3: + raise ValueError("an ELF runtime library must not name an interpreter") + if len(dynamic) != 1 or not 16 <= dynamic[0][1] <= 65536 or dynamic[0][1] % 16: + raise ValueError("invalid ELF dynamic table") + tags = {} + offset, size, address = dynamic[0] + mappings = [ + file_offset + address - start + for start, file_offset, length in loads + if start <= address and address + size <= start + length + ] + if mappings != [offset]: + raise ValueError("ELF dynamic table must have one matching file-backed mapping") + for cursor in range(offset, offset + size, 16): + tag, value = struct.unpack_from(" 1: + raise ValueError("duplicate ELF loader metadata") + for offset in tags.get(tag, []): + end = strings.find(b"\0", offset) + if not 0 <= offset < len(strings) or end < 0: + raise ValueError("invalid ELF loader string") + value = ( + os.fsdecode(strings[offset:end]) + if tag in (15, 29) + else strings[offset:end].decode("ascii") + ) + if tag in (1, 14) and not re.fullmatch( + r"[A-Za-z0-9_+.-]+\.so(?:\.[0-9]+)*", value + ): + raise ValueError("ELF dependencies must be plain library names") + values[tag].append(value) + identity = values[14][0] if values[14] else path.name + return Binary( + identity, + tuple(values[1]), + tuple(f"{tag}={value}" for tag in (15, 29) for value in values[tag]), + ) + + +def finalize_copy(destination, metadata, dependency_paths): + allowed = {"29=$ORIGIN", "29=$ORIGIN:$ORIGIN/.."} + needs_parent = destination.parent.name == "gstreamer-1.0" + required = "29=$ORIGIN:$ORIGIN/.." if needs_parent else "29=$ORIGIN" + if any(path not in allowed for path in metadata.rpaths) or ( + any(name not in SYSTEM_IMPORTS for name in metadata.imports) + and required not in metadata.rpaths + and "29=$ORIGIN:$ORIGIN/.." not in metadata.rpaths + ): + raise ValueError("rebuild native libraries with package-relative runtime paths") + return metadata + + +def project(prefix, receipts, target, output): + if sys.platform != "linux" or target not in ( + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + ): + raise ValueError( + "runtime preparation requires GNU Linux and an explicit GNU target" + ) + format = RuntimeFormat( + tuple(Path(f"lib/gstreamer-1.0/libgst{name}.so") for name in sorted(PLUGINS)), + SYSTEM_IMPORTS, + inspect, + finalize_copy, + plugin_dir="lib/gstreamer-1.0", + ) + prepare(prefix, receipts, target, output, format) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prefix", type=Path, required=True) + parser.add_argument("--receipts", type=Path, required=True) + parser.add_argument("--target", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + project(args.prefix, args.receipts, args.target, args.output) diff --git a/third_party/voice/runtime.py b/third_party/voice/runtime.py index c50055e18b..cbc51855bb 100644 --- a/third_party/voice/runtime.py +++ b/third_party/voice/runtime.py @@ -142,7 +142,7 @@ def prepare(prefix, receipts, target, output, format): for relative in sorted(selected): record, metadata = binaries[relative] destination = output / destinations[relative] - destination.parent.mkdir(exist_ok=True) + destination.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(prefix / relative, destination) if ( digest(destination) != record["sha256"] diff --git a/third_party/voice/test_build_native.py b/third_party/voice/test_build_native.py index a0d1d54467..7da0ce2b56 100644 --- a/third_party/voice/test_build_native.py +++ b/third_party/voice/test_build_native.py @@ -119,6 +119,42 @@ class NativeBuildTests(unittest.TestCase): ) self.assertEqual(result.stdout, "linked") + @unittest.skipIf(os.name == "nt", "Unix install paths; Windows layout is unchanged") + def test_cmake_installs_relative_library_paths(self): + for argument, tool in ( + ("cc", "cc"), + ("cxx", "c++"), + ("cmake", "cmake"), + ("make", "make"), + ): + setattr(self.args, argument, Path(shutil.which(tool))) + source = self.root / "library-source" + source.mkdir() + (source / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.15)\nproject(relative_paths LANGUAGES C)\n" + "add_library(fixture SHARED fixture.c)\ninstall(TARGETS fixture DESTINATION lib)\n" + ) + (source / "fixture.c").write_text("int fixture(void) { return 42; }\n") + build = NativeBuild(self.args, self.environment) + build.output.mkdir() + build.sources = {"fixture": source} + build.cmake("fixture", [], bootstrap=True) + if sys.platform == "darwin": + from macos_runtime import inspect + + metadata = inspect(build.tools / "lib/libfixture.dylib", self.args.target) + self.assertEqual( + (metadata.identity, metadata.rpaths), + ("@rpath/libfixture.dylib", ("@loader_path",)), + ) + else: + from linux_runtime import inspect + + metadata = inspect(build.tools / "lib/libfixture.so", self.args.target) + self.assertEqual( + (metadata.identity, metadata.rpaths), ("libfixture.so", ("29=$ORIGIN",)) + ) + def test_build_refuses_existing_output_before_reading_sources(self): self.args.output.mkdir() (self.args.output / "keep").write_text("untouched") @@ -178,7 +214,12 @@ class NativeBuildTests(unittest.TestCase): else: self.assertEqual( shlex.split(build.environment["LDFLAGS"]), - [f"-L{build.prefix / 'lib'}", f"-Wl,-rpath,{build.prefix / 'lib'}"], + [ + f"-L{build.prefix / 'lib'}", + "-Wl,-rpath,$ORIGIN:$ORIGIN/.." + if build.args.target.endswith("unknown-linux-gnu") + else f"-Wl,-rpath,{build.prefix / 'lib'}", + ], ) def test_windows_paths_use_cygpath_and_propagate_failure(self): diff --git a/third_party/voice/test_linux_runtime.py b/third_party/voice/test_linux_runtime.py new file mode 100644 index 0000000000..7cf8a8c44f --- /dev/null +++ b/third_party/voice/test_linux_runtime.py @@ -0,0 +1,255 @@ +"""Exercise real ELF relocation and rejection of unsafe native build inputs.""" + +import json +import os +from pathlib import Path +import platform +import shutil +import struct +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from linux_runtime import inspect, project +from runtime import PLUGINS, digest + + +@unittest.skipUnless(sys.platform == "linux", "GNU Linux native runtime preparation") +class RuntimeTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="native voice ") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.prefix, self.receipts, self.output = ( + self.root / name for name in ("input é", "receipts", "runtime") + ) + self.target = ( + "aarch64" if platform.machine() == "aarch64" else "x86_64" + ) + "-unknown-linux-gnu" + (self.prefix / "lib/gstreamer-1.0").mkdir(parents=True) + (self.receipts / "inspection").mkdir(parents=True) + source = self.root / "fixture.c" + source.write_text("int voice_fixture(void) { return 42; }\n") + self.library = self.prefix / "lib/libfixture.so.1.2" + subprocess.run( + [ + "cc", + "-shared", + "-fPIC", + str(source), + "-o", + str(self.library), + "-Wl,-soname,libfixture.so.1", + ], + check=True, + capture_output=True, + ) + source.write_text( + "extern int voice_fixture(void); int voice_plugin(void) { return voice_fixture(); }\n" + ) + for name in PLUGINS: + plugin = self.prefix / f"lib/gstreamer-1.0/libgst{name}.so" + subprocess.run( + [ + "cc", + "-shared", + "-fPIC", + str(source), + str(self.library), + "-o", + str(plugin), + f"-Wl,-soname,{plugin.name}", + "-Wl,-rpath,$ORIGIN:$ORIGIN/..", + ], + check=True, + capture_output=True, + ) + self.inventory_path = self.receipts / "inspection/binaries.json" + self.inventory = [ + { + "path": p.relative_to(self.prefix).as_posix(), + "sha256": digest(p), + "target": self.target, + } + for p in sorted(self.prefix.rglob("*.so*")) + if p.is_file() + ] + self.inventory_path.write_text(json.dumps(self.inventory)) + (self.receipts / "ci.json").write_text( + json.dumps( + { + "commit": "a" * 40, + "target": self.target, + "build_complete": True, + "inspection_complete": True, + "manifest_sha256": digest(Path(__file__).with_name("sources.json")), + } + ) + ) + + def test_relocated_libraries_load_without_original_prefix(self): + subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name("linux_runtime.py").resolve()), + "--prefix", + str(self.prefix), + "--receipts", + str(self.receipts), + "--target", + self.target, + "--output", + str(self.output), + ], + check=True, + capture_output=True, + cwd=self.root, + env={**os.environ, "PYTHONSAFEPATH": "1"}, + ) + self.assertEqual( + {r["path"]: digest(self.prefix / r["path"]) for r in self.inventory}, + {r["path"]: r["sha256"] for r in self.inventory}, + ) + moved = self.root / "moved runtime" + self.output.rename(moved) + shutil.rmtree(self.prefix) + environment = {k: v for k, v in os.environ.items() if not k.startswith("LD_")} + result = subprocess.run( + [ + sys.executable, + "-c", + "import ctypes,json,pathlib,sys; root=pathlib.Path(sys.argv[1]); " + "manifest=json.loads((root/'runtime.json').read_text()); " + "print([ctypes.CDLL(str(root/p)).voice_plugin() for p in manifest['plugins']])", + str(moved), + ], + check=True, + capture_output=True, + text=True, + env=environment, + cwd=self.root, + ) + self.assertEqual(json.loads(result.stdout), [42] * len(PLUGINS)) + manifest = json.loads((moved / "runtime.json").read_text()) + self.assertEqual(len(manifest["libraries"]), len(PLUGINS) + 1) + for record in manifest["libraries"]: + path = moved / record["path"] + expected = ( + ("29=$ORIGIN:$ORIGIN/..",) + if path.parent.name == "gstreamer-1.0" + else () + ) + self.assertEqual(inspect(path, self.target).rpaths, expected) + self.assertEqual(digest(path), record["sourceSha256"]) + self.assertEqual(record["sha256"], record["sourceSha256"]) + + def test_absolute_build_paths_require_a_new_native_build(self): + plugin = self.prefix / "lib/gstreamer-1.0/libgstapp.so" + subprocess.run( + ["patchelf", "--set-rpath", str(self.prefix / "lib"), str(plugin)], + check=True, + capture_output=True, + ) + for record in self.inventory: + record["sha256"] = digest(self.prefix / record["path"]) + self.inventory_path.write_text(json.dumps(self.inventory)) + with self.assertRaisesRegex(ValueError, "rebuild native libraries"): + project(self.prefix, self.receipts, self.target, self.output) + self.assertFalse(self.output.exists()) + + def test_missing_dependency_and_digest_mismatch_leave_no_output(self): + for records in ( + [r for r in self.inventory if "libfixture" not in r["path"]], + [{**r, "sha256": "b" * 64} for r in self.inventory], + ): + with self.subTest(records=records), self.assertRaises(ValueError): + self.inventory_path.write_text(json.dumps(records)) + project(self.prefix, self.receipts, self.target, self.output) + self.assertFalse(self.output.exists()) + + def test_path_import_is_not_treated_as_a_system_library(self): + plugin = self.prefix / "lib/gstreamer-1.0/libgstapp.so" + subprocess.run( + [ + "patchelf", + "--replace-needed", + "libfixture.so.1", + "outside\nlibc.so.6", + str(plugin), + ], + check=True, + capture_output=True, + ) + with self.assertRaisesRegex(ValueError, "plain library names"): + inspect(plugin, self.target) + + def test_malformed_header_and_segment_bounds_are_rejected(self): + original = self.library.read_bytes() + for offset, replacement in ( + (18, b"\x00\x00"), + (32, struct.pack("