mirror of
https://github.com/openai/codex.git
synced 2026-09-10 20:26:47 +00:00
Add Windows voice runtime preparation (#42209)
## What changed - Add an x64 and ARM64 MSVC runtime preparer that uses `dumpbin` to validate PE32+ DLL metadata, selects the declared GStreamer plugins and dependency closure, and copies them unchanged into a private `bin/` directory. - Reject malformed or unsupported PE metadata, path-bearing imports, delayed imports, managed DLLs, forwarded exports, undeclared dependencies, and case-insensitive DLL identity conflicts. - Keep `third_party/voice/sources.json` line endings stable across Windows checkouts so native build receipts remain valid. ## Testing Add native Windows tests for relocated DLL loading with a restricted search path, receipt and digest failures, duplicate identities, malformed PE headers, unsupported loader features, and cleanup after failed preparation. GitOrigin-RevId: 5375f21fc54f5d597f3d4e5f8fbca3df307fa2a8
This commit is contained in:
committed by
copyberry
parent
8d01cd42fa
commit
eb10d91e48
1
.gitattributes
vendored
1
.gitattributes
vendored
@@ -1,2 +1,3 @@
|
||||
codex-rs/app-server-protocol/schema/** linguist-generated
|
||||
codex-rs/hooks/schema/generated/** linguist-generated
|
||||
third_party/voice/sources.json text eol=lf
|
||||
|
||||
1
third_party/voice/BUILD.bazel
vendored
1
third_party/voice/BUILD.bazel
vendored
@@ -36,6 +36,7 @@ filegroup(
|
||||
"linux_runtime.py",
|
||||
"macos_runtime.py",
|
||||
"runtime.py",
|
||||
"windows_runtime.py",
|
||||
":source_inputs",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
|
||||
16
third_party/voice/README.md
vendored
16
third_party/voice/README.md
vendored
@@ -131,3 +131,19 @@ and `patchelf`; the latter constructs malformed inputs and is not needed during
|
||||
preparation or shipped in the runtime. The output is development-only, uses the
|
||||
host glibc, and does not establish musl or minimum-glibc support, dynamic-only
|
||||
dependency closure, helper loading policy or working voice.
|
||||
|
||||
## Private Windows runtime preparation
|
||||
|
||||
`windows_runtime.py` takes the same arguments for x64/ARM64 MSVC build prefixes.
|
||||
MSVC's existing `dumpbin` reads PE headers, dependencies and exports; the Python
|
||||
adapter applies package policy without walking binary structures.
|
||||
It checks bounded PE32+ import tables, uses case-insensitive DLL identities, and
|
||||
copies the seven plugins and their declared dependencies into one private `bin/`
|
||||
directory without changing DLL bytes. Delayed imports, managed DLLs and forwarded
|
||||
exports are unsupported and rejected. Native tests require MSVC and Python 3.12;
|
||||
they load the moved DLLs using only the DLL directory and System32 search flags.
|
||||
This development payload expects the Windows Universal CRT and the matching
|
||||
Microsoft Visual C++ runtime (`VCRUNTIME140.dll`) already installed. The latter
|
||||
is not a guaranteed OS component. Release redistribution/licensing, Authenticode
|
||||
policy and actual helper loading remain separate requirements; this script does
|
||||
not install or redistribute Microsoft runtime files or enable voice.
|
||||
|
||||
241
third_party/voice/test_windows_runtime.py
vendored
Normal file
241
third_party/voice/test_windows_runtime.py
vendored
Normal file
@@ -0,0 +1,241 @@
|
||||
"""Exercise native MSVC DLL preparation and restricted-search loading."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from runtime import PLUGINS, digest
|
||||
from windows_runtime import inspect, project
|
||||
|
||||
|
||||
@unittest.skipUnless(sys.platform == "win32", "Windows native runtime preparation")
|
||||
class RuntimeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory(prefix="native voice ")
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
self.prefix, self.receipts, self.output = (
|
||||
self.root / name for name in ("input", "receipts", "runtime")
|
||||
)
|
||||
self.target = (
|
||||
"aarch64"
|
||||
if platform.machine().lower() in ("arm64", "aarch64")
|
||||
else "x86_64"
|
||||
) + "-pc-windows-msvc"
|
||||
(self.prefix / "lib/gstreamer-1.0").mkdir(parents=True)
|
||||
(self.prefix / "bin").mkdir()
|
||||
(self.receipts / "inspection").mkdir(parents=True)
|
||||
source = self.root / "fixture.c"
|
||||
source.write_text(
|
||||
"__declspec(dllexport) int voice_fixture(void) { return 42; }\n"
|
||||
)
|
||||
self.library = self.prefix / "bin/fixture.dll"
|
||||
import_library = self.root / "fixture.lib"
|
||||
subprocess.run(
|
||||
[
|
||||
"cl",
|
||||
"/nologo",
|
||||
"/LD",
|
||||
"/MD",
|
||||
str(source),
|
||||
"/link",
|
||||
f"/OUT:{self.library}",
|
||||
f"/IMPLIB:{import_library}",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
cwd=self.root,
|
||||
)
|
||||
source = self.root / "plugin.c"
|
||||
source.write_text(
|
||||
"__declspec(dllimport) int voice_fixture(void); "
|
||||
"__declspec(dllexport) int voice_plugin(void) { return voice_fixture(); }\n"
|
||||
)
|
||||
for name in PLUGINS:
|
||||
plugin = self.prefix / f"lib/gstreamer-1.0/gst{name}.dll"
|
||||
subprocess.run(
|
||||
[
|
||||
"cl",
|
||||
"/nologo",
|
||||
"/LD",
|
||||
"/MD",
|
||||
str(source),
|
||||
str(import_library),
|
||||
"/link",
|
||||
f"/OUT:{plugin}",
|
||||
f"/IMPLIB:{self.root / (name + '.lib')}",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
cwd=self.root,
|
||||
)
|
||||
self.inventory_path = self.receipts / "inspection/binaries.json"
|
||||
self.inventory = [
|
||||
{
|
||||
"path": str(p.relative_to(self.prefix)),
|
||||
"sha256": digest(p),
|
||||
"target": self.target,
|
||||
}
|
||||
for p in sorted(self.prefix.rglob("*.dll"))
|
||||
]
|
||||
self.inventory_path.write_text(json.dumps(self.inventory))
|
||||
(self.receipts / "ci.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"commit": "a" * 40,
|
||||
"target": self.target,
|
||||
"build_complete": True,
|
||||
"inspection_complete": True,
|
||||
"manifest_sha256": digest(Path(__file__).with_name("sources.json")),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def test_receipts_match_across_checkout_line_endings(self):
|
||||
source = Path(__file__).with_name("sources.json")
|
||||
checkout = self.root / "checkout"
|
||||
manifest = checkout / "third_party/voice/sources.json"
|
||||
manifest.parent.mkdir(parents=True)
|
||||
manifest.write_bytes(source.read_bytes().replace(b"\r\n", b"\n"))
|
||||
shutil.copy2(source.parents[2] / ".gitattributes", checkout)
|
||||
git = ["git", "-C", str(checkout), "-c", "core.autocrlf=false"]
|
||||
subprocess.run([*git, "init", "-q"], check=True, capture_output=True)
|
||||
subprocess.run([*git, "add", "."], check=True, capture_output=True)
|
||||
hashes = []
|
||||
for autocrlf in ("true", "false"):
|
||||
manifest.unlink()
|
||||
subprocess.run(
|
||||
[*git, "-c", f"core.autocrlf={autocrlf}", "checkout-index", "-a", "-f"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
hashes.append(digest(manifest))
|
||||
self.assertEqual(hashes, [digest(source), digest(source)])
|
||||
receipt_path = self.receipts / "ci.json"
|
||||
receipt = json.loads(receipt_path.read_text())
|
||||
receipt["manifest_sha256"] = hashes[0]
|
||||
receipt_path.write_text(json.dumps(receipt))
|
||||
project(self.prefix, self.receipts, self.target, self.output)
|
||||
|
||||
def test_moved_dlls_load_with_private_directory_and_system_search_only(self):
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(Path(__file__).with_name("windows_runtime.py").resolve()),
|
||||
"--prefix",
|
||||
str(self.prefix),
|
||||
"--receipts",
|
||||
str(self.receipts),
|
||||
"--target",
|
||||
self.target,
|
||||
"--output",
|
||||
str(self.output),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
cwd=self.root,
|
||||
env={**os.environ, "PYTHONSAFEPATH": "1"},
|
||||
)
|
||||
self.assertEqual(
|
||||
{r["path"]: digest(self.prefix / r["path"]) for r in self.inventory},
|
||||
{r["path"]: r["sha256"] for r in self.inventory},
|
||||
)
|
||||
moved = self.root / "moved runtime"
|
||||
self.output.rename(moved)
|
||||
shutil.rmtree(self.prefix)
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import ctypes,json,pathlib,sys; root=pathlib.Path(sys.argv[1]); "
|
||||
"manifest=json.loads((root/'runtime.json').read_text()); "
|
||||
"print([ctypes.CDLL(str(root/p),winmode=0x100|0x800).voice_plugin() for p in manifest['plugins']])",
|
||||
str(moved),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=self.root,
|
||||
)
|
||||
self.assertEqual(json.loads(result.stdout), [42] * len(PLUGINS))
|
||||
manifest = json.loads((moved / "runtime.json").read_text())
|
||||
self.assertEqual(len(manifest["libraries"]), len(PLUGINS) + 1)
|
||||
for record in manifest["libraries"]:
|
||||
self.assertEqual(digest(moved / record["path"]), record["sourceSha256"])
|
||||
|
||||
def test_missing_dependency_and_digest_mismatch_leave_no_output(self):
|
||||
for records in (
|
||||
[r for r in self.inventory if "fixture.dll" not in r["path"]],
|
||||
[{**r, "sha256": "b" * 64} for r in self.inventory],
|
||||
):
|
||||
with self.subTest(records=records), self.assertRaises(ValueError):
|
||||
self.inventory_path.write_text(json.dumps(records))
|
||||
project(self.prefix, self.receipts, self.target, self.output)
|
||||
self.assertFalse(self.output.exists())
|
||||
|
||||
def test_case_alias_dll_identities_are_rejected(self):
|
||||
duplicate = self.prefix / "lib/FIXTURE.dll"
|
||||
shutil.copy2(self.library, duplicate)
|
||||
records = self.inventory + [
|
||||
{
|
||||
"path": str(duplicate.relative_to(self.prefix)),
|
||||
"sha256": digest(duplicate),
|
||||
"target": self.target,
|
||||
}
|
||||
]
|
||||
self.inventory_path.write_text(json.dumps(records))
|
||||
with self.assertRaisesRegex(ValueError, "duplicate"):
|
||||
project(self.prefix, self.receipts, self.target, self.output)
|
||||
self.assertFalse(self.output.exists())
|
||||
|
||||
def test_malformed_headers_and_delayed_imports_are_rejected(self):
|
||||
original = self.library.read_bytes()
|
||||
pe = struct.unpack_from("<I", original, 60)[0]
|
||||
for offset, replacement in (
|
||||
(60, struct.pack("<I", len(original))),
|
||||
(pe + 4, b"\0\0"),
|
||||
(pe + 6, b"\xff\xff"),
|
||||
(pe + 24 + 112 + 13 * 8, struct.pack("<II", 1, 32)),
|
||||
):
|
||||
data = bytearray(original)
|
||||
data[offset : offset + len(replacement)] = replacement
|
||||
self.library.write_bytes(data)
|
||||
with self.subTest(offset=offset), self.assertRaises(ValueError):
|
||||
inspect(self.library, self.target)
|
||||
|
||||
def test_path_bearing_import_is_rejected(self):
|
||||
plugin = self.prefix / "lib/gstreamer-1.0/gstapp.dll"
|
||||
original = plugin.read_bytes()
|
||||
self.assertIn(b"fixture.dll\0", original)
|
||||
plugin.write_bytes(original.replace(b"fixture.dll\0", b"/ixture.dll\0"))
|
||||
with self.assertRaisesRegex(ValueError, "plain DLL names"):
|
||||
inspect(plugin, self.target)
|
||||
|
||||
def test_forwarded_exports_are_rejected(self):
|
||||
definition = self.root / "forward.def"
|
||||
definition.write_text("EXPORTS\nforwarded=KERNEL32.Sleep\n")
|
||||
destination = self.root / "forward.dll"
|
||||
machine = "ARM64" if self.target.startswith("aarch64") else "X64"
|
||||
subprocess.run(
|
||||
[
|
||||
"link",
|
||||
"/NOLOGO",
|
||||
"/DLL",
|
||||
"/NOENTRY",
|
||||
f"/MACHINE:{machine}",
|
||||
f"/DEF:{definition}",
|
||||
f"/OUT:{destination}",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
cwd=self.root,
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "forwarded DLL exports"):
|
||||
inspect(destination, self.target)
|
||||
145
third_party/voice/windows_runtime.py
vendored
Normal file
145
third_party/voice/windows_runtime.py
vendored
Normal file
@@ -0,0 +1,145 @@
|
||||
"""Prepare verified Windows audio DLLs in one private development runtime directory."""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Import only this script's siblings, including under PYTHONSAFEPATH.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from runtime import Binary, PLUGINS, RuntimeFormat, prepare
|
||||
|
||||
# VCRUNTIME140 is a development prerequisite, not a guaranteed Windows component.
|
||||
EXTERNAL_IMPORTS = frozenset(
|
||||
{
|
||||
"advapi32.dll",
|
||||
"dnsapi.dll",
|
||||
"iphlpapi.dll",
|
||||
"kernel32.dll",
|
||||
"ole32.dll",
|
||||
"shell32.dll",
|
||||
"shlwapi.dll",
|
||||
"user32.dll",
|
||||
"ws2_32.dll",
|
||||
"vcruntime140.dll",
|
||||
*(
|
||||
f"api-ms-win-crt-{part}-l1-1-0.dll"
|
||||
for part in (
|
||||
"conio",
|
||||
"convert",
|
||||
"environment",
|
||||
"filesystem",
|
||||
"heap",
|
||||
"locale",
|
||||
"math",
|
||||
"process",
|
||||
"runtime",
|
||||
"stdio",
|
||||
"string",
|
||||
"time",
|
||||
"utility",
|
||||
)
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def inspect(path, target):
|
||||
machine = {"x86_64-pc-windows-msvc": "8664", "aarch64-pc-windows-msvc": "AA64"}[
|
||||
target
|
||||
]
|
||||
if not 64 <= path.stat().st_size <= 64 * 1024 * 1024:
|
||||
raise ValueError("invalid PE file size")
|
||||
result = subprocess.run(
|
||||
["dumpbin", "/nologo", "/headers", "/dependents", "/exports", str(path)],
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
)
|
||||
output = result.stdout.decode("ascii", errors="backslashreplace").replace(
|
||||
"\r\n", "\n"
|
||||
)
|
||||
if (
|
||||
result.returncode
|
||||
or result.stderr
|
||||
or re.search(r"(?:fatal error|warning) LNK[0-9]+", output)
|
||||
):
|
||||
raise ValueError("dumpbin rejected the PE library")
|
||||
headers = output.split("SECTION HEADER #", 1)[0]
|
||||
if (
|
||||
"File Type: DLL\n" not in headers
|
||||
or not re.search(r"^ +" + machine + r" machine \(", headers, re.M)
|
||||
or not re.search(r"^ +20B magic # \(PE32\+\)$", headers, re.M)
|
||||
):
|
||||
raise ValueError(f"expected a {target} PE32+ DLL")
|
||||
count = re.search(r"^ +([0-9A-F]+) number of sections$", headers, re.M)
|
||||
if not count or not 1 <= int(count[1], 16) <= 96:
|
||||
raise ValueError("invalid PE section table")
|
||||
directories = {
|
||||
name: (int(address, 16), int(size, 16))
|
||||
for address, size, name in re.findall(
|
||||
r"^ +([0-9A-F]+) \[ *([0-9A-F]+)\] RVA \[size\] of ([^\n]+ Directory)$",
|
||||
headers,
|
||||
re.M,
|
||||
)
|
||||
}
|
||||
if len(directories) != 16:
|
||||
raise ValueError("unsupported PE data directories")
|
||||
if any(
|
||||
directories.get(name) != (0, 0)
|
||||
for name in ("Delay Import Directory", "COM Descriptor Directory")
|
||||
):
|
||||
raise ValueError("delay-load and managed DLL dependencies are unsupported")
|
||||
if "(forwarded to " in output:
|
||||
raise ValueError("forwarded DLL exports are unsupported loader dependencies")
|
||||
groups = re.findall(
|
||||
r"\n Image has the following dependencies:\n\n(.*?)(?:\n\n|\Z)", output, re.S
|
||||
)
|
||||
if len(groups) > 1:
|
||||
raise ValueError("ambiguous dumpbin dependencies")
|
||||
imports = (
|
||||
tuple(line.strip().lower() for line in groups[0].splitlines()) if groups else ()
|
||||
)
|
||||
if any(not re.fullmatch(r"[a-z0-9_+.-]+\.dll", name) for name in imports):
|
||||
raise ValueError("PE dependencies must be plain DLL names")
|
||||
address, size = directories["Import Directory"]
|
||||
# A descriptor per DLL plus its null terminator; reject truncated/injected output.
|
||||
if (address, size) != (0, 0) and (
|
||||
not address or size != 20 * (len(imports) + 1) or len(imports) > 128
|
||||
):
|
||||
raise ValueError("inconsistent PE import directory")
|
||||
return Binary(path.name.lower(), imports, ())
|
||||
|
||||
|
||||
def finalize_copy(destination, metadata, dependency_paths):
|
||||
# DLL import names already resolve among siblings; do not alter signed bytes.
|
||||
return metadata
|
||||
|
||||
|
||||
def project(prefix, receipts, target, output):
|
||||
if sys.platform != "win32" or target not in (
|
||||
"x86_64-pc-windows-msvc",
|
||||
"aarch64-pc-windows-msvc",
|
||||
):
|
||||
raise ValueError(
|
||||
"runtime preparation requires Windows and an explicit MSVC target"
|
||||
)
|
||||
format = RuntimeFormat(
|
||||
tuple(Path(f"lib/gstreamer-1.0/gst{name}.dll") for name in sorted(PLUGINS)),
|
||||
EXTERNAL_IMPORTS,
|
||||
inspect,
|
||||
finalize_copy,
|
||||
library_dir="bin",
|
||||
plugin_dir="bin",
|
||||
)
|
||||
prepare(prefix, receipts, target, output, format)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--prefix", type=Path, required=True)
|
||||
parser.add_argument("--receipts", type=Path, required=True)
|
||||
parser.add_argument("--target", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
project(args.prefix, args.receipts, args.target, args.output)
|
||||
Reference in New Issue
Block a user