mirror of
https://github.com/openai/codex.git
synced 2026-11-01 19:33:35 +00:00
Wire native voice dependencies into full Rust CI (#58719)
## What changed - Build native voice SDK and runtime artifacts with Bazel for Linux GNU, macOS, and Windows on x64 and ARM64, then configure Cargo jobs to consume them. Keep musl builds in a separate job without these inputs. - Validate artifact receipts against the target, source commit, and source manifest; verify SDK file hashes and relocate pkg-config metadata. Package the runtime and plugins with nextest archives and restore `CODEX_TEST_VOICE_RUNTIME` when running shards. - Prepare native Windows voice tools for argument-comment linting and install ARM64 DotSlash on Windows ARM64. Reduce archive disk usage, bound Linux ARM64 build concurrency, and correct the JUnit artifact path. - Recognize TLS errors wrapped directly in `io::Error`. Make TLS fallback fixtures select the native backend explicitly, allow the MCP test server's non-loopback address, and isolate skill-root tests from ancestor Git markers. Install `git` in remote test containers and filter MCP discovery assertions to POST requests. ## Testing Add tests for relocated voice inputs, Windows ARM64 import libraries with host-native pkgconf, and rejection of mismatched receipts or corrupt SDK files. Extend untrusted-certificate coverage to an explicitly selected rustls backend. GitOrigin-RevId: da3ff690b94906edc1f8f60a3c7feda1bf66134a
This commit is contained in:
Benjamin Carlsson
committed by
copyberry
copyberry
parent
8c811e8d93
commit
e02a32cd51
@@ -18,6 +18,26 @@ runs:
|
||||
target: ${{ inputs.target }}
|
||||
cache-scope: bazel-argument-comment-lint
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
if: ${{ runner.os == 'Windows' && inputs.buildbuddy-api-key == '' }}
|
||||
with:
|
||||
python-version: "3.12"
|
||||
architecture: x64
|
||||
|
||||
- name: Prepare native Windows audio dependencies
|
||||
if: ${{ runner.os == 'Windows' && inputs.buildbuddy-api-key == '' }}
|
||||
uses: ./third_party/voice/cargo-ci/windows-setup
|
||||
with:
|
||||
target: x86_64-pc-windows-msvc
|
||||
|
||||
- name: Verify native Windows audio dependencies
|
||||
if: ${{ runner.os == 'Windows' && inputs.buildbuddy-api-key == '' }}
|
||||
shell: bash
|
||||
run: |
|
||||
: "${VOICE_WINDOWS_BAZEL_REPOSITORY:?Windows voice tools are required}"
|
||||
: "${VOICE_WINDOWS_SYSTEM_ROOT:?Windows SystemRoot is required}"
|
||||
: "${VOICE_WINDOWS_HOST_ARCH:?Windows host architecture is required}"
|
||||
|
||||
- name: Install Linux sandbox build dependencies
|
||||
if: ${{ runner.os == 'Linux' }}
|
||||
shell: bash
|
||||
|
||||
@@ -81,10 +81,28 @@ runs:
|
||||
run: mkdir -p "$RUNNER_TEMP/dotslash-curl" && echo retry-all-errors > "$RUNNER_TEMP/dotslash-curl/.curlrc"
|
||||
|
||||
- name: Install DotSlash
|
||||
if: runner.os != 'Windows' || runner.arch != 'ARM64'
|
||||
uses: meta/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2
|
||||
env:
|
||||
CURL_HOME: ${{ runner.temp }}/dotslash-curl
|
||||
|
||||
- name: Install native DotSlash (Windows ARM64)
|
||||
if: runner.os == 'Windows' && runner.arch == 'ARM64'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# DotSlash selects resource architecture from its own executable.
|
||||
destination="$RUNNER_TEMP/install-dotslash-arm64/bin"
|
||||
archive="$RUNNER_TEMP/dotslash-windows-arm64.v0.5.9.tar.gz"
|
||||
mkdir -p "$destination"
|
||||
curl --location --silent --show-error --fail --retry 5 --retry-all-errors \
|
||||
https://github.com/facebook/dotslash/releases/download/v0.5.9/dotslash-windows-arm64.v0.5.9.tar.gz \
|
||||
--output "$archive"
|
||||
echo "43d49d3472808738993ccf95a36dda8cc2869996d9a397bb02bde1e39b16146b $archive" | sha256sum --check
|
||||
tar xzf "$archive" -O --force-local > "$destination/dotslash.exe"
|
||||
echo "$destination" >> "$GITHUB_PATH"
|
||||
"$destination/dotslash.exe" --version
|
||||
|
||||
- name: Install just
|
||||
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
|
||||
with:
|
||||
|
||||
@@ -9,6 +9,21 @@ if [[ "${RUNNER_OS:-}" == "Windows" ]]; then
|
||||
# lint aspect can run across the compatible crate graph instead of failing the
|
||||
# whole build after analysis.
|
||||
bazel_lint_args+=("--skip_incompatible_explicit_targets")
|
||||
if [[ -z "${BUILDBUDDY_API_KEY:-}" && -n "${VOICE_WINDOWS_BAZEL_REPOSITORY:-}" ]]; then
|
||||
: "${VOICE_WINDOWS_BAZEL_REPOSITORY:?Windows voice tools are required}"
|
||||
: "${VOICE_WINDOWS_SYSTEM_ROOT:?Windows SystemRoot is required}"
|
||||
: "${VOICE_WINDOWS_HOST_ARCH:?Windows host architecture is required}"
|
||||
bazel_lint_args+=(
|
||||
"--extra_toolchains=//third_party/voice:windows_pkg_config_toolchain,//third_party/voice:windows_cmake_toolchain"
|
||||
"--repo_env=BAZEL_MSVC_RUNTIME_VISUAL_STUDIO_EULA=1"
|
||||
"--inject_repository=voice_windows_tools=${VOICE_WINDOWS_BAZEL_REPOSITORY}"
|
||||
"--//third_party/voice:windows_installed_tools=@voice_windows_tools//:tools"
|
||||
"--action_env=SystemRoot=${VOICE_WINDOWS_SYSTEM_ROOT}"
|
||||
"--host_action_env=SystemRoot=${VOICE_WINDOWS_SYSTEM_ROOT}"
|
||||
"--action_env=PROCESSOR_ARCHITECTURE=${VOICE_WINDOWS_HOST_ARCH}"
|
||||
"--host_action_env=PROCESSOR_ARCHITECTURE=${VOICE_WINDOWS_HOST_ARCH}"
|
||||
)
|
||||
fi
|
||||
fi
|
||||
|
||||
read_query_labels() {
|
||||
|
||||
@@ -65,6 +65,8 @@ jobs:
|
||||
CODEX_CI_PROFILE: ${{ inputs.profile }}
|
||||
CODEX_CI_TARGET: ${{ inputs.target }}
|
||||
CARGO_INCREMENTAL: "0"
|
||||
# Debug symbols for hundreds of test binaries can exhaust the archive runner's disk.
|
||||
CARGO_PROFILE_CI_TEST_DEBUG: "0"
|
||||
SCCACHE_CACHE_SIZE: 10G
|
||||
NEXTEST_ARCHIVE_FILE: nextest-${{ inputs.artifact_id }}.tar.zst
|
||||
TEST_HELPERS_ARTIFACT: nextest-test-helpers-${{ inputs.artifact_id }}
|
||||
@@ -73,6 +75,15 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Free disk space for the macOS test archive
|
||||
if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64' && runner.environment == 'github-hosted' }}
|
||||
shell: bash
|
||||
run: |
|
||||
# The macOS build does not use the hosted image's Android SDK or NDKs.
|
||||
df -h "$RUNNER_TEMP"
|
||||
sudo rm -rf /Users/runner/Library/Android/sdk
|
||||
df -h "$RUNNER_TEMP"
|
||||
|
||||
- uses: ./.github/actions/setup-ci
|
||||
|
||||
- name: Install Linux build dependencies
|
||||
@@ -165,13 +176,25 @@ jobs:
|
||||
tool: nextest
|
||||
version: 0.9.103
|
||||
|
||||
- name: Set up native voice dependencies
|
||||
uses: ./third_party/voice/cargo-ci/setup
|
||||
with:
|
||||
target: ${{ inputs.target }}
|
||||
|
||||
- name: Build nextest archive
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
archive_dir="${RUNNER_TEMP}/nextest-archive"
|
||||
mkdir -p "${archive_dir}"
|
||||
cargo nextest archive \
|
||||
if [[ "${RUNNER_OS}" == "Linux" && "${RUNNER_ARCH}" == "ARM64" && "${CODEX_CI_TARGET}" == "aarch64-unknown-linux-gnu" ]]; then
|
||||
# Bound concurrency for full GNU ARM archives on native Linux runners.
|
||||
nproc
|
||||
free -h
|
||||
export CARGO_BUILD_JOBS=4
|
||||
fi
|
||||
printf '%s\n' "$CODEX_VOICE_RUNTIME_RELATIVE" > "$archive_dir/voice-runtime-path.txt"
|
||||
cargo nextest --tool-config-file "codex-voice:$CODEX_VOICE_NEXTEST_CONFIG" archive \
|
||||
--target "${CODEX_CI_TARGET}" \
|
||||
--cargo-profile "${CODEX_CI_PROFILE}" \
|
||||
--timings \
|
||||
@@ -215,7 +238,7 @@ jobs:
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: nextest-archive-${{ inputs.artifact_id }}
|
||||
path: ${{ runner.temp }}/nextest-archive/${{ env.NEXTEST_ARCHIVE_FILE }}
|
||||
path: ${{ runner.temp }}/nextest-archive/
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
@@ -341,10 +364,16 @@ jobs:
|
||||
set -euo pipefail
|
||||
archive_file="${RUNNER_TEMP}/nextest-archive/${NEXTEST_ARCHIVE_FILE}"
|
||||
workspace_root="$(pwd)"
|
||||
extract_dir="${RUNNER_TEMP}/nextest-extracted"
|
||||
mkdir -p "$extract_dir"
|
||||
runtime_relative="$(cat "${RUNNER_TEMP}/nextest-archive/voice-runtime-path.txt")"
|
||||
export CODEX_TEST_VOICE_RUNTIME="$extract_dir/target/$runtime_relative"
|
||||
|
||||
if [[ "${RUNNER_OS}" == "Windows" ]]; then
|
||||
archive_file="$(cygpath -w "${archive_file}")"
|
||||
workspace_root="$(cygpath -w "${workspace_root}")"
|
||||
extract_dir="$(cygpath -w "$extract_dir")"
|
||||
CODEX_TEST_VOICE_RUNTIME="$(cygpath -w "$CODEX_TEST_VOICE_RUNTIME")"
|
||||
fi
|
||||
|
||||
if [[ "${RUNNER_OS}" == "Linux" ]]; then
|
||||
@@ -365,6 +394,7 @@ jobs:
|
||||
run
|
||||
--no-fail-fast
|
||||
--archive-file "${archive_file}"
|
||||
--extract-to "$extract_dir"
|
||||
--workspace-remap "${workspace_root}"
|
||||
--partition "hash:${{ matrix.shard }}/4"
|
||||
)
|
||||
@@ -395,6 +425,8 @@ jobs:
|
||||
"${test_command[@]}"
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
# Keep panic backtraces without changing the CLI errors asserted by tests.
|
||||
RUST_LIB_BACKTRACE: "0"
|
||||
RUST_MIN_STACK: "8388608" # 8 MiB
|
||||
NEXTEST_STATUS_LEVEL: leak
|
||||
|
||||
@@ -403,7 +435,7 @@ jobs:
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: nextest-junit-rust-ci-${{ inputs.artifact_id }}-shard-${{ matrix.shard }}
|
||||
path: ${{ env.CARGO_TARGET_DIR }}/nextest/default/junit.xml
|
||||
path: codex-rs/target/nextest/default/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Tear down remote test env
|
||||
|
||||
@@ -92,7 +92,7 @@ jobs:
|
||||
argument_comment_lint_prebuilt:
|
||||
name: Argument comment lint - ${{ matrix.name }}
|
||||
runs-on: ${{ matrix.runs_on || matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 120
|
||||
environment:
|
||||
name: bazel
|
||||
deployment: false
|
||||
@@ -117,17 +117,51 @@ jobs:
|
||||
uses: ./.github/actions/run-argument-comment-lint
|
||||
with:
|
||||
target: ${{ runner.os }}
|
||||
buildbuddy-api-key: ${{ secrets.BUILDBUDDY_API_KEY }}
|
||||
|
||||
# --- CI to validate on different os/targets --------------------------------
|
||||
cargo_voice_inputs:
|
||||
name: Cargo voice inputs - ${{ matrix.target }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
environment:
|
||||
name: bazel
|
||||
deployment: false
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 120
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { target: x86_64-unknown-linux-gnu, runner: ubuntu-24.04 }
|
||||
- { target: aarch64-unknown-linux-gnu, runner: ubuntu-24.04-arm }
|
||||
- { target: x86_64-apple-darwin, runner: macos-15-intel }
|
||||
- { target: aarch64-apple-darwin, runner: macos-15 }
|
||||
- target: x86_64-pc-windows-msvc
|
||||
runner:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-windows-x64
|
||||
- target: aarch64-pc-windows-msvc
|
||||
runner:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-windows-arm64
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./third_party/voice/cargo-ci/produce
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
lint_build:
|
||||
name: Lint/Build — ${{ matrix.runner }} - ${{ matrix.target }}${{ matrix.profile == 'release' && ' (release)' || '' }}
|
||||
runs-on: ${{ matrix.runs_on || matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
defaults:
|
||||
needs: cargo_voice_inputs
|
||||
if: &lint_build_if ${{ !cancelled() }}
|
||||
name: &lint_build_name Lint/Build — ${{ matrix.runner }} - ${{ matrix.target }}${{ matrix.profile == 'release' && ' (release)' || '' }}
|
||||
runs-on: &lint_build_runner ${{ matrix.runs_on || matrix.runner }}
|
||||
timeout-minutes: &lint_build_timeout 30
|
||||
defaults: &lint_build_defaults
|
||||
run:
|
||||
working-directory: codex-rs
|
||||
env:
|
||||
env: &lint_build_env
|
||||
# Speed up repeated builds across CI runs by caching compiled objects, except on
|
||||
# arm64 macOS runners cross-targeting x86_64 where ring/cc-rs can produce
|
||||
# mixed-architecture archives under sccache.
|
||||
@@ -145,24 +179,12 @@ jobs:
|
||||
- runner: macos-15-xlarge
|
||||
target: x86_64-apple-darwin
|
||||
profile: dev
|
||||
- runner: ubuntu-24.04
|
||||
target: x86_64-unknown-linux-musl
|
||||
profile: dev
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-x64
|
||||
- runner: ubuntu-24.04
|
||||
target: x86_64-unknown-linux-gnu
|
||||
profile: dev
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-x64
|
||||
- runner: ubuntu-24.04-arm
|
||||
target: aarch64-unknown-linux-musl
|
||||
profile: dev
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-arm64
|
||||
- runner: ubuntu-24.04-arm
|
||||
target: aarch64-unknown-linux-gnu
|
||||
profile: dev
|
||||
@@ -189,18 +211,6 @@ jobs:
|
||||
- runner: macos-15-xlarge
|
||||
target: aarch64-apple-darwin
|
||||
profile: release
|
||||
- runner: ubuntu-24.04
|
||||
target: x86_64-unknown-linux-musl
|
||||
profile: release
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-x64
|
||||
- runner: ubuntu-24.04-arm
|
||||
target: aarch64-unknown-linux-musl
|
||||
profile: release
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-arm64
|
||||
- runner: windows-x64
|
||||
target: x86_64-pc-windows-msvc
|
||||
profile: release
|
||||
@@ -214,7 +224,7 @@ jobs:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-windows-arm64
|
||||
|
||||
steps:
|
||||
steps: &lint_build_steps
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
@@ -348,6 +358,12 @@ jobs:
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Set up native voice dependencies
|
||||
if: ${{ !endsWith(matrix.target, '-musl') }}
|
||||
uses: ./third_party/voice/cargo-ci/setup
|
||||
with:
|
||||
target: ${{ matrix.target }}
|
||||
|
||||
- name: Install cargo-chef
|
||||
if: ${{ matrix.profile == 'release' }}
|
||||
uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49
|
||||
@@ -427,7 +443,46 @@ jobs:
|
||||
/var/cache/apt
|
||||
key: apt-${{ matrix.runner }}-${{ matrix.target }}-v1
|
||||
|
||||
lint_build_musl:
|
||||
if: *lint_build_if
|
||||
name: *lint_build_name
|
||||
runs-on: *lint_build_runner
|
||||
timeout-minutes: *lint_build_timeout
|
||||
defaults: *lint_build_defaults
|
||||
env: *lint_build_env
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: ubuntu-24.04
|
||||
target: x86_64-unknown-linux-musl
|
||||
profile: dev
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-x64
|
||||
- runner: ubuntu-24.04-arm
|
||||
target: aarch64-unknown-linux-musl
|
||||
profile: dev
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-arm64
|
||||
- runner: ubuntu-24.04
|
||||
target: x86_64-unknown-linux-musl
|
||||
profile: release
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-x64
|
||||
- runner: ubuntu-24.04-arm
|
||||
target: aarch64-unknown-linux-musl
|
||||
profile: release
|
||||
runs_on:
|
||||
group: ${{ github.event.repository.name }}-runners
|
||||
labels: ${{ github.event.repository.name }}-linux-arm64
|
||||
steps: *lint_build_steps
|
||||
|
||||
tests_macos_aarch64:
|
||||
needs: cargo_voice_inputs
|
||||
if: ${{ !cancelled() }}
|
||||
name: Tests — macos-15-xlarge - aarch64-apple-darwin
|
||||
uses: ./.github/workflows/rust-ci-full-nextest-platform.yml
|
||||
with:
|
||||
@@ -439,6 +494,8 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
tests_linux_x64_remote:
|
||||
needs: cargo_voice_inputs
|
||||
if: ${{ !cancelled() }}
|
||||
name: Tests — ubuntu-24.04 - x86_64-unknown-linux-gnu (remote)
|
||||
uses: ./.github/workflows/rust-ci-full-nextest-platform.yml
|
||||
with:
|
||||
@@ -453,6 +510,8 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
tests_linux_arm64:
|
||||
needs: cargo_voice_inputs
|
||||
if: ${{ !cancelled() }}
|
||||
name: Tests — ubuntu-24.04-arm - aarch64-unknown-linux-gnu
|
||||
uses: ./.github/workflows/rust-ci-full-nextest-platform.yml
|
||||
with:
|
||||
@@ -466,6 +525,8 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
tests_windows_x64:
|
||||
needs: cargo_voice_inputs
|
||||
if: ${{ !cancelled() }}
|
||||
name: Tests — windows-x64 - x86_64-pc-windows-msvc
|
||||
uses: ./.github/workflows/rust-ci-full-nextest-platform.yml
|
||||
with:
|
||||
@@ -479,6 +540,8 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
tests_windows_arm64:
|
||||
needs: cargo_voice_inputs
|
||||
if: ${{ !cancelled() }}
|
||||
name: Tests — windows-arm64 - aarch64-pc-windows-msvc
|
||||
uses: ./.github/workflows/rust-ci-full-nextest-platform.yml
|
||||
with:
|
||||
@@ -504,7 +567,9 @@ jobs:
|
||||
cargo_shear,
|
||||
argument_comment_lint_package,
|
||||
argument_comment_lint_prebuilt,
|
||||
cargo_voice_inputs,
|
||||
lint_build,
|
||||
lint_build_musl,
|
||||
tests_macos_aarch64,
|
||||
tests_linux_x64_remote,
|
||||
tests_linux_arm64,
|
||||
@@ -522,6 +587,7 @@ jobs:
|
||||
echo "general: ${{ needs.general.result }}"
|
||||
echo "shear : ${{ needs.cargo_shear.result }}"
|
||||
echo "lint : ${{ needs.lint_build.result }}"
|
||||
echo "musl : ${{ needs.lint_build_musl.result }}"
|
||||
echo "test macos : ${{ needs.tests_macos_aarch64.result }}"
|
||||
echo "test linux : ${{ needs.tests_linux_x64_remote.result }}"
|
||||
echo "test arm64 : ${{ needs.tests_linux_arm64.result }}"
|
||||
@@ -531,7 +597,9 @@ jobs:
|
||||
[[ '${{ needs.argument_comment_lint_prebuilt.result }}' == 'success' ]] || { echo 'argument_comment_lint_prebuilt failed'; exit 1; }
|
||||
[[ '${{ needs.general.result }}' == 'success' ]] || { echo 'general failed'; exit 1; }
|
||||
[[ '${{ needs.cargo_shear.result }}' == 'success' ]] || { echo 'cargo_shear failed'; exit 1; }
|
||||
[[ '${{ needs.cargo_voice_inputs.result }}' == 'success' ]] || { echo 'cargo_voice_inputs failed'; exit 1; }
|
||||
[[ '${{ needs.lint_build.result }}' == 'success' ]] || { echo 'lint_build failed'; exit 1; }
|
||||
[[ '${{ needs.lint_build_musl.result }}' == 'success' ]] || { echo 'lint_build_musl failed'; exit 1; }
|
||||
[[ '${{ needs.tests_macos_aarch64.result }}' == 'success' ]] || { echo 'tests_macos_aarch64 failed'; exit 1; }
|
||||
[[ '${{ needs.tests_linux_x64_remote.result }}' == 'success' ]] || { echo 'tests_linux_x64_remote failed'; exit 1; }
|
||||
[[ '${{ needs.tests_linux_arm64.result }}' == 'success' ]] || { echo 'tests_linux_arm64 failed'; exit 1; }
|
||||
|
||||
@@ -445,9 +445,15 @@ async fn repo_ancestry_without_project_marker_does_not_walk_parents() {
|
||||
fs::create_dir_all(outer.join(".agents/skills")).expect("create outer skills");
|
||||
fs::create_dir_all(cwd.join(".agents/skills")).expect("create cwd skills");
|
||||
|
||||
// Sandbox runners can put a .git marker above the temporary directory.
|
||||
// Probe for an absent fixture marker while retaining the real ancestor search.
|
||||
let config_stack = stack(vec![ConfigLayerEntry::new(
|
||||
ConfigLayerSource::SessionFlags,
|
||||
toml::toml! { project_root_markers = [".missing-skill-test-project"] }.into(),
|
||||
)]);
|
||||
let roots = repo_agents_skill_roots(
|
||||
Some(Arc::clone(&LOCAL_FS)),
|
||||
&stack(Vec::new()),
|
||||
&config_stack,
|
||||
&cwd,
|
||||
/*empty_remote_roots*/ None,
|
||||
)
|
||||
|
||||
@@ -110,13 +110,6 @@ impl RouteAwareRequestError {
|
||||
if self.status() == Some(StatusCode::PROXY_AUTHENTICATION_REQUIRED) {
|
||||
return Some(RouteFailureClass::ProxyAuthenticationRequired);
|
||||
}
|
||||
if let Self::Route(RouteAwareClientPoolError::Resolve(error)) = self
|
||||
&& let Some(source) = error.get_ref()
|
||||
&& source.is::<rustls::Error>()
|
||||
{
|
||||
return Some(RouteFailureClass::TlsError);
|
||||
}
|
||||
|
||||
let mut source: Option<&(dyn std::error::Error + 'static)> = Some(self);
|
||||
while let Some(error) = source {
|
||||
if error.downcast_ref::<rustls::Error>().is_some()
|
||||
@@ -127,7 +120,14 @@ impl RouteAwareRequestError {
|
||||
if error.to_string() == "tunnel error: proxy authorization required" {
|
||||
return Some(RouteFailureClass::ProxyAuthenticationRequired);
|
||||
}
|
||||
source = error.source();
|
||||
// io::Error::source skips its inner error, which may itself be the TLS error.
|
||||
source = match error
|
||||
.downcast_ref::<io::Error>()
|
||||
.and_then(io::Error::get_ref)
|
||||
{
|
||||
Some(inner) => Some(inner),
|
||||
None => error.source(),
|
||||
};
|
||||
}
|
||||
|
||||
match self {
|
||||
|
||||
@@ -227,6 +227,15 @@ async fn direct_and_routed_clients_build_equivalent_requests() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_failures_classify_real_untrusted_certificate_handshakes() {
|
||||
assert_untrusted_certificate_classified(HttpClientBuilder::new()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_failures_classify_real_untrusted_rustls_certificate_handshakes() {
|
||||
assert_untrusted_certificate_classified(HttpClientBuilder::new().with_rustls_tls()).await;
|
||||
}
|
||||
|
||||
async fn assert_untrusted_certificate_classified(client_builder: HttpClientBuilder) {
|
||||
codex_utils_rustls_provider::ensure_rustls_crypto_provider();
|
||||
let certificate = rcgen::generate_simple_self_signed(vec!["localhost".to_string()])
|
||||
.expect("self-signed certificate should generate");
|
||||
@@ -252,9 +261,10 @@ async fn request_failures_classify_real_untrusted_certificate_handshakes() {
|
||||
.expect("TLS server connection should be created");
|
||||
let _ = connection.complete_io(&mut stream);
|
||||
});
|
||||
let pool = RouteAwareClientPool::new_without_request_logging(
|
||||
let pool = RouteAwareClientPool::with_builder(
|
||||
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||
ClientRouteClass::Api,
|
||||
client_builder.without_request_logging(),
|
||||
);
|
||||
|
||||
let mut request = reqwest::Request::new(
|
||||
|
||||
@@ -29,6 +29,13 @@ use crate::tls_backend_fallback::should_retry_with_rustls;
|
||||
|
||||
const PROTOCOL_VERSION_TLS_ALERT: &[u8] = &[21, 3, 3, 0, 2, 2, 70];
|
||||
|
||||
// Schannel ARM needs TLS 1.1 to expose the protocol-version alert; the x64 CI image
|
||||
// disables TLS 1.1, so x64 and other platforms retain TLS 1.2.
|
||||
#[cfg(all(windows, target_arch = "aarch64"))]
|
||||
const NATIVE_TLS_TEST_VERSION: reqwest::tls::Version = reqwest::tls::Version::TLS_1_1;
|
||||
#[cfg(not(all(windows, target_arch = "aarch64")))]
|
||||
const NATIVE_TLS_TEST_VERSION: reqwest::tls::Version = reqwest::tls::Version::TLS_1_2;
|
||||
|
||||
type SuccessfulTlsFallbackServer = (
|
||||
String,
|
||||
TransportClient,
|
||||
@@ -44,6 +51,7 @@ async fn default_pool_does_not_retry_a_native_tls_protocol_failure() {
|
||||
HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault),
|
||||
ClientRouteClass::Other,
|
||||
);
|
||||
use_native_tls_for_direct_route(&pool);
|
||||
let request = reqwest::Request::new(
|
||||
Method::POST,
|
||||
reqwest::Url::parse(&url).expect("valid HTTPS URL"),
|
||||
@@ -75,6 +83,7 @@ async fn retries_a_native_tls_protocol_failure_once_with_rustls() {
|
||||
ClientRouteClass::Other,
|
||||
)
|
||||
.with_tls_backend_fallback();
|
||||
use_native_tls_for_direct_route(&pool);
|
||||
let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
|
||||
let mut request = reqwest::Request::new(Method::POST, destination.clone());
|
||||
*request.body_mut() = Some(Bytes::from_static(b"mcp-initialize").into());
|
||||
@@ -116,6 +125,7 @@ async fn retries_a_native_tls_failure_after_another_request_caches_rustls() {
|
||||
ClientRouteClass::Other,
|
||||
)
|
||||
.with_tls_backend_fallback();
|
||||
use_native_tls_for_direct_route(&pool);
|
||||
let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
|
||||
let (route, native_client, selected_tls_backend) = pool
|
||||
.client_for_url_with_resolver(destination.as_str(), |_| async {
|
||||
@@ -211,6 +221,7 @@ async fn successful_rustls_fallback_replays_the_request_and_reuses_the_destinati
|
||||
ClientRouteClass::Other,
|
||||
)
|
||||
.with_tls_backend_fallback();
|
||||
use_native_tls_for_direct_route(&pool);
|
||||
let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
|
||||
let existing_destination =
|
||||
reqwest::Url::parse("https://another-mcp.example.com/mcp").expect("valid HTTPS URL");
|
||||
@@ -276,6 +287,7 @@ async fn retries_a_tls_protocol_failure_when_request_url_contains_certificate_ma
|
||||
ClientRouteClass::Other,
|
||||
)
|
||||
.with_tls_backend_fallback();
|
||||
use_native_tls_for_direct_route(&pool);
|
||||
let mut destination = reqwest::Url::parse(&url).expect("valid HTTPS URL");
|
||||
destination.set_path("/certificate/hostname/expired/revoked/mcp");
|
||||
let request = reqwest::Request::new(Method::POST, destination);
|
||||
@@ -307,6 +319,7 @@ async fn does_not_retry_a_non_replayable_streaming_request() {
|
||||
ClientRouteClass::Other,
|
||||
)
|
||||
.with_tls_backend_fallback();
|
||||
use_native_tls_for_direct_route(&pool);
|
||||
let mut request = reqwest::Request::new(
|
||||
Method::POST,
|
||||
reqwest::Url::parse(&url).expect("valid HTTPS URL"),
|
||||
@@ -341,6 +354,20 @@ async fn does_not_retry_a_non_replayable_streaming_request() {
|
||||
);
|
||||
}
|
||||
|
||||
fn use_native_tls_for_direct_route(pool: &RouteAwareClientPool) {
|
||||
// Start with reqwest's default native backend even when the environment supplies a CA bundle.
|
||||
let client = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.min_tls_version(NATIVE_TLS_TEST_VERSION)
|
||||
.max_tls_version(NATIVE_TLS_TEST_VERSION)
|
||||
.build()
|
||||
.expect("native TLS client should build");
|
||||
pool.clients.lock().expect("client cache lock").insert(
|
||||
OutboundProxyRoute::Direct,
|
||||
TransportClient::new(client, RequestLogging::Enabled, http::HeaderMap::new()),
|
||||
);
|
||||
}
|
||||
|
||||
fn spawn_successful_tls_fallback_server() -> io::Result<SuccessfulTlsFallbackServer> {
|
||||
codex_utils_rustls_provider::ensure_rustls_crypto_provider();
|
||||
let CertifiedKey { cert, signing_key } =
|
||||
|
||||
@@ -165,6 +165,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
}
|
||||
eprintln!("starting rmcp streamable http test server on http://{actual_bind_addr}/mcp");
|
||||
|
||||
let mut transport_config = StreamableHttpServerConfig::default();
|
||||
if !actual_bind_addr.ip().is_loopback() {
|
||||
transport_config
|
||||
.allowed_hosts
|
||||
.push(actual_bind_addr.to_string());
|
||||
transport_config =
|
||||
transport_config.with_allowed_origins([format!("http://{actual_bind_addr}")]);
|
||||
}
|
||||
|
||||
let router = Router::new()
|
||||
.route(
|
||||
SESSION_POST_FAILURE_CONTROL_PATH,
|
||||
@@ -220,7 +229,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
StreamableHttpService::new(
|
||||
|| Ok(TestToolServer::new()),
|
||||
Arc::new(LocalSessionManager::default()),
|
||||
StreamableHttpServerConfig::default(),
|
||||
transport_config,
|
||||
),
|
||||
)
|
||||
.layer(middleware::from_fn_with_state(
|
||||
|
||||
@@ -165,6 +165,7 @@ async fn read_only_tool_requests_preserve_host_policy_and_caller_metadata() -> a
|
||||
.await
|
||||
.expect("recorded requests")
|
||||
.into_iter()
|
||||
.filter(|request| request.method == "POST")
|
||||
.map(|request| request.body_json::<Value>().expect("JSON-RPC request"))
|
||||
.filter(|body| matches!(body["method"].as_str(), Some("tools/list" | "tools/call")))
|
||||
.map(|body| {
|
||||
|
||||
@@ -59,7 +59,7 @@ setup_remote_env() {
|
||||
--privileged \
|
||||
--security-opt seccomp=unconfined \
|
||||
ubuntu:24.04 sleep infinity >/dev/null
|
||||
if ! docker exec "${container_name}" sh -lc "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 zsh bubblewrap"; then
|
||||
if ! docker exec "${container_name}" sh -lc "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 zsh bubblewrap git"; then
|
||||
docker rm -f "${container_name}" >/dev/null 2>&1 || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
name: Build Cargo voice inputs
|
||||
description: Build and upload same-commit native voice SDK and runtime inputs.
|
||||
inputs:
|
||||
target:
|
||||
description: Native target triple matching the runner
|
||||
required: true
|
||||
windows-tools:
|
||||
description: Directory containing the public Windows voice bootstrap
|
||||
default: .github/scripts
|
||||
buildbuddy-api-key:
|
||||
description: Optional BuildBuddy credential for trusted CI builds
|
||||
default: ""
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
architecture: ${{ contains(inputs.target, 'aarch64') && 'arm64' || 'x64' }}
|
||||
- uses: ./.github/actions/prepare-bazel-ci
|
||||
with:
|
||||
target: ${{ inputs.target }}
|
||||
cache-scope: cargo-voice-native
|
||||
- name: Set up Windows voice build tools
|
||||
if: runner.os == 'Windows'
|
||||
uses: ./third_party/voice/cargo-ci/windows-setup
|
||||
with:
|
||||
target: ${{ inputs.target }}
|
||||
windows-tools: ${{ inputs.windows-tools }}
|
||||
- name: Build and stage native dependencies
|
||||
shell: bash
|
||||
env:
|
||||
BUILDBUDDY_API_KEY: ${{ inputs.buildbuddy-api-key }}
|
||||
TARGET: ${{ inputs.target }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
arch="${TARGET%%-*}"
|
||||
case "$RUNNER_OS" in
|
||||
Windows) prefix="windows_$arch"; temp="$(cygpath -u "$RUNNER_TEMP")" ;;
|
||||
macOS) prefix="macos_$arch"; temp="$RUNNER_TEMP" ;;
|
||||
Linux) prefix="linux_$arch"; temp="$RUNNER_TEMP" ;;
|
||||
*) echo "Unsupported runner OS: $RUNNER_OS" >&2; exit 1 ;;
|
||||
esac
|
||||
wrapper=(--remote-download-toplevel --print-failed-action-summary)
|
||||
flags=(--config=release --build_metadata="COMMIT_SHA=$GITHUB_SHA")
|
||||
if [[ "$RUNNER_OS" == Windows ]]; then
|
||||
wrapper+=(--windows-msvc-host-platform)
|
||||
flags+=(--platforms=//:local_windows_msvc
|
||||
--extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows,@local_config_cc//:cc-toolchain-arm64_windows,//third_party/voice:windows_pkg_config_toolchain,//third_party/voice:windows_cmake_toolchain
|
||||
--repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0 --repo_env=BAZEL_MSVC_RUNTIME_VISUAL_STUDIO_EULA=1
|
||||
--inject_repository="voice_windows_tools=$VOICE_WINDOWS_BAZEL_REPOSITORY"
|
||||
--//third_party/voice:windows_installed_tools=@voice_windows_tools//:tools
|
||||
--action_env="SystemRoot=$VOICE_WINDOWS_SYSTEM_ROOT" --host_action_env="SystemRoot=$VOICE_WINDOWS_SYSTEM_ROOT"
|
||||
--action_env="PROCESSOR_ARCHITECTURE=$VOICE_WINDOWS_HOST_ARCH" --host_action_env="PROCESSOR_ARCHITECTURE=$VOICE_WINDOWS_HOST_ARCH"
|
||||
--workspace_status_command=./scripts/workspace-status.cmd)
|
||||
elif [[ "$RUNNER_OS" == macOS ]]; then
|
||||
flags+=(--macos_minimum_os=14.0)
|
||||
fi
|
||||
./.github/scripts/run-bazel-ci.sh "${wrapper[@]}" -- build "${flags[@]}" -- \
|
||||
//third_party/voice:native_sdk //third_party/voice:native_runtime
|
||||
runtime="bazel-bin/third_party/voice/native_runtime_${prefix}"
|
||||
output="$temp/cargo-voice-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
|
||||
python third_party/voice/cargo_ci.py stage --sdk "${runtime}_sdk" --runtime "$runtime" --output "$output" --target "$TARGET"
|
||||
tar -czf "$temp/cargo-voice.tar.gz" -C "$output" sdk runtime
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: cargo-voice-${{ inputs.target }}
|
||||
path: ${{ runner.temp }}/cargo-voice.tar.gz
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
- name: Stage verified native pkgconf, support DLLs, and license
|
||||
if: runner.os == 'Windows'
|
||||
id: pkgconf
|
||||
shell: pwsh
|
||||
run: |
|
||||
$directory = Split-Path -Parent $env:VOICE_PKG_CONFIG
|
||||
$license = Join-Path $directory "COPYING"
|
||||
Invoke-WebRequest -Uri "https://raw.githubusercontent.com/pkgconf/pkgconf/d97cae4e2a169916f77d9ee8a46c011b69e42a7b/COPYING" -OutFile $license
|
||||
if ((Get-FileHash $license -Algorithm SHA256).Hash.ToLowerInvariant() -ne "055a265b839672a62f590869b0a1f8e4f5147e8f073c9ba77b17b8d7ad653966") {
|
||||
throw "Pinned pkgconf license mismatch"
|
||||
}
|
||||
"directory=$directory" | Out-File $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
- uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
if: runner.os == 'Windows'
|
||||
with:
|
||||
name: cargo-voice-pkgconf-${{ runner.arch }}
|
||||
path: ${{ steps.pkgconf.outputs.directory }}
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
name: Set up Cargo voice inputs
|
||||
description: Use the target SDK and prepared runtime from this workflow run.
|
||||
inputs:
|
||||
target:
|
||||
description: Cargo target triple
|
||||
required: true
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Ensure CMake is available for the Opus build
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: command -v cmake || sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cmake
|
||||
- name: Ensure host pkg-config is available
|
||||
if: runner.os == 'macOS'
|
||||
shell: bash
|
||||
run: command -v pkg-config || brew install pkgconf
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: cargo-voice-${{ inputs.target }}
|
||||
path: ${{ runner.temp }}/cargo-voice-inputs
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
if: runner.os == 'Windows'
|
||||
with:
|
||||
# The x64 archive builder must execute x64 pkgconf even for ARM64.
|
||||
name: cargo-voice-pkgconf-${{ runner.arch }}
|
||||
path: ${{ runner.temp }}/cargo-voice-pkgconf
|
||||
- name: Configure target libraries for Cargo
|
||||
env:
|
||||
VOICE_TARGET: ${{ inputs.target }}
|
||||
shell: python
|
||||
run: |
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
temp = Path(os.environ["RUNNER_TEMP"])
|
||||
target_dir = Path(os.environ["CARGO_TARGET_DIR"])
|
||||
root = Path(tempfile.mkdtemp(prefix="cargo-voice-", dir=target_dir))
|
||||
with tarfile.open(temp / "cargo-voice-inputs/cargo-voice.tar.gz") as archive:
|
||||
archive.extractall(root, filter="data")
|
||||
if os.name == "nt":
|
||||
os.environ["VOICE_PKG_CONFIG"] = str(temp / "cargo-voice-pkgconf/pkgconf.exe")
|
||||
script = Path(os.environ["GITHUB_WORKSPACE"]) / "third_party/voice/cargo_ci.py"
|
||||
subprocess.run([sys.executable, str(script), "configure", "--root", str(root),
|
||||
"--target", os.environ["VOICE_TARGET"]], check=True)
|
||||
# nextest collects linked libraries itself. Include the runtime manifest
|
||||
# and plugins too so the whole runtime can move with the test binaries.
|
||||
relative = (root / "runtime").relative_to(target_dir).as_posix()
|
||||
config = root / "nextest.toml"
|
||||
config.write_text('[profile.default]\narchive.include = [{ path = ' + json.dumps(relative)
|
||||
+ ', relative-to = "target", on-missing = "error" }]\n')
|
||||
with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env:
|
||||
print(f"CODEX_VOICE_NEXTEST_CONFIG={config}", file=env)
|
||||
print(f"CODEX_VOICE_RUNTIME_RELATIVE={relative}", file=env)
|
||||
@@ -0,0 +1,33 @@
|
||||
name: Set up Windows voice build tools
|
||||
description: Prepare verified native Windows voice tools for Bazel.
|
||||
inputs:
|
||||
target:
|
||||
description: Native Windows target triple
|
||||
required: true
|
||||
windows-tools:
|
||||
description: Directory containing the public Windows voice bootstrap
|
||||
default: .github/scripts
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: ./.github/actions/setup-msvc-env
|
||||
with:
|
||||
target: ${{ inputs.target }}
|
||||
- name: Install verified public Windows tools
|
||||
shell: pwsh
|
||||
env:
|
||||
WINDOWS_TOOLS: ${{ inputs.windows-tools }}
|
||||
TARGET: ${{ inputs.target }}
|
||||
run: |
|
||||
./.github/scripts/compute-bazel-windows-path.ps1
|
||||
$manifest = Get-Content -Raw "$env:WINDOWS_TOOLS/voice-cygwin-snapshot.json" | ConvertFrom-Json
|
||||
$pin = $manifest.archive
|
||||
$archive = Join-Path $env:RUNNER_TEMP $pin.name
|
||||
Invoke-WebRequest "https://github.com/openai/codex/releases/download/$($pin.tag)/$($pin.name)" -OutFile $archive
|
||||
& "$env:WINDOWS_TOOLS/setup-voice-windows.ps1" -Target $env:TARGET -SnapshotArchive $archive
|
||||
if (-not $env:SystemRoot) { throw "Windows SystemRoot is required" }
|
||||
"VOICE_WINDOWS_SYSTEM_ROOT=$env:SystemRoot" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
$hostArch = $env:PROCESSOR_ARCHITEW6432
|
||||
if (-not $hostArch) { $hostArch = $env:PROCESSOR_ARCHITECTURE }
|
||||
if (-not $hostArch) { throw "Windows host architecture is required" }
|
||||
"VOICE_WINDOWS_HOST_ARCH=$hostArch" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
Vendored
+153
@@ -0,0 +1,153 @@
|
||||
"""Stage receipt-checked voice build inputs for Cargo and configure a shard."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from release_runtime import stage as stage_runtime
|
||||
from runtime import digest
|
||||
from sdk import MODULES
|
||||
|
||||
|
||||
def read_receipt(path: Path, target: str, commit: str) -> dict:
|
||||
receipt = json.loads(path.read_text())
|
||||
if (
|
||||
receipt["target"] != target
|
||||
or receipt["sourceCommit"] != commit
|
||||
or receipt["sourceManifestSha256"]
|
||||
!= digest(Path(__file__).with_name("sources.json"))
|
||||
):
|
||||
raise ValueError("native inputs do not match target, commit and sources")
|
||||
return receipt
|
||||
|
||||
|
||||
def stage(sdk: Path, runtime: Path, output: Path, target: str, commit: str) -> None:
|
||||
sdk, runtime, output = (
|
||||
sdk.resolve(strict=True),
|
||||
runtime.resolve(strict=True),
|
||||
output.resolve(),
|
||||
)
|
||||
if any(output.is_relative_to(source) for source in (sdk, runtime)):
|
||||
raise ValueError("output must be outside the native inputs")
|
||||
files = read_receipt(sdk / "sdk.json", target, commit)["files"]
|
||||
native = read_receipt(runtime / "runtime.json", target, commit)
|
||||
library_dir = output / "runtime" / ("bin" if target.endswith("msvc") else "lib")
|
||||
prepared = {
|
||||
entry.get("sourceSha256"): runtime / entry["path"]
|
||||
for entry in native["libraries"]
|
||||
if Path(entry["path"]).parent.name == library_dir.name
|
||||
}
|
||||
|
||||
output.mkdir(parents=True)
|
||||
try:
|
||||
stage_runtime(runtime, output / "runtime", target)
|
||||
for entry in files:
|
||||
source = (sdk / entry["path"]).resolve(strict=True)
|
||||
relative = source.relative_to(sdk)
|
||||
if digest(source) != entry["sha256"]:
|
||||
raise ValueError(f"SDK file does not match its receipt: {relative}")
|
||||
dest = output / "sdk" / relative
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if source.suffix == ".pc":
|
||||
# Some transitive libraries retain the producer's absolute prefix.
|
||||
dest.write_text(
|
||||
re.sub(
|
||||
r"^prefix=.*$",
|
||||
"prefix=${pcfiledir}/../..",
|
||||
source.read_text(),
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
)
|
||||
else:
|
||||
shutil.copy2(source, dest)
|
||||
|
||||
# Link with the prepared runtime bytes and put MSVC import libraries
|
||||
# beside their DLLs so nextest collects a self-contained search path.
|
||||
if relative.parent == Path("lib"):
|
||||
alias = library_dir / relative.name
|
||||
payload = (
|
||||
source if source.suffix == ".lib" else prepared.get(entry["sha256"])
|
||||
)
|
||||
if payload and not alias.exists():
|
||||
shutil.copy2(payload, alias)
|
||||
except BaseException:
|
||||
# Windows cannot delete read-only files copied from Bazel.
|
||||
if os.name == "nt":
|
||||
for copied in output.rglob("*"):
|
||||
copied.chmod(copied.stat().st_mode | stat.S_IWRITE)
|
||||
shutil.rmtree(output)
|
||||
raise
|
||||
|
||||
|
||||
def environment(root: Path, target: str, pkg_config: Path) -> dict[str, str]:
|
||||
root, pkg_config = root.resolve(strict=True), pkg_config.resolve(strict=True)
|
||||
read_receipt(root / "runtime/runtime.json", target, os.environ["GITHUB_SHA"])
|
||||
suffix = target.replace("-", "_")
|
||||
values = {
|
||||
f"PKG_CONFIG_{suffix}": str(pkg_config),
|
||||
f"PKG_CONFIG_PATH_{suffix}": str(root / "sdk/lib/pkgconfig"),
|
||||
"CODEX_TEST_VOICE_RUNTIME": str(root / "runtime"),
|
||||
}
|
||||
for module in MODULES:
|
||||
key = re.sub(r"[^A-Z0-9]", "_", module.upper())
|
||||
values[f"SYSTEM_DEPS_{key}_SEARCH_NATIVE"] = str(
|
||||
root / "runtime" / ("bin" if target.endswith("msvc") else "lib")
|
||||
)
|
||||
if module.startswith("gstreamer"):
|
||||
values[f"SYSTEM_DEPS_{key}_LDFLAGS"] = ""
|
||||
if not target.endswith("windows-msvc"):
|
||||
origin = "@loader_path" if target.endswith("darwin") else "$ORIGIN"
|
||||
values[f"CARGO_TARGET_{suffix.upper()}_RUSTFLAGS"] = (
|
||||
f"-C link-arg=-Wl,-rpath,{origin}/../lib"
|
||||
)
|
||||
return values
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
producer = sub.add_parser("stage")
|
||||
producer.add_argument("--sdk", type=Path, required=True)
|
||||
producer.add_argument("--runtime", type=Path, required=True)
|
||||
producer.add_argument("--output", type=Path, required=True)
|
||||
consumer = sub.add_parser("configure")
|
||||
consumer.add_argument("--root", type=Path, required=True)
|
||||
for command in (producer, consumer):
|
||||
command.add_argument("--target", required=True)
|
||||
args = parser.parse_args()
|
||||
if args.command == "stage":
|
||||
stage(
|
||||
args.sdk, args.runtime, args.output, args.target, os.environ["GITHUB_SHA"]
|
||||
)
|
||||
return
|
||||
pkg_config = os.environ.get("VOICE_PKG_CONFIG") or shutil.which("pkg-config")
|
||||
if not pkg_config:
|
||||
parser.error("pkg-config is required on the build host")
|
||||
values = environment(args.root, args.target, Path(pkg_config))
|
||||
# Probe only the staged SDK so a missing native module cannot resolve from
|
||||
# the host. Cargo keeps its usual system search for unrelated dependencies.
|
||||
probe_env = {
|
||||
**os.environ,
|
||||
"PKG_CONFIG_PATH": "",
|
||||
"PKG_CONFIG_LIBDIR": str(args.root.resolve() / "sdk/lib/pkgconfig"),
|
||||
}
|
||||
subprocess.run(
|
||||
[pkg_config, "--exists", *MODULES, "gstreamer-1.0 >= 1.28"],
|
||||
env=probe_env,
|
||||
check=True,
|
||||
)
|
||||
with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as output:
|
||||
for key, value in values.items():
|
||||
output.write(f"{key}={value}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Vendored
+158
@@ -0,0 +1,158 @@
|
||||
"""Exercise how full Cargo CI consumes prepared native inputs."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from cargo_ci import environment, main, stage
|
||||
from package_runtime import runtime_files
|
||||
from runtime import PLUGINS, digest, required_library_paths
|
||||
from sdk import MODULES
|
||||
|
||||
|
||||
def write_files(root, contents):
|
||||
records = []
|
||||
for name, data in contents.items():
|
||||
path = root / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
path.chmod(0o444) # Bazel materializes these inputs read-only.
|
||||
records.append({"path": name, "sha256": digest(path)})
|
||||
return records
|
||||
|
||||
|
||||
class CargoCiTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temp = tempfile.TemporaryDirectory(prefix="voice cargo ")
|
||||
self.addCleanup(temp.cleanup)
|
||||
self.root = Path(temp.name).resolve()
|
||||
self.sdk = self.root / "sdk"
|
||||
self.runtime = self.root / "runtime"
|
||||
self.output = self.root / "stage"
|
||||
self.commit = "a" * 40
|
||||
|
||||
def inputs(self, target, alias, plugin):
|
||||
canonical = required_library_paths(target)[0]
|
||||
plugins = [plugin.format(name) for name in PLUGINS]
|
||||
metadata = (
|
||||
"prefix=/producer\nName: {name}\nDescription: fixture\nVersion: 1.28\n"
|
||||
)
|
||||
sdk = {alias: b"original", "lib/glib-2.0/include/glibconfig.h": b"header"}
|
||||
for name in MODULES:
|
||||
sdk[f"lib/pkgconfig/{name}.pc"] = metadata.format(name=name).encode()
|
||||
if not target.endswith("msvc"):
|
||||
sdk[canonical] = sdk[alias]
|
||||
sdk_files = write_files(self.sdk, sdk)
|
||||
libraries = write_files(
|
||||
self.runtime, {name: b"prepared" for name in [canonical, *plugins]}
|
||||
)
|
||||
libraries[0]["sourceSha256"] = digest(self.sdk / alias)
|
||||
identity = dict(
|
||||
schemaVersion=1,
|
||||
target=target,
|
||||
sourceCommit=self.commit,
|
||||
sourceManifestSha256=digest(Path(__file__).with_name("sources.json")),
|
||||
)
|
||||
(self.sdk / "sdk.json").write_text(json.dumps(dict(identity, files=sdk_files)))
|
||||
receipt = dict(
|
||||
identity, developmentOnly=True, libraries=libraries, plugins=plugins
|
||||
)
|
||||
(self.runtime / "runtime.json").write_text(json.dumps(receipt))
|
||||
return canonical
|
||||
|
||||
def test_relocated_metadata_runtime_and_cargo_probe(self):
|
||||
target = "aarch64-apple-darwin"
|
||||
alias = "lib/libgio-2.0.dylib"
|
||||
canonical = self.inputs(target, alias, "plugins/libgst{}.dylib")
|
||||
stage(self.sdk, self.runtime, self.output, target, self.commit)
|
||||
moved = self.root / "moved native inputs"
|
||||
self.output.rename(moved)
|
||||
native = moved / "runtime"
|
||||
self.assertEqual(
|
||||
(native / alias).read_bytes(), (self.runtime / canonical).read_bytes()
|
||||
)
|
||||
self.assertEqual(
|
||||
runtime_files(native, target), runtime_files(self.runtime, target)
|
||||
)
|
||||
|
||||
host = (
|
||||
os.environ.get("VOICE_PKG_CONFIG")
|
||||
or shutil.which("pkg-config")
|
||||
or shutil.which("pkgconf")
|
||||
)
|
||||
if not host:
|
||||
self.skipTest("host pkg-config is required to probe relocated metadata")
|
||||
pkgdir = moved / "sdk/lib/pkgconfig"
|
||||
probe_env = dict(os.environ, PKG_CONFIG_PATH="", PKG_CONFIG_LIBDIR=str(pkgdir))
|
||||
result = subprocess.check_output(
|
||||
[host, "--variable=prefix", "gio-2.0"], env=probe_env, text=True
|
||||
)
|
||||
# pkgconf can escape spaces while pkg-config can print the raw path.
|
||||
raw = result.strip()
|
||||
words = shlex.split(raw)
|
||||
self.assertEqual(
|
||||
Path(words[0] if len(words) == 1 else raw).resolve(), moved / "sdk"
|
||||
)
|
||||
|
||||
env_file = self.root / "github env"
|
||||
ci_env = dict(
|
||||
GITHUB_SHA=self.commit, GITHUB_ENV=str(env_file), VOICE_PKG_CONFIG=host
|
||||
)
|
||||
argv = ["cargo_ci", "configure", "--root", str(moved), "--target", target]
|
||||
with patch.dict(os.environ, ci_env), patch.object(sys, "argv", argv):
|
||||
main()
|
||||
written = env_file.read_text()
|
||||
values = dict(line.split("=", 1) for line in written.splitlines())
|
||||
self.assertEqual(
|
||||
values["PKG_CONFIG_PATH_aarch64_apple_darwin"], str(pkgdir)
|
||||
)
|
||||
self.assertNotIn("PKG_CONFIG_LIBDIR_aarch64_apple_darwin", values)
|
||||
(pkgdir / "gio-2.0.pc").unlink()
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
main()
|
||||
self.assertEqual(env_file.read_text(), written)
|
||||
|
||||
def test_arm64_import_library_and_dll_with_x64_host_pkgconf(self):
|
||||
target = "aarch64-pc-windows-msvc"
|
||||
alias = "lib/gio-2.0.lib"
|
||||
canonical = self.inputs(target, alias, "bin/gst{}.dll")
|
||||
stage(self.sdk, self.runtime, self.output, target, self.commit)
|
||||
native = self.output / "runtime"
|
||||
self.assertEqual(
|
||||
(native / "bin/gio-2.0.lib").read_bytes(), (self.sdk / alias).read_bytes()
|
||||
)
|
||||
self.assertEqual(
|
||||
(native / canonical).read_bytes(), (self.runtime / canonical).read_bytes()
|
||||
)
|
||||
tool = self.root / "x64 pkgconf.exe"
|
||||
tool.touch()
|
||||
with patch.dict(os.environ, GITHUB_SHA=self.commit):
|
||||
values = environment(self.output, target, tool)
|
||||
self.assertEqual(values["PKG_CONFIG_aarch64_pc_windows_msvc"], str(tool))
|
||||
self.assertEqual(
|
||||
values["SYSTEM_DEPS_GSTREAMER_1_0_SEARCH_NATIVE"], str(native / "bin")
|
||||
)
|
||||
|
||||
def test_mismatched_receipt_and_corrupt_sdk_fail_staging(self):
|
||||
target = "x86_64-unknown-linux-gnu"
|
||||
alias = "lib/libgio-2.0.so"
|
||||
self.inputs(target, alias, "lib/gstreamer-1.0/libgst{}.so")
|
||||
for triple, commit in [
|
||||
(target, "b" * 40),
|
||||
("aarch64-unknown-linux-gnu", self.commit),
|
||||
]:
|
||||
with self.assertRaisesRegex(ValueError, "do not match"):
|
||||
stage(self.sdk, self.runtime, self.output, triple, commit)
|
||||
source = self.sdk / alias
|
||||
source.chmod(0o644)
|
||||
source.write_bytes(b"corrupt")
|
||||
with self.assertRaisesRegex(ValueError, "receipt"):
|
||||
stage(self.sdk, self.runtime, self.output, target, self.commit)
|
||||
self.assertFalse(self.output.exists())
|
||||
Reference in New Issue
Block a user