From e02a32cd51ab40b503e2e6eac42b47585851134d Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Sun, 1 Nov 2026 03:43:23 +0000 Subject: [PATCH] Wire native voice dependencies into full Rust CI (#58719) ## What changed - Build native voice SDK and runtime artifacts with Bazel for Linux GNU, macOS, and Windows on x64 and ARM64, then configure Cargo jobs to consume them. Keep musl builds in a separate job without these inputs. - Validate artifact receipts against the target, source commit, and source manifest; verify SDK file hashes and relocate pkg-config metadata. Package the runtime and plugins with nextest archives and restore `CODEX_TEST_VOICE_RUNTIME` when running shards. - Prepare native Windows voice tools for argument-comment linting and install ARM64 DotSlash on Windows ARM64. Reduce archive disk usage, bound Linux ARM64 build concurrency, and correct the JUnit artifact path. - Recognize TLS errors wrapped directly in `io::Error`. Make TLS fallback fixtures select the native backend explicitly, allow the MCP test server's non-loopback address, and isolate skill-root tests from ancestor Git markers. Install `git` in remote test containers and filter MCP discovery assertions to POST requests. ## Testing Add tests for relocated voice inputs, Windows ARM64 import libraries with host-native pkgconf, and rejection of mismatched receipts or corrupt SDK files. Extend untrusted-certificate coverage to an explicitly selected rustls backend. GitOrigin-RevId: da3ff690b94906edc1f8f60a3c7feda1bf66134a --- .../run-argument-comment-lint/action.yml | 20 +++ .github/actions/setup-ci/action.yml | 18 ++ .../run-argument-comment-lint-bazel.sh | 15 ++ .../rust-ci-full-nextest-platform.yml | 38 ++++- .github/workflows/rust-ci-full.yml | 132 +++++++++++---- codex-rs/ext/skills/src/host_roots_tests.rs | 8 +- .../src/route_aware_client_pool.rs | 16 +- .../src/route_aware_client_pool_tests.rs | 12 +- .../src/route_aware_tls_fallback_tests.rs | 27 +++ .../src/bin/test_streamable_http_server.rs | 11 +- .../rmcp-client/tests/mcp_2026_discovery.rs | 1 + scripts/test-remote-env.sh | 2 +- third_party/voice/cargo-ci/produce/action.yml | 89 ++++++++++ third_party/voice/cargo-ci/setup/action.yml | 62 +++++++ .../voice/cargo-ci/windows-setup/action.yml | 33 ++++ third_party/voice/cargo_ci.py | 153 +++++++++++++++++ third_party/voice/test_cargo_ci.py | 158 ++++++++++++++++++ 17 files changed, 748 insertions(+), 47 deletions(-) create mode 100644 third_party/voice/cargo-ci/produce/action.yml create mode 100644 third_party/voice/cargo-ci/setup/action.yml create mode 100644 third_party/voice/cargo-ci/windows-setup/action.yml create mode 100644 third_party/voice/cargo_ci.py create mode 100644 third_party/voice/test_cargo_ci.py diff --git a/.github/actions/run-argument-comment-lint/action.yml b/.github/actions/run-argument-comment-lint/action.yml index 16cdd54eb4..5bcc77bdcf 100644 --- a/.github/actions/run-argument-comment-lint/action.yml +++ b/.github/actions/run-argument-comment-lint/action.yml @@ -18,6 +18,26 @@ runs: target: ${{ inputs.target }} cache-scope: bazel-argument-comment-lint + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + if: ${{ runner.os == 'Windows' && inputs.buildbuddy-api-key == '' }} + with: + python-version: "3.12" + architecture: x64 + + - name: Prepare native Windows audio dependencies + if: ${{ runner.os == 'Windows' && inputs.buildbuddy-api-key == '' }} + uses: ./third_party/voice/cargo-ci/windows-setup + with: + target: x86_64-pc-windows-msvc + + - name: Verify native Windows audio dependencies + if: ${{ runner.os == 'Windows' && inputs.buildbuddy-api-key == '' }} + shell: bash + run: | + : "${VOICE_WINDOWS_BAZEL_REPOSITORY:?Windows voice tools are required}" + : "${VOICE_WINDOWS_SYSTEM_ROOT:?Windows SystemRoot is required}" + : "${VOICE_WINDOWS_HOST_ARCH:?Windows host architecture is required}" + - name: Install Linux sandbox build dependencies if: ${{ runner.os == 'Linux' }} shell: bash diff --git a/.github/actions/setup-ci/action.yml b/.github/actions/setup-ci/action.yml index 6e145eb763..09d58affcc 100644 --- a/.github/actions/setup-ci/action.yml +++ b/.github/actions/setup-ci/action.yml @@ -81,10 +81,28 @@ runs: run: mkdir -p "$RUNNER_TEMP/dotslash-curl" && echo retry-all-errors > "$RUNNER_TEMP/dotslash-curl/.curlrc" - name: Install DotSlash + if: runner.os != 'Windows' || runner.arch != 'ARM64' uses: meta/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 env: CURL_HOME: ${{ runner.temp }}/dotslash-curl + - name: Install native DotSlash (Windows ARM64) + if: runner.os == 'Windows' && runner.arch == 'ARM64' + shell: bash + run: | + set -euo pipefail + # DotSlash selects resource architecture from its own executable. + destination="$RUNNER_TEMP/install-dotslash-arm64/bin" + archive="$RUNNER_TEMP/dotslash-windows-arm64.v0.5.9.tar.gz" + mkdir -p "$destination" + curl --location --silent --show-error --fail --retry 5 --retry-all-errors \ + https://github.com/facebook/dotslash/releases/download/v0.5.9/dotslash-windows-arm64.v0.5.9.tar.gz \ + --output "$archive" + echo "43d49d3472808738993ccf95a36dda8cc2869996d9a397bb02bde1e39b16146b $archive" | sha256sum --check + tar xzf "$archive" -O --force-local > "$destination/dotslash.exe" + echo "$destination" >> "$GITHUB_PATH" + "$destination/dotslash.exe" --version + - name: Install just uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 with: diff --git a/.github/scripts/run-argument-comment-lint-bazel.sh b/.github/scripts/run-argument-comment-lint-bazel.sh index 7b73ca2f2e..b506227c78 100755 --- a/.github/scripts/run-argument-comment-lint-bazel.sh +++ b/.github/scripts/run-argument-comment-lint-bazel.sh @@ -9,6 +9,21 @@ if [[ "${RUNNER_OS:-}" == "Windows" ]]; then # lint aspect can run across the compatible crate graph instead of failing the # whole build after analysis. bazel_lint_args+=("--skip_incompatible_explicit_targets") + if [[ -z "${BUILDBUDDY_API_KEY:-}" && -n "${VOICE_WINDOWS_BAZEL_REPOSITORY:-}" ]]; then + : "${VOICE_WINDOWS_BAZEL_REPOSITORY:?Windows voice tools are required}" + : "${VOICE_WINDOWS_SYSTEM_ROOT:?Windows SystemRoot is required}" + : "${VOICE_WINDOWS_HOST_ARCH:?Windows host architecture is required}" + bazel_lint_args+=( + "--extra_toolchains=//third_party/voice:windows_pkg_config_toolchain,//third_party/voice:windows_cmake_toolchain" + "--repo_env=BAZEL_MSVC_RUNTIME_VISUAL_STUDIO_EULA=1" + "--inject_repository=voice_windows_tools=${VOICE_WINDOWS_BAZEL_REPOSITORY}" + "--//third_party/voice:windows_installed_tools=@voice_windows_tools//:tools" + "--action_env=SystemRoot=${VOICE_WINDOWS_SYSTEM_ROOT}" + "--host_action_env=SystemRoot=${VOICE_WINDOWS_SYSTEM_ROOT}" + "--action_env=PROCESSOR_ARCHITECTURE=${VOICE_WINDOWS_HOST_ARCH}" + "--host_action_env=PROCESSOR_ARCHITECTURE=${VOICE_WINDOWS_HOST_ARCH}" + ) + fi fi read_query_labels() { diff --git a/.github/workflows/rust-ci-full-nextest-platform.yml b/.github/workflows/rust-ci-full-nextest-platform.yml index 474535267a..03050c6159 100644 --- a/.github/workflows/rust-ci-full-nextest-platform.yml +++ b/.github/workflows/rust-ci-full-nextest-platform.yml @@ -65,6 +65,8 @@ jobs: CODEX_CI_PROFILE: ${{ inputs.profile }} CODEX_CI_TARGET: ${{ inputs.target }} CARGO_INCREMENTAL: "0" + # Debug symbols for hundreds of test binaries can exhaust the archive runner's disk. + CARGO_PROFILE_CI_TEST_DEBUG: "0" SCCACHE_CACHE_SIZE: 10G NEXTEST_ARCHIVE_FILE: nextest-${{ inputs.artifact_id }}.tar.zst TEST_HELPERS_ARTIFACT: nextest-test-helpers-${{ inputs.artifact_id }} @@ -73,6 +75,15 @@ jobs: with: persist-credentials: false + - name: Free disk space for the macOS test archive + if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64' && runner.environment == 'github-hosted' }} + shell: bash + run: | + # The macOS build does not use the hosted image's Android SDK or NDKs. + df -h "$RUNNER_TEMP" + sudo rm -rf /Users/runner/Library/Android/sdk + df -h "$RUNNER_TEMP" + - uses: ./.github/actions/setup-ci - name: Install Linux build dependencies @@ -165,13 +176,25 @@ jobs: tool: nextest version: 0.9.103 + - name: Set up native voice dependencies + uses: ./third_party/voice/cargo-ci/setup + with: + target: ${{ inputs.target }} + - name: Build nextest archive shell: bash run: | set -euo pipefail archive_dir="${RUNNER_TEMP}/nextest-archive" mkdir -p "${archive_dir}" - cargo nextest archive \ + if [[ "${RUNNER_OS}" == "Linux" && "${RUNNER_ARCH}" == "ARM64" && "${CODEX_CI_TARGET}" == "aarch64-unknown-linux-gnu" ]]; then + # Bound concurrency for full GNU ARM archives on native Linux runners. + nproc + free -h + export CARGO_BUILD_JOBS=4 + fi + printf '%s\n' "$CODEX_VOICE_RUNTIME_RELATIVE" > "$archive_dir/voice-runtime-path.txt" + cargo nextest --tool-config-file "codex-voice:$CODEX_VOICE_NEXTEST_CONFIG" archive \ --target "${CODEX_CI_TARGET}" \ --cargo-profile "${CODEX_CI_PROFILE}" \ --timings \ @@ -215,7 +238,7 @@ jobs: uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: nextest-archive-${{ inputs.artifact_id }} - path: ${{ runner.temp }}/nextest-archive/${{ env.NEXTEST_ARCHIVE_FILE }} + path: ${{ runner.temp }}/nextest-archive/ if-no-files-found: error retention-days: 1 @@ -341,10 +364,16 @@ jobs: set -euo pipefail archive_file="${RUNNER_TEMP}/nextest-archive/${NEXTEST_ARCHIVE_FILE}" workspace_root="$(pwd)" + extract_dir="${RUNNER_TEMP}/nextest-extracted" + mkdir -p "$extract_dir" + runtime_relative="$(cat "${RUNNER_TEMP}/nextest-archive/voice-runtime-path.txt")" + export CODEX_TEST_VOICE_RUNTIME="$extract_dir/target/$runtime_relative" if [[ "${RUNNER_OS}" == "Windows" ]]; then archive_file="$(cygpath -w "${archive_file}")" workspace_root="$(cygpath -w "${workspace_root}")" + extract_dir="$(cygpath -w "$extract_dir")" + CODEX_TEST_VOICE_RUNTIME="$(cygpath -w "$CODEX_TEST_VOICE_RUNTIME")" fi if [[ "${RUNNER_OS}" == "Linux" ]]; then @@ -365,6 +394,7 @@ jobs: run --no-fail-fast --archive-file "${archive_file}" + --extract-to "$extract_dir" --workspace-remap "${workspace_root}" --partition "hash:${{ matrix.shard }}/4" ) @@ -395,6 +425,8 @@ jobs: "${test_command[@]}" env: RUST_BACKTRACE: 1 + # Keep panic backtraces without changing the CLI errors asserted by tests. + RUST_LIB_BACKTRACE: "0" RUST_MIN_STACK: "8388608" # 8 MiB NEXTEST_STATUS_LEVEL: leak @@ -403,7 +435,7 @@ jobs: uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: nextest-junit-rust-ci-${{ inputs.artifact_id }}-shard-${{ matrix.shard }} - path: ${{ env.CARGO_TARGET_DIR }}/nextest/default/junit.xml + path: codex-rs/target/nextest/default/junit.xml if-no-files-found: warn - name: Tear down remote test env diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index b353f578bc..8e045e040f 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -92,7 +92,7 @@ jobs: argument_comment_lint_prebuilt: name: Argument comment lint - ${{ matrix.name }} runs-on: ${{ matrix.runs_on || matrix.runner }} - timeout-minutes: 30 + timeout-minutes: 120 environment: name: bazel deployment: false @@ -117,17 +117,51 @@ jobs: uses: ./.github/actions/run-argument-comment-lint with: target: ${{ runner.os }} - buildbuddy-api-key: ${{ secrets.BUILDBUDDY_API_KEY }} # --- CI to validate on different os/targets -------------------------------- + cargo_voice_inputs: + name: Cargo voice inputs - ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + environment: + name: bazel + deployment: false + permissions: + contents: read + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - { target: x86_64-unknown-linux-gnu, runner: ubuntu-24.04 } + - { target: aarch64-unknown-linux-gnu, runner: ubuntu-24.04-arm } + - { target: x86_64-apple-darwin, runner: macos-15-intel } + - { target: aarch64-apple-darwin, runner: macos-15 } + - target: x86_64-pc-windows-msvc + runner: + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-x64 + - target: aarch64-pc-windows-msvc + runner: + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-windows-arm64 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: ./third_party/voice/cargo-ci/produce + with: + target: ${{ matrix.target }} + lint_build: - name: Lint/Build — ${{ matrix.runner }} - ${{ matrix.target }}${{ matrix.profile == 'release' && ' (release)' || '' }} - runs-on: ${{ matrix.runs_on || matrix.runner }} - timeout-minutes: 30 - defaults: + needs: cargo_voice_inputs + if: &lint_build_if ${{ !cancelled() }} + name: &lint_build_name Lint/Build — ${{ matrix.runner }} - ${{ matrix.target }}${{ matrix.profile == 'release' && ' (release)' || '' }} + runs-on: &lint_build_runner ${{ matrix.runs_on || matrix.runner }} + timeout-minutes: &lint_build_timeout 30 + defaults: &lint_build_defaults run: working-directory: codex-rs - env: + env: &lint_build_env # Speed up repeated builds across CI runs by caching compiled objects, except on # arm64 macOS runners cross-targeting x86_64 where ring/cc-rs can produce # mixed-architecture archives under sccache. @@ -145,24 +179,12 @@ jobs: - runner: macos-15-xlarge target: x86_64-apple-darwin profile: dev - - runner: ubuntu-24.04 - target: x86_64-unknown-linux-musl - profile: dev - runs_on: - group: ${{ github.event.repository.name }}-runners - labels: ${{ github.event.repository.name }}-linux-x64 - runner: ubuntu-24.04 target: x86_64-unknown-linux-gnu profile: dev runs_on: group: ${{ github.event.repository.name }}-runners labels: ${{ github.event.repository.name }}-linux-x64 - - runner: ubuntu-24.04-arm - target: aarch64-unknown-linux-musl - profile: dev - runs_on: - group: ${{ github.event.repository.name }}-runners - labels: ${{ github.event.repository.name }}-linux-arm64 - runner: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu profile: dev @@ -189,18 +211,6 @@ jobs: - runner: macos-15-xlarge target: aarch64-apple-darwin profile: release - - runner: ubuntu-24.04 - target: x86_64-unknown-linux-musl - profile: release - runs_on: - group: ${{ github.event.repository.name }}-runners - labels: ${{ github.event.repository.name }}-linux-x64 - - runner: ubuntu-24.04-arm - target: aarch64-unknown-linux-musl - profile: release - runs_on: - group: ${{ github.event.repository.name }}-runners - labels: ${{ github.event.repository.name }}-linux-arm64 - runner: windows-x64 target: x86_64-pc-windows-msvc profile: release @@ -214,7 +224,7 @@ jobs: group: ${{ github.event.repository.name }}-runners labels: ${{ github.event.repository.name }}-windows-arm64 - steps: + steps: &lint_build_steps - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false @@ -348,6 +358,12 @@ jobs: with: target: ${{ matrix.target }} + - name: Set up native voice dependencies + if: ${{ !endsWith(matrix.target, '-musl') }} + uses: ./third_party/voice/cargo-ci/setup + with: + target: ${{ matrix.target }} + - name: Install cargo-chef if: ${{ matrix.profile == 'release' }} uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2.62.49 @@ -427,7 +443,46 @@ jobs: /var/cache/apt key: apt-${{ matrix.runner }}-${{ matrix.target }}-v1 + lint_build_musl: + if: *lint_build_if + name: *lint_build_name + runs-on: *lint_build_runner + timeout-minutes: *lint_build_timeout + defaults: *lint_build_defaults + env: *lint_build_env + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + profile: dev + runs_on: + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-x64 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + profile: dev + runs_on: + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-arm64 + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-musl + profile: release + runs_on: + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-x64 + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + profile: release + runs_on: + group: ${{ github.event.repository.name }}-runners + labels: ${{ github.event.repository.name }}-linux-arm64 + steps: *lint_build_steps + tests_macos_aarch64: + needs: cargo_voice_inputs + if: ${{ !cancelled() }} name: Tests — macos-15-xlarge - aarch64-apple-darwin uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: @@ -439,6 +494,8 @@ jobs: secrets: inherit tests_linux_x64_remote: + needs: cargo_voice_inputs + if: ${{ !cancelled() }} name: Tests — ubuntu-24.04 - x86_64-unknown-linux-gnu (remote) uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: @@ -453,6 +510,8 @@ jobs: secrets: inherit tests_linux_arm64: + needs: cargo_voice_inputs + if: ${{ !cancelled() }} name: Tests — ubuntu-24.04-arm - aarch64-unknown-linux-gnu uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: @@ -466,6 +525,8 @@ jobs: secrets: inherit tests_windows_x64: + needs: cargo_voice_inputs + if: ${{ !cancelled() }} name: Tests — windows-x64 - x86_64-pc-windows-msvc uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: @@ -479,6 +540,8 @@ jobs: secrets: inherit tests_windows_arm64: + needs: cargo_voice_inputs + if: ${{ !cancelled() }} name: Tests — windows-arm64 - aarch64-pc-windows-msvc uses: ./.github/workflows/rust-ci-full-nextest-platform.yml with: @@ -504,7 +567,9 @@ jobs: cargo_shear, argument_comment_lint_package, argument_comment_lint_prebuilt, + cargo_voice_inputs, lint_build, + lint_build_musl, tests_macos_aarch64, tests_linux_x64_remote, tests_linux_arm64, @@ -522,6 +587,7 @@ jobs: echo "general: ${{ needs.general.result }}" echo "shear : ${{ needs.cargo_shear.result }}" echo "lint : ${{ needs.lint_build.result }}" + echo "musl : ${{ needs.lint_build_musl.result }}" echo "test macos : ${{ needs.tests_macos_aarch64.result }}" echo "test linux : ${{ needs.tests_linux_x64_remote.result }}" echo "test arm64 : ${{ needs.tests_linux_arm64.result }}" @@ -531,7 +597,9 @@ jobs: [[ '${{ needs.argument_comment_lint_prebuilt.result }}' == 'success' ]] || { echo 'argument_comment_lint_prebuilt failed'; exit 1; } [[ '${{ needs.general.result }}' == 'success' ]] || { echo 'general failed'; exit 1; } [[ '${{ needs.cargo_shear.result }}' == 'success' ]] || { echo 'cargo_shear failed'; exit 1; } + [[ '${{ needs.cargo_voice_inputs.result }}' == 'success' ]] || { echo 'cargo_voice_inputs failed'; exit 1; } [[ '${{ needs.lint_build.result }}' == 'success' ]] || { echo 'lint_build failed'; exit 1; } + [[ '${{ needs.lint_build_musl.result }}' == 'success' ]] || { echo 'lint_build_musl failed'; exit 1; } [[ '${{ needs.tests_macos_aarch64.result }}' == 'success' ]] || { echo 'tests_macos_aarch64 failed'; exit 1; } [[ '${{ needs.tests_linux_x64_remote.result }}' == 'success' ]] || { echo 'tests_linux_x64_remote failed'; exit 1; } [[ '${{ needs.tests_linux_arm64.result }}' == 'success' ]] || { echo 'tests_linux_arm64 failed'; exit 1; } diff --git a/codex-rs/ext/skills/src/host_roots_tests.rs b/codex-rs/ext/skills/src/host_roots_tests.rs index c6fb5ec096..e7b46f71d9 100644 --- a/codex-rs/ext/skills/src/host_roots_tests.rs +++ b/codex-rs/ext/skills/src/host_roots_tests.rs @@ -445,9 +445,15 @@ async fn repo_ancestry_without_project_marker_does_not_walk_parents() { fs::create_dir_all(outer.join(".agents/skills")).expect("create outer skills"); fs::create_dir_all(cwd.join(".agents/skills")).expect("create cwd skills"); + // Sandbox runners can put a .git marker above the temporary directory. + // Probe for an absent fixture marker while retaining the real ancestor search. + let config_stack = stack(vec![ConfigLayerEntry::new( + ConfigLayerSource::SessionFlags, + toml::toml! { project_root_markers = [".missing-skill-test-project"] }.into(), + )]); let roots = repo_agents_skill_roots( Some(Arc::clone(&LOCAL_FS)), - &stack(Vec::new()), + &config_stack, &cwd, /*empty_remote_roots*/ None, ) diff --git a/codex-rs/http-client/src/route_aware_client_pool.rs b/codex-rs/http-client/src/route_aware_client_pool.rs index f327b1045f..122a982434 100644 --- a/codex-rs/http-client/src/route_aware_client_pool.rs +++ b/codex-rs/http-client/src/route_aware_client_pool.rs @@ -110,13 +110,6 @@ impl RouteAwareRequestError { if self.status() == Some(StatusCode::PROXY_AUTHENTICATION_REQUIRED) { return Some(RouteFailureClass::ProxyAuthenticationRequired); } - if let Self::Route(RouteAwareClientPoolError::Resolve(error)) = self - && let Some(source) = error.get_ref() - && source.is::() - { - return Some(RouteFailureClass::TlsError); - } - let mut source: Option<&(dyn std::error::Error + 'static)> = Some(self); while let Some(error) = source { if error.downcast_ref::().is_some() @@ -127,7 +120,14 @@ impl RouteAwareRequestError { if error.to_string() == "tunnel error: proxy authorization required" { return Some(RouteFailureClass::ProxyAuthenticationRequired); } - source = error.source(); + // io::Error::source skips its inner error, which may itself be the TLS error. + source = match error + .downcast_ref::() + .and_then(io::Error::get_ref) + { + Some(inner) => Some(inner), + None => error.source(), + }; } match self { diff --git a/codex-rs/http-client/src/route_aware_client_pool_tests.rs b/codex-rs/http-client/src/route_aware_client_pool_tests.rs index 0c4a453976..5d6512a2cf 100644 --- a/codex-rs/http-client/src/route_aware_client_pool_tests.rs +++ b/codex-rs/http-client/src/route_aware_client_pool_tests.rs @@ -227,6 +227,15 @@ async fn direct_and_routed_clients_build_equivalent_requests() { #[tokio::test] async fn request_failures_classify_real_untrusted_certificate_handshakes() { + assert_untrusted_certificate_classified(HttpClientBuilder::new()).await; +} + +#[tokio::test] +async fn request_failures_classify_real_untrusted_rustls_certificate_handshakes() { + assert_untrusted_certificate_classified(HttpClientBuilder::new().with_rustls_tls()).await; +} + +async fn assert_untrusted_certificate_classified(client_builder: HttpClientBuilder) { codex_utils_rustls_provider::ensure_rustls_crypto_provider(); let certificate = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]) .expect("self-signed certificate should generate"); @@ -252,9 +261,10 @@ async fn request_failures_classify_real_untrusted_certificate_handshakes() { .expect("TLS server connection should be created"); let _ = connection.complete_io(&mut stream); }); - let pool = RouteAwareClientPool::new_without_request_logging( + let pool = RouteAwareClientPool::with_builder( HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), ClientRouteClass::Api, + client_builder.without_request_logging(), ); let mut request = reqwest::Request::new( diff --git a/codex-rs/http-client/src/route_aware_tls_fallback_tests.rs b/codex-rs/http-client/src/route_aware_tls_fallback_tests.rs index 8cdfae6663..be6f9b1c9f 100644 --- a/codex-rs/http-client/src/route_aware_tls_fallback_tests.rs +++ b/codex-rs/http-client/src/route_aware_tls_fallback_tests.rs @@ -29,6 +29,13 @@ use crate::tls_backend_fallback::should_retry_with_rustls; const PROTOCOL_VERSION_TLS_ALERT: &[u8] = &[21, 3, 3, 0, 2, 2, 70]; +// Schannel ARM needs TLS 1.1 to expose the protocol-version alert; the x64 CI image +// disables TLS 1.1, so x64 and other platforms retain TLS 1.2. +#[cfg(all(windows, target_arch = "aarch64"))] +const NATIVE_TLS_TEST_VERSION: reqwest::tls::Version = reqwest::tls::Version::TLS_1_1; +#[cfg(not(all(windows, target_arch = "aarch64")))] +const NATIVE_TLS_TEST_VERSION: reqwest::tls::Version = reqwest::tls::Version::TLS_1_2; + type SuccessfulTlsFallbackServer = ( String, TransportClient, @@ -44,6 +51,7 @@ async fn default_pool_does_not_retry_a_native_tls_protocol_failure() { HttpClientFactory::new(OutboundProxyPolicy::ReqwestDefault), ClientRouteClass::Other, ); + use_native_tls_for_direct_route(&pool); let request = reqwest::Request::new( Method::POST, reqwest::Url::parse(&url).expect("valid HTTPS URL"), @@ -75,6 +83,7 @@ async fn retries_a_native_tls_protocol_failure_once_with_rustls() { ClientRouteClass::Other, ) .with_tls_backend_fallback(); + use_native_tls_for_direct_route(&pool); let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL"); let mut request = reqwest::Request::new(Method::POST, destination.clone()); *request.body_mut() = Some(Bytes::from_static(b"mcp-initialize").into()); @@ -116,6 +125,7 @@ async fn retries_a_native_tls_failure_after_another_request_caches_rustls() { ClientRouteClass::Other, ) .with_tls_backend_fallback(); + use_native_tls_for_direct_route(&pool); let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL"); let (route, native_client, selected_tls_backend) = pool .client_for_url_with_resolver(destination.as_str(), |_| async { @@ -211,6 +221,7 @@ async fn successful_rustls_fallback_replays_the_request_and_reuses_the_destinati ClientRouteClass::Other, ) .with_tls_backend_fallback(); + use_native_tls_for_direct_route(&pool); let destination = reqwest::Url::parse(&url).expect("valid HTTPS URL"); let existing_destination = reqwest::Url::parse("https://another-mcp.example.com/mcp").expect("valid HTTPS URL"); @@ -276,6 +287,7 @@ async fn retries_a_tls_protocol_failure_when_request_url_contains_certificate_ma ClientRouteClass::Other, ) .with_tls_backend_fallback(); + use_native_tls_for_direct_route(&pool); let mut destination = reqwest::Url::parse(&url).expect("valid HTTPS URL"); destination.set_path("/certificate/hostname/expired/revoked/mcp"); let request = reqwest::Request::new(Method::POST, destination); @@ -307,6 +319,7 @@ async fn does_not_retry_a_non_replayable_streaming_request() { ClientRouteClass::Other, ) .with_tls_backend_fallback(); + use_native_tls_for_direct_route(&pool); let mut request = reqwest::Request::new( Method::POST, reqwest::Url::parse(&url).expect("valid HTTPS URL"), @@ -341,6 +354,20 @@ async fn does_not_retry_a_non_replayable_streaming_request() { ); } +fn use_native_tls_for_direct_route(pool: &RouteAwareClientPool) { + // Start with reqwest's default native backend even when the environment supplies a CA bundle. + let client = reqwest::Client::builder() + .no_proxy() + .min_tls_version(NATIVE_TLS_TEST_VERSION) + .max_tls_version(NATIVE_TLS_TEST_VERSION) + .build() + .expect("native TLS client should build"); + pool.clients.lock().expect("client cache lock").insert( + OutboundProxyRoute::Direct, + TransportClient::new(client, RequestLogging::Enabled, http::HeaderMap::new()), + ); +} + fn spawn_successful_tls_fallback_server() -> io::Result { codex_utils_rustls_provider::ensure_rustls_crypto_provider(); let CertifiedKey { cert, signing_key } = diff --git a/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs b/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs index 31af242ed1..7696dacd15 100644 --- a/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs +++ b/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs @@ -165,6 +165,15 @@ async fn main() -> Result<(), Box> { } eprintln!("starting rmcp streamable http test server on http://{actual_bind_addr}/mcp"); + let mut transport_config = StreamableHttpServerConfig::default(); + if !actual_bind_addr.ip().is_loopback() { + transport_config + .allowed_hosts + .push(actual_bind_addr.to_string()); + transport_config = + transport_config.with_allowed_origins([format!("http://{actual_bind_addr}")]); + } + let router = Router::new() .route( SESSION_POST_FAILURE_CONTROL_PATH, @@ -220,7 +229,7 @@ async fn main() -> Result<(), Box> { StreamableHttpService::new( || Ok(TestToolServer::new()), Arc::new(LocalSessionManager::default()), - StreamableHttpServerConfig::default(), + transport_config, ), ) .layer(middleware::from_fn_with_state( diff --git a/codex-rs/rmcp-client/tests/mcp_2026_discovery.rs b/codex-rs/rmcp-client/tests/mcp_2026_discovery.rs index 0adfe630b4..950100cd7b 100644 --- a/codex-rs/rmcp-client/tests/mcp_2026_discovery.rs +++ b/codex-rs/rmcp-client/tests/mcp_2026_discovery.rs @@ -165,6 +165,7 @@ async fn read_only_tool_requests_preserve_host_policy_and_caller_metadata() -> a .await .expect("recorded requests") .into_iter() + .filter(|request| request.method == "POST") .map(|request| request.body_json::().expect("JSON-RPC request")) .filter(|body| matches!(body["method"].as_str(), Some("tools/list" | "tools/call"))) .map(|body| { diff --git a/scripts/test-remote-env.sh b/scripts/test-remote-env.sh index 7fcdd1ca37..2eee963ba9 100755 --- a/scripts/test-remote-env.sh +++ b/scripts/test-remote-env.sh @@ -59,7 +59,7 @@ setup_remote_env() { --privileged \ --security-opt seccomp=unconfined \ ubuntu:24.04 sleep infinity >/dev/null - if ! docker exec "${container_name}" sh -lc "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 zsh bubblewrap"; then + if ! docker exec "${container_name}" sh -lc "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python3 zsh bubblewrap git"; then docker rm -f "${container_name}" >/dev/null 2>&1 || true return 1 fi diff --git a/third_party/voice/cargo-ci/produce/action.yml b/third_party/voice/cargo-ci/produce/action.yml new file mode 100644 index 0000000000..2265833070 --- /dev/null +++ b/third_party/voice/cargo-ci/produce/action.yml @@ -0,0 +1,89 @@ +name: Build Cargo voice inputs +description: Build and upload same-commit native voice SDK and runtime inputs. +inputs: + target: + description: Native target triple matching the runner + required: true + windows-tools: + description: Directory containing the public Windows voice bootstrap + default: .github/scripts + buildbuddy-api-key: + description: Optional BuildBuddy credential for trusted CI builds + default: "" +runs: + using: composite + steps: + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + architecture: ${{ contains(inputs.target, 'aarch64') && 'arm64' || 'x64' }} + - uses: ./.github/actions/prepare-bazel-ci + with: + target: ${{ inputs.target }} + cache-scope: cargo-voice-native + - name: Set up Windows voice build tools + if: runner.os == 'Windows' + uses: ./third_party/voice/cargo-ci/windows-setup + with: + target: ${{ inputs.target }} + windows-tools: ${{ inputs.windows-tools }} + - name: Build and stage native dependencies + shell: bash + env: + BUILDBUDDY_API_KEY: ${{ inputs.buildbuddy-api-key }} + TARGET: ${{ inputs.target }} + run: | + set -euo pipefail + arch="${TARGET%%-*}" + case "$RUNNER_OS" in + Windows) prefix="windows_$arch"; temp="$(cygpath -u "$RUNNER_TEMP")" ;; + macOS) prefix="macos_$arch"; temp="$RUNNER_TEMP" ;; + Linux) prefix="linux_$arch"; temp="$RUNNER_TEMP" ;; + *) echo "Unsupported runner OS: $RUNNER_OS" >&2; exit 1 ;; + esac + wrapper=(--remote-download-toplevel --print-failed-action-summary) + flags=(--config=release --build_metadata="COMMIT_SHA=$GITHUB_SHA") + if [[ "$RUNNER_OS" == Windows ]]; then + wrapper+=(--windows-msvc-host-platform) + flags+=(--platforms=//:local_windows_msvc + --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows,@local_config_cc//:cc-toolchain-arm64_windows,//third_party/voice:windows_pkg_config_toolchain,//third_party/voice:windows_cmake_toolchain + --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0 --repo_env=BAZEL_MSVC_RUNTIME_VISUAL_STUDIO_EULA=1 + --inject_repository="voice_windows_tools=$VOICE_WINDOWS_BAZEL_REPOSITORY" + --//third_party/voice:windows_installed_tools=@voice_windows_tools//:tools + --action_env="SystemRoot=$VOICE_WINDOWS_SYSTEM_ROOT" --host_action_env="SystemRoot=$VOICE_WINDOWS_SYSTEM_ROOT" + --action_env="PROCESSOR_ARCHITECTURE=$VOICE_WINDOWS_HOST_ARCH" --host_action_env="PROCESSOR_ARCHITECTURE=$VOICE_WINDOWS_HOST_ARCH" + --workspace_status_command=./scripts/workspace-status.cmd) + elif [[ "$RUNNER_OS" == macOS ]]; then + flags+=(--macos_minimum_os=14.0) + fi + ./.github/scripts/run-bazel-ci.sh "${wrapper[@]}" -- build "${flags[@]}" -- \ + //third_party/voice:native_sdk //third_party/voice:native_runtime + runtime="bazel-bin/third_party/voice/native_runtime_${prefix}" + output="$temp/cargo-voice-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + python third_party/voice/cargo_ci.py stage --sdk "${runtime}_sdk" --runtime "$runtime" --output "$output" --target "$TARGET" + tar -czf "$temp/cargo-voice.tar.gz" -C "$output" sdk runtime + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: cargo-voice-${{ inputs.target }} + path: ${{ runner.temp }}/cargo-voice.tar.gz + if-no-files-found: error + overwrite: true + - name: Stage verified native pkgconf, support DLLs, and license + if: runner.os == 'Windows' + id: pkgconf + shell: pwsh + run: | + $directory = Split-Path -Parent $env:VOICE_PKG_CONFIG + $license = Join-Path $directory "COPYING" + Invoke-WebRequest -Uri "https://raw.githubusercontent.com/pkgconf/pkgconf/d97cae4e2a169916f77d9ee8a46c011b69e42a7b/COPYING" -OutFile $license + if ((Get-FileHash $license -Algorithm SHA256).Hash.ToLowerInvariant() -ne "055a265b839672a62f590869b0a1f8e4f5147e8f073c9ba77b17b8d7ad653966") { + throw "Pinned pkgconf license mismatch" + } + "directory=$directory" | Out-File $env:GITHUB_OUTPUT -Encoding utf8 -Append + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + if: runner.os == 'Windows' + with: + name: cargo-voice-pkgconf-${{ runner.arch }} + path: ${{ steps.pkgconf.outputs.directory }} + if-no-files-found: error + overwrite: true diff --git a/third_party/voice/cargo-ci/setup/action.yml b/third_party/voice/cargo-ci/setup/action.yml new file mode 100644 index 0000000000..ffc2760a38 --- /dev/null +++ b/third_party/voice/cargo-ci/setup/action.yml @@ -0,0 +1,62 @@ +name: Set up Cargo voice inputs +description: Use the target SDK and prepared runtime from this workflow run. +inputs: + target: + description: Cargo target triple + required: true +runs: + using: composite + steps: + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + - name: Ensure CMake is available for the Opus build + if: runner.os == 'Linux' + shell: bash + run: command -v cmake || sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cmake + - name: Ensure host pkg-config is available + if: runner.os == 'macOS' + shell: bash + run: command -v pkg-config || brew install pkgconf + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: cargo-voice-${{ inputs.target }} + path: ${{ runner.temp }}/cargo-voice-inputs + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + if: runner.os == 'Windows' + with: + # The x64 archive builder must execute x64 pkgconf even for ARM64. + name: cargo-voice-pkgconf-${{ runner.arch }} + path: ${{ runner.temp }}/cargo-voice-pkgconf + - name: Configure target libraries for Cargo + env: + VOICE_TARGET: ${{ inputs.target }} + shell: python + run: | + import json + import os + from pathlib import Path + import subprocess + import sys + import tarfile + import tempfile + + temp = Path(os.environ["RUNNER_TEMP"]) + target_dir = Path(os.environ["CARGO_TARGET_DIR"]) + root = Path(tempfile.mkdtemp(prefix="cargo-voice-", dir=target_dir)) + with tarfile.open(temp / "cargo-voice-inputs/cargo-voice.tar.gz") as archive: + archive.extractall(root, filter="data") + if os.name == "nt": + os.environ["VOICE_PKG_CONFIG"] = str(temp / "cargo-voice-pkgconf/pkgconf.exe") + script = Path(os.environ["GITHUB_WORKSPACE"]) / "third_party/voice/cargo_ci.py" + subprocess.run([sys.executable, str(script), "configure", "--root", str(root), + "--target", os.environ["VOICE_TARGET"]], check=True) + # nextest collects linked libraries itself. Include the runtime manifest + # and plugins too so the whole runtime can move with the test binaries. + relative = (root / "runtime").relative_to(target_dir).as_posix() + config = root / "nextest.toml" + config.write_text('[profile.default]\narchive.include = [{ path = ' + json.dumps(relative) + + ', relative-to = "target", on-missing = "error" }]\n') + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env: + print(f"CODEX_VOICE_NEXTEST_CONFIG={config}", file=env) + print(f"CODEX_VOICE_RUNTIME_RELATIVE={relative}", file=env) diff --git a/third_party/voice/cargo-ci/windows-setup/action.yml b/third_party/voice/cargo-ci/windows-setup/action.yml new file mode 100644 index 0000000000..c5b747175c --- /dev/null +++ b/third_party/voice/cargo-ci/windows-setup/action.yml @@ -0,0 +1,33 @@ +name: Set up Windows voice build tools +description: Prepare verified native Windows voice tools for Bazel. +inputs: + target: + description: Native Windows target triple + required: true + windows-tools: + description: Directory containing the public Windows voice bootstrap + default: .github/scripts +runs: + using: composite + steps: + - uses: ./.github/actions/setup-msvc-env + with: + target: ${{ inputs.target }} + - name: Install verified public Windows tools + shell: pwsh + env: + WINDOWS_TOOLS: ${{ inputs.windows-tools }} + TARGET: ${{ inputs.target }} + run: | + ./.github/scripts/compute-bazel-windows-path.ps1 + $manifest = Get-Content -Raw "$env:WINDOWS_TOOLS/voice-cygwin-snapshot.json" | ConvertFrom-Json + $pin = $manifest.archive + $archive = Join-Path $env:RUNNER_TEMP $pin.name + Invoke-WebRequest "https://github.com/openai/codex/releases/download/$($pin.tag)/$($pin.name)" -OutFile $archive + & "$env:WINDOWS_TOOLS/setup-voice-windows.ps1" -Target $env:TARGET -SnapshotArchive $archive + if (-not $env:SystemRoot) { throw "Windows SystemRoot is required" } + "VOICE_WINDOWS_SYSTEM_ROOT=$env:SystemRoot" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append + $hostArch = $env:PROCESSOR_ARCHITEW6432 + if (-not $hostArch) { $hostArch = $env:PROCESSOR_ARCHITECTURE } + if (-not $hostArch) { throw "Windows host architecture is required" } + "VOICE_WINDOWS_HOST_ARCH=$hostArch" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/third_party/voice/cargo_ci.py b/third_party/voice/cargo_ci.py new file mode 100644 index 0000000000..265c62b915 --- /dev/null +++ b/third_party/voice/cargo_ci.py @@ -0,0 +1,153 @@ +"""Stage receipt-checked voice build inputs for Cargo and configure a shard.""" + +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from release_runtime import stage as stage_runtime +from runtime import digest +from sdk import MODULES + + +def read_receipt(path: Path, target: str, commit: str) -> dict: + receipt = json.loads(path.read_text()) + if ( + receipt["target"] != target + or receipt["sourceCommit"] != commit + or receipt["sourceManifestSha256"] + != digest(Path(__file__).with_name("sources.json")) + ): + raise ValueError("native inputs do not match target, commit and sources") + return receipt + + +def stage(sdk: Path, runtime: Path, output: Path, target: str, commit: str) -> None: + sdk, runtime, output = ( + sdk.resolve(strict=True), + runtime.resolve(strict=True), + output.resolve(), + ) + if any(output.is_relative_to(source) for source in (sdk, runtime)): + raise ValueError("output must be outside the native inputs") + files = read_receipt(sdk / "sdk.json", target, commit)["files"] + native = read_receipt(runtime / "runtime.json", target, commit) + library_dir = output / "runtime" / ("bin" if target.endswith("msvc") else "lib") + prepared = { + entry.get("sourceSha256"): runtime / entry["path"] + for entry in native["libraries"] + if Path(entry["path"]).parent.name == library_dir.name + } + + output.mkdir(parents=True) + try: + stage_runtime(runtime, output / "runtime", target) + for entry in files: + source = (sdk / entry["path"]).resolve(strict=True) + relative = source.relative_to(sdk) + if digest(source) != entry["sha256"]: + raise ValueError(f"SDK file does not match its receipt: {relative}") + dest = output / "sdk" / relative + dest.parent.mkdir(parents=True, exist_ok=True) + if source.suffix == ".pc": + # Some transitive libraries retain the producer's absolute prefix. + dest.write_text( + re.sub( + r"^prefix=.*$", + "prefix=${pcfiledir}/../..", + source.read_text(), + flags=re.MULTILINE, + ) + ) + else: + shutil.copy2(source, dest) + + # Link with the prepared runtime bytes and put MSVC import libraries + # beside their DLLs so nextest collects a self-contained search path. + if relative.parent == Path("lib"): + alias = library_dir / relative.name + payload = ( + source if source.suffix == ".lib" else prepared.get(entry["sha256"]) + ) + if payload and not alias.exists(): + shutil.copy2(payload, alias) + except BaseException: + # Windows cannot delete read-only files copied from Bazel. + if os.name == "nt": + for copied in output.rglob("*"): + copied.chmod(copied.stat().st_mode | stat.S_IWRITE) + shutil.rmtree(output) + raise + + +def environment(root: Path, target: str, pkg_config: Path) -> dict[str, str]: + root, pkg_config = root.resolve(strict=True), pkg_config.resolve(strict=True) + read_receipt(root / "runtime/runtime.json", target, os.environ["GITHUB_SHA"]) + suffix = target.replace("-", "_") + values = { + f"PKG_CONFIG_{suffix}": str(pkg_config), + f"PKG_CONFIG_PATH_{suffix}": str(root / "sdk/lib/pkgconfig"), + "CODEX_TEST_VOICE_RUNTIME": str(root / "runtime"), + } + for module in MODULES: + key = re.sub(r"[^A-Z0-9]", "_", module.upper()) + values[f"SYSTEM_DEPS_{key}_SEARCH_NATIVE"] = str( + root / "runtime" / ("bin" if target.endswith("msvc") else "lib") + ) + if module.startswith("gstreamer"): + values[f"SYSTEM_DEPS_{key}_LDFLAGS"] = "" + if not target.endswith("windows-msvc"): + origin = "@loader_path" if target.endswith("darwin") else "$ORIGIN" + values[f"CARGO_TARGET_{suffix.upper()}_RUSTFLAGS"] = ( + f"-C link-arg=-Wl,-rpath,{origin}/../lib" + ) + return values + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + producer = sub.add_parser("stage") + producer.add_argument("--sdk", type=Path, required=True) + producer.add_argument("--runtime", type=Path, required=True) + producer.add_argument("--output", type=Path, required=True) + consumer = sub.add_parser("configure") + consumer.add_argument("--root", type=Path, required=True) + for command in (producer, consumer): + command.add_argument("--target", required=True) + args = parser.parse_args() + if args.command == "stage": + stage( + args.sdk, args.runtime, args.output, args.target, os.environ["GITHUB_SHA"] + ) + return + pkg_config = os.environ.get("VOICE_PKG_CONFIG") or shutil.which("pkg-config") + if not pkg_config: + parser.error("pkg-config is required on the build host") + values = environment(args.root, args.target, Path(pkg_config)) + # Probe only the staged SDK so a missing native module cannot resolve from + # the host. Cargo keeps its usual system search for unrelated dependencies. + probe_env = { + **os.environ, + "PKG_CONFIG_PATH": "", + "PKG_CONFIG_LIBDIR": str(args.root.resolve() / "sdk/lib/pkgconfig"), + } + subprocess.run( + [pkg_config, "--exists", *MODULES, "gstreamer-1.0 >= 1.28"], + env=probe_env, + check=True, + ) + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as output: + for key, value in values.items(): + output.write(f"{key}={value}\n") + + +if __name__ == "__main__": + main() diff --git a/third_party/voice/test_cargo_ci.py b/third_party/voice/test_cargo_ci.py new file mode 100644 index 0000000000..cad781a604 --- /dev/null +++ b/third_party/voice/test_cargo_ci.py @@ -0,0 +1,158 @@ +"""Exercise how full Cargo CI consumes prepared native inputs.""" + +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from cargo_ci import environment, main, stage +from package_runtime import runtime_files +from runtime import PLUGINS, digest, required_library_paths +from sdk import MODULES + + +def write_files(root, contents): + records = [] + for name, data in contents.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(0o444) # Bazel materializes these inputs read-only. + records.append({"path": name, "sha256": digest(path)}) + return records + + +class CargoCiTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory(prefix="voice cargo ") + self.addCleanup(temp.cleanup) + self.root = Path(temp.name).resolve() + self.sdk = self.root / "sdk" + self.runtime = self.root / "runtime" + self.output = self.root / "stage" + self.commit = "a" * 40 + + def inputs(self, target, alias, plugin): + canonical = required_library_paths(target)[0] + plugins = [plugin.format(name) for name in PLUGINS] + metadata = ( + "prefix=/producer\nName: {name}\nDescription: fixture\nVersion: 1.28\n" + ) + sdk = {alias: b"original", "lib/glib-2.0/include/glibconfig.h": b"header"} + for name in MODULES: + sdk[f"lib/pkgconfig/{name}.pc"] = metadata.format(name=name).encode() + if not target.endswith("msvc"): + sdk[canonical] = sdk[alias] + sdk_files = write_files(self.sdk, sdk) + libraries = write_files( + self.runtime, {name: b"prepared" for name in [canonical, *plugins]} + ) + libraries[0]["sourceSha256"] = digest(self.sdk / alias) + identity = dict( + schemaVersion=1, + target=target, + sourceCommit=self.commit, + sourceManifestSha256=digest(Path(__file__).with_name("sources.json")), + ) + (self.sdk / "sdk.json").write_text(json.dumps(dict(identity, files=sdk_files))) + receipt = dict( + identity, developmentOnly=True, libraries=libraries, plugins=plugins + ) + (self.runtime / "runtime.json").write_text(json.dumps(receipt)) + return canonical + + def test_relocated_metadata_runtime_and_cargo_probe(self): + target = "aarch64-apple-darwin" + alias = "lib/libgio-2.0.dylib" + canonical = self.inputs(target, alias, "plugins/libgst{}.dylib") + stage(self.sdk, self.runtime, self.output, target, self.commit) + moved = self.root / "moved native inputs" + self.output.rename(moved) + native = moved / "runtime" + self.assertEqual( + (native / alias).read_bytes(), (self.runtime / canonical).read_bytes() + ) + self.assertEqual( + runtime_files(native, target), runtime_files(self.runtime, target) + ) + + host = ( + os.environ.get("VOICE_PKG_CONFIG") + or shutil.which("pkg-config") + or shutil.which("pkgconf") + ) + if not host: + self.skipTest("host pkg-config is required to probe relocated metadata") + pkgdir = moved / "sdk/lib/pkgconfig" + probe_env = dict(os.environ, PKG_CONFIG_PATH="", PKG_CONFIG_LIBDIR=str(pkgdir)) + result = subprocess.check_output( + [host, "--variable=prefix", "gio-2.0"], env=probe_env, text=True + ) + # pkgconf can escape spaces while pkg-config can print the raw path. + raw = result.strip() + words = shlex.split(raw) + self.assertEqual( + Path(words[0] if len(words) == 1 else raw).resolve(), moved / "sdk" + ) + + env_file = self.root / "github env" + ci_env = dict( + GITHUB_SHA=self.commit, GITHUB_ENV=str(env_file), VOICE_PKG_CONFIG=host + ) + argv = ["cargo_ci", "configure", "--root", str(moved), "--target", target] + with patch.dict(os.environ, ci_env), patch.object(sys, "argv", argv): + main() + written = env_file.read_text() + values = dict(line.split("=", 1) for line in written.splitlines()) + self.assertEqual( + values["PKG_CONFIG_PATH_aarch64_apple_darwin"], str(pkgdir) + ) + self.assertNotIn("PKG_CONFIG_LIBDIR_aarch64_apple_darwin", values) + (pkgdir / "gio-2.0.pc").unlink() + with self.assertRaises(subprocess.CalledProcessError): + main() + self.assertEqual(env_file.read_text(), written) + + def test_arm64_import_library_and_dll_with_x64_host_pkgconf(self): + target = "aarch64-pc-windows-msvc" + alias = "lib/gio-2.0.lib" + canonical = self.inputs(target, alias, "bin/gst{}.dll") + stage(self.sdk, self.runtime, self.output, target, self.commit) + native = self.output / "runtime" + self.assertEqual( + (native / "bin/gio-2.0.lib").read_bytes(), (self.sdk / alias).read_bytes() + ) + self.assertEqual( + (native / canonical).read_bytes(), (self.runtime / canonical).read_bytes() + ) + tool = self.root / "x64 pkgconf.exe" + tool.touch() + with patch.dict(os.environ, GITHUB_SHA=self.commit): + values = environment(self.output, target, tool) + self.assertEqual(values["PKG_CONFIG_aarch64_pc_windows_msvc"], str(tool)) + self.assertEqual( + values["SYSTEM_DEPS_GSTREAMER_1_0_SEARCH_NATIVE"], str(native / "bin") + ) + + def test_mismatched_receipt_and_corrupt_sdk_fail_staging(self): + target = "x86_64-unknown-linux-gnu" + alias = "lib/libgio-2.0.so" + self.inputs(target, alias, "lib/gstreamer-1.0/libgst{}.so") + for triple, commit in [ + (target, "b" * 40), + ("aarch64-unknown-linux-gnu", self.commit), + ]: + with self.assertRaisesRegex(ValueError, "do not match"): + stage(self.sdk, self.runtime, self.output, triple, commit) + source = self.sdk / alias + source.chmod(0o644) + source.write_bytes(b"corrupt") + with self.assertRaisesRegex(ValueError, "receipt"): + stage(self.sdk, self.runtime, self.output, target, self.commit) + self.assertFalse(self.output.exists())