diff --git a/.github/scripts/windows-io-trace.ps1 b/.github/scripts/windows-io-trace.ps1 new file mode 100644 index 0000000000..94a11f15c7 --- /dev/null +++ b/.github/scripts/windows-io-trace.ps1 @@ -0,0 +1,41 @@ +param( + [Parameter(Mandatory = $true)] + [ValidateSet("mark", "transition", "stop")] + [string]$Action, + + [string]$Label, + + [string]$From, + + [string]$To +) + +$ErrorActionPreference = "Stop" + +function Invoke-Wpr { + param([string[]]$Arguments) + + & wpr @Arguments + if ($LASTEXITCODE -ne 0) { + throw "wpr $($Arguments -join ' ') failed with exit code $LASTEXITCODE" + } +} + +switch ($Action) { + "mark" { + Invoke-Wpr @("-marker", $Label) + } + "transition" { + Invoke-Wpr @("-marker", "step:end:$From") + Invoke-Wpr @("-marker", "step:start:$To") + } + "stop" { + Invoke-Wpr @("-marker", "step:end:$Label") + + $traceRoot = Join-Path $env:RUNNER_TEMP "codex-io-trace" + New-Item -ItemType Directory -Force -Path $traceRoot | Out-Null + $tracePath = Join-Path $traceRoot "windows-io-$($env:GITHUB_JOB)-$($env:GITHUB_RUN_ID)-$($env:GITHUB_RUN_ATTEMPT).etl" + Invoke-Wpr @("-stop", $tracePath) + Write-Output "saved Windows I/O trace to $tracePath" + } +} diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 3e48b741b7..91f2d3a3bb 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -152,16 +152,41 @@ jobs: deployment: false steps: + - name: Start Windows I/O trace + shell: pwsh + run: | + $traceRoot = Join-Path $env:RUNNER_TEMP "codex-io-trace" + New-Item -ItemType Directory -Force -Path $traceRoot | Out-Null + $null = & wpr -cancel 2>&1 + & wpr -start FileIO -start DiskIO -filemode -recordtempto $traceRoot + if ($LASTEXITCODE -ne 0) { + throw "failed to start Windows I/O trace" + } + & wpr -marker "step:start:checkout" + if ($LASTEXITCODE -ne 0) { + throw "failed to mark checkout start" + } + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false + - name: Mark checkout end and wrapper test start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From checkout -To test-buildbuddy-wrapper + - name: Test BuildBuddy Bazel wrapper if: matrix.shard == 1 shell: pwsh run: python .github/scripts/test_run_bazel_with_buildbuddy.py + - name: Mark wrapper test end and Bazel setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From test-buildbuddy-wrapper -To prepare-bazel-ci + - name: Prepare Bazel CI id: prepare_bazel uses: ./.github/actions/prepare-bazel-ci @@ -172,6 +197,11 @@ jobs: # the PR-blocking critical path after the useful test work is done. cache-scope: bazel-test + - name: Mark Bazel setup end and test shard start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From prepare-bazel-ci -To bazel-test-shard + - name: bazel test shard env: BAZEL_TEST_SHARD: ${{ matrix.shard }} @@ -211,6 +241,10 @@ jobs: --skip_incompatible_explicit_targets --test_tag_filters=-argument-comment-lint --test_verbose_timeout_warnings + "--action_env=CODEX_IO_TRACE_CACHE_BUST=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${BAZEL_TEST_SHARD}" + "--host_action_env=CODEX_IO_TRACE_CACHE_BUST=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${BAZEL_TEST_SHARD}" + "--test_env=CODEX_IO_TRACE_CACHE_BUST=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${BAZEL_TEST_SHARD}" + --nocache_test_results --build_metadata=COMMIT_SHA=${GITHUB_SHA} --build_metadata=TAG_windows_test_shard=${BAZEL_TEST_SHARD} ) @@ -225,6 +259,11 @@ jobs: -- \ "${selected_targets[@]}" + - name: Mark test shard end and execution-log upload start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From bazel-test-shard -To upload-bazel-execution-logs + - name: Upload Bazel execution logs if: always() && !cancelled() continue-on-error: true @@ -234,10 +273,29 @@ jobs: path: ${{ runner.temp }}/bazel-execution-logs if-no-files-found: ignore + - name: Mark execution-log upload end and clean-worktree check start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From upload-bazel-execution-logs -To check-clean-worktree + - name: Check for a clean worktree if: always() && !cancelled() uses: ./.github/actions/check-clean-worktree + - name: Stop Windows I/O trace + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action stop -Label check-clean-worktree + + - name: Upload Windows I/O trace + if: always() && !cancelled() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: windows-io-bazel-shard-${{ matrix.shard }} + path: ${{ runner.temp }}/codex-io-trace + if-no-files-found: error + test-windows: # Preserve the existing required-check surface while the real work happens # in the sharded Windows jobs above. diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index d5f0133528..2a05aabe7a 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -320,14 +320,44 @@ jobs: target: aarch64-pc-windows-msvc steps: + - name: Start Windows I/O trace + shell: pwsh + run: | + $traceRoot = Join-Path $env:RUNNER_TEMP "codex-io-trace" + New-Item -ItemType Directory -Force -Path $traceRoot | Out-Null + $null = & wpr -cancel 2>&1 + & wpr -start FileIO -start DiskIO -filemode -recordtempto $traceRoot + if ($LASTEXITCODE -ne 0) { + throw "failed to start Windows I/O trace" + } + & wpr -marker "step:start:checkout-codex" + if ($LASTEXITCODE -ne 0) { + throw "failed to mark Codex checkout start" + } + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Mark Codex checkout end and CI setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From checkout-codex -To setup-ci + - uses: ./.github/actions/setup-ci + - name: Mark CI setup end and git configuration start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From setup-ci -To configure-git + - name: Configure git for upstream checkout shell: bash run: git config --global core.symlinks true + - name: Mark git configuration end and upstream checkout start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From configure-git -To checkout-upstream + - name: Check out upstream rusty_v8 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -336,18 +366,33 @@ jobs: path: upstream-rusty-v8 submodules: recursive + - name: Mark upstream checkout end and Python setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From checkout-upstream -To setup-python + - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.11" architecture: x64 + - name: Mark Python setup end and Codex Rust setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From setup-python -To setup-codex-rust + - name: Set up Codex Rust toolchain for Cargo smoke uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 with: toolchain: "1.95.0" targets: ${{ matrix.target }} + - name: Mark Codex Rust setup end and rusty_v8 Rust setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From setup-codex-rust -To setup-rusty-v8-rust + - name: Install rusty_v8 Rust toolchain env: TARGET: ${{ matrix.target }} @@ -357,20 +402,33 @@ jobs: rustup toolchain install 1.91.0 --profile minimal --no-self-update rustup target add --toolchain 1.91.0 "${TARGET}" + - name: Mark rusty_v8 Rust setup end and submodule-status write start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From setup-rusty-v8-rust -To write-submodule-status + - name: Write upstream submodule status shell: bash working-directory: upstream-rusty-v8 run: git submodule status --recursive > git_submodule_status.txt + - name: Mark submodule-status write end and source-cache restore start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From write-submodule-status -To restore-source-cache + - name: Restore upstream source-build cache - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 with: path: | ${{ env.CARGO_TARGET_DIR }}/sccache ${{ env.CARGO_TARGET_DIR }}/${{ matrix.target }}/release/gn_out - key: rusty-v8-source-${{ matrix.target }}-${{ matrix.runner }}-sandbox-${{ hashFiles('upstream-rusty-v8/Cargo.lock', 'upstream-rusty-v8/build.rs', 'upstream-rusty-v8/git_submodule_status.txt') }} - restore-keys: | - rusty-v8-source-${{ matrix.target }}-${{ matrix.runner }}-sandbox- + key: rusty-v8-source-io-trace-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Mark source-cache restore end and sccache setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From restore-source-cache -To setup-sccache - name: Install and start sccache shell: pwsh @@ -389,12 +447,35 @@ jobs: . $basename/sccache --start-server echo "$(pwd)/$basename" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Mark sccache setup end and Chromium clang setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From setup-sccache -To setup-chromium-clang + - name: Install Chromium clang for ARM64 MSVC cross build if: matrix.target == 'aarch64-pc-windows-msvc' shell: bash working-directory: upstream-rusty-v8 run: python3 tools/clang/scripts/update.py + - name: Mark Chromium clang setup end and cold-graph setup start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From setup-chromium-clang -To force-cold-source-graph + + - name: Force cold source-build graph + shell: pwsh + env: + TARGET: ${{ matrix.target }} + run: | + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue (Join-Path $env:CARGO_TARGET_DIR "sccache") + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue (Join-Path $env:CARGO_TARGET_DIR "$env:TARGET/release/gn_out") + + - name: Mark cold-graph setup end and upstream build start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From force-cold-source-graph -To build-upstream-rusty-v8 + - name: Build upstream rusty_v8 sandbox release pair env: SCCACHE_IDLE_TIMEOUT: 0 @@ -404,6 +485,11 @@ jobs: working-directory: upstream-rusty-v8 run: cargo +1.91.0 build --locked --release --target "${TARGET}" --features v8_enable_sandbox + - name: Mark upstream build end and staging start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From build-upstream-rusty-v8 -To stage-upstream-pair + - name: Stage upstream sandbox release pair env: TARGET: ${{ matrix.target }} @@ -416,6 +502,11 @@ jobs: --output-dir "dist/${TARGET}" \ --sandbox + - name: Mark staging end and Cargo smoke link start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From stage-upstream-pair -To smoke-link-cargo + - name: Smoke link staged artifact with Cargo env: TARGET: ${{ matrix.target }} @@ -437,12 +528,36 @@ jobs: cargo +1.95.0 test -p codex-v8-poc --target "${TARGET}" --features sandbox --no-run ) + - name: Mark Cargo smoke link end and staged-artifact upload start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From smoke-link-cargo -To upload-staged-artifacts + - name: Upload staged artifacts uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: name: v8-canary-${{ needs.metadata.outputs.v8_version }}-ptrcomp-sandbox-${{ matrix.target }} path: dist/${{ matrix.target }}/* + - name: Mark staged-artifact upload end and clean-worktree check start + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action transition -From upload-staged-artifacts -To check-clean-worktree + - name: Check for a clean worktree if: always() && !cancelled() uses: ./.github/actions/check-clean-worktree + + - name: Stop Windows I/O trace + if: always() && !cancelled() + shell: pwsh + run: ./.github/scripts/windows-io-trace.ps1 -Action stop -Label check-clean-worktree + + - name: Upload Windows I/O trace + if: always() && !cancelled() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: windows-io-v8-source-${{ matrix.target }} + path: ${{ runner.temp }}/codex-io-trace + if-no-files-found: error