mirror of
https://github.com/openai/codex.git
synced 2026-09-06 15:29:32 +00:00
## Why Macrobenchmarks benefit from having a way to exercise remote-executor latency without depending on Docker. This is a very minimal first cut, if we find that simulating network conditions is useful we can always expand this scope or switch to a more robust network shaping approach. ## What - add a package-local exec-server binary for Cargo and Bazel test fixtures - add a host-local WebSocket exec-server fixture and fixed-delay interposer - let TestAppServer route its auto environment through that delayed WebSocket transport - cover the delayed thread/start path through the public app-server API ## Stack 1. [#31425 test: add TestAppServer builder](https://github.com/openai/codex/pull/31425) 2. [#31427 test: add delayed exec-server transport](https://github.com/openai/codex/pull/31427) 3. [#31295 bench: add cold skill load macrobenchmark](https://github.com/openai/codex/pull/31295) 4. [#31428 bench: add e2e benchmark entrypoints](https://github.com/openai/codex/pull/31428) 5. [#31429 ci: smoke Bazel e2e benchmarks](https://github.com/openai/codex/pull/31429)
172 lines
5.5 KiB
Rust
172 lines
5.5 KiB
Rust
use anyhow::Context;
|
|
use anyhow::Result;
|
|
use app_test_support::TestAppServer;
|
|
use app_test_support::to_response;
|
|
use app_test_support::write_mock_responses_config_toml;
|
|
use codex_app_server_protocol::JSONRPCResponse;
|
|
use codex_app_server_protocol::RequestId;
|
|
use codex_app_server_protocol::ThreadStartParams;
|
|
use codex_app_server_protocol::ThreadStartResponse;
|
|
use codex_app_server_protocol::TurnStartParams;
|
|
use codex_app_server_protocol::UserInput as V2UserInput;
|
|
use core_test_support::responses;
|
|
use core_test_support::skip_if_host_windows;
|
|
use core_test_support::skip_if_remote;
|
|
use pretty_assertions::assert_eq;
|
|
use std::collections::BTreeMap;
|
|
use std::time::Duration;
|
|
use std::time::Instant;
|
|
use tempfile::TempDir;
|
|
use tokio::time::timeout;
|
|
|
|
const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10);
|
|
|
|
#[tokio::test]
|
|
async fn builder_interposes_fixed_delay_for_auto_env() -> Result<()> {
|
|
skip_if_host_windows!(Ok(()));
|
|
skip_if_remote!(Ok(()), "the fixed-delay fixture is local-only");
|
|
|
|
let codex_home = TempDir::new()?;
|
|
let requested_delay = Duration::from_secs(1);
|
|
let mut mcp = TestAppServer::builder()
|
|
.with_codex_home(codex_home.path())
|
|
.with_exec_server_delay(requested_delay)
|
|
.build()
|
|
.await?;
|
|
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
|
assert_eq!(
|
|
mcp.auto_env_params()?.environment_id,
|
|
codex_exec_server::REMOTE_ENVIRONMENT_ID
|
|
);
|
|
|
|
let thread_start = Instant::now();
|
|
let request_id = mcp
|
|
.send_thread_start_request_with_auto_env(ThreadStartParams::default())
|
|
.await?;
|
|
let _: JSONRPCResponse = timeout(
|
|
Duration::from_secs(60),
|
|
mcp.read_stream_until_response_message(RequestId::Integer(request_id)),
|
|
)
|
|
.await??;
|
|
let elapsed = thread_start.elapsed();
|
|
assert!(
|
|
elapsed >= requested_delay,
|
|
"thread/start completed in {elapsed:?}, below the requested {requested_delay:?} delay"
|
|
);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn thread_start_with_auto_env_exposes_fixture_cwd_to_model() -> Result<()> {
|
|
let server = responses::start_mock_server().await;
|
|
let response_mock = responses::mount_sse_once(
|
|
&server,
|
|
responses::sse(vec![
|
|
responses::ev_response_created("resp-1"),
|
|
responses::ev_assistant_message("msg-1", "done"),
|
|
responses::ev_completed("resp-1"),
|
|
]),
|
|
)
|
|
.await;
|
|
let codex_home = TempDir::new()?;
|
|
write_mock_responses_config_toml(
|
|
codex_home.path(),
|
|
&server.uri(),
|
|
&BTreeMap::new(),
|
|
/*auto_compact_limit*/ 100_000,
|
|
/*requires_openai_auth*/ None,
|
|
"mock_provider",
|
|
"compact",
|
|
)?;
|
|
|
|
let mut mcp = TestAppServer::builder()
|
|
.with_codex_home(codex_home.path())
|
|
.build()
|
|
.await?;
|
|
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
|
let expected_environment = mcp.auto_env_params()?;
|
|
|
|
let err = mcp
|
|
.send_thread_start_request_with_auto_env(ThreadStartParams {
|
|
environments: Some(Vec::new()),
|
|
..Default::default()
|
|
})
|
|
.await
|
|
.expect_err("the auto-env helper should reject caller-supplied environments");
|
|
assert_eq!(
|
|
err.to_string(),
|
|
"send_thread_start_request_with_auto_env requires params.environments to be omitted"
|
|
);
|
|
|
|
let request_id = mcp
|
|
.send_thread_start_request_with_auto_env(ThreadStartParams::default())
|
|
.await?;
|
|
let response: JSONRPCResponse = timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.read_stream_until_response_message(RequestId::Integer(request_id)),
|
|
)
|
|
.await??;
|
|
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(response)?;
|
|
|
|
let request_id = mcp
|
|
.send_turn_start_request(TurnStartParams {
|
|
thread_id: thread.id,
|
|
input: vec![V2UserInput::Text {
|
|
text: "report the current directory".to_string(),
|
|
text_elements: Vec::new(),
|
|
}],
|
|
..Default::default()
|
|
})
|
|
.await?;
|
|
let _: JSONRPCResponse = timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.read_stream_until_response_message(RequestId::Integer(request_id)),
|
|
)
|
|
.await??;
|
|
|
|
timeout(
|
|
DEFAULT_READ_TIMEOUT,
|
|
mcp.read_stream_until_notification_message("turn/completed"),
|
|
)
|
|
.await??;
|
|
|
|
let environment_context = response_mock
|
|
.single_request()
|
|
.message_input_texts("user")
|
|
.into_iter()
|
|
.find(|text| text.starts_with("<environment_context>"))
|
|
.context("environment context should be model visible")?;
|
|
let model_cwd = environment_context
|
|
.lines()
|
|
.find(|line| line.trim_start().starts_with("<cwd>"))
|
|
.map(str::trim);
|
|
let expected_cwd = format!("<cwd>{}</cwd>", expected_environment.cwd);
|
|
assert_eq!(model_cwd, Some(expected_cwd.as_str()));
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn auto_env_rejects_explicit_environment_config() -> Result<()> {
|
|
let codex_home = TempDir::new()?;
|
|
std::fs::write(codex_home.path().join("environments.toml"), "")?;
|
|
|
|
let result = TestAppServer::builder()
|
|
.with_codex_home(codex_home.path())
|
|
.build()
|
|
.await;
|
|
let Err(err) = result else {
|
|
anyhow::bail!("auto-env construction unexpectedly succeeded");
|
|
};
|
|
assert_eq!(
|
|
err.to_string(),
|
|
format!(
|
|
"automatic environment cannot be used when {} exists",
|
|
codex_home.path().join("environments.toml").display()
|
|
)
|
|
);
|
|
|
|
Ok(())
|
|
}
|