mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
388 lines
13 KiB
Rust
388 lines
13 KiB
Rust
use anyhow::Result;
|
|
use codex_core::AuthManager;
|
|
use codex_core::ModelClient;
|
|
use codex_core::Prompt;
|
|
use codex_core::ResponseEvent;
|
|
use codex_core::auth::AuthCredentialsStoreMode;
|
|
use codex_core::auth::OPENAI_API_KEY_ENV_VAR;
|
|
use codex_core::built_in_model_providers;
|
|
use codex_otel::SessionTelemetry;
|
|
use codex_otel::TelemetryAuthMode;
|
|
use codex_protocol::ThreadId;
|
|
use codex_protocol::config_types::ReasoningSummary;
|
|
use codex_protocol::models::ContentItem;
|
|
use codex_protocol::models::ResponseItem;
|
|
use codex_protocol::openai_models::ReasoningEffort;
|
|
use codex_protocol::protocol::AskForApproval;
|
|
use codex_protocol::protocol::EventMsg;
|
|
use codex_protocol::protocol::Op;
|
|
use codex_protocol::protocol::SandboxPolicy;
|
|
use codex_protocol::protocol::SessionSource;
|
|
use codex_protocol::user_input::UserInput;
|
|
use core_test_support::load_default_config_for_test;
|
|
use core_test_support::responses;
|
|
use core_test_support::responses::ev_completed;
|
|
use core_test_support::responses::ev_response_created;
|
|
use core_test_support::responses::mount_sse_once;
|
|
use core_test_support::responses::mount_sse_once_match;
|
|
use core_test_support::responses::mount_sse_sequence;
|
|
use core_test_support::responses::sse;
|
|
use core_test_support::skip_if_no_network;
|
|
use core_test_support::test_codex::TestCodex;
|
|
use core_test_support::test_codex::test_codex;
|
|
use futures::StreamExt;
|
|
use pretty_assertions::assert_eq;
|
|
use std::ffi::OsStr;
|
|
use std::ffi::OsString;
|
|
use std::sync::Arc;
|
|
use tempfile::TempDir;
|
|
use tokio::time::Duration;
|
|
use tokio::time::timeout;
|
|
use wiremock::Mock;
|
|
use wiremock::ResponseTemplate;
|
|
use wiremock::http::Method;
|
|
use wiremock::matchers::method;
|
|
use wiremock::matchers::path_regex;
|
|
|
|
const MODEL: &str = "gpt-5.3-codex";
|
|
|
|
struct EnvVarGuard {
|
|
key: &'static str,
|
|
original: Option<OsString>,
|
|
}
|
|
|
|
impl EnvVarGuard {
|
|
fn set(key: &'static str, value: &OsStr) -> Self {
|
|
let original = std::env::var_os(key);
|
|
unsafe {
|
|
std::env::set_var(key, value);
|
|
}
|
|
Self { key, original }
|
|
}
|
|
}
|
|
|
|
impl Drop for EnvVarGuard {
|
|
fn drop(&mut self) {
|
|
unsafe {
|
|
match &self.original {
|
|
Some(value) => std::env::set_var(self.key, value),
|
|
None => std::env::remove_var(self.key),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn websocket_fallback_switches_to_http_on_upgrade_required_connect() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
Mock::given(method("GET"))
|
|
.and(path_regex(".*/responses$"))
|
|
.respond_with(ResponseTemplate::new(426))
|
|
.mount(&server)
|
|
.await;
|
|
|
|
let response_mock = mount_sse_once(
|
|
&server,
|
|
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config({
|
|
let base_url = format!("{}/v1", server.uri());
|
|
move |config| {
|
|
config.model_provider.base_url = Some(base_url);
|
|
config.model_provider.wire_api = codex_core::WireApi::Responses;
|
|
config.model_provider.supports_websockets = true;
|
|
// If we don't treat 426 specially, the sampling loop would retry the WebSocket
|
|
// handshake before switching to the HTTP transport.
|
|
config.model_provider.stream_max_retries = Some(2);
|
|
config.model_provider.request_max_retries = Some(0);
|
|
}
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
test.submit_turn("hello").await?;
|
|
|
|
let requests = server.received_requests().await.unwrap_or_default();
|
|
let websocket_attempts = requests
|
|
.iter()
|
|
.filter(|req| req.method == Method::GET && req.url.path().ends_with("/responses"))
|
|
.count();
|
|
let http_attempts = requests
|
|
.iter()
|
|
.filter(|req| req.method == Method::POST && req.url.path().ends_with("/responses"))
|
|
.count();
|
|
|
|
// The startup prewarm request sees 426 and immediately switches the session to HTTP fallback,
|
|
// so the first turn goes straight to HTTP with no additional websocket connect attempt.
|
|
assert_eq!(websocket_attempts, 1);
|
|
assert_eq!(http_attempts, 1);
|
|
assert_eq!(response_mock.requests().len(), 1);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn websocket_fallback_uses_openai_api_key_env_for_http_replay() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let _api_key_guard = EnvVarGuard::set(OPENAI_API_KEY_ENV_VAR, OsStr::new("dummy"));
|
|
let home = TempDir::new()?;
|
|
let server = responses::start_mock_server().await;
|
|
Mock::given(method("GET"))
|
|
.and(path_regex(".*/responses$"))
|
|
.respond_with(ResponseTemplate::new(426))
|
|
.expect(1)
|
|
.mount(&server)
|
|
.await;
|
|
|
|
let response_mock = mount_sse_once_match(
|
|
&server,
|
|
wiremock::matchers::header("authorization", "Bearer dummy"),
|
|
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
|
)
|
|
.await;
|
|
|
|
let mut config = load_default_config_for_test(&home).await;
|
|
config.model = Some(MODEL.to_string());
|
|
let model_info = codex_core::test_support::construct_model_info_offline(MODEL, &config);
|
|
|
|
let mut provider = built_in_model_providers(/* openai_base_url */ None)["openai"].clone();
|
|
provider.base_url = Some(format!("{}/v1", server.uri()));
|
|
provider.supports_websockets = true;
|
|
provider.request_max_retries = Some(0);
|
|
provider.stream_max_retries = Some(0);
|
|
|
|
let auth_manager = Arc::new(AuthManager::new(
|
|
home.path().to_path_buf(),
|
|
/*enable_codex_api_key_env*/ false,
|
|
AuthCredentialsStoreMode::File,
|
|
));
|
|
let conversation_id = ThreadId::new();
|
|
let session_telemetry = SessionTelemetry::new(
|
|
conversation_id,
|
|
MODEL,
|
|
model_info.slug.as_str(),
|
|
None,
|
|
Some("test@test.com".to_string()),
|
|
auth_manager.auth_mode().map(TelemetryAuthMode::from),
|
|
"test_originator".to_string(),
|
|
false,
|
|
"test".to_string(),
|
|
SessionSource::Exec,
|
|
);
|
|
let client = ModelClient::new(
|
|
Some(auth_manager),
|
|
conversation_id,
|
|
provider,
|
|
SessionSource::Exec,
|
|
config.model_verbosity,
|
|
false,
|
|
false,
|
|
None,
|
|
);
|
|
let mut client_session = client.new_session();
|
|
let mut prompt = Prompt::default();
|
|
prompt.input = vec![ResponseItem::Message {
|
|
id: None,
|
|
role: "user".to_string(),
|
|
content: vec![ContentItem::InputText {
|
|
text: "hello".to_string(),
|
|
}],
|
|
end_turn: None,
|
|
phase: None,
|
|
}];
|
|
|
|
let mut stream = client_session
|
|
.stream(
|
|
&prompt,
|
|
&model_info,
|
|
&session_telemetry,
|
|
/*effort*/ None::<ReasoningEffort>,
|
|
ReasoningSummary::Auto,
|
|
None,
|
|
None,
|
|
)
|
|
.await?;
|
|
|
|
while let Some(event) = stream.next().await {
|
|
if matches!(event?, ResponseEvent::Completed { .. }) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
let request = response_mock.single_request();
|
|
assert_eq!(request.path(), "/v1/responses");
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn websocket_fallback_switches_to_http_after_retries_exhausted() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let response_mock = mount_sse_once(
|
|
&server,
|
|
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config({
|
|
let base_url = format!("{}/v1", server.uri());
|
|
move |config| {
|
|
config.model_provider.base_url = Some(base_url);
|
|
config.model_provider.wire_api = codex_core::WireApi::Responses;
|
|
config.model_provider.supports_websockets = true;
|
|
config.model_provider.stream_max_retries = Some(2);
|
|
config.model_provider.request_max_retries = Some(0);
|
|
}
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
test.submit_turn("hello").await?;
|
|
|
|
let requests = server.received_requests().await.unwrap_or_default();
|
|
let websocket_attempts = requests
|
|
.iter()
|
|
.filter(|req| req.method == Method::GET && req.url.path().ends_with("/responses"))
|
|
.count();
|
|
let http_attempts = requests
|
|
.iter()
|
|
.filter(|req| req.method == Method::POST && req.url.path().ends_with("/responses"))
|
|
.count();
|
|
|
|
// Deferred request prewarm is attempted at startup.
|
|
// The first turn then makes 3 websocket stream attempts (initial try + 2 retries),
|
|
// after which fallback activates and the request is replayed over HTTP.
|
|
assert_eq!(websocket_attempts, 4);
|
|
assert_eq!(http_attempts, 1);
|
|
assert_eq!(response_mock.requests().len(), 1);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn websocket_fallback_hides_first_websocket_retry_stream_error() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let response_mock = mount_sse_once(
|
|
&server,
|
|
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config({
|
|
let base_url = format!("{}/v1", server.uri());
|
|
move |config| {
|
|
config.model_provider.base_url = Some(base_url);
|
|
config.model_provider.wire_api = codex_core::WireApi::Responses;
|
|
config.model_provider.supports_websockets = true;
|
|
config.model_provider.stream_max_retries = Some(2);
|
|
config.model_provider.request_max_retries = Some(0);
|
|
}
|
|
});
|
|
let TestCodex {
|
|
codex,
|
|
session_configured,
|
|
cwd,
|
|
..
|
|
} = builder.build(&server).await?;
|
|
|
|
codex
|
|
.submit(Op::UserTurn {
|
|
items: vec![UserInput::Text {
|
|
text: "hello".into(),
|
|
text_elements: Vec::new(),
|
|
}],
|
|
final_output_json_schema: None,
|
|
cwd: cwd.path().to_path_buf(),
|
|
approval_policy: AskForApproval::Never,
|
|
sandbox_policy: SandboxPolicy::DangerFullAccess,
|
|
model: session_configured.model.clone(),
|
|
effort: None,
|
|
summary: None,
|
|
service_tier: None,
|
|
collaboration_mode: None,
|
|
personality: None,
|
|
})
|
|
.await?;
|
|
|
|
let mut stream_error_messages = Vec::new();
|
|
loop {
|
|
let event = timeout(Duration::from_secs(10), codex.next_event())
|
|
.await
|
|
.expect("timeout waiting for event")
|
|
.expect("event stream ended unexpectedly")
|
|
.msg;
|
|
match event {
|
|
EventMsg::StreamError(e) => stream_error_messages.push(e.message),
|
|
EventMsg::TurnComplete(_) => break,
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
let expected_stream_errors = if cfg!(debug_assertions) {
|
|
vec!["Reconnecting... 1/2", "Reconnecting... 2/2"]
|
|
} else {
|
|
vec!["Reconnecting... 2/2"]
|
|
};
|
|
assert_eq!(stream_error_messages, expected_stream_errors);
|
|
assert_eq!(response_mock.requests().len(), 1);
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn websocket_fallback_is_sticky_across_turns() -> Result<()> {
|
|
skip_if_no_network!(Ok(()));
|
|
|
|
let server = responses::start_mock_server().await;
|
|
let response_mock = mount_sse_sequence(
|
|
&server,
|
|
vec![
|
|
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
|
sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
let mut builder = test_codex().with_config({
|
|
let base_url = format!("{}/v1", server.uri());
|
|
move |config| {
|
|
config.model_provider.base_url = Some(base_url);
|
|
config.model_provider.wire_api = codex_core::WireApi::Responses;
|
|
config.model_provider.supports_websockets = true;
|
|
config.model_provider.stream_max_retries = Some(2);
|
|
config.model_provider.request_max_retries = Some(0);
|
|
}
|
|
});
|
|
let test = builder.build(&server).await?;
|
|
|
|
test.submit_turn("first").await?;
|
|
test.submit_turn("second").await?;
|
|
|
|
let requests = server.received_requests().await.unwrap_or_default();
|
|
let websocket_attempts = requests
|
|
.iter()
|
|
.filter(|req| req.method == Method::GET && req.url.path().ends_with("/responses"))
|
|
.count();
|
|
let http_attempts = requests
|
|
.iter()
|
|
.filter(|req| req.method == Method::POST && req.url.path().ends_with("/responses"))
|
|
.count();
|
|
|
|
// WebSocket attempts all happen on the first turn:
|
|
// 1 deferred request prewarm attempt (startup) + 3 stream attempts
|
|
// (initial try + 2 retries) before fallback.
|
|
// Fallback is sticky, so the second turn stays on HTTP and adds no websocket attempts.
|
|
assert_eq!(websocket_attempts, 4);
|
|
assert_eq!(http_attempts, 2);
|
|
assert_eq!(response_mock.requests().len(), 2);
|
|
|
|
Ok(())
|
|
}
|