openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-13 11:47:17 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
6d377e96eb5e57504c8387ebbbbbac38825a1fa8
codex/codex-rs/core/tests/suite/review.rs
Owen Lin 69cebb5d15 Route rollout reads through the canonical JSON decoder (#42378) 
## Why

Directly deserializing the flattened `RolloutLine` envelope can reject nested
decimal values, preventing affected paginated sessions from resuming.

## What changed

- Add canonical string, byte, and reverse-scanner helpers that decode rollout
  records through `serde_json::Value` before decoding the flattened item.
- Route rollout readers across session discovery, history, migration, search,
  thread storage, and transcript previews through those helpers.
- Remove `Deserialize` from `RolloutLine` so new readers cannot bypass the
  canonical persistence decoder.

## Testing

Add coverage that resumes a paginated rollout after a token-count record with
a decimal rate-limit value and verifies that ordinal sequencing continues.

GitOrigin-RevId: 49abac1e0751c073daa5a93a840d8a483fd2d013
2026-09-02 22:45:19 +00:00

1522 lines
55 KiB
Rust
Raw Blame History

use codex_core::CodexThread;
use codex_core::REVIEW_PROMPT;
use codex_core::TurnInputRequest;
use codex_core::config::Config;
use codex_core::config::Constrained;
use codex_core::find_thread_path_by_id_str;
use codex_exec_server::CreateDirectoryOptions;
use codex_features::Feature;
use codex_history::RolloutItem;
use codex_login::CodexAuth;
use codex_protocol::config_types::ApprovalsReviewer;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::items::TurnItem;
use codex_protocol::models::ContentItem;
use codex_protocol::models::PermissionProfile;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelServiceTier;
use codex_protocol::openai_models::ModelTokenBudgetConfig;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::openai_models::ReasoningEffortPreset;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::ENVIRONMENT_CONTEXT_OPEN_TAG;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ExitedReviewModeEvent;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::ReviewCodeLocation;
use codex_protocol::protocol::ReviewFinding;
use codex_protocol::protocol::ReviewLineRange;
use codex_protocol::protocol::ReviewOutputEvent;
use codex_protocol::protocol::ReviewRequest;
use codex_protocol::protocol::ReviewTarget;
use codex_protocol::protocol::ThreadSettingsOverrides;
use codex_protocol::protocol::TurnEnvironmentSelections;
use codex_protocol::review_format::render_review_output_text;
use codex_protocol::user_input::UserInput;
use core_test_support::PathBufExt;
use core_test_support::responses;
use core_test_support::responses::ResponseMock;
use core_test_support::responses::mount_sse_sequence;
use core_test_support::responses::start_mock_server;
use core_test_support::skip_if_no_network;
use core_test_support::test_codex::local_selections;
use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tempfile::TempDir;
use tokio::io::AsyncWriteExt as _;
use uuid::Uuid;
use wiremock::MockServer;
/// Verify that submitting `Op::Review` emits review item lifecycle,
/// legacy review events, and TurnComplete when the model returns a structured review payload.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_op_emits_lifecycle_and_review_output() {
// Skip under Codex sandbox network restrictions.
skip_if_no_network!();
// Start mock Responses API server. Return a single assistant message whose
// text is a JSON-encoded ReviewOutputEvent.
let review_json = serde_json::json!({
"findings": [
{
"title": "Prefer Stylize helpers",
"body": "Use .dim()/.bold() chaining instead of manual Style where possible.",
"confidence_score": 0.9,
"priority": 1,
"code_location": {
"absolute_file_path": "/tmp/file.rs",
"line_range": {"start": 10, "end": 20}
}
}
],
"overall_correctness": "good",
"overall_explanation": "All good with some improvements suggested.",
"overall_confidence_score": 0.8
})
.to_string();
let (server, request_log) = start_responses_server_with_sse(
assistant_message_sse(&review_json),
/*expected_requests*/ 1,
)
.await;
let codex_home = Arc::new(TempDir::new().unwrap());
let codex = new_conversation_for_server(&server, codex_home.clone(), |_| {}).await;
// Submit review request.
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "Please review my changes".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
// Item lifecycle events are emitted first, then the legacy review event is fanned out
// with the same stable IDs for compatibility consumers.
let entered_started = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ItemStarted(event)
if matches!(event.item, TurnItem::EnteredReviewMode(_))
)
})
.await;
let (review_turn_id, entered_item_id) = match entered_started {
EventMsg::ItemStarted(event) => (event.turn_id, event.item.id()),
other => panic!("expected entered review item start, got {other:?}"),
};
let entered_completed = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ItemCompleted(event)
if matches!(event.item, TurnItem::EnteredReviewMode(_))
)
})
.await;
match entered_completed {
EventMsg::ItemCompleted(event) => {
assert_eq!(event.turn_id, review_turn_id);
assert_eq!(event.item.id(), entered_item_id);
}
other => panic!("expected entered review item completion, got {other:?}"),
}
let entered = wait_for_event(&codex, |ev| matches!(ev, EventMsg::EnteredReviewMode(_))).await;
match entered {
EventMsg::EnteredReviewMode(event) => {
assert_eq!(event.turn_id.as_deref(), Some(review_turn_id.as_str()));
assert_eq!(event.item_id.as_deref(), Some(entered_item_id.as_str()));
}
other => panic!("expected EnteredReviewMode(..), got {other:?}"),
}
let exited_started = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ItemStarted(event)
if matches!(event.item, TurnItem::ExitedReviewMode(_))
)
})
.await;
let exited_item_id = match exited_started {
EventMsg::ItemStarted(event) => {
assert_eq!(event.turn_id, review_turn_id);
event.item.id()
}
other => panic!("expected exited review item start, got {other:?}"),
};
let exited_completed = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ItemCompleted(event)
if matches!(event.item, TurnItem::ExitedReviewMode(_))
)
})
.await;
match exited_completed {
EventMsg::ItemCompleted(event) => {
assert_eq!(event.turn_id, review_turn_id);
assert_eq!(event.item.id(), exited_item_id);
}
other => panic!("expected exited review item completion, got {other:?}"),
}
let closed = wait_for_event(&codex, |ev| matches!(ev, EventMsg::ExitedReviewMode(_))).await;
let review = match closed {
EventMsg::ExitedReviewMode(ev) => {
assert_eq!(ev.turn_id.as_deref(), Some(review_turn_id.as_str()));
assert_eq!(ev.item_id.as_deref(), Some(exited_item_id.as_str()));
ev.review_output
.expect("expected ExitedReviewMode with Some(review_output)")
}
other => panic!("expected ExitedReviewMode(..), got {other:?}"),
};
// Deep compare full structure using PartialEq (floats are f32 on both sides).
let expected = ReviewOutputEvent {
findings: vec![ReviewFinding {
title: "Prefer Stylize helpers".to_string(),
body: "Use .dim()/.bold() chaining instead of manual Style where possible.".to_string(),
confidence_score: 0.9,
priority: 1,
code_location: ReviewCodeLocation {
absolute_file_path: PathBuf::from("/tmp/file.rs"),
line_range: ReviewLineRange { start: 10, end: 20 },
},
}],
overall_correctness: "good".to_string(),
overall_explanation: "All good with some improvements suggested.".to_string(),
overall_confidence_score: 0.8,
};
assert_eq!(expected, review);
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let path = codex.rollout_path().expect("rollout path");
let text = std::fs::read_to_string(&path).expect("read rollout file");
let parent_thread_id = text
.lines()
.filter(|line| !line.trim().is_empty())
.find_map(|line| {
let rollout_line = codex_rollout::parse_rollout_line(line).expect("rollout line");
match rollout_line.item {
RolloutItem::SessionMeta(session_meta) => Some(session_meta.meta.id.to_string()),
_ => None,
}
})
.expect("parent session meta");
let request = request_log.single_request();
assert_eq!(
request.header("x-openai-subagent").as_deref(),
Some("review")
);
let turn_metadata: serde_json::Value = serde_json::from_str(
&request
.header("x-codex-turn-metadata")
.expect("review request turn metadata"),
)
.expect("review request turn metadata json");
assert!(turn_metadata.get("forked_from_thread_id").is_none());
assert_eq!(
turn_metadata["parent_thread_id"].as_str(),
Some(parent_thread_id.as_str())
);
let request_body = request.body_json();
responses::assert_root_turn(&request_body, Some(review_turn_id.as_str()))
.expect("review request root turn metadata");
responses::assert_parent_turn(&request_body, Some(review_turn_id.as_str()))
.expect("review request parent turn metadata");
// Also verify that a user message with the header and a formatted finding
// was recorded back in the parent session's rollout.
let mut saw_header = false;
let mut saw_finding_line = false;
let expected_assistant_text = render_review_output_text(&expected);
let mut saw_assistant_plain = false;
let mut saw_assistant_xml = false;
for line in text.lines() {
if line.trim().is_empty() {
continue;
}
let v: serde_json::Value = serde_json::from_str(line).expect("jsonl line");
let rl = codex_rollout::decode_rollout_line(v).expect("rollout line");
if let RolloutItem::ResponseItem(envelope) = rl.item
&& let ResponseItem::Message { role, content, .. } = envelope.item
{
if role == "user" {
for c in content {
if let ContentItem::InputText { text } = c {
if text.contains("full review output from reviewer model") {
saw_header = true;
}
if text.contains("- Prefer Stylize helpers — /tmp/file.rs:10-20") {
saw_finding_line = true;
}
}
}
} else if role == "assistant" {
for c in content {
if let ContentItem::OutputText { text } = c {
if text.contains("<user_action>") {
saw_assistant_xml = true;
}
if text == expected_assistant_text {
saw_assistant_plain = true;
}
}
}
}
}
}
assert!(saw_header, "user header missing from rollout");
assert!(
saw_finding_line,
"formatted finding line missing from rollout"
);
assert!(
saw_assistant_plain,
"assistant review output missing from rollout"
);
assert!(
!saw_assistant_xml,
"assistant review output contains user_action markup"
);
let _codex_home_guard = codex_home;
server.verify().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn cancelled_review_does_not_forward_delegate_mcp_startup() {
skip_if_no_network!();
let server = start_mock_server().await;
let request_log = responses::mount_response_once(
&server,
responses::sse_response(responses::sse(vec![responses::ev_response_created(
"resp-1",
)]))
.set_delay(Duration::from_secs(30)),
)
.await;
let codex_home = Arc::new(TempDir::new().unwrap());
let codex = new_conversation_for_server(&server, codex_home.clone(), |_| {}).await;
// Consume the parent session's own empty startup round before starting the review.
wait_for_event(&codex, |event| {
matches!(event, EventMsg::McpStartupComplete(_))
})
.await;
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "Cancel this review".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), async {
loop {
match codex.next_event().await.expect("review event").msg {
event @ (EventMsg::McpStartupUpdate(_) | EventMsg::McpStartupComplete(_)) => {
panic!("review forwarded delegate MCP startup: {event:?}")
}
EventMsg::EnteredReviewMode(_) => break,
_ => {}
}
}
})
.await
.expect("timed out waiting for review entry");
tokio::time::timeout(Duration::from_secs(5), async {
while request_log.requests().is_empty() {
tokio::task::yield_now().await;
}
})
.await
.expect("review request did not reach the server");
codex.submit(Op::Interrupt).await.unwrap();
let mut exited_review = false;
tokio::time::timeout(Duration::from_secs(5), async {
loop {
match codex
.next_event()
.await
.expect("review cancellation event")
.msg
{
event @ (EventMsg::McpStartupUpdate(_) | EventMsg::McpStartupComplete(_)) => {
panic!("cancelled review forwarded delegate MCP startup: {event:?}")
}
EventMsg::ExitedReviewMode(ExitedReviewModeEvent { review_output, .. }) => {
assert_eq!(review_output, None);
exited_review = true;
}
EventMsg::TurnAborted(_) if exited_review => break,
EventMsg::TurnAborted(_) => panic!("review turn aborted before review mode exited"),
_ => {}
}
}
})
.await
.expect("timed out waiting for review cancellation");
assert_eq!(request_log.requests().len(), 1);
let _codex_home_guard = codex_home;
server.verify().await;
}
/// When the model returns plain text that is not JSON, ensure the child
/// lifecycle still occurs and the plain text is surfaced via
/// ExitedReviewMode(Some(..)) as the overall_explanation.
// Windows CI only: bump to 4 workers to prevent SSE/event starvation and test timeouts.
#[cfg_attr(windows, tokio::test(flavor = "multi_thread", worker_threads = 4))]
#[cfg_attr(not(windows), tokio::test(flavor = "multi_thread", worker_threads = 2))]
async fn review_op_with_plain_text_emits_review_fallback() {
skip_if_no_network!();
let (server, _request_log) = start_responses_server_with_sse(
assistant_message_sse("just plain text"),
/*expected_requests*/ 1,
)
.await;
let codex_home = Arc::new(TempDir::new().unwrap());
let codex = new_conversation_for_server(&server, codex_home.clone(), |_| {}).await;
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "Plain text review".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
let _entered = wait_for_event(&codex, |ev| matches!(ev, EventMsg::EnteredReviewMode(_))).await;
let closed = wait_for_event(&codex, |ev| matches!(ev, EventMsg::ExitedReviewMode(_))).await;
let review = match closed {
EventMsg::ExitedReviewMode(ev) => ev
.review_output
.expect("expected ExitedReviewMode with Some(review_output)"),
other => panic!("expected ExitedReviewMode(..), got {other:?}"),
};
// Expect a structured fallback carrying the plain text.
let expected = ReviewOutputEvent {
overall_explanation: "just plain text".to_string(),
..Default::default()
};
assert_eq!(expected, review);
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let _codex_home_guard = codex_home;
server.verify().await;
}
/// Ensure review flow suppresses assistant-specific streaming/completion events:
/// - AgentMessageContentDelta
/// - ItemCompleted for TurnItem::AgentMessage
// Windows CI only: bump to 4 workers to prevent SSE/event starvation and test timeouts.
#[cfg_attr(windows, tokio::test(flavor = "multi_thread", worker_threads = 4))]
#[cfg_attr(not(windows), tokio::test(flavor = "multi_thread", worker_threads = 2))]
async fn review_filters_agent_message_related_events() {
skip_if_no_network!();
let (server, _request_log) = start_responses_server_with_sse(
vec![
responses::ev_message_item_added("msg-1", ""),
responses::ev_output_text_delta("Hi"),
responses::ev_output_text_delta(" there"),
responses::ev_assistant_message("msg-1", "Hi there"),
responses::ev_completed("resp-1"),
],
/*expected_requests*/ 1,
)
.await;
let codex_home = Arc::new(TempDir::new().unwrap());
let codex = new_conversation_for_server(&server, codex_home.clone(), |_| {}).await;
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "Filter streaming events".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
let mut saw_entered = false;
let mut saw_exited = false;
// Drain until TurnComplete; assert streaming-related events never surface.
wait_for_event(&codex, |event| match event {
EventMsg::TurnComplete(_) => true,
EventMsg::EnteredReviewMode(_) => {
saw_entered = true;
false
}
EventMsg::ExitedReviewMode(_) => {
saw_exited = true;
false
}
// The following must be filtered by review flow
EventMsg::AgentMessageContentDelta(_) => {
panic!("unexpected AgentMessageContentDelta surfaced during review")
}
_ => false,
})
.await;
assert!(saw_entered && saw_exited, "missing review lifecycle events");
let _codex_home_guard = codex_home;
server.verify().await;
}
/// When the model returns structured JSON in a review, ensure only a single
/// non-streaming AgentMessage is emitted; the UI consumes the structured
/// result via ExitedReviewMode plus a final assistant message.
// Windows CI only: bump to 4 workers to prevent SSE/event starvation and test timeouts.
#[cfg_attr(windows, tokio::test(flavor = "multi_thread", worker_threads = 4))]
#[cfg_attr(not(windows), tokio::test(flavor = "multi_thread", worker_threads = 2))]
async fn review_does_not_emit_agent_message_on_structured_output() {
skip_if_no_network!();
let review_json = serde_json::json!({
"findings": [
{
"title": "Example",
"body": "Structured review output.",
"confidence_score": 0.5,
"priority": 1,
"code_location": {
"absolute_file_path": "/tmp/file.rs",
"line_range": {"start": 1, "end": 2}
}
}
],
"overall_correctness": "ok",
"overall_explanation": "ok",
"overall_confidence_score": 0.5
})
.to_string();
let (server, _request_log) = start_responses_server_with_sse(
assistant_message_sse(&review_json),
/*expected_requests*/ 1,
)
.await;
let codex_home = Arc::new(TempDir::new().unwrap());
let codex = new_conversation_for_server(&server, codex_home.clone(), |_| {}).await;
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "check structured".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
// Drain events until TurnComplete; ensure we only see a final
// AgentMessage (no streaming assistant messages).
let mut saw_entered = false;
let mut saw_exited = false;
let mut agent_messages = 0;
wait_for_event(&codex, |event| match event {
EventMsg::TurnComplete(_) => true,
EventMsg::AgentMessage(_) => {
agent_messages += 1;
false
}
EventMsg::EnteredReviewMode(_) => {
saw_entered = true;
false
}
EventMsg::ExitedReviewMode(_) => {
saw_exited = true;
false
}
_ => false,
})
.await;
assert_eq!(1, agent_messages, "expected exactly one AgentMessage event");
assert!(saw_entered && saw_exited, "missing review lifecycle events");
let _codex_home_guard = codex_home;
server.verify().await;
}
/// Reviews inherit current session settings without inheriting another model's defaults.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_uses_updated_turn_permissions_and_approval_policy() {
skip_if_no_network!();
fn model_defaults(guidance_message: &str) -> ModelTokenBudgetConfig {
ModelTokenBudgetConfig {
enabled: false,
use_history_notes_extension: false,
reminder_threshold_tokens: 6_144,
reminder_message_template: "Reminder: {n_remaining} tokens remain.".to_string(),
guidance_message: guidance_message.to_string(),
auto_compact_fallback_prompt: "Preserve the important context.".to_string(),
auto_compact_fallback_buffer_tokens: 16_384,
}
}
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
let codex_home = Arc::new(TempDir::new().unwrap());
let test = test_codex()
.with_home(codex_home.clone())
.with_model_info_override("gpt-5.2", |model_info| {
model_info.service_tiers.clear();
model_info
.model_messages
.as_mut()
.expect("parent model should have model messages")
.token_budget = Some(model_defaults("PARENT MODEL ONLY"));
})
.with_model_info_override("gpt-5.4", |model_info| {
model_info.service_tiers = vec![ModelServiceTier {
id: ServiceTier::Fast.request_value().to_string(),
name: "Fast".to_string(),
description: "Priority processing".to_string(),
}];
model_info.supported_reasoning_levels = [
ReasoningEffort::Low,
ReasoningEffort::Medium,
ReasoningEffort::High,
]
.into_iter()
.map(|effort| ReasoningEffortPreset {
description: effort.to_string(),
effort,
})
.collect();
model_info.default_reasoning_level = Some(ReasoningEffort::High);
model_info
.model_messages
.as_mut()
.expect("review model should have model messages")
.token_budget = Some(model_defaults("REVIEW MODEL ONLY"));
})
.with_model("gpt-5.2")
.with_config(|config| {
config.review_model = Some("gpt-5.4".to_string());
config.model_context_window = Some(128_000);
config.service_tier = Some(ServiceTier::Fast.request_value().to_string());
config
.features
.enable(Feature::FastMode)
.expect("enable FastMode");
config
.features
.enable(Feature::TokenBudget)
.expect("token budget should be available");
config.approvals_reviewer = ApprovalsReviewer::AutoReview;
config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest);
config
.permissions
.set_permission_profile(PermissionProfile::read_only())
.expect("initial permission profile should be valid");
})
.build_with_auto_env(&server)
.await
.expect("review conversation should be created");
let codex = Arc::clone(&test.codex);
let updated_cwd = test.config.cwd.join("updated-review-workspace");
let mut selection = test.executor_environment().selection().clone();
selection.cwd = selection
.cwd
.join("updated-review-workspace")
.expect("updated execution directory should be valid");
selection.workspace_roots = vec![selection.cwd.clone()];
test.fs()
.create_directory(
&selection.cwd,
CreateDirectoryOptions {
recursive: true,
follow_symlinks: true,
},
/*sandbox*/ None,
)
.await
.expect("updated review workspace should be created");
core_test_support::submit_thread_settings(
&codex,
ThreadSettingsOverrides {
environments: Some(TurnEnvironmentSelections::new(
updated_cwd.clone(),
vec![selection],
)),
approval_policy: Some(AskForApproval::Never),
approvals_reviewer: Some(ApprovalsReviewer::User),
permission_profile: Some(PermissionProfile::Disabled),
personality: Some(Personality::Friendly),
collaboration_mode: Some(CollaborationMode {
mode: ModeKind::Plan,
settings: Settings {
model: "gpt-5.2".to_string(),
reasoning_effort: Some(ReasoningEffort::XHigh),
developer_instructions: Some("Parent planning instructions".to_string()),
},
}),
..Default::default()
},
)
.await
.expect("updated thread permissions should be accepted");
let stored_settings = codex.thread_settings_snapshot().await;
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "review current permissions".to_string(),
},
user_facing_hint: None,
},
})
.await
.expect("review should start");
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
assert_eq!(codex.thread_settings_snapshot().await, stored_settings);
let request = request_log.single_request();
assert_eq!(request.body_json()["reasoning"]["effort"], "medium");
assert_eq!(
request.body_json()["service_tier"],
ServiceTier::Fast.request_value()
);
assert!(
request
.message_input_texts("developer")
.iter()
.any(|text| text.contains("Approval policy is currently never")),
"review should use the updated approval policy"
);
assert!(
request
.message_input_texts("developer")
.iter()
.any(|text| text.contains("REVIEW MODEL ONLY")),
"review should use its own model's token-budget defaults"
);
assert!(
!request
.message_input_texts("developer")
.iter()
.any(|text| text.contains("PARENT MODEL ONLY")),
"review should not inherit the parent model's token-budget defaults"
);
assert!(
request
.message_input_texts("user")
.iter()
.any(|text| text.contains("<permission_profile type=\"disabled\">")),
"review should use the updated permission profile"
);
let review_thread_id = request.body_json()["client_metadata"]["thread_id"]
.as_str()
.expect("review request should include its thread ID")
.to_string();
let review_rollout_path = find_thread_path_by_id_str(
codex_home.path(),
&review_thread_id,
/*state_db_ctx*/ None,
)
.await
.expect("review rollout lookup should succeed")
.expect("review thread should have a rollout");
let review_rollout =
std::fs::read_to_string(review_rollout_path).expect("review rollout should be readable");
let review_session_cwd = review_rollout
.lines()
.find_map(|line| {
let rollout_line = codex_rollout::parse_rollout_line(line)
.expect("review rollout line should be valid");
match rollout_line.item {
RolloutItem::SessionMeta(session_meta) => Some(session_meta.meta.cwd),
_ => None,
}
})
.expect("review rollout should contain session metadata");
assert_eq!(review_session_cwd, updated_cwd.as_path());
let review_context = review_rollout
.lines()
.filter_map(|line| {
let rollout_line = codex_rollout::parse_rollout_line(line)
.expect("review rollout line should be valid");
match rollout_line.item {
RolloutItem::TurnContext(turn_context) => Some(turn_context),
_ => None,
}
})
.next_back()
.expect("review rollout should contain turn context");
assert_eq!(
review_context.approvals_reviewer,
Some(ApprovalsReviewer::User)
);
assert_eq!(review_context.personality, Some(Personality::Friendly));
// The review delegate still starts in its own default mode, not the parent's Plan mode.
assert_eq!(
review_context.collaboration_mode,
Some(CollaborationMode {
mode: ModeKind::Default,
settings: Settings {
model: "gpt-5.4".to_string(),
reasoning_effort: Some(ReasoningEffort::Medium),
developer_instructions: None,
},
})
);
let _codex_home_guard = codex_home;
server.verify().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_omits_retained_tier_when_fast_mode_disabled() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
let test = test_codex()
.with_model_info_override("gpt-5.4", |model| {
model.service_tiers = vec![ModelServiceTier {
id: ServiceTier::Flex.request_value().to_string(),
name: "Flex".to_string(),
description: "Flexible processing".to_string(),
}];
})
.with_config(|config| {
config
.features
.disable(Feature::FastMode)
.expect("disable FastMode");
})
.build_with_auto_env(&server)
.await?;
core_test_support::submit_thread_settings(
&test.codex,
ThreadSettingsOverrides {
service_tier: Some(Some(ServiceTier::Flex.request_value().to_string())),
..Default::default()
},
)
.await?;
test.codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "review the changes".to_string(),
},
user_facing_hint: None,
},
})
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
assert_eq!(
test.codex
.thread_settings_snapshot()
.await
.service_tier
.as_deref(),
Some("flex")
);
assert_eq!(
request_log.single_request().body_json().get("service_tier"),
None
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_resolves_inherited_summary_preferences() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 2).await;
let test = test_codex()
.with_model_info_override("gpt-5.2", |model| {
model.default_reasoning_summary = ReasoningSummary::Auto;
})
.with_model_info_override("gpt-5.4", |model| {
model.default_reasoning_summary = ReasoningSummary::Detailed;
})
.with_model("gpt-5.2")
.with_config(|config| {
config.review_model = Some("gpt-5.4".to_string());
config.model_reasoning_summary = None;
})
.build_with_auto_env(&server)
.await?;
// First follow the review model's default, then use a preference updated on the thread.
for summary in [None, Some(ReasoningSummary::Concise)] {
if let Some(summary) = summary {
core_test_support::submit_thread_settings(
&test.codex,
ThreadSettingsOverrides {
summary: Some(summary),
..Default::default()
},
)
.await?;
}
let stored_settings = test.codex.thread_settings_snapshot().await;
test.codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "review the changes".to_string(),
},
user_facing_hint: None,
},
})
.await?;
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
assert_eq!(test.codex.thread_settings_snapshot().await, stored_settings);
}
let actual = request_log
.requests()
.iter()
.map(|request| {
let body = request.body_json();
serde_json::json!([body["model"], body["reasoning"]["summary"]])
})
.collect::<Vec<_>>();
assert_eq!(
actual,
vec![
serde_json::json!(["gpt-5.4", "detailed"]),
serde_json::json!(["gpt-5.4", "concise"]),
]
);
Ok(())
}
/// Ensure that when a custom `review_model` is set in the config, the review
/// request uses that model (and not the main chat model).
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_uses_custom_review_model_from_config() {
skip_if_no_network!();
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
let codex_home = Arc::new(TempDir::new().unwrap());
let test = test_codex()
.with_home(Arc::clone(&codex_home))
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
.with_config(|config| {
config.model = Some("gpt-4.1".to_string());
config.review_model = Some("custom-review-model".to_string());
config.model_reasoning_effort = Some(ReasoningEffort::Max);
})
.build_with_auto_env(&server)
.await
.expect("custom review conversation should be created");
let codex = Arc::clone(&test.codex);
std::fs::remove_file(codex_home.path().join("models_cache.json"))
.expect("initial empty model catalog should be cached");
let mut models = codex_models_manager::bundled_models_response()
.expect("bundled model catalog should parse");
let model = models
.models
.iter_mut()
.find(|model| model.slug == "gpt-5.6-sol")
.expect("bundled model should exist");
model.slug = "custom-review-model".to_string();
model.node_repl_auto_review_required = true;
model.node_repl_disabled = true;
let models_mock = responses::mount_models_once(&server, models).await;
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "use custom model".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
// Wait for completion
let _entered = wait_for_event(&codex, |ev| matches!(ev, EventMsg::EnteredReviewMode(_))).await;
let _closed = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ExitedReviewMode(ExitedReviewModeEvent {
review_output: None,
..
})
)
})
.await;
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
// Assert the request body model equals the configured review model
let request = request_log.single_request();
assert_eq!(request.path(), "/v1/responses");
let body = request.body_json();
assert_eq!(body["model"].as_str().unwrap(), "custom-review-model");
assert_eq!(body["reasoning"]["effort"].as_str(), Some("max"));
let turn_metadata: serde_json::Value = serde_json::from_str(
&request
.header("x-codex-turn-metadata")
.expect("review request turn metadata"),
)
.expect("review request turn metadata json");
assert_eq!(turn_metadata["node_repl_auto_review_required"], true);
assert_eq!(turn_metadata["node_repl_disabled"], true);
assert_eq!(models_mock.requests().len(), 1);
let _codex_home_guard = codex_home;
server.verify().await;
}
/// Ensure that when `review_model` is not set in the config, the review request
/// uses the session model without exposing disabled clock tools or reminders.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_uses_session_model_when_review_model_unset() {
skip_if_no_network!();
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
let codex_home = Arc::new(TempDir::new().unwrap());
let test = test_codex()
.with_home(Arc::clone(&codex_home))
.with_config(|config| {
config.model = Some("gpt-5.4".to_string());
config.review_model = None;
config.model_reasoning_effort = Some(ReasoningEffort::Persistent);
})
.build_with_auto_env(&server)
.await
.expect("same-model review conversation should be created");
let codex = Arc::clone(&test.codex);
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "use session model".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
let _entered = wait_for_event(&codex, |ev| matches!(ev, EventMsg::EnteredReviewMode(_))).await;
let _closed = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ExitedReviewMode(ExitedReviewModeEvent {
review_output: None,
..
})
)
})
.await;
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let request = request_log.single_request();
assert_eq!(request.path(), "/v1/responses");
let body = request.body_json();
assert_eq!(body["model"].as_str().unwrap(), "gpt-5.4");
assert_eq!(body["reasoning"]["effort"].as_str(), Some("disabled"));
assert_eq!(
["curr_time", "sleep"].map(|name| request.tool_by_name("clock", name).is_some()),
[false, false]
);
assert!(!request.has_content_kinds(&["current_time.reminder"]));
let _codex_home_guard = codex_home;
server.verify().await;
}
/// When a review session begins, it must not prepend prior chat history from
/// the parent session. The request `input` should contain only the review
/// prompt from the user.
// Windows CI only: bump to 4 workers to prevent SSE/event starvation and test timeouts.
#[cfg_attr(windows, tokio::test(flavor = "multi_thread", worker_threads = 4))]
#[cfg_attr(not(windows), tokio::test(flavor = "multi_thread", worker_threads = 2))]
async fn review_input_isolated_from_parent_history() {
skip_if_no_network!();
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
// Seed a parent session history via resume file with both user + assistant items.
let codex_home = Arc::new(TempDir::new().unwrap());
let session_file = codex_home.path().join("resume.jsonl");
{
let mut f = tokio::fs::File::create(&session_file).await.unwrap();
let convo_id = Uuid::new_v4();
// Proper session_meta line (enveloped) with a conversation id
let meta_line = serde_json::json!({
"timestamp": "2024-01-01T00:00:00.000Z",
"type": "session_meta",
"payload": {
"session_id": convo_id,
"id": convo_id,
"timestamp": "2024-01-01T00:00:00Z",
"cwd": ".",
"originator": "test_originator",
"cli_version": "test_version",
"model_provider": "test-provider"
}
});
f.write_all(format!("{meta_line}\n").as_bytes())
.await
.unwrap();
// Prior user message (enveloped response_item)
let user = codex_protocol::models::ResponseItem::Message {
id: None,
role: "user".to_string(),
content: vec![codex_protocol::models::ContentItem::InputText {
text: "parent: earlier user message".to_string(),
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
let user_json = serde_json::to_value(&user).unwrap();
let user_line = serde_json::json!({
"timestamp": "2024-01-01T00:00:01.000Z",
"type": "response_item",
"payload": user_json
});
f.write_all(format!("{user_line}\n").as_bytes())
.await
.unwrap();
// Prior assistant message (enveloped response_item)
let assistant = codex_protocol::models::ResponseItem::Message {
id: None,
role: "assistant".to_string(),
content: vec![codex_protocol::models::ContentItem::OutputText {
text: "parent: assistant reply".to_string(),
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
let assistant_json = serde_json::to_value(&assistant).unwrap();
let assistant_line = serde_json::json!({
"timestamp": "2024-01-01T00:00:02.000Z",
"type": "response_item",
"payload": assistant_json
});
f.write_all(format!("{assistant_line}\n").as_bytes())
.await
.unwrap();
}
let codex =
resume_conversation_for_server(&server, codex_home.clone(), session_file.clone(), |_| {})
.await;
// Submit review request; it must start fresh (no parent history in `input`).
let review_prompt = "Please review only this".to_string();
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: review_prompt.clone(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
let _entered = wait_for_event(&codex, |ev| matches!(ev, EventMsg::EnteredReviewMode(_))).await;
let _closed = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ExitedReviewMode(ExitedReviewModeEvent {
review_output: None,
..
})
)
})
.await;
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
// Assert the request `input` contains the environment context followed by the user review prompt.
let request = request_log.single_request();
assert_eq!(request.path(), "/v1/responses");
let body = request.body_json();
let input = body["input"].as_array().expect("input array");
assert!(
input.len() >= 2,
"expected at least environment context and review prompt"
);
let env_text = input
.iter()
.filter_map(|msg| msg.get("content").and_then(|content| content.as_array()))
.flat_map(|content| content.iter())
.filter_map(|entry| entry.get("text").and_then(|text| text.as_str()))
.find(|text| text.starts_with(ENVIRONMENT_CONTEXT_OPEN_TAG))
.expect("env text");
assert!(
env_text.contains("<cwd>"),
"environment context should include cwd"
);
let review_text = input
.iter()
.filter_map(|msg| msg.get("content").and_then(|content| content.as_array()))
.flat_map(|content| content.iter())
.filter_map(|entry| entry.get("text").and_then(|text| text.as_str()))
.find(|text| *text == review_prompt)
.expect("review prompt text");
assert_eq!(
review_text, review_prompt,
"user message should only contain the raw review prompt"
);
// Ensure the REVIEW_PROMPT rubric is sent via instructions.
let instructions = body["instructions"].as_str().expect("instructions string");
assert_eq!(instructions, REVIEW_PROMPT);
// Also verify that a user interruption note was recorded in the rollout.
let path = codex.rollout_path().expect("rollout path");
let text = std::fs::read_to_string(&path).expect("read rollout file");
let mut saw_interruption_message = false;
for line in text.lines() {
if line.trim().is_empty() {
continue;
}
let v: serde_json::Value = serde_json::from_str(line).expect("jsonl line");
let rl = codex_rollout::decode_rollout_line(v).expect("rollout line");
if let RolloutItem::ResponseItem(envelope) = rl.item
&& let ResponseItem::Message { role, content, .. } = envelope.item
&& role == "user"
{
for c in content {
if let ContentItem::InputText { text } = c
&& text.contains("User initiated a review task, but was interrupted.")
{
saw_interruption_message = true;
break;
}
}
}
if saw_interruption_message {
break;
}
}
assert!(
saw_interruption_message,
"expected user interruption message in rollout"
);
let _codex_home_guard = codex_home;
server.verify().await;
}
/// After a review thread finishes, its conversation should be visible in the
/// parent session so later turns can reference the results.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_history_surfaces_in_parent_session() {
skip_if_no_network!();
let (server, request_log) = start_responses_server_with_sse(
assistant_message_sse("review assistant output"),
/*expected_requests*/ 2,
)
.await;
let codex_home = Arc::new(TempDir::new().unwrap());
let codex = new_conversation_for_server(&server, codex_home.clone(), |_| {}).await;
// 1) Run a review turn that produces an assistant message (isolated in child).
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::Custom {
instructions: "Start a review".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
let _entered = wait_for_event(&codex, |ev| matches!(ev, EventMsg::EnteredReviewMode(_))).await;
let _closed = wait_for_event(&codex, |ev| {
matches!(
ev,
EventMsg::ExitedReviewMode(ExitedReviewModeEvent {
review_output: Some(_),
..
})
)
})
.await;
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
// 2) Continue in the parent session; request input must not include any review items.
let followup = "back to parent".to_string();
codex
.start_or_steer_turn(TurnInputRequest::user_input(vec![UserInput::Text {
text: followup.clone(),
text_elements: Vec::new(),
}]))
.await
.unwrap();
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
// Inspect the second request (parent turn) input contents.
// Parent turns include session initial messages (user_instructions, environment_context).
// Critically, no messages from the review thread should appear.
let requests = request_log.requests();
assert_eq!(requests.len(), 2);
for request in &requests {
assert_eq!(request.path(), "/v1/responses");
}
let body = requests[1].body_json();
let input = body["input"].as_array().expect("input array");
// Must include the followup as the last item for this turn
let last = input.last().expect("at least one item in input");
assert_eq!(last["role"].as_str().unwrap(), "user");
let last_text = last["content"][0]["text"].as_str().unwrap();
assert_eq!(last_text, followup);
// Ensure review-thread content is present for downstream turns.
let contains_review_rollout_user = input.iter().any(|msg| {
msg["content"][0]["text"]
.as_str()
.unwrap_or_default()
.contains("User initiated a review task.")
});
let contains_review_assistant = input.iter().any(|msg| {
msg["content"][0]["text"]
.as_str()
.unwrap_or_default()
.contains("review assistant output")
});
assert!(
contains_review_rollout_user,
"review rollout user message missing from parent turn input"
);
assert!(
contains_review_assistant,
"review assistant output missing from parent turn input"
);
let _codex_home_guard = codex_home;
server.verify().await;
}
/// `/review` should use the session's current cwd (including runtime overrides)
/// when resolving base-branch review prompts (merge-base computation).
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn review_uses_overridden_cwd_for_base_branch_merge_base() {
skip_if_no_network!();
let (server, request_log) =
start_responses_server_with_sse(completed_sse(), /*expected_requests*/ 1).await;
let initial_cwd = TempDir::new().unwrap();
let repo_dir = TempDir::new().unwrap();
let repo_path = repo_dir.path();
fn run_git(repo_path: &std::path::Path, args: &[&str]) {
let output = std::process::Command::new("git")
.arg("-C")
.arg(repo_path)
.args(args)
.output()
.expect("spawn git");
assert!(
output.status.success(),
"git {:?} failed: stdout={:?} stderr={:?}",
args,
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
run_git(repo_path, &["init", "-b", "main"]);
run_git(repo_path, &["config", "user.email", "test@example.com"]);
run_git(repo_path, &["config", "user.name", "Test User"]);
std::fs::write(repo_path.join("file.txt"), "hello\n").unwrap();
run_git(repo_path, &["add", "."]);
run_git(repo_path, &["commit", "-m", "initial"]);
let head_sha = std::process::Command::new("git")
.arg("-C")
.arg(repo_path)
.args(["rev-parse", "HEAD"])
.output()
.expect("rev-parse HEAD");
assert!(head_sha.status.success());
let head_sha = String::from_utf8(head_sha.stdout)
.expect("utf8 sha")
.trim()
.to_string();
let codex_home = Arc::new(TempDir::new().unwrap());
let initial_cwd_path = initial_cwd.path().to_path_buf();
let codex = new_conversation_for_server(&server, codex_home.clone(), move |config| {
config.cwd = initial_cwd_path.abs();
})
.await;
core_test_support::submit_thread_settings(
&codex,
ThreadSettingsOverrides {
environments: Some(local_selections(repo_path.to_path_buf().abs())),
..Default::default()
},
)
.await
.unwrap();
codex
.submit(Op::Review {
review_request: ReviewRequest {
target: ReviewTarget::BaseBranch {
branch: "main".to_string(),
},
user_facing_hint: None,
},
})
.await
.unwrap();
let _entered = wait_for_event(&codex, |ev| matches!(ev, EventMsg::EnteredReviewMode(_))).await;
let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let requests = request_log.requests();
assert_eq!(requests.len(), 1);
for request in &requests {
assert_eq!(request.path(), "/v1/responses");
}
let body = requests[0].body_json();
let input = body["input"].as_array().expect("input array");
let saw_merge_base_sha = input
.iter()
.filter_map(|msg| msg["content"][0]["text"].as_str())
.any(|text| text.contains(&head_sha));
assert!(
saw_merge_base_sha,
"expected review prompt to include merge-base sha {head_sha}"
);
let _codex_home_guard = codex_home;
server.verify().await;
}
fn assistant_message_sse(text: &str) -> Vec<serde_json::Value> {
vec![
responses::ev_assistant_message("msg-1", text),
responses::ev_completed("resp-1"),
]
}
fn completed_sse() -> Vec<serde_json::Value> {
vec![responses::ev_completed("resp-1")]
}
/// Start a mock Responses API server and mount the given SSE events.
async fn start_responses_server_with_sse(
events: Vec<serde_json::Value>,
expected_requests: usize,
) -> (MockServer, ResponseMock) {
let server = start_mock_server().await;
let sse = responses::sse(events);
let responses = vec![sse; expected_requests];
let request_log = mount_sse_sequence(&server, responses).await;
(server, request_log)
}
/// Create a conversation configured to talk to the provided mock server.
async fn new_conversation_for_server<F>(
server: &MockServer,
codex_home: Arc<TempDir>,
mutator: F,
) -> Arc<CodexThread>
where
F: FnOnce(&mut Config) + Send + 'static,
{
let base_url = format!("{}/v1", server.uri());
let mut builder = test_codex()
.with_home(codex_home)
.with_config(move |config| {
config.model_provider.base_url = Some(base_url.clone());
mutator(config);
});
builder
.build(server)
.await
.expect("create conversation")
.codex
}
/// Create a conversation resuming from a rollout file, configured to talk to the provided mock server.
async fn resume_conversation_for_server<F>(
server: &MockServer,
codex_home: Arc<TempDir>,
resume_path: std::path::PathBuf,
mutator: F,
) -> Arc<CodexThread>
where
F: FnOnce(&mut Config) + Send + 'static,
{
let base_url = format!("{}/v1", server.uri());
let mut builder = test_codex()
.with_home(codex_home.clone())
.with_config(move |config| {
config.model_provider.base_url = Some(base_url.clone());
mutator(config);
});
builder
.resume(server, codex_home, resume_path)
.await
.expect("resume conversation")
.codex
}
Reference in New Issue View Git Blame Copy Permalink