mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Route reviews through the review agent skill
This commit is contained in:
@@ -1,22 +1,15 @@
|
||||
use anyhow::Result;
|
||||
use app_test_support::TestAppServer;
|
||||
use app_test_support::create_final_assistant_message_sse_response;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::create_mock_responses_server_sequence;
|
||||
use app_test_support::create_shell_command_sse_response;
|
||||
use app_test_support::to_response;
|
||||
use codex_app_server_protocol::ItemCompletedNotification;
|
||||
use codex_app_server_protocol::ItemStartedNotification;
|
||||
use codex_app_server_protocol::JSONRPCError;
|
||||
use codex_app_server_protocol::JSONRPCMessage;
|
||||
use codex_app_server_protocol::JSONRPCNotification;
|
||||
use codex_app_server_protocol::JSONRPCResponse;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ReviewDelivery;
|
||||
use codex_app_server_protocol::ReviewStartParams;
|
||||
use codex_app_server_protocol::ReviewStartResponse;
|
||||
use codex_app_server_protocol::ReviewTarget;
|
||||
use codex_app_server_protocol::ServerRequest;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::ThreadStartParams;
|
||||
use codex_app_server_protocol::ThreadStartResponse;
|
||||
@@ -26,8 +19,8 @@ use codex_app_server_protocol::TurnItemsView;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
use codex_app_server_protocol::TurnStatus;
|
||||
use codex_app_server_protocol::UserInput as V2UserInput;
|
||||
use core_test_support::responses;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use tempfile::TempDir;
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -35,26 +28,17 @@ const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs
|
||||
const INVALID_REQUEST_ERROR_CODE: i64 = -32600;
|
||||
|
||||
#[tokio::test]
|
||||
async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<()> {
|
||||
let review_payload = json!({
|
||||
"findings": [
|
||||
{
|
||||
"title": "Prefer Stylize helpers",
|
||||
"body": "Use .dim()/.bold() chaining instead of manual Style.",
|
||||
"confidence_score": 0.9,
|
||||
"priority": 1,
|
||||
"code_location": {
|
||||
"absolute_file_path": "/tmp/file.rs",
|
||||
"line_range": {"start": 10, "end": 20}
|
||||
}
|
||||
}
|
||||
],
|
||||
"overall_correctness": "good",
|
||||
"overall_explanation": "Looks solid overall with minor polish suggested.",
|
||||
"overall_confidence_score": 0.75
|
||||
})
|
||||
.to_string();
|
||||
let server = create_mock_responses_server_repeating_assistant(&review_payload).await;
|
||||
async fn review_start_runs_regular_turn_with_review_agent_skill() -> Result<()> {
|
||||
let server = responses::start_mock_server().await;
|
||||
let response_mock = responses::mount_sse_once(
|
||||
&server,
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_assistant_message("msg-1", "Done"),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
@@ -90,169 +74,42 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<()
|
||||
let turn_id = turn.id.clone();
|
||||
assert_eq!(turn.status, TurnStatus::InProgress);
|
||||
assert_eq!(turn.items_view, TurnItemsView::NotLoaded);
|
||||
assert_eq!(
|
||||
turn.items,
|
||||
vec![ThreadItem::UserMessage {
|
||||
id: turn_id.clone(),
|
||||
client_id: None,
|
||||
content: vec![V2UserInput::Text {
|
||||
text: "commit 1234567: Tidy UI colors".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
}]
|
||||
);
|
||||
|
||||
// Confirm we see the EnteredReviewMode marker on the main thread.
|
||||
let mut saw_entered_review_mode = false;
|
||||
for _ in 0..10 {
|
||||
let item_started: JSONRPCNotification = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("item/started"),
|
||||
)
|
||||
.await??;
|
||||
let started: ItemStartedNotification =
|
||||
serde_json::from_value(item_started.params.expect("params must be present"))?;
|
||||
match started.item {
|
||||
ThreadItem::EnteredReviewMode { review, .. } => {
|
||||
assert_eq!(started.turn_id, turn_id);
|
||||
assert_eq!(review, "commit 1234567: Tidy UI colors");
|
||||
saw_entered_review_mode = true;
|
||||
break;
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_entered_review_mode,
|
||||
"did not observe enteredReviewMode item"
|
||||
);
|
||||
|
||||
// Confirm we see the ExitedReviewMode marker (with review text)
|
||||
// on the same turn. Ignore any other items the stream surfaces.
|
||||
let mut review_body: Option<String> = None;
|
||||
for _ in 0..10 {
|
||||
let review_notif: JSONRPCNotification = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("item/completed"),
|
||||
)
|
||||
.await??;
|
||||
let completed: ItemCompletedNotification =
|
||||
serde_json::from_value(review_notif.params.expect("params must be present"))?;
|
||||
match completed.item {
|
||||
ThreadItem::ExitedReviewMode { review, .. } => {
|
||||
assert_eq!(completed.turn_id, turn_id);
|
||||
review_body = Some(review);
|
||||
break;
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
}
|
||||
|
||||
let review = review_body.expect("did not observe a code review item");
|
||||
assert!(review.contains("Prefer Stylize helpers"));
|
||||
assert!(review.contains("/tmp/file.rs:10-20"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "TODO(owenlin0): flaky"]
|
||||
async fn review_start_exec_approval_item_id_matches_command_execution_item() -> Result<()> {
|
||||
let responses = vec![
|
||||
create_shell_command_sse_response(
|
||||
vec![
|
||||
"git".to_string(),
|
||||
"rev-parse".to_string(),
|
||||
"HEAD".to_string(),
|
||||
],
|
||||
/*workdir*/ None,
|
||||
Some(5000),
|
||||
"review-call-1",
|
||||
)?,
|
||||
create_final_assistant_message_sse_response("done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence(responses).await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml_with_approval_policy(codex_home.path(), &server.uri(), "untrusted")?;
|
||||
|
||||
let mut mcp = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.build()
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_id = start_default_thread(&mut mcp).await?;
|
||||
|
||||
let review_req = mcp
|
||||
.send_review_start_request(ReviewStartParams {
|
||||
thread_id,
|
||||
delivery: Some(ReviewDelivery::Inline),
|
||||
target: ReviewTarget::Commit {
|
||||
sha: "1234567deadbeef".to_string(),
|
||||
title: Some("Check review approvals".to_string()),
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
let review_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(review_req)),
|
||||
)
|
||||
.await??;
|
||||
let ReviewStartResponse { turn, .. } = to_response::<ReviewStartResponse>(review_resp)?;
|
||||
let turn_id = turn.id.clone();
|
||||
assert_eq!(turn.items_view, TurnItemsView::NotLoaded);
|
||||
assert_eq!(
|
||||
turn.items,
|
||||
vec![ThreadItem::UserMessage {
|
||||
id: turn_id.clone(),
|
||||
client_id: None,
|
||||
content: vec![V2UserInput::Text {
|
||||
text: "commit 1234567: Check review approvals".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
}]
|
||||
);
|
||||
|
||||
let server_req = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_request_message(),
|
||||
)
|
||||
.await??;
|
||||
let ServerRequest::CommandExecutionRequestApproval { request_id, params } = server_req else {
|
||||
panic!("expected CommandExecutionRequestApproval request");
|
||||
};
|
||||
assert_eq!(params.item_id, "review-call-1");
|
||||
assert_eq!(params.turn_id, turn_id);
|
||||
|
||||
let mut command_item_id = None;
|
||||
for _ in 0..10 {
|
||||
let item_started: JSONRPCNotification = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("item/started"),
|
||||
)
|
||||
.await??;
|
||||
let started: ItemStartedNotification =
|
||||
serde_json::from_value(item_started.params.expect("params must be present"))?;
|
||||
if let ThreadItem::CommandExecution { id, .. } = started.item {
|
||||
command_item_id = Some(id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let command_item_id = command_item_id.expect("did not observe command execution item");
|
||||
assert_eq!(command_item_id, params.item_id);
|
||||
|
||||
mcp.send_response(
|
||||
request_id,
|
||||
serde_json::json!({ "decision": codex_protocol::protocol::ReviewDecision::Approved }),
|
||||
)
|
||||
.await?;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let request = response_mock.single_request();
|
||||
let user_messages = request.message_input_texts("user");
|
||||
let prompt = user_messages
|
||||
.iter()
|
||||
.find(|text| text.contains("Spawn one sub-agent"))
|
||||
.expect("review delegation prompt should be model-visible");
|
||||
assert!(prompt.contains("$review-agent"));
|
||||
assert!(prompt.contains("1234567deadbeef"));
|
||||
assert!(prompt.contains("Tidy UI colors"));
|
||||
|
||||
let skill_fragments = user_messages
|
||||
.iter()
|
||||
.filter(|text| text.starts_with("<skill>"))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(skill_fragments.len(), 1);
|
||||
assert!(skill_fragments[0].contains("<name>review-agent</name>"));
|
||||
assert!(skill_fragments[0].contains("Do not modify files"));
|
||||
|
||||
assert_eq!(
|
||||
turn.items,
|
||||
vec![ThreadItem::UserMessage {
|
||||
id: turn_id,
|
||||
client_id: None,
|
||||
content: vec![V2UserInput::Text {
|
||||
text: prompt.to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
}]
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -296,14 +153,7 @@ async fn review_start_rejects_empty_base_branch() -> Result<()> {
|
||||
#[cfg_attr(target_os = "windows", ignore = "flaky on windows CI")]
|
||||
#[tokio::test]
|
||||
async fn review_start_with_detached_delivery_returns_new_thread_id() -> Result<()> {
|
||||
let review_payload = json!({
|
||||
"findings": [],
|
||||
"overall_correctness": "ok",
|
||||
"overall_explanation": "detached review",
|
||||
"overall_confidence_score": 0.5
|
||||
})
|
||||
.to_string();
|
||||
let server = create_mock_responses_server_repeating_assistant(&review_payload).await;
|
||||
let server = create_mock_responses_server_repeating_assistant("Done").await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
@@ -338,17 +188,22 @@ async fn review_start_with_detached_delivery_returns_new_thread_id() -> Result<(
|
||||
|
||||
assert_eq!(turn.status, TurnStatus::InProgress);
|
||||
assert_eq!(turn.items_view, TurnItemsView::NotLoaded);
|
||||
assert_eq!(
|
||||
turn.items,
|
||||
vec![ThreadItem::UserMessage {
|
||||
id: turn.id.clone(),
|
||||
client_id: None,
|
||||
content: vec![V2UserInput::Text {
|
||||
text: "detached review".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
}]
|
||||
);
|
||||
let [ThreadItem::UserMessage { content, .. }] = turn.items.as_slice() else {
|
||||
panic!("expected synthesized review user message");
|
||||
};
|
||||
let [
|
||||
V2UserInput::Text {
|
||||
text,
|
||||
text_elements,
|
||||
},
|
||||
] = content.as_slice()
|
||||
else {
|
||||
panic!("expected synthesized review text");
|
||||
};
|
||||
assert!(text.contains("Spawn one sub-agent"));
|
||||
assert!(text.contains("$review-agent"));
|
||||
assert!(text.ends_with("Review target:\ndetached review"));
|
||||
assert!(text_elements.is_empty());
|
||||
assert_ne!(
|
||||
review_thread_id, thread_id,
|
||||
"detached review should run on a different thread"
|
||||
@@ -508,21 +363,13 @@ async fn materialize_thread_rollout(mcp: &mut TestAppServer, thread_id: &str) ->
|
||||
}
|
||||
|
||||
fn create_config_toml(codex_home: &std::path::Path, server_uri: &str) -> std::io::Result<()> {
|
||||
create_config_toml_with_approval_policy(codex_home, server_uri, "never")
|
||||
}
|
||||
|
||||
fn create_config_toml_with_approval_policy(
|
||||
codex_home: &std::path::Path,
|
||||
server_uri: &str,
|
||||
approval_policy: &str,
|
||||
) -> std::io::Result<()> {
|
||||
let config_toml = codex_home.join("config.toml");
|
||||
std::fs::write(
|
||||
config_toml,
|
||||
format!(
|
||||
r#"
|
||||
model = "mock-model"
|
||||
approval_policy = "{approval_policy}"
|
||||
approval_policy = "never"
|
||||
sandbox_mode = "read-only"
|
||||
|
||||
model_provider = "mock_provider"
|
||||
|
||||
Reference in New Issue
Block a user