From 619d36f810f89fcbf0fd91c1fd8dce23d3851d0a Mon Sep 17 00:00:00 2001 From: Alex Kira Date: Sat, 21 Nov 2026 01:12:48 +0000 Subject: [PATCH] Tag user reports with the latest reasoning removal status (#62883) ## What changed Set the `cot_removed` tag from the latest completed sampling request on the live thread. Use `true` for reported reasoning removals and `false` for an explicitly empty report. Omit the tag when no report is available and discard any caller-supplied value. Clear the stored result when thread history is replaced. A resumed session starts without a stored result. ## Testing Cover tag values in saved reports across tool calls, compaction, and session resume. Verify that failed compaction preserves the last sampling result. GitOrigin-RevId: f1568567a6b4a62685569478fc7194f4357cb605 --- .../request_processors/feedback_processor.rs | 7 ++ .../app-server/tests/suite/v2/feedback.rs | 98 +++++++++++++++++++ codex-rs/core/src/codex_thread.rs | 6 ++ codex-rs/core/src/session/mod.rs | 4 + codex-rs/core/src/session/turn.rs | 4 +- codex-rs/core/src/state/session.rs | 3 + codex-rs/core/tests/suite/compact_remote.rs | 15 ++- 7 files changed, 132 insertions(+), 5 deletions(-) diff --git a/codex-rs/app-server/src/request_processors/feedback_processor.rs b/codex-rs/app-server/src/request_processors/feedback_processor.rs index 297a405be4..a4220474fc 100644 --- a/codex-rs/app-server/src/request_processors/feedback_processor.rs +++ b/codex-rs/app-server/src/request_processors/feedback_processor.rs @@ -128,6 +128,13 @@ impl FeedbackRequestProcessor { let http_client_factory = self.config.http_client_factory(); let auth = self.auth_manager.auth_cached(); + upload_tags.remove("cot_removed"); + if let Some(id) = conversation_id + && let Ok(thread) = self.thread_manager.get_thread(id).await + && let Some(removed) = thread.last_sampling_cot_removed().await + { + upload_tags.insert("cot_removed".to_string(), removed.to_string()); + } let turn_metadata = if let Some(conversation_id) = conversation_id && let Some(rollout_path) = self .resolve_rollout_path(conversation_id, self.state_db.as_ref(), save_path.as_ref()) diff --git a/codex-rs/app-server/tests/suite/v2/feedback.rs b/codex-rs/app-server/tests/suite/v2/feedback.rs index 2bd9237705..8d41596142 100644 --- a/codex-rs/app-server/tests/suite/v2/feedback.rs +++ b/codex-rs/app-server/tests/suite/v2/feedback.rs @@ -11,6 +11,104 @@ use tokio::io::BufReader; use tokio::net::TcpListener; use tokio::time::timeout; +#[tokio::test] +async fn feedback_reports_cot_removal_from_latest_completed_sampling_request() -> Result<()> { + use core_test_support::responses; + + let server = responses::start_mock_server().await; + let removed = + json!([{"type":"reasoning", "id":"rs-1", "action":"removed", "reason":"prefix_mismatch"}]); + let streams = [ + ("drop", Some(removed.clone())), + ("replayed", Some(removed.clone())), + ("tool", Some(removed.clone())), + ("clean", Some(json!([]))), + ("unreported", None), + ("before_compaction", Some(removed.clone())), + ("summary", Some(removed.clone())), + ("after", Some(removed)), + ] + .into_iter() + .map(|(id, transformations)| { + let mut completed = responses::ev_completed(id); + if let Some(transformations) = transformations { + completed["response"]["input_transformations"] = transformations; + } + // The tool call and clean final response belong to the same user turn. + let output = if id == "tool" { + responses::ev_function_call("call", "unknown_test_tool", "{}") + } else { + responses::ev_assistant_message(&format!("msg-{id}"), id) + }; + responses::sse(vec![output, completed]) + }) + .collect(); + responses::mount_sse_sequence(&server, streams).await; + let home = tempfile::tempdir()?; + app_test_support::MockResponsesConfig::new(&server.uri()).write(home.path())?; + app_test_support::write_models_cache(home.path()).await?; + let mut app = TestAppServer::builder() + .with_codex_home(home.path()) + .build_initialized() + .await?; + let thread = app.start_thread(Default::default()).await?.thread; + for (step, expected) in [ + ("drop", Some("true")), + ("replayed", Some("true")), + ("tool_then_clean", Some("false")), + ("unreported", None), + ("before_compaction", Some("true")), + ("compact", None), + ("after", Some("true")), + ("resume", None), + ] { + let mut params = json!({"threadId": thread.id}); + let method = match step { + "compact" => "thread/compact/start", + "resume" => { + app.shutdown_gracefully().await?; + app = TestAppServer::builder() + .with_codex_home(home.path()) + .build_initialized() + .await?; + "thread/resume" + } + text => { + params["input"] = json!([{"type": "text", "text": text}]); + "turn/start" + } + }; + let id = app.send_raw_request(method, Some(params)).await?; + app.read_response::(id).await?; + if matches!(method, "turn/start" | "thread/compact/start") { + app.read_stream_until_notification_message("turn/completed") + .await?; + } + + let destination = home.path().join(format!("{step}.zip")); + let id = app + .send_raw_request( + "feedback/save", + Some(json!({ + "classification": "bad_result", + "threadId": thread.id, + "tags": {"cot_removed": "untrusted"}, + "savePath": destination, + })), + ) + .await?; + app.read_response::(id).await?; + let mut archive = zip::ZipArchive::new(std::fs::File::open(destination)?)?; + let report: serde_json::Value = serde_json::from_reader(archive.by_name("report.json")?)?; + assert_eq!( + report["tags"].get("cot_removed").cloned(), + expected.map(serde_json::Value::from), + "after {step}" + ); + } + Ok(()) +} + #[tokio::test] async fn feedback_upload_limits_concurrency_and_releases_failed_uploads() -> Result<()> { let proxy = TcpListener::bind("127.0.0.1:0").await?; diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index ba115d3b37..03583c3436 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -1319,6 +1319,12 @@ impl CodexThread { self.rollout_path.clone() } + /// Whether the latest completed sampling request in this live context reported a removal. + /// Returns None before sampling or when that response omitted the report. + pub async fn last_sampling_cot_removed(&self) -> Option { + self.session.last_sampling_cot_removed().await + } + /// Selects a local feedback rollout without materializing an untouched thread. pub async fn feedback_rollout_path(&self) -> ThreadStoreResult> { match self.session.live_thread() { diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index c7e3b03cd4..72b9bd49df 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1744,6 +1744,10 @@ impl Session { state.token_info() } + pub(crate) async fn last_sampling_cot_removed(&self) -> Option { + self.state.lock().await.last_sampling_cot_removed + } + pub(crate) async fn get_estimated_token_count( &self, turn_context: &TurnContext, diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index b56e065296..69cf16e774 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -3742,7 +3742,7 @@ async fn try_run_sampling_request( response_id, token_usage, usage_metadata, - reasoning_removals: _, + reasoning_removals, end_turn, } => { let retry_prompt = resampling.finish_response(end_turn, token_usage.is_some()); @@ -3754,6 +3754,8 @@ async fn try_run_sampling_request( ) .await; } + sess.state.lock().await.last_sampling_cot_removed = + reasoning_removals.as_ref().map(|counts| !counts.is_empty()); response_completed = true; sess.services .analytics_events_client diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index b368b2aef9..d35606a56f 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -109,6 +109,7 @@ pub(crate) struct SessionState { pub(crate) restored_turn_routing: Option<(String, Vec)>, /// Runtime accounting state for the active auto-compaction window. auto_compact_window: AutoCompactWindow, + pub(crate) last_sampling_cot_removed: Option, pub(crate) async_compaction: crate::session::async_compaction::AsyncCompactionState, /// Original request effort for the current model while configuration updates remain active. pub(crate) reasoning_effort_pin: ReasoningEffortPin, @@ -155,6 +156,7 @@ impl SessionState { history_reset: CancellationToken::new(), latest_rate_limits: None, latest_token_usage_record: None, + last_sampling_cot_removed: None, mcp_dependency_prompted: HashSet::new(), additional_context: AdditionalContextStore::default(), previous_turn_settings: None, @@ -229,6 +231,7 @@ impl SessionState { reference_context_item: Option, replacement: HistoryReplacement, ) { + self.last_sampling_cot_removed = None; let invalidate_reviews = match replacement { HistoryReplacement::Compaction { reviewer_compaction_hash, diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index b49cccb801..596224fab5 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -475,13 +475,15 @@ async fn remote_compact_v2_records_usage_before_output_validation() -> Result<() .await?; let codex = &harness.test().codex; let rollout_path = codex.rollout_path().context("rollout path")?; + let mut sample = + responses::ev_completed_with_tokens("before-compact", /*total_tokens*/ 500); + sample["response"]["input_transformations"] = json!([{ + "type": "reasoning", "id": "rs-1", "action": "removed", "reason": "prefix_mismatch", + }]); let responses_mock = responses::mount_sse_sequence( harness.server(), vec![ - sse(vec![responses::ev_completed_with_tokens( - "before-compact", - /*total_tokens*/ 500, - )]), + sse(vec![sample]), sse(vec![ json!({ "type": "response.output_item.done", @@ -540,6 +542,11 @@ async fn remote_compact_v2_records_usage_before_output_validation() -> Result<() 1, "the accepted prompt should be saved exactly once after compaction fails" ); + assert_eq!( + codex.last_sampling_cot_removed().await, + Some(true), + "a failed compaction should preserve the last sampling report", + ); codex.shutdown_and_wait().await?; let record = fs::read_to_string(&rollout_path)?