openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-30 19:18:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

tui: improve eval capture UX

Browse Source 
Add eval capture flow to /feedback (bad result), including start picker with relative timestamps, updated notes prompts, and upload consent UI + snapshots.
This commit is contained in:
Noah MacCallum
2026-01-01 03:05:26 -08:00
parent 13c42a077c
commit 602e5fca05
16 changed files with 1268 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/tui/Cargo.toml
+1
View File
@@ -36,6 +36,7 @@ codex-common = { workspace = true, features = [
"sandbox_summary",
] }
codex-core = { workspace = true }
codex-eval-case = { workspace = true }
codex-feedback = { workspace = true }
codex-file-search = { workspace = true }
codex-login = { workspace = true }
codex-rs/tui/src/app.rs
+29
View File
@@ -806,6 +806,35 @@ impl App {
AppEvent::OpenFeedbackConsent { category } => {
self.chat_widget.open_feedback_consent(category);
}
AppEvent::OpenFeedbackBadResultFork => {
self.chat_widget.open_feedback_bad_result_fork();
}
AppEvent::OpenEvalCaptureIntro => {
self.chat_widget.open_eval_capture_intro();
}
AppEvent::EvalCaptureIntroContinue => {
self.chat_widget.on_eval_capture_intro_continue();
}
AppEvent::OpenEvalCaptureNotes { start_marker } => {
self.chat_widget.open_eval_capture_notes(start_marker);
}
AppEvent::CreateEvalCaptureBundle {
start_marker,
what_went_wrong,
what_good_looks_like,
} => {
self.chat_widget.create_eval_capture_bundle(
start_marker,
what_went_wrong,
what_good_looks_like,
);
}
AppEvent::EvalCaptureUpload { case_id, path } => {
self.chat_widget.upload_eval_capture_bundle(case_id, path);
}
AppEvent::EvalCaptureUploadSkipped { case_id, path } => {
self.chat_widget.eval_capture_upload_skipped(case_id, path);
}
AppEvent::LaunchExternalEditor => {
if self.chat_widget.external_editor_state() == ExternalEditorState::Active {
self.launch_external_editor(tui).await;
codex-rs/tui/src/app_event.rs
+57
View File
@@ -182,6 +182,39 @@ pub(crate) enum AppEvent {
category: FeedbackCategory,
},
/// Open the bad-result fork chooser ("quick feedback" vs "capture eval sample").
OpenFeedbackBadResultFork,
/// Start the "capture eval sample" flow from /feedback.
OpenEvalCaptureIntro,
/// Dismiss the eval capture intro screen and continue to the start picker.
EvalCaptureIntroContinue,
/// Open the freeform notes entry for eval capture.
OpenEvalCaptureNotes {
start_marker: EvalCaptureStartMarker,
},
/// Create the eval bundle and report the output path/id in history.
CreateEvalCaptureBundle {
start_marker: EvalCaptureStartMarker,
what_went_wrong: String,
what_good_looks_like: String,
},
/// Upload the previously captured eval case bundle to the team.
EvalCaptureUpload {
case_id: String,
path: String,
},
/// Skip uploading the previously captured eval case bundle.
EvalCaptureUploadSkipped {
case_id: String,
path: String,
},
/// Launch the external editor after a normal draw has completed.
LaunchExternalEditor,
}
@@ -193,3 +226,27 @@ pub(crate) enum FeedbackCategory {
Bug,
Other,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct EvalCaptureRepoSnapshot {
pub(crate) ghost_commit: String,
pub(crate) base_sha: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum EvalCaptureStartMarker {
RolloutLineIndex {
index: usize,
timestamp: String,
display: String,
message: String,
repo_snapshot: Option<EvalCaptureRepoSnapshot>,
},
#[allow(dead_code)]
RolloutLineTimestamp {
timestamp: String,
display: String,
message: String,
repo_snapshot: Option<EvalCaptureRepoSnapshot>,
},
}
codex-rs/tui/src/bottom_pane/eval_capture_view.rs
+403
View File
@@ -0,0 +1,403 @@
use std::cell::RefCell;
use crossterm::event::KeyCode;
use crossterm::event::KeyEvent;
use crossterm::event::KeyModifiers;
use ratatui::buffer::Buffer;
use ratatui::layout::Rect;
use ratatui::style::Stylize;
use ratatui::text::Line;
use ratatui::widgets::Clear;
use ratatui::widgets::Paragraph;
use ratatui::widgets::StatefulWidgetRef;
use ratatui::widgets::Widget;
use crate::app_event::AppEvent;
use crate::app_event::EvalCaptureStartMarker;
use crate::app_event_sender::AppEventSender;
use crate::render::renderable::Renderable;
use super::CancellationEvent;
use super::bottom_pane_view::BottomPaneView;
use super::popup_consts::standard_popup_hint_line;
use super::textarea::TextArea;
use super::textarea::TextAreaState;
pub(crate) fn eval_capture_intro_params(
app_event_tx: AppEventSender,
) -> super::SelectionViewParams {
let continue_action: super::SelectionAction = Box::new({
let tx = app_event_tx.clone();
move |_sender: &AppEventSender| {
tx.send(AppEvent::EvalCaptureIntroContinue);
}
});
let basic_feedback_action: super::SelectionAction = Box::new({
let tx = app_event_tx;
move |_sender: &AppEventSender| {
tx.send(AppEvent::OpenFeedbackConsent {
category: crate::app_event::FeedbackCategory::BadResult,
});
}
});
let header_lines: Vec<Box<dyn crate::render::renderable::Renderable>> = vec![
Line::from("Capture eval sample".bold()).into(),
Line::from("").into(),
Line::from("Here's everything we capture as an eval sample.".dim()).into(),
Line::from("It's stored locally on your computer:".dim()).into(),
Line::from(vec![" • ".into(), "manifest.json".into()]).into(),
Line::from(vec![" • ".into(), "rollout.jsonl".into()]).into(),
Line::from(vec![" • ".into(), "repo.patch".into()]).into(),
Line::from(vec![" • ".into(), "codex-logs.log".into()]).into(),
Line::from("").into(),
Line::from("Next, you can optionally upload this bundle to the team.".dim()).into(),
];
super::SelectionViewParams {
footer_hint: Some(standard_popup_hint_line()),
title: None,
header: Box::new(crate::render::renderable::ColumnRenderable::with(
header_lines,
)),
items: vec![
super::SelectionItem {
name: "Continue".to_string(),
description: Some("Create an eval sample bundle locally.".to_string()),
actions: vec![continue_action],
dismiss_on_select: true,
..Default::default()
},
super::SelectionItem {
name: "Send basic feedback instead".to_string(),
description: Some(
"Skip eval capture and send basic feedback to the team.".to_string(),
),
actions: vec![basic_feedback_action],
dismiss_on_select: true,
..Default::default()
},
],
..Default::default()
}
}
pub(crate) fn eval_capture_upload_consent_params(
app_event_tx: AppEventSender,
case_id: String,
path: String,
) -> super::SelectionViewParams {
let upload_action: super::SelectionAction = Box::new({
let tx = app_event_tx.clone();
let case_id = case_id.clone();
let path = path.clone();
move |_sender: &AppEventSender| {
tx.send(AppEvent::EvalCaptureUpload {
case_id: case_id.clone(),
path: path.clone(),
});
}
});
let skip_action: super::SelectionAction = Box::new({
let tx = app_event_tx;
let case_id = case_id.clone();
let path = path.clone();
move |_sender: &AppEventSender| {
tx.send(AppEvent::EvalCaptureUploadSkipped {
case_id: case_id.clone(),
path: path.clone(),
});
}
});
let header_lines: Vec<Box<dyn crate::render::renderable::Renderable>> = vec![
Line::from("Upload eval sample?".bold()).into(),
Line::from("").into(),
Line::from("If you choose Yes, it will upload the full bundle".dim()).into(),
Line::from("to the team:".dim()).into(),
Line::from(vec![" • ".into(), "manifest.json".into()]).into(),
Line::from(vec![" • ".into(), "rollout.jsonl".into()]).into(),
Line::from(vec![" • ".into(), "repo.patch".into()]).into(),
Line::from(vec![" • ".into(), "codex-logs.log".into()]).into(),
Line::from("").into(),
Line::from("This may include file paths, code snippets, and tool outputs.".dim()).into(),
];
super::SelectionViewParams {
footer_hint: Some(standard_popup_hint_line()),
title: None,
header: Box::new(crate::render::renderable::ColumnRenderable::with(
header_lines,
)),
items: vec![
super::SelectionItem {
name: "Yes".to_string(),
description: Some(
"Upload the full bundle to the team for troubleshooting and model improvement."
.to_string(),
),
actions: vec![upload_action],
dismiss_on_select: true,
..Default::default()
},
super::SelectionItem {
name: "No".to_string(),
description: Some("Keep it local only.".to_string()),
actions: vec![skip_action],
dismiss_on_select: true,
..Default::default()
},
],
..Default::default()
}
}
pub(crate) fn eval_capture_start_picker_params(
app_event_tx: AppEventSender,
options: Vec<EvalCaptureStartMarker>,
initial_selected_idx: Option<usize>,
) -> super::SelectionViewParams {
let items = options
.into_iter()
.map(|marker| {
let action: super::SelectionAction = Box::new({
let tx = app_event_tx.clone();
let marker = marker.clone();
move |_sender: &AppEventSender| {
tx.send(AppEvent::OpenEvalCaptureNotes {
start_marker: marker.clone(),
});
}
});
let disabled_reason = (!marker_has_repo_snapshot(&marker))
.then_some("No repo snapshot available for this message.".to_string());
super::SelectionItem {
name: marker_display_name(&marker),
// For the eval-capture start picker, we render the full message directly as the
// row name (including a relative timestamp) to avoid duplicating it in the
// description column.
actions: vec![action],
dismiss_on_select: true,
disabled_reason,
..Default::default()
}
})
.collect();
super::SelectionViewParams {
footer_hint: Some(standard_popup_hint_line()),
title: Some("Pick the start message".to_string()),
subtitle: Some("Default is your last message; scroll for earlier messages.".to_string()),
items,
initial_selected_idx,
show_numbers: false,
..Default::default()
}
}
fn marker_display_name(marker: &EvalCaptureStartMarker) -> String {
match marker {
EvalCaptureStartMarker::RolloutLineIndex { display, .. }
| EvalCaptureStartMarker::RolloutLineTimestamp { display, .. } => display.clone(),
}
}
fn marker_has_repo_snapshot(marker: &EvalCaptureStartMarker) -> bool {
match marker {
EvalCaptureStartMarker::RolloutLineIndex { repo_snapshot, .. }
| EvalCaptureStartMarker::RolloutLineTimestamp { repo_snapshot, .. } => {
repo_snapshot.is_some()
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EvalCaptureNotesStage {
WhatWentWrong,
WhatGoodLooksLike,
}
pub(crate) struct EvalCaptureNotesView {
stage: EvalCaptureNotesStage,
start_marker: EvalCaptureStartMarker,
app_event_tx: AppEventSender,
textarea: TextArea,
textarea_state: RefCell<TextAreaState>,
what_went_wrong: String,
complete: bool,
}
impl EvalCaptureNotesView {
pub(crate) fn new(start_marker: EvalCaptureStartMarker, app_event_tx: AppEventSender) -> Self {
Self {
stage: EvalCaptureNotesStage::WhatWentWrong,
start_marker,
app_event_tx,
textarea: TextArea::new(),
textarea_state: RefCell::new(TextAreaState::default()),
what_went_wrong: String::new(),
complete: false,
}
}
fn submit(&mut self) {
let text = self.textarea.text().trim().to_string();
match self.stage {
EvalCaptureNotesStage::WhatWentWrong => {
self.what_went_wrong = text;
self.textarea.set_text("");
*self.textarea_state.borrow_mut() = TextAreaState::default();
self.stage = EvalCaptureNotesStage::WhatGoodLooksLike;
}
EvalCaptureNotesStage::WhatGoodLooksLike => {
let what_good_looks_like = text;
self.app_event_tx.send(AppEvent::CreateEvalCaptureBundle {
start_marker: self.start_marker.clone(),
what_went_wrong: std::mem::take(&mut self.what_went_wrong),
what_good_looks_like,
});
self.complete = true;
}
}
}
}
impl BottomPaneView for EvalCaptureNotesView {
fn handle_key_event(&mut self, key_event: KeyEvent) {
match key_event {
KeyEvent {
code: KeyCode::Esc, ..
} => {
self.on_ctrl_c();
}
KeyEvent {
code: KeyCode::Enter,
modifiers: KeyModifiers::NONE,
..
} => {
self.submit();
}
other => {
self.textarea.input(other);
}
}
}
fn on_ctrl_c(&mut self) -> CancellationEvent {
self.complete = true;
CancellationEvent::Handled
}
fn is_complete(&self) -> bool {
self.complete
}
fn handle_paste(&mut self, pasted: String) -> bool {
if pasted.is_empty() {
return false;
}
self.textarea.insert_str(&pasted);
true
}
}
impl Renderable for EvalCaptureNotesView {
fn desired_height(&self, width: u16) -> u16 {
// 2 title lines + input + (blank spacer + hint)
2u16 + self.input_height(width) + 2u16
}
fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> {
if area.height < 2 || area.width <= 2 {
return None;
}
let text_area_height = self.input_height(area.width).saturating_sub(1);
if text_area_height == 0 {
return None;
}
let top_line_count = 2u16; // title + subtitle
let textarea_rect = Rect {
x: area.x.saturating_add(2),
y: area.y.saturating_add(top_line_count).saturating_add(1),
width: area.width.saturating_sub(2),
height: text_area_height,
};
let state = *self.textarea_state.borrow();
self.textarea.cursor_pos_with_state(textarea_rect, state)
}
fn render(&self, area: Rect, buf: &mut Buffer) {
if area.height == 0 || area.width == 0 {
return;
}
let (title, subtitle, placeholder) = match self.stage {
EvalCaptureNotesStage::WhatWentWrong => (
"What was wrong about this rollout?",
"Please be specific.",
"Type here.",
),
EvalCaptureNotesStage::WhatGoodLooksLike => (
"With respect to that issue, what would the ideal behavior be?",
"Again, please be specific.",
"Type here.",
),
};
let input_height = self.input_height(area.width);
let title_area = Rect {
x: area.x,
y: area.y,
width: area.width,
height: 2,
};
let input_area = Rect {
x: area.x,
y: area.y.saturating_add(2),
width: area.width,
height: input_height,
};
Clear.render(area, buf);
Paragraph::new(vec![Line::from(title.bold()), Line::from(subtitle.dim())])
.render(title_area, buf);
let textarea_rect = Rect {
x: input_area.x.saturating_add(2),
y: input_area.y.saturating_add(1),
width: input_area.width.saturating_sub(2),
height: input_area.height.saturating_sub(1),
};
let mut state = self.textarea_state.borrow_mut();
StatefulWidgetRef::render_ref(&(&self.textarea), textarea_rect, buf, &mut state);
if self.textarea.text().is_empty() {
Paragraph::new(Line::from(placeholder.dim())).render(textarea_rect, buf);
}
let hint_area = Rect {
x: area.x,
y: area
.y
.saturating_add(2)
.saturating_add(input_height)
.saturating_add(1),
width: area.width,
height: 1,
};
Paragraph::new(Line::from("Enter to continue • Esc to cancel".dim()))
.render(hint_area, buf);
}
}
impl EvalCaptureNotesView {
fn input_height(&self, width: u16) -> u16 {
let usable_width = width.saturating_sub(2);
let text_height = self.textarea.desired_height(usable_width).clamp(1, 8);
text_height.saturating_add(1).min(9)
}
}
codex-rs/tui/src/bottom_pane/feedback_view.rs
+51 -1
View File
@@ -387,7 +387,11 @@ fn make_feedback_item(
category: FeedbackCategory,
) -> super::SelectionItem {
let action: super::SelectionAction = Box::new(move |_sender: &AppEventSender| {
app_event_tx.send(AppEvent::OpenFeedbackConsent { category });
if category == FeedbackCategory::BadResult {
app_event_tx.send(AppEvent::OpenFeedbackBadResultFork);
} else {
app_event_tx.send(AppEvent::OpenFeedbackConsent { category });
}
});
super::SelectionItem {
name: name.to_string(),
@@ -398,6 +402,52 @@ fn make_feedback_item(
}
}
pub(crate) fn feedback_bad_result_fork_params(
app_event_tx: AppEventSender,
) -> super::SelectionViewParams {
let quick_action: super::SelectionAction = Box::new({
let tx = app_event_tx.clone();
move |_sender: &AppEventSender| {
tx.send(AppEvent::OpenFeedbackConsent {
category: FeedbackCategory::BadResult,
});
}
});
let capture_action: super::SelectionAction = Box::new({
let tx = app_event_tx;
move |_sender: &AppEventSender| {
tx.send(AppEvent::OpenEvalCaptureIntro);
}
});
super::SelectionViewParams {
title: Some("Bad result".to_string()),
subtitle: Some("Choose the fastest way to help us improve.".to_string()),
items: vec![
super::SelectionItem {
name: "Quick feedback".to_string(),
description: Some(
"Send a short note (and optionally logs) to maintainers.".to_string(),
),
actions: vec![quick_action],
dismiss_on_select: true,
..Default::default()
},
super::SelectionItem {
name: "Capture eval sample".to_string(),
description: Some(
"Create a local bundle you can rerun as an eval case later.".to_string(),
),
actions: vec![capture_action],
dismiss_on_select: true,
..Default::default()
},
],
..Default::default()
}
}
/// Build the upload consent popup params for a given feedback category.
pub(crate) fn feedback_upload_consent_params(
app_event_tx: AppEventSender,
codex-rs/tui/src/bottom_pane/list_selection_view.rs
+8 -7
View File
@@ -54,6 +54,7 @@ pub(crate) struct SelectionViewParams {
pub items: Vec<SelectionItem>,
pub is_searchable: bool,
pub search_placeholder: Option<String>,
pub show_numbers: bool,
pub header: Box<dyn Renderable>,
pub initial_selected_idx: Option<usize>,
}
@@ -67,6 +68,7 @@ impl Default for SelectionViewParams {
items: Vec::new(),
is_searchable: false,
search_placeholder: None,
show_numbers: true,
header: Box::new(()),
initial_selected_idx: None,
}
@@ -80,6 +82,7 @@ pub(crate) struct ListSelectionView {
complete: bool,
app_event_tx: AppEventSender,
is_searchable: bool,
show_numbers: bool,
search_query: String,
search_placeholder: Option<String>,
filtered_indices: Vec<usize>,
@@ -107,6 +110,7 @@ impl ListSelectionView {
complete: false,
app_event_tx,
is_searchable: params.is_searchable,
show_numbers: params.show_numbers,
search_query: String::new(),
search_placeholder: if params.is_searchable {
params.search_placeholder
@@ -198,13 +202,9 @@ impl ListSelectionView {
};
let name_with_marker = format!("{name}{marker}");
let n = visible_idx + 1;
let wrap_prefix = if self.is_searchable {
// The number keys don't work when search is enabled (since we let the
// numbers be used for the search query).
format!("{prefix} ")
} else {
format!("{prefix} {n}. ")
};
let wrap_prefix = (self.show_numbers && !self.is_searchable)
.then(|| format!("{prefix} {n}. "))
.unwrap_or_else(|| format!("{prefix} "));
let wrap_prefix_width = UnicodeWidthStr::width(wrap_prefix.as_str());
let display_name = format!("{wrap_prefix}{name_with_marker}");
let description = is_selected
@@ -386,6 +386,7 @@ impl BottomPaneView for ListSelectionView {
.to_digit(10)
.map(|d| d as usize)
.and_then(|d| d.checked_sub(1))
.filter(|_| self.show_numbers)
&& idx < self.items.len()
&& self
.items
codex-rs/tui/src/bottom_pane/mod.rs
+6
View File
@@ -26,6 +26,7 @@ mod chat_composer;
mod chat_composer_history;
mod command_popup;
pub mod custom_prompt_view;
mod eval_capture_view;
mod experimental_features_view;
mod file_search_popup;
mod footer;
@@ -34,6 +35,11 @@ mod prompt_args;
mod skill_popup;
pub(crate) use list_selection_view::SelectionViewParams;
mod feedback_view;
pub(crate) use eval_capture_view::EvalCaptureNotesView;
pub(crate) use eval_capture_view::eval_capture_intro_params;
pub(crate) use eval_capture_view::eval_capture_start_picker_params;
pub(crate) use eval_capture_view::eval_capture_upload_consent_params;
pub(crate) use feedback_view::feedback_bad_result_fork_params;
pub(crate) use feedback_view::feedback_selection_params;
pub(crate) use feedback_view::feedback_upload_consent_params;
mod paste_burst;
codex-rs/tui/src/bottom_pane/selection_popup_common.rs
+4 -1
View File
@@ -110,7 +110,10 @@ fn compute_desc_col(
let max_name_width = rows_all
.iter()
.enumerate()
.filter(|(i, _)| visible_range.contains(i))
.filter(|(i, row)| {
visible_range.contains(i)
&& (row.description.is_some() || row.disabled_reason.is_some())
})
.map(|(_, r)| {
let mut spans: Vec<Span> = vec![r.name.clone().into()];
if r.disabled_reason.is_some() {
codex-rs/tui/src/chatwidget.rs
+518
View File
@@ -132,7 +132,9 @@ use self::session_header::SessionHeader;
use crate::streaming::controller::StreamController;
use std::path::Path;
use chrono::DateTime;
use chrono::Local;
use chrono::Utc;
use codex_common::approval_presets::ApprovalPreset;
use codex_common::approval_presets::builtin_approval_presets;
use codex_core::AuthManager;
@@ -438,6 +440,9 @@ impl ChatWidget {
self.set_skills(None);
self.conversation_id = Some(event.session_id);
self.current_rollout_path = Some(event.rollout_path.clone());
// Repo snapshots are stored locally and are required for capturing eval cases
// from arbitrary earlier user messages.
self.submit_op(Op::SetRepoSnapshotting { enabled: true });
let initial_messages = event.initial_messages.clone();
let model_for_header = event.model.clone();
self.session_header.set_model(&model_for_header);
@@ -506,6 +511,465 @@ impl ChatWidget {
self.request_redraw();
}
pub(crate) fn open_feedback_bad_result_fork(&mut self) {
let params = crate::bottom_pane::feedback_bad_result_fork_params(self.app_event_tx.clone());
self.bottom_pane.show_selection_view(params);
self.request_redraw();
}
pub(crate) fn open_eval_capture_intro(&mut self) {
if !crate::eval_capture_state::should_show_eval_capture_intro(&self.config.codex_home) {
self.open_eval_capture_start_picker();
return;
}
let params = crate::bottom_pane::eval_capture_intro_params(self.app_event_tx.clone());
self.bottom_pane.show_selection_view(params);
self.request_redraw();
}
pub(crate) fn on_eval_capture_intro_continue(&mut self) {
if let Err(err) =
crate::eval_capture_state::persist_eval_capture_intro_dismissed(&self.config.codex_home)
{
self.add_error_message(format!("Failed to persist eval capture preference: {err}"));
}
self.open_eval_capture_start_picker();
}
fn open_eval_capture_start_picker(&mut self) {
let Some(rollout_path) = self.current_rollout_path.as_ref() else {
self.add_error_message(
"No rollout file for this session; cannot capture eval sample.".to_string(),
);
return;
};
let text = match std::fs::read_to_string(rollout_path) {
Ok(text) => text,
Err(err) => {
self.add_error_message(format!(
"Failed to read rollout file {}: {err}",
rollout_path.display()
));
return;
}
};
// List every user message as a candidate start point, and attach the next
// ghost snapshot to the most recent user message. Markers without snapshots
// will be disabled in the picker.
let mut options = Vec::new();
let mut last_user_marker_idx: Option<usize> = None;
let mut max_timestamp: Option<DateTime<Utc>> = None;
for (idx, line) in text.lines().enumerate() {
let Ok(rollout_line) =
serde_json::from_str::<codex_protocol::protocol::RolloutLine>(line)
else {
continue;
};
if let Some(ts) = parse_rollout_timestamp_utc(&rollout_line.timestamp) {
max_timestamp = Some(max_timestamp.map_or(ts, |cur| cur.max(ts)));
}
match rollout_line.item {
codex_protocol::protocol::RolloutItem::ResponseItem(
codex_protocol::models::ResponseItem::Message { role, content, .. },
) if role == "user" => {
let Some(message) = extract_user_message_text(&content) else {
continue;
};
options.push(crate::app_event::EvalCaptureStartMarker::RolloutLineIndex {
index: idx,
timestamp: rollout_line.timestamp.clone(),
// Filled in below once we know the latest timestamp in the rollout.
display: String::new(),
message,
repo_snapshot: None,
});
last_user_marker_idx = Some(options.len().saturating_sub(1));
}
codex_protocol::protocol::RolloutItem::ResponseItem(
codex_protocol::models::ResponseItem::GhostSnapshot { ghost_commit },
) => {
if let Some(marker_idx) = last_user_marker_idx
&& let Some(base_sha) = ghost_commit.parent()
{
if let Some(crate::app_event::EvalCaptureStartMarker::RolloutLineIndex {
repo_snapshot,
..
}) = options.get_mut(marker_idx)
{
*repo_snapshot = Some(crate::app_event::EvalCaptureRepoSnapshot {
ghost_commit: ghost_commit.id().to_string(),
base_sha: Some(base_sha.to_string()),
});
}
}
}
_ => {}
}
}
if let Some(now) = max_timestamp {
for marker in &mut options {
match marker {
crate::app_event::EvalCaptureStartMarker::RolloutLineIndex {
timestamp,
display,
message,
..
}
| crate::app_event::EvalCaptureStartMarker::RolloutLineTimestamp {
timestamp,
display,
message,
..
} => {
let Some(then) = parse_rollout_timestamp_utc(timestamp) else {
*display = message.clone();
continue;
};
let delta = format_relative_time_ago(now, then);
*display = format!("[{delta}] {message}");
}
}
}
} else {
for marker in &mut options {
match marker {
crate::app_event::EvalCaptureStartMarker::RolloutLineIndex {
display,
message,
..
}
| crate::app_event::EvalCaptureStartMarker::RolloutLineTimestamp {
display,
message,
..
} => {
*display = message.clone();
}
}
}
}
options.sort_by(|a, b| match (a, b) {
(
crate::app_event::EvalCaptureStartMarker::RolloutLineIndex { index: a, .. },
crate::app_event::EvalCaptureStartMarker::RolloutLineIndex { index: b, .. },
) => b.cmp(a),
_ => std::cmp::Ordering::Equal,
});
if !options.iter().any(|marker| match marker {
crate::app_event::EvalCaptureStartMarker::RolloutLineIndex {
repo_snapshot, ..
}
| crate::app_event::EvalCaptureStartMarker::RolloutLineTimestamp {
repo_snapshot,
..
} => repo_snapshot.is_some(),
}) {
self.add_error_message(
"No repo snapshots are available yet in this session. Try again after another message/turn (or in a new session)."
.to_string(),
);
self.open_feedback_consent(crate::app_event::FeedbackCategory::BadResult);
return;
}
let initial_selected_idx = options.iter().position(|marker| match marker {
crate::app_event::EvalCaptureStartMarker::RolloutLineIndex {
repo_snapshot, ..
}
| crate::app_event::EvalCaptureStartMarker::RolloutLineTimestamp {
repo_snapshot,
..
} => repo_snapshot.is_some(),
});
let params = crate::bottom_pane::eval_capture_start_picker_params(
self.app_event_tx.clone(),
options,
initial_selected_idx,
);
self.bottom_pane.show_selection_view(params);
self.request_redraw();
}
pub(crate) fn open_eval_capture_notes(
&mut self,
start_marker: crate::app_event::EvalCaptureStartMarker,
) {
let view =
crate::bottom_pane::EvalCaptureNotesView::new(start_marker, self.app_event_tx.clone());
self.bottom_pane.show_view(Box::new(view));
self.request_redraw();
}
pub(crate) fn create_eval_capture_bundle(
&mut self,
start_marker: crate::app_event::EvalCaptureStartMarker,
what_went_wrong: String,
what_good_looks_like: String,
) {
let Some(conversation_id) = self.conversation_id else {
self.add_error_message(
"No active conversation; cannot capture eval sample.".to_string(),
);
return;
};
let Some(rollout_path) = self.current_rollout_path.clone() else {
self.add_error_message(
"No rollout file for this session; cannot capture eval sample.".to_string(),
);
return;
};
let (start, repo_snapshot) = match start_marker {
crate::app_event::EvalCaptureStartMarker::RolloutLineIndex {
index,
timestamp: _,
display,
message: _,
repo_snapshot,
} => {
let Some(snapshot) = repo_snapshot else {
self.add_error_message(
"No repo snapshot available for the selected start marker; cannot capture eval sample."
.to_string(),
);
return;
};
let Some(base_sha) = snapshot.base_sha else {
self.add_error_message(
"No base sha for selected repo snapshot; cannot capture eval sample."
.to_string(),
);
return;
};
let repo_snapshot = Some(codex_eval_case::RepoSnapshot {
base_sha,
commit_sha: snapshot.ghost_commit,
});
(
codex_eval_case::StartMarker {
kind: codex_eval_case::StartMarkerKind::RolloutLineIndex,
value: codex_eval_case::StartMarkerValue::LineIndex(index as u64),
display,
},
repo_snapshot,
)
}
crate::app_event::EvalCaptureStartMarker::RolloutLineTimestamp {
timestamp,
display,
message: _,
repo_snapshot,
} => {
let Some(snapshot) = repo_snapshot else {
self.add_error_message(
"No repo snapshot available for the selected start marker; cannot capture eval sample."
.to_string(),
);
return;
};
let Some(base_sha) = snapshot.base_sha else {
self.add_error_message(
"No base sha for selected repo snapshot; cannot capture eval sample."
.to_string(),
);
return;
};
let repo_snapshot = Some(codex_eval_case::RepoSnapshot {
base_sha,
commit_sha: snapshot.ghost_commit,
});
(
codex_eval_case::StartMarker {
kind: codex_eval_case::StartMarkerKind::RolloutLineTimestamp,
value: codex_eval_case::StartMarkerValue::Timestamp(timestamp),
display,
},
repo_snapshot,
)
}
};
let logs_bytes = Some(
self.feedback
.snapshot(Some(conversation_id))
.as_bytes()
.to_vec(),
);
let args = codex_eval_case::CreateEvalCaseArgs {
codex_home: self.config.codex_home.clone(),
conversation_id: conversation_id.to_string(),
rollout_path,
start,
repo_cwd: self.config.cwd.clone(),
repo_snapshot,
notes: codex_eval_case::Notes {
what_went_wrong,
what_good_looks_like,
},
include_logs: true,
logs_bytes,
};
match codex_eval_case::create_eval_case_bundle(&args) {
Ok(result) => {
use ratatui::style::Stylize as _;
use ratatui::text::Line;
let case_id = result.case_id;
let path = result.path.display().to_string();
let lines = vec![
Line::from("• Eval sample captured (local only).".to_string()),
"".into(),
Line::from(vec![" Case ID: ".into(), case_id.clone().bold()]),
Line::from(vec![" Path: ".into(), path.clone().cyan().underlined()]),
];
self.app_event_tx
.send(crate::app_event::AppEvent::InsertHistoryCell(Box::new(
crate::history_cell::PlainHistoryCell::new(lines),
)));
let params = crate::bottom_pane::eval_capture_upload_consent_params(
self.app_event_tx.clone(),
case_id,
path,
);
self.bottom_pane.show_selection_view(params);
self.request_redraw();
}
Err(err) => {
self.app_event_tx
.send(crate::app_event::AppEvent::InsertHistoryCell(Box::new(
crate::history_cell::new_error_event(format!(
"Failed to create eval sample bundle: {err}"
)),
)));
}
}
}
pub(crate) fn upload_eval_capture_bundle(&mut self, case_id: String, path: String) {
let Some(conversation_id) = self.conversation_id else {
self.add_error_message(
"No active conversation; cannot upload eval sample.".to_string(),
);
return;
};
let bundle_dir = std::path::PathBuf::from(&path);
let manifest_path = bundle_dir.join("manifest.json");
let rollout_path = bundle_dir.join("rollout.jsonl");
let repo_patch_path = bundle_dir.join("repo.patch");
let logs_path = bundle_dir.join("codex-logs.log");
let read_bytes = |p: &std::path::Path| std::fs::read(p).map_err(anyhow::Error::from);
let logs_bytes = match read_bytes(&logs_path) {
Ok(bytes) => bytes,
Err(err) => {
self.add_error_message(format!(
"Failed to read eval sample logs {}: {err}",
logs_path.display()
));
return;
}
};
let snapshot = self.feedback.snapshot(Some(conversation_id));
let snapshot =
codex_feedback::CodexLogSnapshot::from_bytes(snapshot.thread_id.clone(), logs_bytes);
let extra_attachments = match (read_bytes(&manifest_path), read_bytes(&repo_patch_path)) {
(Ok(manifest_bytes), Ok(repo_patch_bytes)) => vec![
codex_feedback::FeedbackAttachment::new(
"manifest.json".to_string(),
Some("application/json".to_string()),
manifest_bytes,
),
codex_feedback::FeedbackAttachment::new(
"repo.patch".to_string(),
Some("text/plain".to_string()),
repo_patch_bytes,
),
],
(manifest_err, repo_err) => {
if let Err(err) = manifest_err {
self.add_error_message(format!(
"Failed to read eval sample manifest {}: {err}",
manifest_path.display()
));
}
if let Err(err) = repo_err {
self.add_error_message(format!(
"Failed to read eval sample repo.patch {}: {err}",
repo_patch_path.display()
));
}
return;
}
};
let reason = format!("Eval sample {case_id}");
let result = snapshot.upload_feedback_with_attachments(
"bad_result",
Some(reason.as_str()),
true,
Some(rollout_path.as_path()),
Some(codex_core::protocol::SessionSource::Cli),
extra_attachments,
);
match result {
Ok(()) => {
use ratatui::style::Stylize as _;
use ratatui::text::Line;
let lines = vec![
Line::from("• Eval sample uploaded.".to_string()),
"".into(),
Line::from(vec![
" Thread ID: ".into(),
snapshot.thread_id.clone().bold(),
]),
];
self.app_event_tx
.send(crate::app_event::AppEvent::InsertHistoryCell(Box::new(
crate::history_cell::PlainHistoryCell::new(lines),
)));
}
Err(err) => {
self.app_event_tx
.send(crate::app_event::AppEvent::InsertHistoryCell(Box::new(
crate::history_cell::new_error_event(format!(
"Failed to upload eval sample: {err}"
)),
)));
}
}
self.request_redraw();
}
pub(crate) fn eval_capture_upload_skipped(&mut self, case_id: String, path: String) {
let _ = (case_id, path);
use ratatui::text::Line;
// The "captured" message already prints the case id/path. Avoid duplicating it here.
let lines = vec![Line::from(
"• Eval sample saved locally (not uploaded).".to_string(),
)];
self.app_event_tx
.send(crate::app_event::AppEvent::InsertHistoryCell(Box::new(
crate::history_cell::PlainHistoryCell::new(lines),
)));
self.request_redraw();
}
fn on_agent_message(&mut self, message: String) {
// If we have a stream_controller, then the final agent message is redundant and will be a
// duplicate of what has already been streamed.
@@ -3695,6 +4159,60 @@ async fn fetch_rate_limits(base_url: String, auth: CodexAuth) -> Option<RateLimi
}
}
fn extract_user_message_text(content: &[codex_protocol::models::ContentItem]) -> Option<String> {
let mut combined = String::new();
for item in content {
let text = match item {
codex_protocol::models::ContentItem::InputText { text }
| codex_protocol::models::ContentItem::OutputText { text } => text.as_str(),
codex_protocol::models::ContentItem::InputImage { .. } => continue,
};
if !combined.is_empty() {
combined.push(' ');
}
combined.push_str(text);
}
let mut normalized = String::new();
for part in combined.split_whitespace() {
if !normalized.is_empty() {
normalized.push(' ');
}
normalized.push_str(part);
}
let trimmed = normalized.trim();
if trimmed.is_empty() {
None
} else {
// Keep the picker readable even for very long user messages.
Some(truncate_text(trimmed, 800))
}
}
fn parse_rollout_timestamp_utc(timestamp: &str) -> Option<DateTime<Utc>> {
DateTime::parse_from_rfc3339(timestamp)
.ok()
.map(|dt| dt.with_timezone(&Utc))
}
fn format_relative_time_ago(now: DateTime<Utc>, then: DateTime<Utc>) -> String {
let secs = now.signed_duration_since(then).num_seconds().max(0);
if secs < 60 {
return format!("{secs}s ago");
}
let mins = secs / 60;
if mins < 60 {
return format!("{mins}m ago");
}
let hours = mins / 60;
if hours < 24 {
return format!("{hours}h ago");
}
let days = hours / 24;
format!("{days}d ago")
}
#[cfg(test)]
pub(crate) fn show_review_commit_picker_with_entries(
chat: &mut ChatWidget,
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__eval_capture_intro_popup.snap
+21
View File
@@ -0,0 +1,21 @@
---
source: tui/src/chatwidget/tests.rs
assertion_line: 2190
expression: popup
---
Capture eval sample
Here's everything we capture as an eval sample.
It's stored locally on your computer:
• manifest.json
• rollout.jsonl
• repo.patch
• codex-logs.log
Next, you can optionally upload this bundle to the team.
› 1. Continue Create an eval sample bundle locally.
2. Send basic feedback instead Skip eval capture and send basic feedback to
the team.
Press enter to confirm or esc to go back
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__eval_capture_start_picker_popup.snap
+11
View File
@@ -0,0 +1,11 @@
---
source: tui/src/chatwidget/tests.rs
assertion_line: 2253
expression: popup
---
Pick the start message
Default is your last message; scroll for earlier messages.
› [0s ago] Please help me fix the build errors
Press enter to confirm or esc to go back
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__eval_capture_upload_consent_popup.snap
+21
View File
@@ -0,0 +1,21 @@
---
source: tui/src/chatwidget/tests.rs
assertion_line: 2206
expression: popup
---
Upload eval sample?
If you choose Yes, it will upload the full bundle
to the team:
• manifest.json
• rollout.jsonl
• repo.patch
• codex-logs.log
This may include file paths, code snippets, and tool outputs.
› 1. Yes Upload the full bundle to the team for troubleshooting and model
improvement.
2. No Keep it local only.
Press enter to confirm or esc to go back
codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__feedback_bad_result_fork_popup.snap
+11
View File
@@ -0,0 +1,11 @@
---
source: tui/src/chatwidget/tests.rs
expression: popup
---
Bad result
Choose the fastest way to help us improve.
› 1. Quick feedback Send a short note (and optionally logs) to
maintainers.
2. Capture eval sample Create a local bundle you can rerun as an eval case
later.
codex-rs/tui/src/chatwidget/tests.rs
+86
View File
@@ -2167,6 +2167,92 @@ async fn feedback_upload_consent_popup_snapshot() {
assert_snapshot!("feedback_upload_consent_popup", popup);
}
#[tokio::test]
async fn feedback_bad_result_fork_popup_snapshot() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await;
chat.open_feedback_bad_result_fork();
let popup = render_bottom_popup(&chat, 80);
assert_snapshot!("feedback_bad_result_fork_popup", popup);
}
#[tokio::test]
async fn eval_capture_intro_popup_snapshot() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await;
// Ensure host state doesn't leak into snapshots.
let _ = std::fs::remove_file(chat.config.codex_home.join("eval_capture.json"));
chat.open_eval_capture_intro();
let popup = render_bottom_popup(&chat, 80);
assert_snapshot!("eval_capture_intro_popup", popup);
}
#[tokio::test]
async fn eval_capture_upload_consent_popup_snapshot() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await;
let params = crate::bottom_pane::eval_capture_upload_consent_params(
chat.app_event_tx.clone(),
"case-2026-01-01T00-00-00-repo-000001".to_string(),
"/tmp/eval-case/case-2026-01-01T00-00-00-repo-000001".to_string(),
);
chat.bottom_pane.show_selection_view(params);
chat.request_redraw();
let popup = render_bottom_popup(&chat, 80);
assert_snapshot!("eval_capture_upload_consent_popup", popup);
}
#[tokio::test]
async fn eval_capture_start_picker_popup_snapshot() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await;
let rollout_file = NamedTempFile::new().unwrap();
let user_line = serde_json::json!({
"timestamp": "2025-09-05T16:53:11.850Z",
"type": "response_item",
"payload": {
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Please help me fix the build errors",
}],
},
});
let ghost_line = serde_json::json!({
"timestamp": "2025-09-05T16:53:12.000Z",
"type": "response_item",
"payload": {
"type": "ghost_snapshot",
"ghost_commit": {
"id": "ghost-sha",
"parent": "base-sha",
"preexisting_untracked_files": [],
"preexisting_untracked_dirs": [],
},
},
});
std::fs::write(
rollout_file.path(),
format!(
"{}\n{}\n",
serde_json::to_string(&user_line).unwrap(),
serde_json::to_string(&ghost_line).unwrap()
),
)
.unwrap();
chat.current_rollout_path = Some(rollout_file.path().to_path_buf());
chat.open_eval_capture_start_picker();
let popup = render_bottom_popup(&chat, 80);
assert_snapshot!("eval_capture_start_picker_popup", popup);
}
#[tokio::test]
async fn reasoning_popup_escape_returns_to_model_popup() {
let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.1-codex-max")).await;
codex-rs/tui/src/eval_capture_state.rs
+40
View File
@@ -0,0 +1,40 @@
use serde::Deserialize;
use serde::Serialize;
use std::path::Path;
use std::path::PathBuf;
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct EvalCaptureState {
#[serde(default)]
intro_dismissed: bool,
}
const STATE_FILENAME: &str = "eval_capture.json";
fn state_path(codex_home: &Path) -> PathBuf {
codex_home.join(STATE_FILENAME)
}
pub(crate) fn should_show_eval_capture_intro(codex_home: &Path) -> bool {
let path = state_path(codex_home);
let Ok(contents) = std::fs::read_to_string(path) else {
return true;
};
serde_json::from_str::<EvalCaptureState>(&contents)
.ok()
.map(|s| !s.intro_dismissed)
.unwrap_or(true)
}
pub(crate) fn persist_eval_capture_intro_dismissed(codex_home: &Path) -> anyhow::Result<()> {
let path = state_path(codex_home);
let state = EvalCaptureState {
intro_dismissed: true,
};
let json_line = format!("{}\n", serde_json::to_string(&state)?);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::write(path, json_line)?;
Ok(())
}
codex-rs/tui/src/lib.rs
+1
View File
@@ -45,6 +45,7 @@ mod clipboard_paste;
mod color;
pub mod custom_terminal;
mod diff_render;
mod eval_capture_state;
mod exec_cell;
mod exec_command;
mod external_editor;