From 798e0a23114be48ea1ad77585e6c1dee4d3773a7 Mon Sep 17 00:00:00 2001 From: Felipe Coury Date: Mon, 6 Jul 2026 22:32:13 -0300 Subject: [PATCH] feat(tui): activate browser control on click --- codex-rs/terminal-browser/src/actions.rs | 41 ++++ .../terminal-browser/src/actions_tests.rs | 43 +++- .../terminal-browser/src/human_control.rs | 230 +++++++++++++++--- .../src/human_control_tests.rs | 119 +++++++++ codex-rs/terminal-browser/src/session.rs | 4 +- codex-rs/tui/src/app.rs | 26 +- codex-rs/tui/src/app/owned_screen_frame.rs | 46 +++- codex-rs/tui/src/app/owned_screen_tests.rs | 88 +++++++ ...owser_preserves_narrow_overlay_chrome.snap | 2 +- ...en_browser_renders_in_wide_right_rail.snap | 2 +- codex-rs/tui/src/app/terminal_browser.rs | 112 ++++++++- codex-rs/tui/src/terminal_browser/input.rs | 22 ++ codex-rs/tui/src/terminal_browser/mod.rs | 2 + codex-rs/tui/src/terminal_browser/panel.rs | 18 +- ...ests__terminal_browser_panel_snapshot.snap | 2 +- ...available_panel_wraps_reason_snapshot.snap | 2 +- codex-rs/tui/src/terminal_browser/tests.rs | 30 +++ justfile | 2 +- 18 files changed, 738 insertions(+), 53 deletions(-) create mode 100644 codex-rs/terminal-browser/src/human_control_tests.rs diff --git a/codex-rs/terminal-browser/src/actions.rs b/codex-rs/terminal-browser/src/actions.rs index 2eb3309070..7f2544c964 100644 --- a/codex-rs/terminal-browser/src/actions.rs +++ b/codex-rs/terminal-browser/src/actions.rs @@ -5,6 +5,7 @@ use serde_json::json; use crate::cdp::CdpClient; use crate::input::BrowserKeyInput; +use crate::input::BrowserMouseButton; use crate::input::BrowserMouseInput; use crate::input::BrowserMouseKind; use crate::scripts; @@ -18,6 +19,7 @@ const MAX_SNAPSHOT_TEXT_CHARS: usize = 6_000; #[derive(Default)] pub(crate) struct HumanMouseDispatchState { viewport: Option<(u16, u16, f64, f64)>, + last_position: Option<(f64, f64)>, buttons: u8, } @@ -136,6 +138,44 @@ pub(crate) async fn dispatch_human_key(client: &CdpClient, input: &BrowserKeyInp Ok(()) } +pub(crate) async fn release_human_mouse_buttons( + client: &CdpClient, + state: &mut HumanMouseDispatchState, +) -> Result<()> { + let Some((x, y)) = state.last_position else { + state.buttons = 0; + return Ok(()); + }; + for button in [ + BrowserMouseButton::Left, + BrowserMouseButton::Middle, + BrowserMouseButton::Right, + ] { + let button_mask = button.cdp_buttons_mask(); + if state.buttons & button_mask == 0 { + continue; + } + state.buttons &= !button_mask; + client + .call( + "Input.dispatchMouseEvent", + json!({ + "type": "mouseReleased", + "x": x, + "y": y, + "button": button.as_cdp(), + "buttons": state.buttons, + "modifiers": 0, + "clickCount": 0, + "deltaX": 0.0, + "deltaY": 0.0, + }), + ) + .await?; + } + Ok(()) +} + pub(crate) async fn insert_human_text(client: &CdpClient, text: &str) -> Result<()> { client .call("Input.insertText", json!({ "text": text })) @@ -202,6 +242,7 @@ pub(crate) async fn dispatch_human_mouse( }), ) .await?; + state.last_position = Some((x, y)); Ok(()) } diff --git a/codex-rs/terminal-browser/src/actions_tests.rs b/codex-rs/terminal-browser/src/actions_tests.rs index c50ca376a3..3c1cd17bf4 100644 --- a/codex-rs/terminal-browser/src/actions_tests.rs +++ b/codex-rs/terminal-browser/src/actions_tests.rs @@ -11,6 +11,7 @@ use tokio_tungstenite::tungstenite::Message; use super::BrowserToolOutput; use super::HumanMouseDispatchState; use super::dispatch_human_mouse; +use super::release_human_mouse_buttons; use crate::accessibility::click; use crate::accessibility::snapshot; use crate::cdp::CdpClient; @@ -186,7 +187,7 @@ async fn native_click_uses_the_accessibility_handle_box_center() { } #[tokio::test] -async fn human_mouse_caches_metrics_and_preserves_drag_buttons() { +async fn human_mouse_preserves_drag_buttons_and_cleans_up_held_buttons() { let (url, server) = test_server(|mut socket| { tokio::spawn(async move { initialize(&mut socket).await; @@ -207,6 +208,22 @@ async fn human_mouse_caches_metrics_and_preserves_drag_buttons() { assert_eq!(input["params"]["buttons"], expected_buttons); respond(&mut socket, &input, json!({})).await; } + for expected_buttons in [1, 3] { + let input = request(&mut socket).await; + assert_eq!(input["params"]["type"], "mousePressed"); + assert_eq!(input["params"]["buttons"], expected_buttons); + respond(&mut socket, &input, json!({})).await; + } + for (expected_button, expected_buttons) in [("left", 2), ("right", 0)] { + let input = request(&mut socket).await; + assert_eq!(input["params"]["type"], "mouseReleased"); + assert_eq!(input["params"]["button"], expected_button); + assert_eq!(input["params"]["buttons"], expected_buttons); + assert_eq!(input["params"]["x"], 200.0); + assert_eq!(input["params"]["y"], 150.0); + assert_eq!(input["params"]["clickCount"], 0); + respond(&mut socket, &input, json!({})).await; + } }) }) .await; @@ -233,6 +250,30 @@ async fn human_mouse_caches_metrics_and_preserves_drag_buttons() { .await .expect("dispatch mouse input"); } + for button in [BrowserMouseButton::Left, BrowserMouseButton::Right] { + dispatch_human_mouse( + &client, + BrowserMouseInput { + kind: BrowserMouseKind::Down, + button, + column: 20, + row: 6, + viewport_cols: 80, + viewport_rows: 24, + modifiers: BrowserInputModifiers::default(), + }, + &mut state, + ) + .await + .expect("dispatch mouse press"); + } + + release_human_mouse_buttons(&client, &mut state) + .await + .expect("release held mouse buttons"); + release_human_mouse_buttons(&client, &mut state) + .await + .expect("repeated cleanup is empty"); server.await.expect("server task"); } diff --git a/codex-rs/terminal-browser/src/human_control.rs b/codex-rs/terminal-browser/src/human_control.rs index 1959267a05..c6d45652dd 100644 --- a/codex-rs/terminal-browser/src/human_control.rs +++ b/codex-rs/terminal-browser/src/human_control.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; use std::sync::Arc; use std::sync::Weak; use std::sync::atomic::Ordering; @@ -5,6 +6,7 @@ use std::sync::atomic::Ordering; use anyhow::Result; use anyhow::anyhow; use tokio::sync::mpsc; +use tokio::sync::oneshot; use crate::actions; use crate::actions::HumanMouseDispatchState; @@ -15,9 +17,18 @@ use crate::session::Inner; use crate::session::TerminalBrowser; const HUMAN_INPUT_CAPACITY: usize = 128; +const HUMAN_CONTROL_INPUT_CAPACITY: usize = 1; const MAX_HUMAN_TEXT_BYTES: usize = 64 * 1024; -pub(crate) struct HumanInputSender(mpsc::Sender); +pub(crate) struct HumanInputSender { + input: mpsc::Sender, + control: mpsc::Sender, +} + +pub(crate) struct HumanInputReceivers { + input: mpsc::Receiver, + control: mpsc::Receiver, +} /// Captures the browser-control epoch at the time a UI transition is requested. /// @@ -38,9 +49,27 @@ enum HumanInput { Key(BrowserKeyInput), Text(String), Mouse(BrowserMouseInput), + ReleaseMouseButtons { + completion_tx: oneshot::Sender>, + after_release: HumanControlAfterRelease, + }, } -pub(crate) struct QueuedHumanInput { +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum HumanControlAfterRelease { + Continue, + End, +} + +impl HumanInput { + fn complete_release(self, result: std::result::Result<(), String>) { + if let Self::ReleaseMouseButtons { completion_tx, .. } = self { + let _ = completion_tx.send(result); + } + } +} + +struct QueuedHumanInput { generation: u64, input: HumanInput, } @@ -89,9 +118,19 @@ impl Inner { } impl TerminalBrowser { - pub(crate) fn human_input_channel() -> (HumanInputSender, mpsc::Receiver) { - let (sender, receiver) = mpsc::channel(HUMAN_INPUT_CAPACITY); - (HumanInputSender(sender), receiver) + pub(crate) fn human_input_channel() -> (HumanInputSender, HumanInputReceivers) { + let (input_tx, input_rx) = mpsc::channel(HUMAN_INPUT_CAPACITY); + let (control_tx, control_rx) = mpsc::channel(HUMAN_CONTROL_INPUT_CAPACITY); + ( + HumanInputSender { + input: input_tx, + control: control_tx, + }, + HumanInputReceivers { + input: input_rx, + control: control_rx, + }, + ) } pub fn is_human_control_active(&self) -> bool { @@ -207,12 +246,14 @@ impl TerminalBrowser { )] pub async fn end_human_control(&self, token: HumanControlToken) -> Result { anyhow::ensure!( - self.inner - .transition_human_control(HumanControlStateTransition::Deactivate { - generation: token.generation, - }), + self.human_control_is_active_for(token.generation), "browser control transition was canceled" ); + self.release_human_mouse_buttons_for_generation( + token.generation, + HumanControlAfterRelease::End, + ) + .await?; let _operation = self.inner.operation.lock().await; self.flush_human_handle_invalidation().await; Ok(HumanControlToken { @@ -259,13 +300,52 @@ impl TerminalBrowser { self.queue_human_input(HumanInput::Mouse(input)) } + /// Releases every mouse button currently held by the browser input worker. + pub async fn release_human_mouse_buttons(&self) -> Result<()> { + let generation = self.inner.human_control_generation.load(Ordering::SeqCst); + self.release_human_mouse_buttons_for_generation( + generation, + HumanControlAfterRelease::Continue, + ) + .await + } + + async fn release_human_mouse_buttons_for_generation( + &self, + generation: u64, + after_release: HumanControlAfterRelease, + ) -> Result<()> { + anyhow::ensure!( + self.human_control_is_active_for(generation), + "browser control transition was canceled" + ); + let (completion_tx, completion_rx) = oneshot::channel(); + self.inner + .human_input_tx + .control + .send(QueuedHumanInput { + generation, + input: HumanInput::ReleaseMouseButtons { + completion_tx, + after_release, + }, + }) + .await + .map_err(|_| anyhow!("terminal browser input worker stopped"))?; + match completion_rx.await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(anyhow!(error)), + Err(_) => Err(anyhow!("terminal browser input worker stopped")), + } + } + fn queue_human_input(&self, input: HumanInput) -> Result<()> { anyhow::ensure!( self.is_human_control_active(), "human control is not active" ); let is_motion = matches!( - input, + &input, HumanInput::Mouse(BrowserMouseInput { kind: BrowserMouseKind::Move, .. @@ -275,7 +355,7 @@ impl TerminalBrowser { generation: self.inner.human_control_generation.load(Ordering::SeqCst), input, }; - match self.inner.human_input_tx.0.try_send(queued) { + match self.inner.human_input_tx.input.try_send(queued) { Ok(()) => Ok(()), Err(mpsc::error::TrySendError::Full(_)) if is_motion => Ok(()), Err(mpsc::error::TrySendError::Full(_)) => { @@ -286,6 +366,11 @@ impl TerminalBrowser { } } } + + fn human_control_is_active_for(&self, generation: u64) -> bool { + self.is_human_control_active() + && self.inner.human_control_generation.load(Ordering::SeqCst) == generation + } } struct HumanControlTransition<'a> { @@ -302,50 +387,135 @@ impl Drop for HumanControlTransition<'_> { clippy::await_holding_invalid_type, reason = "human input must remain serialized with model actions until its CDP call completes" )] -async fn run_human_input_worker( - inner: Weak, - mut receiver: mpsc::Receiver, -) { +async fn run_human_input_worker(inner: Weak, mut receivers: HumanInputReceivers) { let mut mouse_state = HumanMouseDispatchState::default(); let mut mouse_generation = None; - while let Some(queued) = receiver.recv().await { + let mut deferred = VecDeque::new(); + while let Some(queued) = next_human_input(&mut receivers, &mut deferred).await { let Some(inner) = inner.upgrade() else { return; }; + let QueuedHumanInput { generation, input } = queued; if !inner.human_control.load(Ordering::SeqCst) - || inner.human_control_generation.load(Ordering::SeqCst) != queued.generation + || inner.human_control_generation.load(Ordering::SeqCst) != generation { + input.complete_release(Err("browser control transition was canceled".to_string())); continue; } - if mouse_generation != Some(queued.generation) { - mouse_state = HumanMouseDispatchState::default(); - mouse_generation = Some(queued.generation); + if matches!(&input, HumanInput::ReleaseMouseButtons { .. }) { + drain_human_inputs_for_generation(&mut receivers, generation, &mut deferred); } - let result = { + if mouse_generation != Some(generation) { + mouse_state = HumanMouseDispatchState::default(); + mouse_generation = Some(generation); + } + let (result, completion_tx, crash_on_error) = { let _operation = inner.operation.lock().await; if !inner.human_control.load(Ordering::SeqCst) - || inner.human_control_generation.load(Ordering::SeqCst) != queued.generation + || inner.human_control_generation.load(Ordering::SeqCst) != generation { + input.complete_release(Err("browser control transition was canceled".to_string())); continue; } let cdp = { let session = inner.session.lock().await; let Some(session) = session.as_ref() else { + input.complete_release(Err("terminal browser is not open".to_string())); continue; }; session.cdp.clone() }; - match queued.input { - HumanInput::Key(input) => actions::dispatch_human_key(&cdp, &input).await, - HumanInput::Text(text) => actions::insert_human_text(&cdp, &text).await, - HumanInput::Mouse(input) => { - actions::dispatch_human_mouse(&cdp, input, &mut mouse_state).await + match input { + HumanInput::Key(input) => { + (actions::dispatch_human_key(&cdp, &input).await, None, true) + } + HumanInput::Text(text) => { + (actions::insert_human_text(&cdp, &text).await, None, true) + } + HumanInput::Mouse(input) => ( + actions::dispatch_human_mouse(&cdp, input, &mut mouse_state).await, + None, + true, + ), + HumanInput::ReleaseMouseButtons { + completion_tx, + after_release, + } => { + let result = actions::release_human_mouse_buttons(&cdp, &mut mouse_state).await; + let crash_on_error = result.is_err(); + let result = result.and_then(|()| { + finish_human_control_after_release(&inner, generation, after_release) + }); + (result, Some(completion_tx), crash_on_error) } } }; - if let Err(error) = result { - tracing::warn!(%error, "terminal-browser human input failed"); - inner.set_crashed("Terminal browser input connection failed".to_string()); + let completion_result = match result { + Ok(()) => Ok(()), + Err(error) => { + let error = error.to_string(); + tracing::warn!(%error, "terminal-browser human input failed"); + if crash_on_error { + inner.set_crashed("Terminal browser input connection failed".to_string()); + } + Err(error) + } + }; + if let Some(completion_tx) = completion_tx { + let _ = completion_tx.send(completion_result); } } } + +async fn next_human_input( + receivers: &mut HumanInputReceivers, + deferred: &mut VecDeque, +) -> Option { + if let Ok(queued) = receivers.control.try_recv() { + return Some(queued); + } + if let Some(queued) = deferred.pop_front() { + return Some(queued); + } + tokio::select! { + biased; + queued = receivers.control.recv() => queued, + queued = receivers.input.recv() => queued, + } +} + +fn drain_human_inputs_for_generation( + receivers: &mut HumanInputReceivers, + generation: u64, + deferred: &mut VecDeque, +) { + deferred.retain(|queued| queued.generation != generation); + while let Ok(queued) = receivers.input.try_recv() { + if queued.generation != generation { + deferred.push_back(queued); + } + } +} + +fn finish_human_control_after_release( + inner: &Inner, + generation: u64, + after_release: HumanControlAfterRelease, +) -> Result<()> { + match after_release { + HumanControlAfterRelease::Continue => Ok(()), + HumanControlAfterRelease::End => { + anyhow::ensure!( + inner.transition_human_control(HumanControlStateTransition::Deactivate { + generation, + }), + "browser control transition was canceled" + ); + Ok(()) + } + } +} + +#[cfg(test)] +#[path = "human_control_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/human_control_tests.rs b/codex-rs/terminal-browser/src/human_control_tests.rs new file mode 100644 index 0000000000..cbe6e86cfe --- /dev/null +++ b/codex-rs/terminal-browser/src/human_control_tests.rs @@ -0,0 +1,119 @@ +use std::sync::atomic::Ordering; + +use pretty_assertions::assert_eq; +use tokio::sync::oneshot; + +use super::*; + +#[tokio::test] +async fn stale_end_does_not_release_a_newer_control_generation() { + let browser = TerminalBrowser::discover(); + browser + .inner + .human_control + .store(/*val*/ true, Ordering::SeqCst); + browser + .inner + .human_control_generation + .store(/*val*/ 2, Ordering::SeqCst); + + let error = browser + .end_human_control(HumanControlToken { generation: 1 }) + .await + .expect_err("stale control token should be rejected"); + + let generation = browser + .inner + .human_control_generation + .load(Ordering::SeqCst); + assert_eq!(error.to_string(), "browser control transition was canceled"); + assert_eq!((browser.is_human_control_active(), generation), (true, 2)); +} + +#[tokio::test] +async fn release_discards_its_backlog_and_preserves_a_new_generation() { + let (sender, mut receivers) = TerminalBrowser::human_input_channel(); + for _ in 1..HUMAN_INPUT_CAPACITY { + sender + .input + .try_send(QueuedHumanInput { + generation: 1, + input: HumanInput::Text(String::new()), + }) + .expect("normal input should fill its reserved queue"); + } + sender + .input + .try_send(QueuedHumanInput { + generation: 3, + input: HumanInput::Text("new generation".to_string()), + }) + .expect("new-generation input should enter the normal queue"); + let (completion_tx, _completion_rx) = oneshot::channel(); + sender + .control + .send(QueuedHumanInput { + generation: 1, + input: HumanInput::ReleaseMouseButtons { + completion_tx, + after_release: HumanControlAfterRelease::Continue, + }, + }) + .await + .expect("release should use its reserved queue"); + + let mut deferred = VecDeque::new(); + let release = next_human_input(&mut receivers, &mut deferred) + .await + .expect("queued release input"); + drain_human_inputs_for_generation(&mut receivers, release.generation, &mut deferred); + assert_eq!(deferred.front().map(|queued| queued.generation), Some(3)); + let (completion_tx, _completion_rx) = oneshot::channel(); + sender + .control + .send(QueuedHumanInput { + generation: 3, + input: HumanInput::ReleaseMouseButtons { + completion_tx, + after_release: HumanControlAfterRelease::Continue, + }, + }) + .await + .expect("new-generation release should use its reserved queue"); + let release = next_human_input(&mut receivers, &mut deferred) + .await + .expect("queued new-generation release"); + drain_human_inputs_for_generation(&mut receivers, release.generation, &mut deferred); + + assert!(matches!( + release.input, + HumanInput::ReleaseMouseButtons { .. } + )); + assert!(deferred.is_empty()); +} + +#[test] +fn ending_release_deactivates_before_acknowledgement() { + let browser = TerminalBrowser::discover(); + browser + .inner + .human_control + .store(/*val*/ true, Ordering::SeqCst); + browser + .inner + .human_control_generation + .store(/*val*/ 1, Ordering::SeqCst); + + finish_human_control_after_release( + &browser.inner, + /*generation*/ 1, + HumanControlAfterRelease::End, + ) + .expect("ending release should deactivate its generation"); + + let generation = browser + .inner + .human_control_generation + .load(Ordering::SeqCst); + assert_eq!((browser.is_human_control_active(), generation), (false, 2)); +} diff --git a/codex-rs/terminal-browser/src/session.rs b/codex-rs/terminal-browser/src/session.rs index 7abb309bd7..53dd341e2e 100644 --- a/codex-rs/terminal-browser/src/session.rs +++ b/codex-rs/terminal-browser/src/session.rs @@ -13,8 +13,8 @@ use crate::diagnostics::inspect_installation; use crate::diagnostics::installation_report; use crate::diagnostics::unavailable_report; use crate::human_control::HumanControlStateTransition; +use crate::human_control::HumanInputReceivers; use crate::human_control::HumanInputSender; -use crate::human_control::QueuedHumanInput; use crate::network::BrowserNetworkPolicy; use crate::process::BrowserSession; use crate::profile::BrowserProfileName; @@ -58,7 +58,7 @@ pub(crate) struct Inner { pub(crate) human_control_generation: AtomicU64, pub(crate) human_handle_invalidation_pending: AtomicBool, pub(crate) human_input_tx: HumanInputSender, - pub(crate) human_input_rx: Mutex>>, + pub(crate) human_input_rx: Mutex>, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 479344e34c..4b9cc7ab00 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -1313,6 +1313,17 @@ See the Codex keymap documentation for supported actions and examples." self.handle_draw_pre_render(tui)?; } + if let TuiEvent::MousePrimary(mouse_event) = &event + && let Some(click) = self.terminal_browser_control_click(*mouse_event) + { + self.handle_owned_screen_mouse_primary(tui, *mouse_event); + self.begin_terminal_browser_control_from_click(click).await; + let raw_mouse_events = self.sync_terminal_browser_panel(); + tui.set_raw_mouse_events(raw_mouse_events); + tui.frame_requester().schedule_frame(); + return Ok(AppRunControl::Continue); + } + if self.terminal_browser_human_control_active() { let browser_viewport = self .owned_screen_frame @@ -1343,18 +1354,25 @@ See the Codex keymap documentation for supported actions and examples." return Ok(AppRunControl::Continue); } TuiEvent::MouseScroll(mouse_event) => { - self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport); + self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport) + .await; return Ok(AppRunControl::Continue); } TuiEvent::MousePrimary(mouse_event) => { - self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport); + self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport) + .await; return Ok(AppRunControl::Continue); } TuiEvent::MouseOther(mouse_event) => { - self.forward_terminal_browser_mouse(mouse_event, browser_viewport); + self.forward_terminal_browser_mouse(mouse_event, browser_viewport) + .await; return Ok(AppRunControl::Continue); } - TuiEvent::FocusLost | TuiEvent::Draw | TuiEvent::Resize => {} + TuiEvent::FocusLost => { + self.release_terminal_browser_mouse_buttons().await; + return Ok(AppRunControl::Continue); + } + TuiEvent::Draw | TuiEvent::Resize => {} } } diff --git a/codex-rs/tui/src/app/owned_screen_frame.rs b/codex-rs/tui/src/app/owned_screen_frame.rs index 848bb39818..ce996deeb4 100644 --- a/codex-rs/tui/src/app/owned_screen_frame.rs +++ b/codex-rs/tui/src/app/owned_screen_frame.rs @@ -139,7 +139,11 @@ enum FrameInteraction { origin_column: u16, has_resized: bool, }, - PanelPointer(OwnedScreenPanel), + PanelPointer { + panel: OwnedScreenPanel, + press: MousePrimaryEvent, + dragged: bool, + }, } #[derive(Debug)] @@ -243,6 +247,25 @@ impl OwnedScreenFrameState { self.interaction != FrameInteraction::Idle } + pub(super) fn completed_panel_click( + &self, + event: MousePrimaryEvent, + ) -> Option<(OwnedScreenPanel, MousePrimaryEvent)> { + if event.kind != MousePrimaryEventKind::Release { + return None; + } + match self.interaction { + FrameInteraction::PanelPointer { + panel, + press, + dragged: false, + } => Some((panel, press)), + FrameInteraction::Idle + | FrameInteraction::Resize { .. } + | FrameInteraction::PanelPointer { dragged: true, .. } => None, + } + } + pub(super) fn traps_background_input(&self) -> bool { self.layout.and_then(active_overlay).is_some() } @@ -267,7 +290,11 @@ impl OwnedScreenFrameState { }; if overlay.area.contains(position) { self.focus = panel.into(); - self.interaction = FrameInteraction::PanelPointer(panel); + self.interaction = FrameInteraction::PanelPointer { + panel, + press: event, + dragged: false, + }; } else { self.set_preference(panel, OwnedScreenPanelPreference::Hidden); } @@ -290,7 +317,11 @@ impl OwnedScreenFrameState { .is_some_and(|panel_layout| panel_layout.area.contains(position)) { self.focus = panel.into(); - self.interaction = FrameInteraction::PanelPointer(panel); + self.interaction = FrameInteraction::PanelPointer { + panel, + press: event, + dragged: false, + }; return true; } } @@ -315,7 +346,14 @@ impl OwnedScreenFrameState { }; true } - FrameInteraction::PanelPointer(_) => true, + FrameInteraction::PanelPointer { panel, press, .. } => { + self.interaction = FrameInteraction::PanelPointer { + panel, + press, + dragged: true, + }; + true + } FrameInteraction::Idle => false, }, MousePrimaryEventKind::Release => { diff --git a/codex-rs/tui/src/app/owned_screen_tests.rs b/codex-rs/tui/src/app/owned_screen_tests.rs index 433e962473..54898a4e18 100644 --- a/codex-rs/tui/src/app/owned_screen_tests.rs +++ b/codex-rs/tui/src/app/owned_screen_tests.rs @@ -1,4 +1,7 @@ use codex_terminal_browser::BrowserCell; +use codex_terminal_browser::BrowserMouseButton; +use codex_terminal_browser::BrowserMouseInput; +use codex_terminal_browser::BrowserMouseKind; use codex_terminal_browser::BrowserScreen; use codex_terminal_browser::BrowserStatus; use codex_terminal_browser::BrowserView; @@ -383,6 +386,91 @@ async fn owned_screen_browser_preserves_narrow_overlay_chrome() { ); } +#[tokio::test] +async fn terminal_browser_completed_click_preserves_input_and_rejects_non_clicks() -> Result<()> { + let mut app = app_with_owned_parent().await; + app.owned_screen_frame + .select_right_rail_content(OwnedScreenRightRailContent::Browser); + let browser_view = browser_view(/*human_control*/ false, Some((1, 2))); + let _terminal = render_frame_app_with_browser( + &mut app, + /*width*/ 144, + /*height*/ 12, + Some(&browser_view), + ); + let body = app + .owned_screen_frame + .panel_body(OwnedScreenPanel::Summary) + .expect("browser panel body"); + let viewport = browser_viewport(body); + let press = primary_event( + MousePrimaryEventKind::Press, + viewport.x.saturating_add(/*rhs*/ 2), + viewport.y.saturating_add(/*rhs*/ 1), + ); + let release = primary_event( + MousePrimaryEventKind::Release, + viewport.x.saturating_add(/*rhs*/ 3), + viewport.y.saturating_add(/*rhs*/ 2), + ); + let mut tui = crate::tui::test_support::make_test_tui()?; + + assert!(app.handle_owned_screen_mouse_primary(&mut tui, press)); + let inputs = app + .terminal_browser_control_inputs(release, &browser_view) + .expect("completed browser viewport click should request control"); + let expected_press = BrowserMouseInput { + kind: BrowserMouseKind::Down, + button: BrowserMouseButton::Left, + column: 2, + row: 1, + viewport_cols: viewport.width, + viewport_rows: viewport.height, + modifiers: Default::default(), + }; + + assert_eq!( + inputs, + [ + expected_press, + BrowserMouseInput { + kind: BrowserMouseKind::Up, + column: 3, + row: 2, + ..expected_press + }, + ] + ); + let idle_view = BrowserView { + status: BrowserStatus::Idle, + ..browser_view.clone() + }; + assert!( + app.terminal_browser_control_inputs(release, &idle_view) + .is_none(), + "an idle browser should ignore click-to-control" + ); + assert!(app.handle_owned_screen_mouse_primary(&mut tui, release)); + assert!(!app.owned_screen_frame.is_interacting()); + + assert!(app.handle_owned_screen_mouse_primary(&mut tui, press)); + assert!(app.handle_owned_screen_mouse_primary( + &mut tui, + MousePrimaryEvent { + kind: MousePrimaryEventKind::Drag, + ..release + }, + )); + assert!( + app.terminal_browser_control_inputs(release, &browser_view) + .is_none(), + "dragging within the browser should not request control" + ); + assert!(app.handle_owned_screen_mouse_primary(&mut tui, release)); + assert!(!app.owned_screen_frame.is_interacting()); + Ok(()) +} + #[tokio::test] async fn browser_cursor_wins_over_composer_during_human_control() { let mut app = app_with_owned_parent().await; diff --git a/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_preserves_narrow_overlay_chrome.snap b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_preserves_narrow_overlay_chrome.snap index ac88f0bc54..aa57fa3511 100644 --- a/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_preserves_narrow_overlay_chrome.snap +++ b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_preserves_narrow_overlay_chrome.snap @@ -12,5 +12,5 @@ expression: terminal.backend() " │ │ " " │ │ " "› draft sentinel │ │ " -" │ Esc hide │ " +" │ Esc hide click control │ " " gpt-5.5 default · /tmp/project └────────────────────────────────┘ " diff --git a/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_renders_in_wide_right_rail.snap b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_renders_in_wide_right_rail.snap index e70891a120..1b8ed5da1a 100644 --- a/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_renders_in_wide_right_rail.snap +++ b/codex-rs/tui/src/app/snapshots/codex_tui__app__owned_screen__tests__owned_screen_browser_renders_in_wide_right_rail.snap @@ -13,4 +13,4 @@ expression: terminal.backend() " │ ┃ " " │› draft sentinel ┃ " " │ ┃ " -" │ gpt-5.5 default · /tmp/project ┃ Esc hide " +" │ gpt-5.5 default · /tmp/project ┃ Esc hide click control " diff --git a/codex-rs/tui/src/app/terminal_browser.rs b/codex-rs/tui/src/app/terminal_browser.rs index 70c4da1d9a..2ed87cb00a 100644 --- a/codex-rs/tui/src/app/terminal_browser.rs +++ b/codex-rs/tui/src/app/terminal_browser.rs @@ -8,8 +8,10 @@ use std::sync::Arc; use codex_features::Feature; use codex_protocol::ThreadId; use codex_terminal_browser::BrowserLaunchContext; +use codex_terminal_browser::BrowserMouseInput; use codex_terminal_browser::BrowserNetworkPolicy; use codex_terminal_browser::BrowserStatus; +use codex_terminal_browser::BrowserView; use codex_terminal_browser::TerminalBrowser; use codex_utils_absolute_path::AbsolutePathBuf; use crossterm::event::KeyEvent; @@ -20,19 +22,29 @@ use tokio::sync::oneshot; use super::App; use super::owned_screen_frame::OwnedScreenRightRailContent; use crate::app_event::AppEvent; +use crate::app_event::OwnedScreenPanel; use crate::app_event::TerminalBrowserControlTarget; use crate::app_event::TerminalBrowserProfileApproval; use crate::app_event::TerminalBrowserProfileCommand; +use crate::terminal_browser::BrowserMouseRoute; use crate::terminal_browser::TerminalBrowserNetworkAvailability; use crate::terminal_browser::browser_key_input; use crate::terminal_browser::browser_mouse_input; +use crate::terminal_browser::browser_mouse_route; use crate::terminal_browser::profile_approval_view_params; +use crate::tui::MousePrimaryEvent; pub(super) struct ReopenableTerminalBrowser { browser: Arc, closed: oneshot::Receiver<()>, } +pub(super) struct TerminalBrowserControlClick { + browser: Arc, + target: TerminalBrowserControlTarget, + inputs: [BrowserMouseInput; 2], +} + impl ReopenableTerminalBrowser { pub(super) fn terminate(&self) { self.browser.terminate(); @@ -272,7 +284,7 @@ impl App { } } - pub(super) fn forward_terminal_browser_mouse( + pub(super) async fn forward_terminal_browser_mouse( &mut self, mouse_event: MouseEvent, viewport: Option, @@ -280,8 +292,13 @@ impl App { let Some(viewport) = viewport else { return; }; - let Some(input) = browser_mouse_input(mouse_event, viewport) else { - return; + let input = match browser_mouse_route(mouse_event, viewport) { + BrowserMouseRoute::Input(input) => input, + BrowserMouseRoute::ReleaseButtons => { + self.release_terminal_browser_mouse_buttons().await; + return; + } + BrowserMouseRoute::Ignore => return, }; let Some(browser) = self.terminal_browser_for_current_thread() else { return; @@ -292,6 +309,95 @@ impl App { } } + pub(super) async fn release_terminal_browser_mouse_buttons(&mut self) { + let Some(browser) = self.terminal_browser_for_current_thread() else { + return; + }; + if let Err(error) = browser.release_human_mouse_buttons().await { + self.chat_widget + .add_error_message(format!("Failed to release browser mouse buttons: {error}")); + } + } + + pub(super) fn terminal_browser_control_inputs( + &self, + event: MousePrimaryEvent, + view: &BrowserView, + ) -> Option<[BrowserMouseInput; 2]> { + if self.overlay.is_some() + || !self.chat_widget.no_modal_or_popup_active() + || self.owned_screen_frame.right_rail_content() != OwnedScreenRightRailContent::Browser + || !view.visible + || view.human_control + || !matches!(&view.status, BrowserStatus::Running) + { + return None; + } + let (panel, press) = self.owned_screen_frame.completed_panel_click(event)?; + if panel != OwnedScreenPanel::Summary { + return None; + } + let viewport = self + .owned_screen_frame + .panel_body(OwnedScreenPanel::Summary) + .map(crate::terminal_browser::browser_viewport)?; + Some([ + browser_mouse_input(press.into(), viewport)?, + browser_mouse_input(event.into(), viewport)?, + ]) + } + + pub(super) fn terminal_browser_control_click( + &self, + event: MousePrimaryEvent, + ) -> Option { + let browser = self.terminal_browser_for_current_thread()?; + let inputs = self.terminal_browser_control_inputs(event, &browser.view())?; + if browser.is_human_control_active() { + return None; + } + let target = self.terminal_browser_control_target()?; + Some(TerminalBrowserControlClick { + browser, + target, + inputs, + }) + } + + pub(super) async fn begin_terminal_browser_control_from_click( + &mut self, + click: TerminalBrowserControlClick, + ) { + let TerminalBrowserControlClick { + browser, + target, + inputs, + } = click; + let result = browser.toggle_human_control(target.token).await; + let mut completion_target = target; + let error = match result { + Ok(token) => { + completion_target.token = token; + for input in inputs { + if let Err(error) = browser.send_human_mouse(input) { + self.chat_widget.add_error_message(format!( + "Failed to send browser mouse event: {error}" + )); + break; + } + } + None + } + Err(error) => Some(error.to_string()), + }; + self.app_event_tx + .unscoped() + .send(AppEvent::TerminalBrowserControlCompleted { + target: completion_target, + error, + }); + } + pub(super) async fn terminal_browser_for_active_request( &mut self, ) -> Option> { diff --git a/codex-rs/tui/src/terminal_browser/input.rs b/codex-rs/tui/src/terminal_browser/input.rs index c42e26c55d..d76e705296 100644 --- a/codex-rs/tui/src/terminal_browser/input.rs +++ b/codex-rs/tui/src/terminal_browser/input.rs @@ -12,6 +12,13 @@ use crossterm::event::MouseEvent; use crossterm::event::MouseEventKind; use ratatui::layout::Rect; +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) enum BrowserMouseRoute { + Input(BrowserMouseInput), + ReleaseButtons, + Ignore, +} + /// Converts a crossterm key event into Carbonyl input. /// /// Release events and keys without a Chromium keyboard equivalent are ignored. @@ -150,6 +157,21 @@ pub(crate) fn browser_mouse_input(event: MouseEvent, viewport: Rect) -> Option BrowserMouseRoute { + if matches!(event.kind, MouseEventKind::Up(_)) + && (viewport.is_empty() || !viewport.contains((event.column, event.row).into())) + { + BrowserMouseRoute::ReleaseButtons + } else { + browser_mouse_input(event, viewport) + .map_or(BrowserMouseRoute::Ignore, BrowserMouseRoute::Input) + } +} + fn browser_mouse_button(button: MouseButton) -> BrowserMouseButton { match button { MouseButton::Left => BrowserMouseButton::Left, diff --git a/codex-rs/tui/src/terminal_browser/mod.rs b/codex-rs/tui/src/terminal_browser/mod.rs index 7f87b70700..7ea80335ee 100644 --- a/codex-rs/tui/src/terminal_browser/mod.rs +++ b/codex-rs/tui/src/terminal_browser/mod.rs @@ -10,8 +10,10 @@ mod panel; mod profile_approval; mod tools; +pub(crate) use input::BrowserMouseRoute; pub(crate) use input::browser_key_input; pub(crate) use input::browser_mouse_input; +pub(crate) use input::browser_mouse_route; pub(crate) use network::TerminalBrowserNetworkAvailability; #[cfg(test)] pub(crate) use panel::BrowserPanelAreas; diff --git a/codex-rs/tui/src/terminal_browser/panel.rs b/codex-rs/tui/src/terminal_browser/panel.rs index 071357e3b9..28cf2f3fe4 100644 --- a/codex-rs/tui/src/terminal_browser/panel.rs +++ b/codex-rs/tui/src/terminal_browser/panel.rs @@ -154,13 +154,13 @@ fn render_footer(view: &BrowserView, area: Rect, buf: &mut Buffer) { Line::from(" Ctrl+] ".cyan()) }; line.render_ref(area, buf); - } else { + } else if matches!(&view.status, BrowserStatus::Running) { let line = if area.width >= 63 { Line::from(vec![ " Esc ".cyan(), "hide".dim(), " ".into(), - "/browser control".cyan(), + " click page ".cyan(), " take control".dim(), " ".into(), "/browser close".cyan(), @@ -171,13 +171,23 @@ fn render_footer(view: &BrowserView, area: Rect, buf: &mut Buffer) { " Esc ".cyan(), "hide".dim(), " ".into(), - "/browser control".cyan(), + " click page ".cyan(), " take control".dim(), ]) + } else if area.width >= 26 { + Line::from(vec![ + " Esc ".cyan(), + "hide".dim(), + " ".into(), + " click ".cyan(), + "control".dim(), + ]) } else { - Line::from(vec![" Esc ".cyan(), "hide".dim()]) + Line::from(vec![" click ".cyan(), "control".dim()]) }; line.render_ref(area, buf); + } else { + Line::from(vec![" Esc ".cyan(), "hide".dim()]).render_ref(area, buf); } } diff --git a/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_panel_snapshot.snap b/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_panel_snapshot.snap index 89591748b5..d2e9f81584 100644 --- a/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_panel_snapshot.snap +++ b/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_panel_snapshot.snap @@ -11,4 +11,4 @@ x UI - Esc hide /browser control take control + Esc hide click page take control diff --git a/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_unavailable_panel_wraps_reason_snapshot.snap b/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_unavailable_panel_wraps_reason_snapshot.snap index b1443f893e..fc16e65651 100644 --- a/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_unavailable_panel_wraps_reason_snapshot.snap +++ b/codex-rs/tui/src/terminal_browser/snapshots/codex_tui__terminal_browser__tests__terminal_browser_unavailable_panel_wraps_reason_snapshot.snap @@ -11,4 +11,4 @@ PATH; install it or set CODEX_CARBONYL_BINARY - Esc hide /browser control take control + Esc hide diff --git a/codex-rs/tui/src/terminal_browser/tests.rs b/codex-rs/tui/src/terminal_browser/tests.rs index ac8ec0de35..aedc83ef65 100644 --- a/codex-rs/tui/src/terminal_browser/tests.rs +++ b/codex-rs/tui/src/terminal_browser/tests.rs @@ -35,6 +35,8 @@ use crate::app_event::TerminalBrowserProfileCommand; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::ListSelectionView; use crate::render::renderable::Renderable; +use crate::terminal_browser::BrowserMouseRoute; +use crate::terminal_browser::browser_mouse_route; use super::BrowserPanelAreas; use super::browser_key_input; @@ -371,6 +373,34 @@ fn mouse_events_outside_the_browser_viewport_are_ignored() { ); } +#[test] +fn active_mouse_capture_routes_outside_release_to_cleanup() { + let viewport = Rect::new( + /*x*/ 10, /*y*/ 5, /*width*/ 40, /*height*/ 20, + ); + let outside_release = MouseEvent { + kind: MouseEventKind::Up(MouseButton::Left), + column: 9, + row: 30, + modifiers: KeyModifiers::SHIFT, + }; + + assert_eq!( + browser_mouse_route(outside_release, viewport), + BrowserMouseRoute::ReleaseButtons + ); + assert_eq!( + browser_mouse_route( + MouseEvent { + kind: MouseEventKind::Down(MouseButton::Left), + ..outside_release + }, + viewport, + ), + BrowserMouseRoute::Ignore + ); +} + #[test] fn terminal_browser_unavailable_panel_wraps_reason_snapshot() { let view = BrowserView { diff --git a/justfile b/justfile index 0ad45d4f39..1a82f30f49 100644 --- a/justfile +++ b/justfile @@ -101,7 +101,7 @@ test-terminal-browser-panel *args: -p codex-core \ -p codex-app-server \ -p codex-tui \ - -E 'package(codex-utils-pty) | package(codex-terminal-browser) | package(codex-sandboxing) | package(codex-linux-sandbox) | package(codex-features) | package(codex-protocol) | package(codex-app-server-protocol) | (package(codex-exec-server) & test(/sandbox_request_allows_prepared_managed_proxy_port|sandbox_exec_request_carries_helper_env/)) | (package(codex-core) & test(/config_schema_matches_fixture|active_profile_update_rebuilds_network_proxy_config|current_network_proxy_runtime|managed_network_proxy_decider_survives_full_access_start|managed_network_proxy_refresh_failure|new_turn_refreshes_managed_network_proxy_for_sandbox_change|danger_full_access_turns_do_not_expose_managed_network_proxy|workspace_write_turns_continue_to_expose_managed_network_proxy/)) | (package(codex-app-server) & test(/thread_start_returns_live_managed_proxy_in_experimental_extra|thread_settings_update_reports_current_managed_proxy_runtime/)) | (package(codex-tui) & test(/terminal_browser|bare_browser|browser_show_alias|network_reconciliation|inactive_thread_settings_notification|proxy_runtime|thread_read_fallback_uses_active_permission_settings/))' \ + -E 'package(codex-utils-pty) | package(codex-terminal-browser) | package(codex-sandboxing) | package(codex-linux-sandbox) | package(codex-features) | package(codex-protocol) | package(codex-app-server-protocol) | (package(codex-exec-server) & test(/sandbox_request_allows_prepared_managed_proxy_port|sandbox_exec_request_carries_helper_env/)) | (package(codex-core) & test(/config_schema_matches_fixture|active_profile_update_rebuilds_network_proxy_config|current_network_proxy_runtime|managed_network_proxy_decider_survives_full_access_start|managed_network_proxy_refresh_failure|new_turn_refreshes_managed_network_proxy_for_sandbox_change|danger_full_access_turns_do_not_expose_managed_network_proxy|workspace_write_turns_continue_to_expose_managed_network_proxy/)) | (package(codex-app-server) & test(/thread_start_returns_live_managed_proxy_in_experimental_extra|thread_settings_update_reports_current_managed_proxy_runtime/)) | (package(codex-tui) & test(/terminal_browser|owned_screen_browser|bare_browser|browser_show_alias|network_reconciliation|inactive_thread_settings_notification|proxy_runtime|thread_read_fallback_uses_active_permission_settings/))' \ "$@" # Run from the repository root so scripts that resolve paths from `cwd` see