feat(tui): activate browser control on click

This commit is contained in:
Felipe Coury
2026-07-06 22:32:13 -03:00
parent bf14e06924
commit 798e0a2311
18 changed files with 738 additions and 53 deletions
+41
View File
@@ -5,6 +5,7 @@ use serde_json::json;
use crate::cdp::CdpClient;
use crate::input::BrowserKeyInput;
use crate::input::BrowserMouseButton;
use crate::input::BrowserMouseInput;
use crate::input::BrowserMouseKind;
use crate::scripts;
@@ -18,6 +19,7 @@ const MAX_SNAPSHOT_TEXT_CHARS: usize = 6_000;
#[derive(Default)]
pub(crate) struct HumanMouseDispatchState {
viewport: Option<(u16, u16, f64, f64)>,
last_position: Option<(f64, f64)>,
buttons: u8,
}
@@ -136,6 +138,44 @@ pub(crate) async fn dispatch_human_key(client: &CdpClient, input: &BrowserKeyInp
Ok(())
}
pub(crate) async fn release_human_mouse_buttons(
client: &CdpClient,
state: &mut HumanMouseDispatchState,
) -> Result<()> {
let Some((x, y)) = state.last_position else {
state.buttons = 0;
return Ok(());
};
for button in [
BrowserMouseButton::Left,
BrowserMouseButton::Middle,
BrowserMouseButton::Right,
] {
let button_mask = button.cdp_buttons_mask();
if state.buttons & button_mask == 0 {
continue;
}
state.buttons &= !button_mask;
client
.call(
"Input.dispatchMouseEvent",
json!({
"type": "mouseReleased",
"x": x,
"y": y,
"button": button.as_cdp(),
"buttons": state.buttons,
"modifiers": 0,
"clickCount": 0,
"deltaX": 0.0,
"deltaY": 0.0,
}),
)
.await?;
}
Ok(())
}
pub(crate) async fn insert_human_text(client: &CdpClient, text: &str) -> Result<()> {
client
.call("Input.insertText", json!({ "text": text }))
@@ -202,6 +242,7 @@ pub(crate) async fn dispatch_human_mouse(
}),
)
.await?;
state.last_position = Some((x, y));
Ok(())
}
+42 -1
View File
@@ -11,6 +11,7 @@ use tokio_tungstenite::tungstenite::Message;
use super::BrowserToolOutput;
use super::HumanMouseDispatchState;
use super::dispatch_human_mouse;
use super::release_human_mouse_buttons;
use crate::accessibility::click;
use crate::accessibility::snapshot;
use crate::cdp::CdpClient;
@@ -186,7 +187,7 @@ async fn native_click_uses_the_accessibility_handle_box_center() {
}
#[tokio::test]
async fn human_mouse_caches_metrics_and_preserves_drag_buttons() {
async fn human_mouse_preserves_drag_buttons_and_cleans_up_held_buttons() {
let (url, server) = test_server(|mut socket| {
tokio::spawn(async move {
initialize(&mut socket).await;
@@ -207,6 +208,22 @@ async fn human_mouse_caches_metrics_and_preserves_drag_buttons() {
assert_eq!(input["params"]["buttons"], expected_buttons);
respond(&mut socket, &input, json!({})).await;
}
for expected_buttons in [1, 3] {
let input = request(&mut socket).await;
assert_eq!(input["params"]["type"], "mousePressed");
assert_eq!(input["params"]["buttons"], expected_buttons);
respond(&mut socket, &input, json!({})).await;
}
for (expected_button, expected_buttons) in [("left", 2), ("right", 0)] {
let input = request(&mut socket).await;
assert_eq!(input["params"]["type"], "mouseReleased");
assert_eq!(input["params"]["button"], expected_button);
assert_eq!(input["params"]["buttons"], expected_buttons);
assert_eq!(input["params"]["x"], 200.0);
assert_eq!(input["params"]["y"], 150.0);
assert_eq!(input["params"]["clickCount"], 0);
respond(&mut socket, &input, json!({})).await;
}
})
})
.await;
@@ -233,6 +250,30 @@ async fn human_mouse_caches_metrics_and_preserves_drag_buttons() {
.await
.expect("dispatch mouse input");
}
for button in [BrowserMouseButton::Left, BrowserMouseButton::Right] {
dispatch_human_mouse(
&client,
BrowserMouseInput {
kind: BrowserMouseKind::Down,
button,
column: 20,
row: 6,
viewport_cols: 80,
viewport_rows: 24,
modifiers: BrowserInputModifiers::default(),
},
&mut state,
)
.await
.expect("dispatch mouse press");
}
release_human_mouse_buttons(&client, &mut state)
.await
.expect("release held mouse buttons");
release_human_mouse_buttons(&client, &mut state)
.await
.expect("repeated cleanup is empty");
server.await.expect("server task");
}
+200 -30
View File
@@ -1,3 +1,4 @@
use std::collections::VecDeque;
use std::sync::Arc;
use std::sync::Weak;
use std::sync::atomic::Ordering;
@@ -5,6 +6,7 @@ use std::sync::atomic::Ordering;
use anyhow::Result;
use anyhow::anyhow;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use crate::actions;
use crate::actions::HumanMouseDispatchState;
@@ -15,9 +17,18 @@ use crate::session::Inner;
use crate::session::TerminalBrowser;
const HUMAN_INPUT_CAPACITY: usize = 128;
const HUMAN_CONTROL_INPUT_CAPACITY: usize = 1;
const MAX_HUMAN_TEXT_BYTES: usize = 64 * 1024;
pub(crate) struct HumanInputSender(mpsc::Sender<QueuedHumanInput>);
pub(crate) struct HumanInputSender {
input: mpsc::Sender<QueuedHumanInput>,
control: mpsc::Sender<QueuedHumanInput>,
}
pub(crate) struct HumanInputReceivers {
input: mpsc::Receiver<QueuedHumanInput>,
control: mpsc::Receiver<QueuedHumanInput>,
}
/// Captures the browser-control epoch at the time a UI transition is requested.
///
@@ -38,9 +49,27 @@ enum HumanInput {
Key(BrowserKeyInput),
Text(String),
Mouse(BrowserMouseInput),
ReleaseMouseButtons {
completion_tx: oneshot::Sender<std::result::Result<(), String>>,
after_release: HumanControlAfterRelease,
},
}
pub(crate) struct QueuedHumanInput {
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum HumanControlAfterRelease {
Continue,
End,
}
impl HumanInput {
fn complete_release(self, result: std::result::Result<(), String>) {
if let Self::ReleaseMouseButtons { completion_tx, .. } = self {
let _ = completion_tx.send(result);
}
}
}
struct QueuedHumanInput {
generation: u64,
input: HumanInput,
}
@@ -89,9 +118,19 @@ impl Inner {
}
impl TerminalBrowser {
pub(crate) fn human_input_channel() -> (HumanInputSender, mpsc::Receiver<QueuedHumanInput>) {
let (sender, receiver) = mpsc::channel(HUMAN_INPUT_CAPACITY);
(HumanInputSender(sender), receiver)
pub(crate) fn human_input_channel() -> (HumanInputSender, HumanInputReceivers) {
let (input_tx, input_rx) = mpsc::channel(HUMAN_INPUT_CAPACITY);
let (control_tx, control_rx) = mpsc::channel(HUMAN_CONTROL_INPUT_CAPACITY);
(
HumanInputSender {
input: input_tx,
control: control_tx,
},
HumanInputReceivers {
input: input_rx,
control: control_rx,
},
)
}
pub fn is_human_control_active(&self) -> bool {
@@ -207,12 +246,14 @@ impl TerminalBrowser {
)]
pub async fn end_human_control(&self, token: HumanControlToken) -> Result<HumanControlToken> {
anyhow::ensure!(
self.inner
.transition_human_control(HumanControlStateTransition::Deactivate {
generation: token.generation,
}),
self.human_control_is_active_for(token.generation),
"browser control transition was canceled"
);
self.release_human_mouse_buttons_for_generation(
token.generation,
HumanControlAfterRelease::End,
)
.await?;
let _operation = self.inner.operation.lock().await;
self.flush_human_handle_invalidation().await;
Ok(HumanControlToken {
@@ -259,13 +300,52 @@ impl TerminalBrowser {
self.queue_human_input(HumanInput::Mouse(input))
}
/// Releases every mouse button currently held by the browser input worker.
pub async fn release_human_mouse_buttons(&self) -> Result<()> {
let generation = self.inner.human_control_generation.load(Ordering::SeqCst);
self.release_human_mouse_buttons_for_generation(
generation,
HumanControlAfterRelease::Continue,
)
.await
}
async fn release_human_mouse_buttons_for_generation(
&self,
generation: u64,
after_release: HumanControlAfterRelease,
) -> Result<()> {
anyhow::ensure!(
self.human_control_is_active_for(generation),
"browser control transition was canceled"
);
let (completion_tx, completion_rx) = oneshot::channel();
self.inner
.human_input_tx
.control
.send(QueuedHumanInput {
generation,
input: HumanInput::ReleaseMouseButtons {
completion_tx,
after_release,
},
})
.await
.map_err(|_| anyhow!("terminal browser input worker stopped"))?;
match completion_rx.await {
Ok(Ok(())) => Ok(()),
Ok(Err(error)) => Err(anyhow!(error)),
Err(_) => Err(anyhow!("terminal browser input worker stopped")),
}
}
fn queue_human_input(&self, input: HumanInput) -> Result<()> {
anyhow::ensure!(
self.is_human_control_active(),
"human control is not active"
);
let is_motion = matches!(
input,
&input,
HumanInput::Mouse(BrowserMouseInput {
kind: BrowserMouseKind::Move,
..
@@ -275,7 +355,7 @@ impl TerminalBrowser {
generation: self.inner.human_control_generation.load(Ordering::SeqCst),
input,
};
match self.inner.human_input_tx.0.try_send(queued) {
match self.inner.human_input_tx.input.try_send(queued) {
Ok(()) => Ok(()),
Err(mpsc::error::TrySendError::Full(_)) if is_motion => Ok(()),
Err(mpsc::error::TrySendError::Full(_)) => {
@@ -286,6 +366,11 @@ impl TerminalBrowser {
}
}
}
fn human_control_is_active_for(&self, generation: u64) -> bool {
self.is_human_control_active()
&& self.inner.human_control_generation.load(Ordering::SeqCst) == generation
}
}
struct HumanControlTransition<'a> {
@@ -302,50 +387,135 @@ impl Drop for HumanControlTransition<'_> {
clippy::await_holding_invalid_type,
reason = "human input must remain serialized with model actions until its CDP call completes"
)]
async fn run_human_input_worker(
inner: Weak<Inner>,
mut receiver: mpsc::Receiver<QueuedHumanInput>,
) {
async fn run_human_input_worker(inner: Weak<Inner>, mut receivers: HumanInputReceivers) {
let mut mouse_state = HumanMouseDispatchState::default();
let mut mouse_generation = None;
while let Some(queued) = receiver.recv().await {
let mut deferred = VecDeque::new();
while let Some(queued) = next_human_input(&mut receivers, &mut deferred).await {
let Some(inner) = inner.upgrade() else {
return;
};
let QueuedHumanInput { generation, input } = queued;
if !inner.human_control.load(Ordering::SeqCst)
|| inner.human_control_generation.load(Ordering::SeqCst) != queued.generation
|| inner.human_control_generation.load(Ordering::SeqCst) != generation
{
input.complete_release(Err("browser control transition was canceled".to_string()));
continue;
}
if mouse_generation != Some(queued.generation) {
mouse_state = HumanMouseDispatchState::default();
mouse_generation = Some(queued.generation);
if matches!(&input, HumanInput::ReleaseMouseButtons { .. }) {
drain_human_inputs_for_generation(&mut receivers, generation, &mut deferred);
}
let result = {
if mouse_generation != Some(generation) {
mouse_state = HumanMouseDispatchState::default();
mouse_generation = Some(generation);
}
let (result, completion_tx, crash_on_error) = {
let _operation = inner.operation.lock().await;
if !inner.human_control.load(Ordering::SeqCst)
|| inner.human_control_generation.load(Ordering::SeqCst) != queued.generation
|| inner.human_control_generation.load(Ordering::SeqCst) != generation
{
input.complete_release(Err("browser control transition was canceled".to_string()));
continue;
}
let cdp = {
let session = inner.session.lock().await;
let Some(session) = session.as_ref() else {
input.complete_release(Err("terminal browser is not open".to_string()));
continue;
};
session.cdp.clone()
};
match queued.input {
HumanInput::Key(input) => actions::dispatch_human_key(&cdp, &input).await,
HumanInput::Text(text) => actions::insert_human_text(&cdp, &text).await,
HumanInput::Mouse(input) => {
actions::dispatch_human_mouse(&cdp, input, &mut mouse_state).await
match input {
HumanInput::Key(input) => {
(actions::dispatch_human_key(&cdp, &input).await, None, true)
}
HumanInput::Text(text) => {
(actions::insert_human_text(&cdp, &text).await, None, true)
}
HumanInput::Mouse(input) => (
actions::dispatch_human_mouse(&cdp, input, &mut mouse_state).await,
None,
true,
),
HumanInput::ReleaseMouseButtons {
completion_tx,
after_release,
} => {
let result = actions::release_human_mouse_buttons(&cdp, &mut mouse_state).await;
let crash_on_error = result.is_err();
let result = result.and_then(|()| {
finish_human_control_after_release(&inner, generation, after_release)
});
(result, Some(completion_tx), crash_on_error)
}
}
};
if let Err(error) = result {
tracing::warn!(%error, "terminal-browser human input failed");
inner.set_crashed("Terminal browser input connection failed".to_string());
let completion_result = match result {
Ok(()) => Ok(()),
Err(error) => {
let error = error.to_string();
tracing::warn!(%error, "terminal-browser human input failed");
if crash_on_error {
inner.set_crashed("Terminal browser input connection failed".to_string());
}
Err(error)
}
};
if let Some(completion_tx) = completion_tx {
let _ = completion_tx.send(completion_result);
}
}
}
async fn next_human_input(
receivers: &mut HumanInputReceivers,
deferred: &mut VecDeque<QueuedHumanInput>,
) -> Option<QueuedHumanInput> {
if let Ok(queued) = receivers.control.try_recv() {
return Some(queued);
}
if let Some(queued) = deferred.pop_front() {
return Some(queued);
}
tokio::select! {
biased;
queued = receivers.control.recv() => queued,
queued = receivers.input.recv() => queued,
}
}
fn drain_human_inputs_for_generation(
receivers: &mut HumanInputReceivers,
generation: u64,
deferred: &mut VecDeque<QueuedHumanInput>,
) {
deferred.retain(|queued| queued.generation != generation);
while let Ok(queued) = receivers.input.try_recv() {
if queued.generation != generation {
deferred.push_back(queued);
}
}
}
fn finish_human_control_after_release(
inner: &Inner,
generation: u64,
after_release: HumanControlAfterRelease,
) -> Result<()> {
match after_release {
HumanControlAfterRelease::Continue => Ok(()),
HumanControlAfterRelease::End => {
anyhow::ensure!(
inner.transition_human_control(HumanControlStateTransition::Deactivate {
generation,
}),
"browser control transition was canceled"
);
Ok(())
}
}
}
#[cfg(test)]
#[path = "human_control_tests.rs"]
mod tests;
@@ -0,0 +1,119 @@
use std::sync::atomic::Ordering;
use pretty_assertions::assert_eq;
use tokio::sync::oneshot;
use super::*;
#[tokio::test]
async fn stale_end_does_not_release_a_newer_control_generation() {
let browser = TerminalBrowser::discover();
browser
.inner
.human_control
.store(/*val*/ true, Ordering::SeqCst);
browser
.inner
.human_control_generation
.store(/*val*/ 2, Ordering::SeqCst);
let error = browser
.end_human_control(HumanControlToken { generation: 1 })
.await
.expect_err("stale control token should be rejected");
let generation = browser
.inner
.human_control_generation
.load(Ordering::SeqCst);
assert_eq!(error.to_string(), "browser control transition was canceled");
assert_eq!((browser.is_human_control_active(), generation), (true, 2));
}
#[tokio::test]
async fn release_discards_its_backlog_and_preserves_a_new_generation() {
let (sender, mut receivers) = TerminalBrowser::human_input_channel();
for _ in 1..HUMAN_INPUT_CAPACITY {
sender
.input
.try_send(QueuedHumanInput {
generation: 1,
input: HumanInput::Text(String::new()),
})
.expect("normal input should fill its reserved queue");
}
sender
.input
.try_send(QueuedHumanInput {
generation: 3,
input: HumanInput::Text("new generation".to_string()),
})
.expect("new-generation input should enter the normal queue");
let (completion_tx, _completion_rx) = oneshot::channel();
sender
.control
.send(QueuedHumanInput {
generation: 1,
input: HumanInput::ReleaseMouseButtons {
completion_tx,
after_release: HumanControlAfterRelease::Continue,
},
})
.await
.expect("release should use its reserved queue");
let mut deferred = VecDeque::new();
let release = next_human_input(&mut receivers, &mut deferred)
.await
.expect("queued release input");
drain_human_inputs_for_generation(&mut receivers, release.generation, &mut deferred);
assert_eq!(deferred.front().map(|queued| queued.generation), Some(3));
let (completion_tx, _completion_rx) = oneshot::channel();
sender
.control
.send(QueuedHumanInput {
generation: 3,
input: HumanInput::ReleaseMouseButtons {
completion_tx,
after_release: HumanControlAfterRelease::Continue,
},
})
.await
.expect("new-generation release should use its reserved queue");
let release = next_human_input(&mut receivers, &mut deferred)
.await
.expect("queued new-generation release");
drain_human_inputs_for_generation(&mut receivers, release.generation, &mut deferred);
assert!(matches!(
release.input,
HumanInput::ReleaseMouseButtons { .. }
));
assert!(deferred.is_empty());
}
#[test]
fn ending_release_deactivates_before_acknowledgement() {
let browser = TerminalBrowser::discover();
browser
.inner
.human_control
.store(/*val*/ true, Ordering::SeqCst);
browser
.inner
.human_control_generation
.store(/*val*/ 1, Ordering::SeqCst);
finish_human_control_after_release(
&browser.inner,
/*generation*/ 1,
HumanControlAfterRelease::End,
)
.expect("ending release should deactivate its generation");
let generation = browser
.inner
.human_control_generation
.load(Ordering::SeqCst);
assert_eq!((browser.is_human_control_active(), generation), (false, 2));
}
+2 -2
View File
@@ -13,8 +13,8 @@ use crate::diagnostics::inspect_installation;
use crate::diagnostics::installation_report;
use crate::diagnostics::unavailable_report;
use crate::human_control::HumanControlStateTransition;
use crate::human_control::HumanInputReceivers;
use crate::human_control::HumanInputSender;
use crate::human_control::QueuedHumanInput;
use crate::network::BrowserNetworkPolicy;
use crate::process::BrowserSession;
use crate::profile::BrowserProfileName;
@@ -58,7 +58,7 @@ pub(crate) struct Inner {
pub(crate) human_control_generation: AtomicU64,
pub(crate) human_handle_invalidation_pending: AtomicBool,
pub(crate) human_input_tx: HumanInputSender,
pub(crate) human_input_rx: Mutex<Option<tokio::sync::mpsc::Receiver<QueuedHumanInput>>>,
pub(crate) human_input_rx: Mutex<Option<HumanInputReceivers>>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
+22 -4
View File
@@ -1313,6 +1313,17 @@ See the Codex keymap documentation for supported actions and examples."
self.handle_draw_pre_render(tui)?;
}
if let TuiEvent::MousePrimary(mouse_event) = &event
&& let Some(click) = self.terminal_browser_control_click(*mouse_event)
{
self.handle_owned_screen_mouse_primary(tui, *mouse_event);
self.begin_terminal_browser_control_from_click(click).await;
let raw_mouse_events = self.sync_terminal_browser_panel();
tui.set_raw_mouse_events(raw_mouse_events);
tui.frame_requester().schedule_frame();
return Ok(AppRunControl::Continue);
}
if self.terminal_browser_human_control_active() {
let browser_viewport = self
.owned_screen_frame
@@ -1343,18 +1354,25 @@ See the Codex keymap documentation for supported actions and examples."
return Ok(AppRunControl::Continue);
}
TuiEvent::MouseScroll(mouse_event) => {
self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport);
self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport)
.await;
return Ok(AppRunControl::Continue);
}
TuiEvent::MousePrimary(mouse_event) => {
self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport);
self.forward_terminal_browser_mouse(mouse_event.into(), browser_viewport)
.await;
return Ok(AppRunControl::Continue);
}
TuiEvent::MouseOther(mouse_event) => {
self.forward_terminal_browser_mouse(mouse_event, browser_viewport);
self.forward_terminal_browser_mouse(mouse_event, browser_viewport)
.await;
return Ok(AppRunControl::Continue);
}
TuiEvent::FocusLost | TuiEvent::Draw | TuiEvent::Resize => {}
TuiEvent::FocusLost => {
self.release_terminal_browser_mouse_buttons().await;
return Ok(AppRunControl::Continue);
}
TuiEvent::Draw | TuiEvent::Resize => {}
}
}
+42 -4
View File
@@ -139,7 +139,11 @@ enum FrameInteraction {
origin_column: u16,
has_resized: bool,
},
PanelPointer(OwnedScreenPanel),
PanelPointer {
panel: OwnedScreenPanel,
press: MousePrimaryEvent,
dragged: bool,
},
}
#[derive(Debug)]
@@ -243,6 +247,25 @@ impl OwnedScreenFrameState {
self.interaction != FrameInteraction::Idle
}
pub(super) fn completed_panel_click(
&self,
event: MousePrimaryEvent,
) -> Option<(OwnedScreenPanel, MousePrimaryEvent)> {
if event.kind != MousePrimaryEventKind::Release {
return None;
}
match self.interaction {
FrameInteraction::PanelPointer {
panel,
press,
dragged: false,
} => Some((panel, press)),
FrameInteraction::Idle
| FrameInteraction::Resize { .. }
| FrameInteraction::PanelPointer { dragged: true, .. } => None,
}
}
pub(super) fn traps_background_input(&self) -> bool {
self.layout.and_then(active_overlay).is_some()
}
@@ -267,7 +290,11 @@ impl OwnedScreenFrameState {
};
if overlay.area.contains(position) {
self.focus = panel.into();
self.interaction = FrameInteraction::PanelPointer(panel);
self.interaction = FrameInteraction::PanelPointer {
panel,
press: event,
dragged: false,
};
} else {
self.set_preference(panel, OwnedScreenPanelPreference::Hidden);
}
@@ -290,7 +317,11 @@ impl OwnedScreenFrameState {
.is_some_and(|panel_layout| panel_layout.area.contains(position))
{
self.focus = panel.into();
self.interaction = FrameInteraction::PanelPointer(panel);
self.interaction = FrameInteraction::PanelPointer {
panel,
press: event,
dragged: false,
};
return true;
}
}
@@ -315,7 +346,14 @@ impl OwnedScreenFrameState {
};
true
}
FrameInteraction::PanelPointer(_) => true,
FrameInteraction::PanelPointer { panel, press, .. } => {
self.interaction = FrameInteraction::PanelPointer {
panel,
press,
dragged: true,
};
true
}
FrameInteraction::Idle => false,
},
MousePrimaryEventKind::Release => {
@@ -1,4 +1,7 @@
use codex_terminal_browser::BrowserCell;
use codex_terminal_browser::BrowserMouseButton;
use codex_terminal_browser::BrowserMouseInput;
use codex_terminal_browser::BrowserMouseKind;
use codex_terminal_browser::BrowserScreen;
use codex_terminal_browser::BrowserStatus;
use codex_terminal_browser::BrowserView;
@@ -383,6 +386,91 @@ async fn owned_screen_browser_preserves_narrow_overlay_chrome() {
);
}
#[tokio::test]
async fn terminal_browser_completed_click_preserves_input_and_rejects_non_clicks() -> Result<()> {
let mut app = app_with_owned_parent().await;
app.owned_screen_frame
.select_right_rail_content(OwnedScreenRightRailContent::Browser);
let browser_view = browser_view(/*human_control*/ false, Some((1, 2)));
let _terminal = render_frame_app_with_browser(
&mut app,
/*width*/ 144,
/*height*/ 12,
Some(&browser_view),
);
let body = app
.owned_screen_frame
.panel_body(OwnedScreenPanel::Summary)
.expect("browser panel body");
let viewport = browser_viewport(body);
let press = primary_event(
MousePrimaryEventKind::Press,
viewport.x.saturating_add(/*rhs*/ 2),
viewport.y.saturating_add(/*rhs*/ 1),
);
let release = primary_event(
MousePrimaryEventKind::Release,
viewport.x.saturating_add(/*rhs*/ 3),
viewport.y.saturating_add(/*rhs*/ 2),
);
let mut tui = crate::tui::test_support::make_test_tui()?;
assert!(app.handle_owned_screen_mouse_primary(&mut tui, press));
let inputs = app
.terminal_browser_control_inputs(release, &browser_view)
.expect("completed browser viewport click should request control");
let expected_press = BrowserMouseInput {
kind: BrowserMouseKind::Down,
button: BrowserMouseButton::Left,
column: 2,
row: 1,
viewport_cols: viewport.width,
viewport_rows: viewport.height,
modifiers: Default::default(),
};
assert_eq!(
inputs,
[
expected_press,
BrowserMouseInput {
kind: BrowserMouseKind::Up,
column: 3,
row: 2,
..expected_press
},
]
);
let idle_view = BrowserView {
status: BrowserStatus::Idle,
..browser_view.clone()
};
assert!(
app.terminal_browser_control_inputs(release, &idle_view)
.is_none(),
"an idle browser should ignore click-to-control"
);
assert!(app.handle_owned_screen_mouse_primary(&mut tui, release));
assert!(!app.owned_screen_frame.is_interacting());
assert!(app.handle_owned_screen_mouse_primary(&mut tui, press));
assert!(app.handle_owned_screen_mouse_primary(
&mut tui,
MousePrimaryEvent {
kind: MousePrimaryEventKind::Drag,
..release
},
));
assert!(
app.terminal_browser_control_inputs(release, &browser_view)
.is_none(),
"dragging within the browser should not request control"
);
assert!(app.handle_owned_screen_mouse_primary(&mut tui, release));
assert!(!app.owned_screen_frame.is_interacting());
Ok(())
}
#[tokio::test]
async fn browser_cursor_wins_over_composer_during_human_control() {
let mut app = app_with_owned_parent().await;
@@ -12,5 +12,5 @@ expression: terminal.backend()
" │ │ "
" │ │ "
"› draft sentinel │ │ "
" │ Esc hide │ "
" │ Esc hide click control │ "
" gpt-5.5 default · /tmp/project └────────────────────────────────┘ "
@@ -13,4 +13,4 @@ expression: terminal.backend()
" │ ┃ "
" │› draft sentinel ┃ "
" │ ┃ "
" │ gpt-5.5 default · /tmp/project ┃ Esc hide "
" │ gpt-5.5 default · /tmp/project ┃ Esc hide click control "
+109 -3
View File
@@ -8,8 +8,10 @@ use std::sync::Arc;
use codex_features::Feature;
use codex_protocol::ThreadId;
use codex_terminal_browser::BrowserLaunchContext;
use codex_terminal_browser::BrowserMouseInput;
use codex_terminal_browser::BrowserNetworkPolicy;
use codex_terminal_browser::BrowserStatus;
use codex_terminal_browser::BrowserView;
use codex_terminal_browser::TerminalBrowser;
use codex_utils_absolute_path::AbsolutePathBuf;
use crossterm::event::KeyEvent;
@@ -20,19 +22,29 @@ use tokio::sync::oneshot;
use super::App;
use super::owned_screen_frame::OwnedScreenRightRailContent;
use crate::app_event::AppEvent;
use crate::app_event::OwnedScreenPanel;
use crate::app_event::TerminalBrowserControlTarget;
use crate::app_event::TerminalBrowserProfileApproval;
use crate::app_event::TerminalBrowserProfileCommand;
use crate::terminal_browser::BrowserMouseRoute;
use crate::terminal_browser::TerminalBrowserNetworkAvailability;
use crate::terminal_browser::browser_key_input;
use crate::terminal_browser::browser_mouse_input;
use crate::terminal_browser::browser_mouse_route;
use crate::terminal_browser::profile_approval_view_params;
use crate::tui::MousePrimaryEvent;
pub(super) struct ReopenableTerminalBrowser {
browser: Arc<TerminalBrowser>,
closed: oneshot::Receiver<()>,
}
pub(super) struct TerminalBrowserControlClick {
browser: Arc<TerminalBrowser>,
target: TerminalBrowserControlTarget,
inputs: [BrowserMouseInput; 2],
}
impl ReopenableTerminalBrowser {
pub(super) fn terminate(&self) {
self.browser.terminate();
@@ -272,7 +284,7 @@ impl App {
}
}
pub(super) fn forward_terminal_browser_mouse(
pub(super) async fn forward_terminal_browser_mouse(
&mut self,
mouse_event: MouseEvent,
viewport: Option<Rect>,
@@ -280,8 +292,13 @@ impl App {
let Some(viewport) = viewport else {
return;
};
let Some(input) = browser_mouse_input(mouse_event, viewport) else {
return;
let input = match browser_mouse_route(mouse_event, viewport) {
BrowserMouseRoute::Input(input) => input,
BrowserMouseRoute::ReleaseButtons => {
self.release_terminal_browser_mouse_buttons().await;
return;
}
BrowserMouseRoute::Ignore => return,
};
let Some(browser) = self.terminal_browser_for_current_thread() else {
return;
@@ -292,6 +309,95 @@ impl App {
}
}
pub(super) async fn release_terminal_browser_mouse_buttons(&mut self) {
let Some(browser) = self.terminal_browser_for_current_thread() else {
return;
};
if let Err(error) = browser.release_human_mouse_buttons().await {
self.chat_widget
.add_error_message(format!("Failed to release browser mouse buttons: {error}"));
}
}
pub(super) fn terminal_browser_control_inputs(
&self,
event: MousePrimaryEvent,
view: &BrowserView,
) -> Option<[BrowserMouseInput; 2]> {
if self.overlay.is_some()
|| !self.chat_widget.no_modal_or_popup_active()
|| self.owned_screen_frame.right_rail_content() != OwnedScreenRightRailContent::Browser
|| !view.visible
|| view.human_control
|| !matches!(&view.status, BrowserStatus::Running)
{
return None;
}
let (panel, press) = self.owned_screen_frame.completed_panel_click(event)?;
if panel != OwnedScreenPanel::Summary {
return None;
}
let viewport = self
.owned_screen_frame
.panel_body(OwnedScreenPanel::Summary)
.map(crate::terminal_browser::browser_viewport)?;
Some([
browser_mouse_input(press.into(), viewport)?,
browser_mouse_input(event.into(), viewport)?,
])
}
pub(super) fn terminal_browser_control_click(
&self,
event: MousePrimaryEvent,
) -> Option<TerminalBrowserControlClick> {
let browser = self.terminal_browser_for_current_thread()?;
let inputs = self.terminal_browser_control_inputs(event, &browser.view())?;
if browser.is_human_control_active() {
return None;
}
let target = self.terminal_browser_control_target()?;
Some(TerminalBrowserControlClick {
browser,
target,
inputs,
})
}
pub(super) async fn begin_terminal_browser_control_from_click(
&mut self,
click: TerminalBrowserControlClick,
) {
let TerminalBrowserControlClick {
browser,
target,
inputs,
} = click;
let result = browser.toggle_human_control(target.token).await;
let mut completion_target = target;
let error = match result {
Ok(token) => {
completion_target.token = token;
for input in inputs {
if let Err(error) = browser.send_human_mouse(input) {
self.chat_widget.add_error_message(format!(
"Failed to send browser mouse event: {error}"
));
break;
}
}
None
}
Err(error) => Some(error.to_string()),
};
self.app_event_tx
.unscoped()
.send(AppEvent::TerminalBrowserControlCompleted {
target: completion_target,
error,
});
}
pub(super) async fn terminal_browser_for_active_request(
&mut self,
) -> Option<Arc<TerminalBrowser>> {
@@ -12,6 +12,13 @@ use crossterm::event::MouseEvent;
use crossterm::event::MouseEventKind;
use ratatui::layout::Rect;
#[derive(Clone, Copy, Debug, PartialEq)]
pub(crate) enum BrowserMouseRoute {
Input(BrowserMouseInput),
ReleaseButtons,
Ignore,
}
/// Converts a crossterm key event into Carbonyl input.
///
/// Release events and keys without a Chromium keyboard equivalent are ignored.
@@ -150,6 +157,21 @@ pub(crate) fn browser_mouse_input(event: MouseEvent, viewport: Rect) -> Option<B
})
}
/// Classifies mouse input while the user has exclusive browser control.
///
/// An outside release must clear the browser worker's actual pressed-button state instead of being
/// dropped or mapped to surrounding panel chrome.
pub(crate) fn browser_mouse_route(event: MouseEvent, viewport: Rect) -> BrowserMouseRoute {
if matches!(event.kind, MouseEventKind::Up(_))
&& (viewport.is_empty() || !viewport.contains((event.column, event.row).into()))
{
BrowserMouseRoute::ReleaseButtons
} else {
browser_mouse_input(event, viewport)
.map_or(BrowserMouseRoute::Ignore, BrowserMouseRoute::Input)
}
}
fn browser_mouse_button(button: MouseButton) -> BrowserMouseButton {
match button {
MouseButton::Left => BrowserMouseButton::Left,
+2
View File
@@ -10,8 +10,10 @@ mod panel;
mod profile_approval;
mod tools;
pub(crate) use input::BrowserMouseRoute;
pub(crate) use input::browser_key_input;
pub(crate) use input::browser_mouse_input;
pub(crate) use input::browser_mouse_route;
pub(crate) use network::TerminalBrowserNetworkAvailability;
#[cfg(test)]
pub(crate) use panel::BrowserPanelAreas;
+14 -4
View File
@@ -154,13 +154,13 @@ fn render_footer(view: &BrowserView, area: Rect, buf: &mut Buffer) {
Line::from(" Ctrl+] ".cyan())
};
line.render_ref(area, buf);
} else {
} else if matches!(&view.status, BrowserStatus::Running) {
let line = if area.width >= 63 {
Line::from(vec![
" Esc ".cyan(),
"hide".dim(),
" ".into(),
"/browser control".cyan(),
" click page ".cyan(),
" take control".dim(),
" ".into(),
"/browser close".cyan(),
@@ -171,13 +171,23 @@ fn render_footer(view: &BrowserView, area: Rect, buf: &mut Buffer) {
" Esc ".cyan(),
"hide".dim(),
" ".into(),
"/browser control".cyan(),
" click page ".cyan(),
" take control".dim(),
])
} else if area.width >= 26 {
Line::from(vec![
" Esc ".cyan(),
"hide".dim(),
" ".into(),
" click ".cyan(),
"control".dim(),
])
} else {
Line::from(vec![" Esc ".cyan(), "hide".dim()])
Line::from(vec![" click ".cyan(), "control".dim()])
};
line.render_ref(area, buf);
} else {
Line::from(vec![" Esc ".cyan(), "hide".dim()]).render_ref(area, buf);
}
}
@@ -11,4 +11,4 @@ x UI
Esc hide /browser control take control
Esc hide click page take control
@@ -11,4 +11,4 @@ PATH; install it or set CODEX_CARBONYL_BINARY
Esc hide /browser control take control
Esc hide
@@ -35,6 +35,8 @@ use crate::app_event::TerminalBrowserProfileCommand;
use crate::app_event_sender::AppEventSender;
use crate::bottom_pane::ListSelectionView;
use crate::render::renderable::Renderable;
use crate::terminal_browser::BrowserMouseRoute;
use crate::terminal_browser::browser_mouse_route;
use super::BrowserPanelAreas;
use super::browser_key_input;
@@ -371,6 +373,34 @@ fn mouse_events_outside_the_browser_viewport_are_ignored() {
);
}
#[test]
fn active_mouse_capture_routes_outside_release_to_cleanup() {
let viewport = Rect::new(
/*x*/ 10, /*y*/ 5, /*width*/ 40, /*height*/ 20,
);
let outside_release = MouseEvent {
kind: MouseEventKind::Up(MouseButton::Left),
column: 9,
row: 30,
modifiers: KeyModifiers::SHIFT,
};
assert_eq!(
browser_mouse_route(outside_release, viewport),
BrowserMouseRoute::ReleaseButtons
);
assert_eq!(
browser_mouse_route(
MouseEvent {
kind: MouseEventKind::Down(MouseButton::Left),
..outside_release
},
viewport,
),
BrowserMouseRoute::Ignore
);
}
#[test]
fn terminal_browser_unavailable_panel_wraps_reason_snapshot() {
let view = BrowserView {
+1 -1
View File
@@ -101,7 +101,7 @@ test-terminal-browser-panel *args:
-p codex-core \
-p codex-app-server \
-p codex-tui \
-E 'package(codex-utils-pty) | package(codex-terminal-browser) | package(codex-sandboxing) | package(codex-linux-sandbox) | package(codex-features) | package(codex-protocol) | package(codex-app-server-protocol) | (package(codex-exec-server) & test(/sandbox_request_allows_prepared_managed_proxy_port|sandbox_exec_request_carries_helper_env/)) | (package(codex-core) & test(/config_schema_matches_fixture|active_profile_update_rebuilds_network_proxy_config|current_network_proxy_runtime|managed_network_proxy_decider_survives_full_access_start|managed_network_proxy_refresh_failure|new_turn_refreshes_managed_network_proxy_for_sandbox_change|danger_full_access_turns_do_not_expose_managed_network_proxy|workspace_write_turns_continue_to_expose_managed_network_proxy/)) | (package(codex-app-server) & test(/thread_start_returns_live_managed_proxy_in_experimental_extra|thread_settings_update_reports_current_managed_proxy_runtime/)) | (package(codex-tui) & test(/terminal_browser|bare_browser|browser_show_alias|network_reconciliation|inactive_thread_settings_notification|proxy_runtime|thread_read_fallback_uses_active_permission_settings/))' \
-E 'package(codex-utils-pty) | package(codex-terminal-browser) | package(codex-sandboxing) | package(codex-linux-sandbox) | package(codex-features) | package(codex-protocol) | package(codex-app-server-protocol) | (package(codex-exec-server) & test(/sandbox_request_allows_prepared_managed_proxy_port|sandbox_exec_request_carries_helper_env/)) | (package(codex-core) & test(/config_schema_matches_fixture|active_profile_update_rebuilds_network_proxy_config|current_network_proxy_runtime|managed_network_proxy_decider_survives_full_access_start|managed_network_proxy_refresh_failure|new_turn_refreshes_managed_network_proxy_for_sandbox_change|danger_full_access_turns_do_not_expose_managed_network_proxy|workspace_write_turns_continue_to_expose_managed_network_proxy/)) | (package(codex-app-server) & test(/thread_start_returns_live_managed_proxy_in_experimental_extra|thread_settings_update_reports_current_managed_proxy_runtime/)) | (package(codex-tui) & test(/terminal_browser|owned_screen_browser|bare_browser|browser_show_alias|network_reconciliation|inactive_thread_settings_notification|proxy_runtime|thread_read_fallback_uses_active_permission_settings/))' \
"$@"
# Run from the repository root so scripts that resolve paths from `cwd` see