diff --git a/codex-rs/wasm-harness/examples/browser/index.html b/codex-rs/wasm-harness/examples/browser/index.html index e403fc1015..4bda60c630 100644 --- a/codex-rs/wasm-harness/examples/browser/index.html +++ b/codex-rs/wasm-harness/examples/browser/index.html @@ -191,7 +191,11 @@
- +
@@ -209,9 +213,11 @@ const clearApiKeyButton = document.querySelector("#clear-api-key"); const API_KEY_STORAGE_KEY = "codexWasmHarness.apiKey"; const codex = new BrowserCodex(readStoredApiKey()); + const sandbox = createSandboxKernel(); apiKeyInput.value = readStoredApiKey(); updateApiKeyStatus(); + codex.set_code_executor(async (code) => sandbox.execute(code)); apiKeyInput.addEventListener("input", () => { const apiKey = apiKeyInput.value.trim(); @@ -230,7 +236,31 @@ form.addEventListener("submit", async (event) => { event.preventDefault(); events.replaceChildren(); - await codex.submit_turn(input.value, renderEvent); + try { + await codex.submit_turn(input.value, renderEvent); + } catch (error) { + renderEvent({ + type: "page_error", + message: error instanceof Error ? error.message : String(error), + }); + } + }); + + window.addEventListener("error", (event) => { + renderEvent({ + type: "window_error", + message: event.message, + }); + }); + + window.addEventListener("unhandledrejection", (event) => { + renderEvent({ + type: "unhandled_rejection", + message: + event.reason instanceof Error + ? event.reason.message + : String(event.reason), + }); }); function readStoredApiKey() { @@ -270,6 +300,112 @@ card.append(title, body); events.append(card); } + + function createSandboxKernel() { + const iframe = document.createElement("iframe"); + iframe.setAttribute("sandbox", "allow-scripts allow-same-origin"); + iframe.style.display = "none"; + iframe.srcdoc = ` + + + + diff --git a/codex-rs/wasm-harness/src/lib.rs b/codex-rs/wasm-harness/src/lib.rs index a03b19986f..40459c693e 100644 --- a/codex-rs/wasm-harness/src/lib.rs +++ b/codex-rs/wasm-harness/src/lib.rs @@ -1,14 +1,16 @@ //! Browser-facing prototype facade for a future Codex WASM harness. //! //! This crate intentionally starts outside `codex-core`: the first milestone is -//! a working browser boundary that streams Codex-shaped turn events. Later -//! iterations can replace the direct model request with the real Codex -//! `RegularTask` / `run_turn` path as host services become injectable. +//! a working browser boundary that can run a minimal Codex-shaped turn loop in +//! the browser. The loop in this crate now uses the real Responses API and can +//! execute a browser-provided code tool callback, but it still does not call +//! `codex-core::run_turn` or `RegularTask::run`. use js_sys::Function; -use js_sys::Reflect; +use js_sys::Promise; use serde::Deserialize; use serde::Serialize; +use serde_json::Value; use wasm_bindgen::JsCast; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; @@ -17,15 +19,24 @@ use web_sys::RequestInit; use web_sys::RequestMode; use web_sys::Response; -const DEFAULT_INSTRUCTIONS: &str = "You are Codex running in a browser WASM prototype."; +const BASE_INSTRUCTIONS: &str = include_str!("../../core/prompt.md"); +const BROWSER_TOOLING_NOTE: &str = concat!( + "Browser prototype note:\n", + "- Only one host tool is available: `exec_js`.\n", + "- `exec_js` runs JavaScript inside a browser-managed sandbox and returns stdout-like text.\n", + "- Native shell, filesystem, MCP, and plugin tools are unavailable in this prototype." +); const RESPONSES_API_URL: &str = "https://api.openai.com/v1/responses"; const RESPONSES_MODEL: &str = "gpt-5.1"; +const MAX_TOOL_ROUNDS: usize = 8; +const EXEC_JS_TOOL_NAME: &str = "exec_js"; /// Browser entrypoint for the prototype harness. #[wasm_bindgen] pub struct BrowserCodex { api_key: String, next_turn_id: u32, + code_executor: Option, } #[wasm_bindgen] @@ -34,13 +45,14 @@ impl BrowserCodex { /// /// `api_key` may be empty. When it is empty, the harness uses a /// deterministic local demo response. When it is present, the harness makes - /// a browser `fetch` call to the Responses API from Rust/WASM. + /// browser `fetch` calls to the Responses API from Rust/WASM. #[wasm_bindgen(constructor)] #[must_use] pub fn new(api_key: String) -> Self { Self { api_key, next_turn_id: 0, + code_executor: None, } } @@ -49,6 +61,16 @@ impl BrowserCodex { self.api_key = api_key; } + /// Registers the JavaScript callback used for the `exec_js` tool. + pub fn set_code_executor(&mut self, executor: Function) { + self.code_executor = Some(executor); + } + + /// Clears the browser-side `exec_js` tool callback. + pub fn clear_code_executor(&mut self) { + self.code_executor = None; + } + /// Submits one browser turn and calls `on_event` for every emitted event. /// /// The method resolves after `turn_complete` has been emitted. This mirrors @@ -78,22 +100,13 @@ impl BrowserCodex { }, )?; - let agent_message = self.sample(&prompt).await?; + let agent_message = if self.api_key.trim().is_empty() { + default_demo_response(&prompt) + } else { + self.run_turn_with_responses_api(&turn_id, &prompt, &on_event) + .await? + }; - emit_event( - &on_event, - &HarnessEvent::AgentMessageDelta { - turn_id: turn_id.clone(), - delta: agent_message.clone(), - }, - )?; - emit_event( - &on_event, - &HarnessEvent::AgentMessage { - turn_id: turn_id.clone(), - message: agent_message.clone(), - }, - )?; emit_event( &on_event, &HarnessEvent::TurnComplete { @@ -107,21 +120,184 @@ impl BrowserCodex { } impl BrowserCodex { - async fn sample(&self, prompt: &str) -> Result { - if self.api_key.trim().is_empty() { - return Ok(default_demo_response(prompt)); + async fn run_turn_with_responses_api( + &self, + turn_id: &str, + prompt: &str, + on_event: &Function, + ) -> Result { + let mut previous_response_id: Option = None; + let mut input = Value::String(prompt.to_string()); + let instructions = build_browser_instructions(); + let mut last_agent_message: Option = None; + + for round in 0..MAX_TOOL_ROUNDS { + let response = fetch_responses_api( + self.api_key.trim(), + ResponsesRequestBody { + model: RESPONSES_MODEL, + instructions: instructions.clone(), + input, + previous_response_id: previous_response_id.clone(), + tools: self.responses_tools(), + parallel_tool_calls: false, + }, + ) + .await?; + + previous_response_id = response.id.clone(); + + let agent_message = extract_response_text(&response); + if !agent_message.is_empty() { + emit_event( + on_event, + &HarnessEvent::AgentMessageDelta { + turn_id: turn_id.to_string(), + delta: agent_message.clone(), + }, + )?; + emit_event( + on_event, + &HarnessEvent::AgentMessage { + turn_id: turn_id.to_string(), + message: agent_message.clone(), + }, + )?; + last_agent_message = Some(agent_message); + } + + let function_calls = extract_function_calls(&response)?; + if function_calls.is_empty() { + return Ok(last_agent_message.unwrap_or_else(|| { + "Responses API returned no assistant message.".to_string() + })); + } + + let response_id = previous_response_id.clone().ok_or_else(|| { + js_error("Responses API omitted response.id for a tool-calling turn") + })?; + let mut tool_outputs = Vec::with_capacity(function_calls.len()); + for function_call in function_calls { + emit_event( + on_event, + &HarnessEvent::ToolCallStarted { + turn_id: turn_id.to_string(), + response_id: response_id.clone(), + call_id: function_call.call_id.clone(), + name: function_call.name.clone(), + arguments: function_call.arguments.clone(), + }, + )?; + + let output = self.execute_function_call(&function_call).await; + match output { + Ok(output) => { + emit_event( + on_event, + &HarnessEvent::ToolCallCompleted { + turn_id: turn_id.to_string(), + response_id: response_id.clone(), + call_id: function_call.call_id.clone(), + name: function_call.name.clone(), + output: output.clone(), + }, + )?; + tool_outputs.push(tool_output_item(&function_call.call_id, output)); + } + Err(err) => { + emit_event( + on_event, + &HarnessEvent::TurnError { + turn_id: turn_id.to_string(), + message: err.as_string().unwrap_or_else(|| { + "tool execution failed with a non-string JavaScript error" + .to_string() + }), + }, + )?; + return Err(err); + } + } + } + + input = Value::Array(tool_outputs); + + if round + 1 == MAX_TOOL_ROUNDS { + return Err(js_error("turn exceeded the browser tool-round limit")); + } } - sample_with_responses_api(self.api_key.trim(), prompt).await + Err(js_error("browser turn loop exited unexpectedly")) + } + + fn responses_tools(&self) -> Option> { + self.code_executor + .as_ref() + .map(|_| vec![ResponsesTool::exec_js()]) + } + + async fn execute_function_call( + &self, + function_call: &ResponsesFunctionCall, + ) -> Result { + if function_call.name != EXEC_JS_TOOL_NAME { + return Err(js_error(format!( + "browser prototype does not implement tool `{}`", + function_call.name + ))); + } + + let executor = self.code_executor.as_ref().ok_or_else(|| { + js_error("`exec_js` was requested but no browser executor is registered") + })?; + let args: ExecJsArguments = + serde_json::from_str(&function_call.arguments).map_err(js_error)?; + let value = executor.call1(&JsValue::NULL, &JsValue::from_str(&args.code))?; + let value = await_possible_promise(value).await?; + js_value_to_string(value) } } #[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct ResponsesRequest<'a> { +struct ResponsesRequestBody { model: &'static str, - instructions: &'static str, - input: &'a str, + instructions: String, + input: Value, + #[serde(skip_serializing_if = "Option::is_none")] + previous_response_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tools: Option>, + parallel_tool_calls: bool, +} + +#[derive(Debug, Serialize)] +struct ResponsesTool { + #[serde(rename = "type")] + kind: &'static str, + name: &'static str, + description: &'static str, + parameters: Value, +} + +impl ResponsesTool { + fn exec_js() -> Self { + Self { + kind: "function", + name: EXEC_JS_TOOL_NAME, + description: "Execute JavaScript inside the browser sandbox and return the textual result.", + parameters: serde_json::json!({ + "type": "object", + "properties": { + "code": { + "type": "string", + "description": "JavaScript source code to execute in the browser sandbox." + } + }, + "required": ["code"], + "additionalProperties": false + }), + } + } } #[derive(Debug, Serialize)] @@ -144,6 +320,24 @@ enum HarnessEvent<'a> { turn_id: String, message: String, }, + ToolCallStarted { + turn_id: String, + response_id: String, + call_id: String, + name: String, + arguments: String, + }, + ToolCallCompleted { + turn_id: String, + response_id: String, + call_id: String, + name: String, + output: String, + }, + TurnError { + turn_id: String, + message: String, + }, TurnComplete { turn_id: String, last_agent_message: Option, @@ -152,6 +346,7 @@ enum HarnessEvent<'a> { #[derive(Debug, Deserialize)] struct ResponsesBody { + id: Option, output_text: Option, output: Option>, error: Option, @@ -164,6 +359,11 @@ struct ResponsesError { #[derive(Debug, Deserialize)] struct ResponsesOutputItem { + #[serde(rename = "type")] + kind: String, + call_id: Option, + name: Option, + arguments: Option, content: Option>, } @@ -173,12 +373,22 @@ struct ResponsesContentItem { output_text: Option, } -async fn sample_with_responses_api(api_key: &str, prompt: &str) -> Result { - let body = ResponsesRequest { - model: RESPONSES_MODEL, - instructions: DEFAULT_INSTRUCTIONS, - input: prompt, - }; +#[derive(Debug)] +struct ResponsesFunctionCall { + call_id: String, + name: String, + arguments: String, +} + +#[derive(Debug, Deserialize)] +struct ExecJsArguments { + code: String, +} + +async fn fetch_responses_api( + api_key: &str, + body: ResponsesRequestBody, +) -> Result { let body = serde_json::to_string(&body).map_err(js_error)?; let headers = Headers::new()?; @@ -195,13 +405,13 @@ async fn sample_with_responses_api(api_key: &str, prompt: &str) -> Result Result Result { @@ -222,29 +432,87 @@ fn parse_response_body(value: JsValue) -> Result { serde_json::from_str(&json).map_err(js_error) } -fn extract_response_text(response: ResponsesBody) -> String { - if let Some(output_text) = response.output_text +fn extract_response_text(response: &ResponsesBody) -> String { + if let Some(output_text) = &response.output_text && !output_text.is_empty() { - return output_text; + return output_text.clone(); } let mut chunks = Vec::new(); - for item in response.output.unwrap_or_default() { - for content in item.content.unwrap_or_default() { - if let Some(text) = content.text { - chunks.push(text); - } else if let Some(output_text) = content.output_text { - chunks.push(output_text); + for item in response.output.as_ref().into_iter().flatten() { + for content in item.content.as_ref().into_iter().flatten() { + if let Some(text) = &content.text { + chunks.push(text.clone()); + } else if let Some(output_text) = &content.output_text { + chunks.push(output_text.clone()); } } } - if chunks.is_empty() { - "Responses API returned no output text.".to_string() - } else { - chunks.join("\n") + chunks.join("\n") +} + +fn extract_function_calls(response: &ResponsesBody) -> Result, JsValue> { + let mut function_calls = Vec::new(); + for item in response.output.as_ref().into_iter().flatten() { + if item.kind != "function_call" { + continue; + } + let call_id = item + .call_id + .clone() + .ok_or_else(|| js_error("Responses API function_call item omitted call_id"))?; + let name = item + .name + .clone() + .ok_or_else(|| js_error("Responses API function_call item omitted name"))?; + let arguments = item + .arguments + .clone() + .ok_or_else(|| js_error("Responses API function_call item omitted arguments"))?; + function_calls.push(ResponsesFunctionCall { + call_id, + name, + arguments, + }); } + Ok(function_calls) +} + +fn tool_output_item(call_id: &str, output: String) -> Value { + serde_json::json!({ + "type": "function_call_output", + "call_id": call_id, + "output": output, + }) +} + +fn build_browser_instructions() -> String { + format!("{BASE_INSTRUCTIONS}\n\n{BROWSER_TOOLING_NOTE}") +} + +async fn await_possible_promise(value: JsValue) -> Result { + if let Ok(promise) = value.clone().dyn_into::() { + JsFuture::from(promise).await + } else { + Ok(value) + } +} + +fn js_value_to_string(value: JsValue) -> Result { + if let Some(text) = value.as_string() { + return Ok(text); + } + + if value.is_undefined() || value.is_null() { + return Ok(String::new()); + } + + let json = js_sys::JSON::stringify(&value)?; + Ok(json + .as_string() + .unwrap_or_else(|| "[non-string value]".to_string())) } fn emit_event(on_event: &Function, event: &HarnessEvent<'_>) -> Result<(), JsValue> { @@ -259,41 +527,85 @@ fn default_demo_response(prompt: &str) -> String { "Here is a minimal hello world example:\n\n```js\nconsole.log(\"hello world\");\n```" .to_string() } else { - format!( - "Browser Codex prototype received the prompt, but no API key was configured: {prompt}" - ) + format!("Demo mode is active because no API key was provided. Prompt received:\n\n{prompt}") } } -fn response_error(error: JsValue) -> JsValue { - if let Some(message) = error.as_string() { - return js_error(message); - } - - if error.is_object() - && let Ok(value) = Reflect::get(&error, &JsValue::from_str("message")) - && let Some(message) = value.as_string() - { - return js_error(message); - } - - js_error("Responses API request failed") +fn js_error(error: impl ToString) -> JsValue { + JsValue::from_str(&error.to_string()) } -fn js_error(message: impl ToString) -> JsValue { - JsValue::from_str(&message.to_string()) +fn js_fetch_error(error: JsValue) -> JsValue { + JsValue::from_str(&format!( + "browser fetch failed: {}", + js_value_to_string_lossy(&error) + )) +} + +fn js_value_to_string_lossy(value: &JsValue) -> String { + if let Some(text) = value.as_string() { + return text; + } + + js_sys::JSON::stringify(value) + .ok() + .and_then(|text| text.as_string()) + .unwrap_or_else(|| "[non-string javascript error]".to_string()) } #[cfg(test)] mod tests { - use super::default_demo_response; + use super::ResponsesBody; + use super::build_browser_instructions; + use super::extract_function_calls; + use super::extract_response_text; use pretty_assertions::assert_eq; #[test] - fn default_demo_response_handles_hello_world() { + fn browser_instructions_append_browser_note() { + let instructions = build_browser_instructions(); + assert!(instructions.contains("Only one host tool is available: `exec_js`")); + assert!(instructions.contains("You are a coding agent running in the Codex CLI")); + } + + #[test] + fn extracts_output_text_when_present() { + let response: ResponsesBody = serde_json::from_str( + r#"{ + "id": "resp_123", + "output_text": "final answer", + "output": [] + }"#, + ) + .expect("response should deserialize"); + assert_eq!(extract_response_text(&response), "final answer"); + } + + #[test] + fn extracts_function_calls_from_output_items() { + let response: ResponsesBody = serde_json::from_str( + r#"{ + "id": "resp_123", + "output": [ + { + "type": "function_call", + "call_id": "call_123", + "name": "exec_js", + "arguments": "{\"code\":\"console.log('hi')\"}" + } + ] + }"#, + ) + .expect("response should deserialize"); + + let function_calls = + extract_function_calls(&response).expect("function calls should parse"); + assert_eq!(function_calls.len(), 1); + assert_eq!(function_calls[0].call_id, "call_123"); + assert_eq!(function_calls[0].name, "exec_js"); assert_eq!( - default_demo_response("write hello world"), - "Here is a minimal hello world example:\n\n```js\nconsole.log(\"hello world\");\n```" + function_calls[0].arguments, + r#"{"code":"console.log('hi')"}"# ); } }