diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 778dee3ea7..f7b56d7a03 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1457,6 +1457,7 @@ dependencies = [ "codex-utils-pty", "core_test_support", "futures", + "image", "opentelemetry", "opentelemetry_sdk", "owo-colors", diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index c239546e22..0a608471fd 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -384,6 +384,111 @@ } ] }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionArtifacts": { + "properties": { + "replayFrameCount": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayFrameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayGifImageUrl": { + "type": [ + "string", + "null" + ] + }, + "screenshotImageUrl": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "BrowserSessionState": { "properties": { "selectedTabId": { @@ -404,6 +509,16 @@ }, "BrowserSessionUpdatedNotification": { "properties": { + "artifacts": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionArtifacts" + }, + { + "type": "null" + } + ] + }, "browserSessionId": { "type": "string" }, @@ -2760,6 +2875,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 90a0366d41..cb14d466c7 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -5515,6 +5515,111 @@ } ] }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/v2/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/v2/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/v2/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionArtifacts": { + "properties": { + "replayFrameCount": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayFrameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayGifImageUrl": { + "type": [ + "string", + "null" + ] + }, + "screenshotImageUrl": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "BrowserSessionState": { "properties": { "selectedTabId": { @@ -5536,6 +5641,16 @@ "BrowserSessionUpdatedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "artifacts": { + "anyOf": [ + { + "$ref": "#/definitions/v2/BrowserSessionArtifacts" + }, + { + "type": "null" + } + ] + }, "browserSessionId": { "type": "string" }, @@ -12543,6 +12658,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/v2/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/v2/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index e4889c6ee7..76168a31ce 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -715,6 +715,111 @@ } ] }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionArtifacts": { + "properties": { + "replayFrameCount": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayFrameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayGifImageUrl": { + "type": [ + "string", + "null" + ] + }, + "screenshotImageUrl": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "BrowserSessionState": { "properties": { "selectedTabId": { @@ -736,6 +841,16 @@ "BrowserSessionUpdatedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "artifacts": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionArtifacts" + }, + { + "type": "null" + } + ] + }, "browserSessionId": { "type": "string" }, @@ -10303,6 +10418,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json index 7d166342eb..e26d365e0a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json @@ -1,6 +1,39 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BrowserSessionArtifacts": { + "properties": { + "replayFrameCount": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayFrameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "replayGifImageUrl": { + "type": [ + "string", + "null" + ] + }, + "screenshotImageUrl": { + "type": [ + "string", + "null" + ] + } + }, + "type": "object" + }, "BrowserSessionState": { "properties": { "selectedTabId": { @@ -44,6 +77,16 @@ } }, "properties": { + "artifacts": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionArtifacts" + }, + { + "type": "null" + } + ] + }, "browserSessionId": { "type": "string" }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 3964107865..9fa9e76c13 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -1,6 +1,119 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -869,6 +982,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index abb8aee5dc..950d9ff54c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -1,6 +1,119 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -869,6 +982,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index 2e0c3605e7..75ec951fb0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -1,6 +1,119 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1012,6 +1125,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 5ae0c5a122..2adcd0326e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -66,6 +66,119 @@ } ] }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1519,6 +1632,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 126d78603a..ea2a5b6a9f 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -4,6 +4,119 @@ "AgentPath": { "type": "string" }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1277,6 +1390,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index dfdab228df..5b70a55ed4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -4,6 +4,119 @@ "AgentPath": { "type": "string" }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1277,6 +1390,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 8f48dee4b7..a6b9f8e7bb 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -4,6 +4,119 @@ "AgentPath": { "type": "string" }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1277,6 +1390,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index edccc337da..86faf5c733 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -66,6 +66,119 @@ } ] }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1519,6 +1632,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index cc41aac27f..508ac9b53d 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -4,6 +4,119 @@ "AgentPath": { "type": "string" }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1277,6 +1390,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index c3b50fee3b..e7ca5ebf84 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -66,6 +66,119 @@ } ] }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1519,6 +1632,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 2240150394..1d99ff68fd 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -4,6 +4,119 @@ "AgentPath": { "type": "string" }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1277,6 +1390,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 41b0d2d409..6d818d733a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -4,6 +4,119 @@ "AgentPath": { "type": "string" }, + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1277,6 +1390,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 770cc920cf..7c43032bac 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -1,6 +1,119 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1012,6 +1125,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 7f1c3e4948..e116b5011e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -1,6 +1,119 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1012,6 +1125,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 761ddc9a62..5889b1be36 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -1,6 +1,119 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "BrowserReplayFrame": { + "properties": { + "browserState": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserSessionState" + }, + { + "type": "null" + } + ] + }, + "imageUrl": { + "type": [ + "string", + "null" + ] + }, + "sequenceNumber": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "sourceItemId": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/DynamicToolCallStatus" + }, + "textSnapshot": { + "type": [ + "string", + "null" + ] + }, + "textSnapshotFormat": { + "anyOf": [ + { + "$ref": "#/definitions/BrowserReplayTextSnapshotFormat" + }, + { + "type": "null" + } + ] + }, + "tool": { + "type": "string" + } + }, + "required": [ + "sequenceNumber", + "sourceItemId", + "status", + "tool" + ], + "type": "object" + }, + "BrowserReplayRenderMode": { + "enum": [ + "animation", + "frames", + "textOnly" + ], + "type": "string" + }, + "BrowserReplayTextSnapshotFormat": { + "enum": [ + "plain", + "json" + ], + "type": "string" + }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -1012,6 +1125,51 @@ "title": "DynamicToolCallThreadItem", "type": "object" }, + { + "properties": { + "animationImageUrl": { + "type": [ + "string", + "null" + ] + }, + "frameDurationMs": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + }, + "frames": { + "items": { + "$ref": "#/definitions/BrowserReplayFrame" + }, + "type": "array" + }, + "id": { + "type": "string" + }, + "renderMode": { + "$ref": "#/definitions/BrowserReplayRenderMode" + }, + "type": { + "enum": [ + "browserReplay" + ], + "title": "BrowserReplayThreadItemType", + "type": "string" + } + }, + "required": [ + "frames", + "id", + "renderMode", + "type" + ], + "title": "BrowserReplayThreadItem", + "type": "object" + }, { "properties": { "agentsStates": { diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayFrame.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayFrame.ts new file mode 100644 index 0000000000..e2ae18c77d --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayFrame.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BrowserReplayTextSnapshotFormat } from "./BrowserReplayTextSnapshotFormat"; +import type { BrowserSessionState } from "./BrowserSessionState"; +import type { DynamicToolCallStatus } from "./DynamicToolCallStatus"; + +export type BrowserReplayFrame = { sequenceNumber: number, sourceItemId: string, tool: string, status: DynamicToolCallStatus, imageUrl?: string | null, textSnapshot?: string | null, textSnapshotFormat?: BrowserReplayTextSnapshotFormat | null, browserState?: BrowserSessionState | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayRenderMode.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayRenderMode.ts new file mode 100644 index 0000000000..8fb48c5411 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayRenderMode.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type BrowserReplayRenderMode = "animation" | "frames" | "textOnly"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayTextSnapshotFormat.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayTextSnapshotFormat.ts new file mode 100644 index 0000000000..6971d1bacf --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserReplayTextSnapshotFormat.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type BrowserReplayTextSnapshotFormat = "plain" | "json"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionArtifacts.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionArtifacts.ts new file mode 100644 index 0000000000..42538af1dd --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionArtifacts.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type BrowserSessionArtifacts = { screenshotImageUrl?: string | null, replayGifImageUrl?: string | null, replayFrameCount?: number | null, replayFrameDurationMs?: number | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts index 89259f2c25..f84b109547 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts @@ -1,6 +1,7 @@ // GENERATED CODE! DO NOT MODIFY BY HAND! // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts"; import type { BrowserSessionState } from "./BrowserSessionState"; -export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, }; +export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, artifacts?: BrowserSessionArtifacts | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index 9202f3728f..a94c404b95 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -4,6 +4,8 @@ import type { MessagePhase } from "../MessagePhase"; import type { ReasoningEffort } from "../ReasoningEffort"; import type { JsonValue } from "../serde_json/JsonValue"; +import type { BrowserReplayFrame } from "./BrowserReplayFrame"; +import type { BrowserReplayRenderMode } from "./BrowserReplayRenderMode"; import type { CollabAgentState } from "./CollabAgentState"; import type { CollabAgentTool } from "./CollabAgentTool"; import type { CollabAgentToolCallStatus } from "./CollabAgentToolCallStatus"; @@ -22,78 +24,78 @@ import type { PatchApplyStatus } from "./PatchApplyStatus"; import type { UserInput } from "./UserInput"; import type { WebSearchAction } from "./WebSearchAction"; -export type ThreadItem = { "type": "userMessage", id: string, content: Array, } | { "type": "hookPrompt", id: string, fragments: Array, } | { "type": "agentMessage", id: string, text: string, phase: MessagePhase | null, memoryCitation: MemoryCitation | null, } | { "type": "plan", id: string, text: string, } | { "type": "reasoning", id: string, summary: Array, content: Array, } | { "type": "commandExecution", id: string, +export type ThreadItem = { "type": "userMessage", id: string, content: Array, } | { "type": "hookPrompt", id: string, fragments: Array, } | { "type": "agentMessage", id: string, text: string, phase: MessagePhase | null, memoryCitation: MemoryCitation | null, } | { "type": "plan", id: string, text: string, } | { "type": "reasoning", id: string, summary: Array, content: Array, } | { "type": "commandExecution", id: string, /** * The command to be executed. */ -command: string, +command: string, /** * The command's working directory. */ -cwd: string, +cwd: string, /** * Identifier for the underlying PTY process (when available). */ -processId: string | null, source: CommandExecutionSource, status: CommandExecutionStatus, +processId: string | null, source: CommandExecutionSource, status: CommandExecutionStatus, /** * A best-effort parsing of the command to understand the action(s) it will perform. * This returns a list of CommandAction objects because a single shell command may * be composed of many commands piped together. */ -commandActions: Array, +commandActions: Array, /** * The command's output, aggregated from stdout and stderr. */ -aggregatedOutput: string | null, +aggregatedOutput: string | null, /** * The command's exit code. */ -exitCode: number | null, +exitCode: number | null, /** * The duration of the command execution in milliseconds. */ -durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null, +durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null, /** * The duration of the MCP tool call in milliseconds. */ -durationMs: number | null, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array | null, success: boolean | null, +durationMs: number | null, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array | null, success: boolean | null, /** * The duration of the dynamic tool call in milliseconds. */ -durationMs: number | null, } | { "type": "collabAgentToolCall", +durationMs: number | null, } | { "type": "browserReplay", id: string, frames: Array, renderMode: BrowserReplayRenderMode, animationImageUrl?: string | null, frameDurationMs?: number | null, } | { "type": "collabAgentToolCall", /** * Unique identifier for this collab tool call. */ -id: string, +id: string, /** * Name of the collab tool that was invoked. */ -tool: CollabAgentTool, +tool: CollabAgentTool, /** * Current status of the collab tool call. */ -status: CollabAgentToolCallStatus, +status: CollabAgentToolCallStatus, /** * Thread ID of the agent issuing the collab request. */ -senderThreadId: string, +senderThreadId: string, /** * Thread ID of the receiving agent, when applicable. In case of spawn operation, * this corresponds to the newly spawned agent. */ -receiverThreadIds: Array, +receiverThreadIds: Array, /** * Prompt text sent as part of the collab tool call, when available. */ -prompt: string | null, +prompt: string | null, /** * Model requested for the spawned agent, when applicable. */ -model: string | null, +model: string | null, /** * Reasoning effort requested for the spawned agent, when applicable. */ -reasoningEffort: ReasoningEffort | null, +reasoningEffort: ReasoningEffort | null, /** * Last known status of the target agents, when available. */ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index a506c435ff..799d76c530 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -25,6 +25,10 @@ export type { AppsDefaultConfig } from "./AppsDefaultConfig"; export type { AppsListParams } from "./AppsListParams"; export type { AppsListResponse } from "./AppsListResponse"; export type { AskForApproval } from "./AskForApproval"; +export type { BrowserReplayFrame } from "./BrowserReplayFrame"; +export type { BrowserReplayRenderMode } from "./BrowserReplayRenderMode"; +export type { BrowserReplayTextSnapshotFormat } from "./BrowserReplayTextSnapshotFormat"; +export type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts"; export type { BrowserSessionState } from "./BrowserSessionState"; export type { BrowserSessionUpdatedNotification } from "./BrowserSessionUpdatedNotification"; export type { BrowserTabState } from "./BrowserTabState"; diff --git a/codex-rs/app-server-protocol/src/export.rs b/codex-rs/app-server-protocol/src/export.rs index b89f23c666..3ea181ab82 100644 --- a/codex-rs/app-server-protocol/src/export.rs +++ b/codex-rs/app-server-protocol/src/export.rs @@ -179,6 +179,10 @@ pub fn generate_ts_with_options( } } + if !ts_files.is_empty() { + normalize_generated_ts_files(&ts_files)?; + } + Ok(()) } @@ -1907,6 +1911,41 @@ fn prepend_header_if_missing(path: &Path) -> Result<()> { Ok(()) } +pub(crate) fn normalize_typescript_text(text: &str) -> String { + let normalized = text.replace("\r\n", "\n").replace('\r', "\n"); + let had_trailing_newline = normalized.ends_with('\n'); + let mut result = normalized + .split('\n') + .map(|line| line.trim_end_matches([' ', '\t'])) + .collect::>() + .join("\n"); + if had_trailing_newline && !result.ends_with('\n') { + result.push('\n'); + } + result +} + +fn normalize_generated_ts_files(paths: &[PathBuf]) -> Result<()> { + for path in paths { + if !path + .file_name() + .and_then(OsStr::to_str) + .is_some_and(|name| name == "ThreadItem.ts") + { + continue; + } + let content = fs::read_to_string(path) + .with_context(|| format!("Failed to read generated TypeScript {}", path.display()))?; + let normalized = normalize_typescript_text(&content); + if normalized == content { + continue; + } + fs::write(path, normalized) + .with_context(|| format!("Failed to normalize {}", path.display()))?; + } + Ok(()) +} + fn ts_files_in(dir: &Path) -> Result> { let mut files = Vec::new(); for entry in diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 11dfe29769..0cba10eb82 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -1921,6 +1921,57 @@ mod tests { ); } + #[test] + fn reconstructs_dynamic_tool_items_from_response_events_only() { + let events = vec![ + EventMsg::TurnStarted(TurnStartedEvent { + turn_id: "turn-1".into(), + model_context_window: None, + collaboration_mode_kind: Default::default(), + }), + EventMsg::UserMessage(UserMessageEvent { + message: "run dynamic tool".into(), + images: None, + text_elements: Vec::new(), + local_images: Vec::new(), + }), + EventMsg::DynamicToolCallResponse(DynamicToolCallResponseEvent { + call_id: "dyn-1".into(), + turn_id: "turn-1".into(), + tool: "lookup_ticket".into(), + arguments: serde_json::json!({"id":"ABC-123"}), + content_items: vec![CoreDynamicToolCallOutputContentItem::InputText { + text: "Ticket is open".into(), + }], + success: true, + error: None, + duration: Duration::from_millis(42), + }), + ]; + + let items = events + .into_iter() + .map(RolloutItem::EventMsg) + .collect::>(); + let turns = build_turns_from_rollout_items(&items); + assert_eq!(turns.len(), 1); + assert_eq!(turns[0].items.len(), 2); + assert_eq!( + turns[0].items[1], + ThreadItem::DynamicToolCall { + id: "dyn-1".into(), + tool: "lookup_ticket".into(), + arguments: serde_json::json!({"id":"ABC-123"}), + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![DynamicToolCallOutputContentItem::InputText { + text: "Ticket is open".into(), + }]), + success: Some(true), + duration_ms: Some(42), + } + ); + } + #[test] fn reconstructs_declined_exec_and_patch_items() { let events = vec![ diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index e61e8a23ac..9e95006353 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -4335,6 +4335,19 @@ pub enum ThreadItem { }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] + BrowserReplay { + id: String, + frames: Vec, + render_mode: BrowserReplayRenderMode, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + animation_image_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + frame_duration_ms: Option, + }, + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] CollabAgentToolCall { /// Unique identifier for this collab tool call. id: String, @@ -4396,6 +4409,45 @@ pub struct HookPromptFragment { pub hook_run_id: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct BrowserReplayFrame { + pub sequence_number: u32, + pub source_item_id: String, + pub tool: String, + pub status: DynamicToolCallStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub image_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub text_snapshot: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub text_snapshot_format: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub browser_state: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum BrowserReplayRenderMode { + Animation, + Frames, + TextOnly, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum BrowserReplayTextSnapshotFormat { + Plain, + Json, +} + impl ThreadItem { pub fn id(&self) -> &str { match self { @@ -4408,6 +4460,7 @@ impl ThreadItem { | ThreadItem::FileChange { id, .. } | ThreadItem::McpToolCall { id, .. } | ThreadItem::DynamicToolCall { id, .. } + | ThreadItem::BrowserReplay { id, .. } | ThreadItem::CollabAgentToolCall { id, .. } | ThreadItem::WebSearch { id, .. } | ThreadItem::ImageView { id, .. } @@ -8011,6 +8064,104 @@ mod tests { ); } + #[test] + fn browser_replay_thread_item_serializes_animation_fields() { + let value = serde_json::to_value(ThreadItem::BrowserReplay { + id: "browser-replay-turn-1".to_string(), + frames: vec![BrowserReplayFrame { + sequence_number: 1, + source_item_id: "tool-call-1".to_string(), + tool: "create_tab".to_string(), + status: DynamicToolCallStatus::Completed, + image_url: Some("data:image/png;base64,AAA".to_string()), + text_snapshot: Some("Opened Hacker News".to_string()), + text_snapshot_format: Some(BrowserReplayTextSnapshotFormat::Plain), + browser_state: Some(BrowserSessionState { + selected_tab_id: "tab_123".to_string(), + tabs: vec![BrowserTabState { + id: "tab_123".to_string(), + title: "Hacker News".to_string(), + url: "https://news.ycombinator.com/".to_string(), + selected: true, + }], + }), + }], + render_mode: BrowserReplayRenderMode::Animation, + animation_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()), + frame_duration_ms: Some(700), + }) + .unwrap(); + + assert_eq!( + value, + json!({ + "type": "browserReplay", + "id": "browser-replay-turn-1", + "renderMode": "animation", + "frames": [{ + "sequenceNumber": 1, + "sourceItemId": "tool-call-1", + "tool": "create_tab", + "status": "completed", + "imageUrl": "data:image/png;base64,AAA", + "textSnapshot": "Opened Hacker News", + "textSnapshotFormat": "plain", + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": true + }] + } + }], + "animationImageUrl": "data:image/gif;base64,R0lGODlh", + "frameDurationMs": 700 + }) + ); + } + + #[test] + fn browser_replay_thread_item_serializes_text_only_fields() { + let value = serde_json::to_value(ThreadItem::BrowserReplay { + id: "browser-replay-turn-2".to_string(), + frames: vec![BrowserReplayFrame { + sequence_number: 1, + source_item_id: "tool-call-2".to_string(), + tool: "tabs_content".to_string(), + status: DynamicToolCallStatus::Completed, + image_url: None, + text_snapshot: Some( + "{\n \"tree\": {\n \"role\": \"document\"\n }\n}".to_string(), + ), + text_snapshot_format: Some(BrowserReplayTextSnapshotFormat::Json), + browser_state: None, + }], + render_mode: BrowserReplayRenderMode::TextOnly, + animation_image_url: None, + frame_duration_ms: None, + }) + .unwrap(); + + assert_eq!( + value, + json!({ + "type": "browserReplay", + "id": "browser-replay-turn-2", + "renderMode": "textOnly", + "frames": [{ + "sequenceNumber": 1, + "sourceItemId": "tool-call-2", + "tool": "tabs_content", + "status": "completed", + "textSnapshot": "{\n \"tree\": {\n \"role\": \"document\"\n }\n}", + "textSnapshotFormat": "json", + }], + }) + ); + } + #[test] fn dynamic_tool_spec_deserializes_defer_loading() { let value = json!({ diff --git a/codex-rs/app-server-protocol/src/schema_fixtures.rs b/codex-rs/app-server-protocol/src/schema_fixtures.rs index 56dcf33a4d..8b0c6b6694 100644 --- a/codex-rs/app-server-protocol/src/schema_fixtures.rs +++ b/codex-rs/app-server-protocol/src/schema_fixtures.rs @@ -5,6 +5,7 @@ use crate::ServerRequest; use crate::export::GENERATED_TS_HEADER; use crate::export::filter_experimental_ts_tree; use crate::export::generate_index_ts_tree; +use crate::export::normalize_typescript_text; use crate::protocol::common::visit_client_response_types; use crate::protocol::common::visit_server_response_types; use anyhow::Context; @@ -129,11 +130,9 @@ fn read_file_bytes(path: &Path) -> Result> { return Ok(normalized); } if path.extension().is_some_and(|ext| ext == "ts") { - // Windows checkouts (and some generators) may produce CRLF; normalize so the - // fixture test is platform-independent. let text = String::from_utf8(bytes) .with_context(|| format!("expected UTF-8 TypeScript in {}", path.display()))?; - let text = text.replace("\r\n", "\n").replace('\r', "\n"); + let text = normalize_typescript_text(&text); // Fixture comparisons care about schema content, not whether the generator // re-prepended the standard banner to every TypeScript file. let text = text @@ -276,10 +275,7 @@ fn collect_typescript_fixture_file( let contents = T::export_to_string().context("export TypeScript fixture content")?; let output_path = normalize_relative_fixture_path(&output_path); - files.insert( - output_path, - contents.replace("\r\n", "\n").replace('\r', "\n"), - ); + files.insert(output_path, normalize_typescript_text(&contents)); let mut visitor = TypeScriptFixtureCollector { files, diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index c6b0085667..fd0cebb6f5 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -52,6 +52,7 @@ codex-utils-json-to-toml = { workspace = true } chrono = { workspace = true } clap = { workspace = true, features = ["derive"] } futures = { workspace = true } +image = { workspace = true, features = ["gif", "jpeg", "png", "webp"] } owo-colors = { workspace = true, features = ["supports-colors"] } reqwest = { workspace = true, features = ["json", "rustls-tls"] } serde = { workspace = true, features = ["derive"] } diff --git a/codex-rs/app-server/src/atlas_command.rs b/codex-rs/app-server/src/atlas_command.rs new file mode 100644 index 0000000000..ff4be9eecf --- /dev/null +++ b/codex-rs/app-server/src/atlas_command.rs @@ -0,0 +1,400 @@ +use crate::remote_browser_api::RemoteBrowserApi; +use crate::remote_browser_api::RemoteBrowserCommandOutcome; +use codex_app_server_protocol::BrowserSessionArtifacts; +use codex_app_server_protocol::BrowserSessionCommandParams; +use codex_app_server_protocol::BrowserSessionState; +use codex_app_server_protocol::DynamicToolCallOutputContentItem; +use codex_app_server_protocol::DynamicToolCallResponse; +use codex_app_server_protocol::DynamicToolSpec; +use serde_json::Map; +use serde_json::Value as JsonValue; +use serde_json::json; + +const ATLAS_COMMAND_TOOL: &str = "atlas_command"; +const ATLAS_DISPLAY_TRUNCATE_MAX_CHARS: u32 = 4_000; + +#[derive(Debug)] +pub(crate) struct AtlasCommandExecution { + pub(crate) browser_session_id: Option, + pub(crate) browser_state: Option, + pub(crate) artifacts: Option, + pub(crate) response: DynamicToolCallResponse, +} + +pub(crate) fn merge_atlas_command_dynamic_tool( + mut tools: Vec, +) -> Vec { + if !tools.iter().any(|tool| tool.name == ATLAS_COMMAND_TOOL) { + tools.push(DynamicToolSpec { + name: ATLAS_COMMAND_TOOL.to_string(), + description: "Hidden bridge for AgentLib inside js_repl. Call via codex.tool(...) only." + .to_string(), + input_schema: json!({ + "type": "object", + "properties": { + "payload": { + "type": "object", + "additionalProperties": true + } + }, + "required": ["payload"], + "additionalProperties": false + }), + defer_loading: true, + }); + } + + tools +} + +pub(crate) fn is_atlas_command_dynamic_tool(name: &str) -> bool { + name == ATLAS_COMMAND_TOOL +} + +pub(crate) async fn execute_atlas_command( + remote_browser_api: &RemoteBrowserApi, + browser_session_id: Option, + arguments: JsonValue, +) -> AtlasCommandExecution { + match execute_atlas_command_impl(remote_browser_api, browser_session_id, arguments).await { + Ok(result) => result, + Err(message) => AtlasCommandExecution { + browser_session_id: None, + browser_state: None, + artifacts: None, + response: atlas_json_response(json!({ + "type": "codex_repl", + "error": message, + })), + }, + } +} + +async fn execute_atlas_command_impl( + remote_browser_api: &RemoteBrowserApi, + browser_session_id: Option, + arguments: JsonValue, +) -> Result { + let payload = arguments + .as_object() + .and_then(|args| args.get("payload")) + .cloned() + .ok_or_else(|| "atlas_command requires an object payload".to_string())?; + let payload_object = payload + .as_object() + .ok_or_else(|| "atlas_command payload must be an object".to_string())?; + let command_type = payload_object + .get("type") + .and_then(JsonValue::as_str) + .ok_or_else(|| "atlas_command payload.type is required".to_string())?; + + let mut session_id = browser_session_id; + let mut last_browser_state = None; + let mut last_artifacts = None; + + let command_output = match command_type { + "runtime_config" => json!({ + "display_truncate_max_chars": ATLAS_DISPLAY_TRUNCATE_MAX_CHARS, + }), + "create_tab" => { + let outcome = call_remote( + remote_browser_api, + &mut session_id, + "create_tab", + json!({ "select": true }), + ) + .await?; + let result = outcome.result.clone(); + last_browser_state = outcome.browser_state.to_public_state(); + last_artifacts = outcome.to_public_artifacts(); + json!({ + "id": result.get("tab_id").and_then(JsonValue::as_str).unwrap_or_default(), + }) + } + "list_tabs" => { + let outcome = + call_remote(remote_browser_api, &mut session_id, "list_tabs", json!({})).await?; + let tabs = outcome + .result + .get("tabs") + .and_then(JsonValue::as_array) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|tab| { + let tab_object = tab.as_object().cloned().unwrap_or_default(); + json!({ + "id": tab_object + .get("id") + .cloned() + .unwrap_or(JsonValue::String(String::new())), + "title": tab_object.get("title").cloned().unwrap_or(JsonValue::Null), + "url": tab_object.get("url").cloned().unwrap_or(JsonValue::Null), + }) + }) + .collect::>(); + last_browser_state = outcome.browser_state.to_public_state(); + json!({ "tabs": tabs }) + } + "selected_tab" => { + let outcome = + call_remote(remote_browser_api, &mut session_id, "selected_tab", json!({})) + .await?; + let selected_tab_id = outcome + .result + .get("selected_tab_id") + .and_then(JsonValue::as_str); + last_browser_state = outcome.browser_state.to_public_state(); + json!({ + "id": selected_tab_id, + }) + } + "navigate_tab_url" => { + let remote_args = normalize_passthrough_payload(payload_object); + let outcome = + call_remote(remote_browser_api, &mut session_id, "navigate_tab_url", remote_args) + .await?; + last_browser_state = outcome.browser_state.to_public_state(); + last_artifacts = outcome.to_public_artifacts(); + json!({}) + } + "playwright_wait_for_load_state" => { + let remote_args = normalize_passthrough_payload(payload_object); + let outcome = call_remote( + remote_browser_api, + &mut session_id, + "playwright_wait_for_load_state", + remote_args, + ) + .await?; + last_browser_state = outcome.browser_state.to_public_state(); + last_artifacts = outcome.to_public_artifacts(); + json!({}) + } + "playwright_locator_count" + | "playwright_locator_text_content" + | "playwright_locator_inner_text" + | "playwright_locator_get_attribute" + | "playwright_locator_is_visible" + | "playwright_locator_is_enabled" => { + let remote_args = normalize_passthrough_payload(payload_object); + let outcome = call_remote( + remote_browser_api, + &mut session_id, + command_type, + remote_args, + ) + .await?; + last_browser_state = outcome.browser_state.to_public_state(); + outcome.result.clone() + } + "playwright_locator_click" | "playwright_locator_dblclick" => { + let remote_args = normalize_passthrough_payload(payload_object); + let outcome = call_remote( + remote_browser_api, + &mut session_id, + command_type, + remote_args, + ) + .await?; + last_browser_state = outcome.browser_state.to_public_state(); + last_artifacts = outcome.to_public_artifacts(); + json!({}) + } + "playwright_locator_wait_for" => { + let remote_args = normalize_passthrough_payload(payload_object); + let outcome = call_remote( + remote_browser_api, + &mut session_id, + "playwright_locator_wait_for", + remote_args, + ) + .await?; + last_browser_state = outcome.browser_state.to_public_state(); + json!({}) + } + "playwright_screenshot" => { + if has_any_keys( + payload_object, + &[ + "cropX", + "cropY", + "cropWidth", + "cropHeight", + "crop_x", + "crop_y", + "crop_width", + "crop_height", + ], + ) { + return Err( + "atlas_command playwright_screenshot crop options are not supported by the remote browser bridge yet" + .to_string(), + ); + } + let remote_args = normalize_passthrough_payload(payload_object); + let outcome = call_remote( + remote_browser_api, + &mut session_id, + "playwright_screenshot", + remote_args, + ) + .await?; + last_browser_state = outcome.browser_state.to_public_state(); + last_artifacts = outcome.to_public_artifacts(); + json!({ + "data": outcome + .screenshot_base64() + .ok_or_else(|| "remote browser did not return screenshot data".to_string())?, + }) + } + "cua_get_visible_screenshot" => { + let tab_id = payload_object + .get("tab_id") + .cloned() + .ok_or_else(|| "cua_get_visible_screenshot requires tab_id".to_string())?; + let outcome = call_remote( + remote_browser_api, + &mut session_id, + "playwright_screenshot", + json!({ + "tab_id": tab_id, + "full_page": false, + }), + ) + .await?; + last_browser_state = outcome.browser_state.to_public_state(); + last_artifacts = outcome.to_public_artifacts(); + json!({ + "data": outcome + .screenshot_base64() + .ok_or_else(|| "remote browser did not return screenshot data".to_string())?, + }) + } + "tabs_content" => { + let urls = payload_object + .get("urls") + .and_then(JsonValue::as_array) + .ok_or_else(|| "tabs_content requires urls".to_string())?; + let content_type = payload_object + .get("content_type") + .and_then(JsonValue::as_str) + .ok_or_else(|| "tabs_content requires content_type".to_string())?; + if content_type != "text" { + return Err(format!( + "atlas_command tabs_content only supports content_type=\"text\" today; got {content_type:?}" + )); + } + + let timeout_ms = payload_object.get("timeout_ms").cloned(); + let mut tab_ids = Vec::with_capacity(urls.len()); + for url in urls { + let url = url + .as_str() + .ok_or_else(|| "tabs_content urls must be strings".to_string())?; + let mut create_args = Map::new(); + create_args.insert("url".to_string(), JsonValue::String(url.to_string())); + create_args.insert("select".to_string(), JsonValue::Bool(false)); + if let Some(timeout_ms) = timeout_ms.clone() { + create_args.insert("timeout_ms".to_string(), timeout_ms); + } + let outcome = call_remote( + remote_browser_api, + &mut session_id, + "create_tab", + JsonValue::Object(create_args), + ) + .await?; + let tab_id = outcome + .result + .get("tab_id") + .and_then(JsonValue::as_str) + .ok_or_else(|| "remote browser create_tab did not return tab_id".to_string())?; + tab_ids.push(tab_id.to_string()); + } + + let outcome = call_remote( + remote_browser_api, + &mut session_id, + "tabs_content", + json!({ "tab_ids": tab_ids }), + ) + .await?; + let tabs = outcome + .result + .get("tabs") + .and_then(JsonValue::as_array) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|tab| { + let tab_object = tab.as_object().cloned().unwrap_or_default(); + json!({ + "url": tab_object + .get("url") + .cloned() + .unwrap_or(JsonValue::String(String::new())), + "title": tab_object.get("title").cloned().unwrap_or(JsonValue::Null), + "content": tab_object.get("content").cloned().unwrap_or(JsonValue::Null), + }) + }) + .collect::>(); + last_browser_state = outcome.browser_state.to_public_state(); + json!({ "results": tabs }) + } + other => { + return Err(format!( + "atlas_command does not support AgentLib command type {other:?} yet" + )); + } + }; + + Ok(AtlasCommandExecution { + browser_session_id: session_id, + browser_state: last_browser_state, + artifacts: last_artifacts, + response: atlas_json_response(json!({ + "type": "codex_repl", + "command_output": command_output, + })), + }) +} + +fn atlas_json_response(payload: JsonValue) -> DynamicToolCallResponse { + DynamicToolCallResponse { + content_items: vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::to_string(&payload).unwrap_or_else(|_| { + "{\"type\":\"codex_repl\",\"error\":\"serialization failure\"}".to_string() + }), + }], + success: true, + } +} + +async fn call_remote( + remote_browser_api: &RemoteBrowserApi, + browser_session_id: &mut Option, + command: &str, + arguments: JsonValue, +) -> Result { + let outcome = remote_browser_api + .command_with_artifacts(BrowserSessionCommandParams { + browser_session_id: browser_session_id.clone(), + command: command.to_string(), + arguments: Some(arguments), + }) + .await + .map_err(|err| err.message)?; + *browser_session_id = Some(outcome.browser_session_id.clone()); + Ok(outcome) +} + +fn normalize_passthrough_payload(payload: &Map) -> JsonValue { + let mut args = payload.clone(); + args.remove("type"); + JsonValue::Object(args) +} + +fn has_any_keys(payload: &Map, keys: &[&str]) -> bool { + keys.iter().any(|key| payload.contains_key(*key)) +} diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 9179ae536f..ea3d705a38 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -1,3 +1,5 @@ +use crate::atlas_command::execute_atlas_command; +use crate::atlas_command::is_atlas_command_dynamic_tool; use crate::codex_message_processor::ApiVersion; use crate::codex_message_processor::read_rollout_items_from_rollout; use crate::codex_message_processor::read_summary_from_rollout; @@ -286,6 +288,42 @@ async fn handle_remote_browser_dynamic_tool_request( submit_dynamic_tool_response(conversation, call_id, response).await; } +async fn handle_atlas_command_dynamic_tool_request( + call_id: String, + arguments: JsonValue, + conversation: Arc, + thread_state: Arc>, + outgoing: ThreadScopedOutgoingMessageSender, + remote_browser_api: RemoteBrowserApi, +) { + let browser_session_id = { thread_state.lock().await.remote_browser_session_id.clone() }; + let outcome = execute_atlas_command(&remote_browser_api, browser_session_id, arguments).await; + + if let Some(browser_session_id) = outcome.browser_session_id.clone() { + let mut state = thread_state.lock().await; + state.remote_browser_session_id = Some(browser_session_id); + } + + if let Some(browser_state) = outcome.browser_state { + outgoing + .send_server_notification(ServerNotification::BrowserSessionUpdated( + BrowserSessionUpdatedNotification { + browser_session_id: outcome.browser_session_id.unwrap_or_default(), + browser_state, + artifacts: outcome.artifacts, + }, + )) + .await; + } + + submit_dynamic_tool_response( + conversation, + call_id, + to_core_dynamic_tool_response(outcome.response), + ) + .await; +} + fn guardian_auto_approval_review_notification( conversation_id: &ThreadId, event_turn_id: &str, @@ -1013,7 +1051,18 @@ pub(crate) async fn apply_bespoke_event_handling( outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; - if remote_browser_api.is_configured() && is_remote_browser_dynamic_tool(&tool) { + if remote_browser_api.is_configured() && is_atlas_command_dynamic_tool(&tool) { + tokio::spawn(handle_atlas_command_dynamic_tool_request( + call_id, + arguments, + conversation, + thread_state.clone(), + outgoing.clone(), + remote_browser_api.clone(), + )); + } else if remote_browser_api.is_configured() + && is_remote_browser_dynamic_tool(&tool) + { tokio::spawn(handle_remote_browser_dynamic_tool_request( call_id, tool, diff --git a/codex-rs/app-server/src/browser_replay.rs b/codex-rs/app-server/src/browser_replay.rs new file mode 100644 index 0000000000..c1472507d7 --- /dev/null +++ b/codex-rs/app-server/src/browser_replay.rs @@ -0,0 +1,498 @@ +use crate::remote_browser_tools::is_remote_browser_dynamic_tool; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use codex_app_server_protocol::BrowserReplayFrame; +use codex_app_server_protocol::BrowserReplayRenderMode; +use codex_app_server_protocol::BrowserReplayTextSnapshotFormat; +use codex_app_server_protocol::BrowserSessionState; +use codex_app_server_protocol::DynamicToolCallOutputContentItem; +use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::Turn; +use image::Delay; +use image::Frame; +use image::Rgba; +use image::RgbaImage; +use image::codecs::gif::GifEncoder; +use image::codecs::gif::Repeat; +use serde::Deserialize; +use serde_json::Value as JsonValue; +use std::io::Cursor; + +const REPLAY_FRAME_DURATION_MS: u32 = 700; +const MAX_TEXT_SNAPSHOT_CHARS: usize = 2_000; + +pub(crate) fn augment_turns_with_browser_replays(turns: &mut [Turn]) { + for turn in turns { + turn.items + .retain(|item| !matches!(item, ThreadItem::BrowserReplay { .. })); + + if let Some(replay_item) = build_browser_replay_item(&turn.id, &turn.items) { + turn.items.push(replay_item); + } + } +} + +fn build_browser_replay_item(turn_id: &str, items: &[ThreadItem]) -> Option { + let mut frames = Vec::new(); + let mut static_image_urls = Vec::new(); + let mut fallback_animation_url = None; + + for item in items { + let ThreadItem::DynamicToolCall { + id, + tool, + status, + content_items: Some(content_items), + .. + } = item + else { + continue; + }; + + if !is_remote_browser_dynamic_tool(tool) { + continue; + } + + let frame_content = inspect_content_items(content_items); + if let Some(image_url) = frame_content.image_url.clone() { + static_image_urls.push(image_url); + } + if fallback_animation_url.is_none() { + fallback_animation_url = frame_content.animation_image_url.clone(); + } + + if frame_content.image_url.is_none() + && frame_content.text_snapshot.is_none() + && frame_content.browser_state.is_none() + { + continue; + } + + frames.push(BrowserReplayFrame { + sequence_number: (frames.len() + 1) as u32, + source_item_id: id.clone(), + tool: tool.clone(), + status: status.clone(), + image_url: frame_content.image_url, + text_snapshot: frame_content.text_snapshot, + text_snapshot_format: frame_content.text_snapshot_format, + browser_state: frame_content.browser_state, + }); + } + + if frames.is_empty() { + return None; + } + + let animation_image_url = + synthesize_animation_image_url(&static_image_urls).or(fallback_animation_url); + let render_mode = if animation_image_url.is_some() { + BrowserReplayRenderMode::Animation + } else if frames.iter().any(|frame| frame.image_url.is_some()) { + BrowserReplayRenderMode::Frames + } else { + BrowserReplayRenderMode::TextOnly + }; + let frame_duration_ms = + (animation_image_url.is_some() || frames.len() > 1).then_some(REPLAY_FRAME_DURATION_MS); + + Some(ThreadItem::BrowserReplay { + id: format!("browser-replay-{turn_id}"), + frames, + render_mode, + animation_image_url, + frame_duration_ms, + }) +} + +#[derive(Default)] +struct FrameContent { + image_url: Option, + animation_image_url: Option, + text_snapshot: Option, + text_snapshot_format: Option, + browser_state: Option, +} + +#[derive(Debug, Deserialize)] +struct RemoteBrowserToolTextPayload { + #[serde(default)] + result: JsonValue, + #[serde(default)] + browser_state: Option, +} + +fn inspect_content_items(items: &[DynamicToolCallOutputContentItem]) -> FrameContent { + let mut content = FrameContent::default(); + + for item in items { + match item { + DynamicToolCallOutputContentItem::InputText { text } => { + if content.text_snapshot.is_none() { + let snapshot = build_text_snapshot(text); + content.text_snapshot = snapshot.text; + content.text_snapshot_format = snapshot.format; + } + if content.browser_state.is_none() { + content.browser_state = parse_browser_state(text); + } + } + DynamicToolCallOutputContentItem::InputImage { image_url } => { + if is_gif_data_url(image_url) { + if content.animation_image_url.is_none() { + content.animation_image_url = Some(image_url.clone()); + } + } else if is_raster_data_url(image_url) && content.image_url.is_none() { + content.image_url = Some(image_url.clone()); + } + } + } + } + + content +} + +#[derive(Default)] +struct TextSnapshot { + text: Option, + format: Option, +} + +fn build_text_snapshot(text: &str) -> TextSnapshot { + serde_json::from_str::(text) + .ok() + .and_then(|payload| render_result_snapshot(&payload.result)) + .or_else(|| { + truncate_snapshot(text).map(|value| TextSnapshot { + text: Some(value), + format: Some(BrowserReplayTextSnapshotFormat::Plain), + }) + }) + .unwrap_or_default() +} + +fn parse_browser_state(text: &str) -> Option { + serde_json::from_str::(text) + .ok() + .and_then(|payload| payload.browser_state) +} + +fn render_result_snapshot(result: &JsonValue) -> Option { + if result.is_null() { + return None; + } + match result { + JsonValue::String(text) => truncate_snapshot(text).map(|value| TextSnapshot { + text: Some(value), + format: Some(BrowserReplayTextSnapshotFormat::Plain), + }), + _ => serde_json::to_string_pretty(result) + .ok() + .and_then(|text| truncate_snapshot(&text)) + .map(|value| TextSnapshot { + text: Some(value), + format: Some(BrowserReplayTextSnapshotFormat::Json), + }), + } +} + +fn truncate_snapshot(text: &str) -> Option { + let trimmed = text.trim(); + if trimmed.is_empty() { + return None; + } + let char_count = trimmed.chars().count(); + if char_count <= MAX_TEXT_SNAPSHOT_CHARS { + return Some(trimmed.to_string()); + } + let truncated = trimmed + .chars() + .take(MAX_TEXT_SNAPSHOT_CHARS) + .collect::(); + Some(format!("{truncated}...")) +} + +fn synthesize_animation_image_url(image_urls: &[String]) -> Option { + if image_urls.len() < 2 { + return None; + } + + let mut decoded_frames = Vec::new(); + for image_url in image_urls { + let image = decode_raster_data_url(image_url)?; + decoded_frames.push(image.to_rgba8()); + } + + if decoded_frames.len() < 2 { + return None; + } + + let max_width = decoded_frames.iter().map(RgbaImage::width).max()?; + let max_height = decoded_frames.iter().map(RgbaImage::height).max()?; + let delay = Delay::from_numer_denom_ms(REPLAY_FRAME_DURATION_MS, 1); + let frames = decoded_frames + .into_iter() + .map(|frame| Frame::from_parts(normalize_frame(frame, max_width, max_height), 0, 0, delay)); + + let mut bytes = Cursor::new(Vec::new()); + { + let mut encoder = GifEncoder::new(&mut bytes); + encoder.set_repeat(Repeat::Infinite).ok()?; + encoder.encode_frames(frames).ok()?; + } + + Some(format!( + "data:image/gif;base64,{}", + BASE64_STANDARD.encode(bytes.into_inner()) + )) +} + +fn normalize_frame(frame: RgbaImage, width: u32, height: u32) -> RgbaImage { + if frame.width() == width && frame.height() == height { + return frame; + } + + let mut canvas = RgbaImage::from_pixel(width, height, Rgba([255, 255, 255, 255])); + image::imageops::overlay(&mut canvas, &frame, 0, 0); + canvas +} + +fn decode_raster_data_url(image_url: &str) -> Option { + if !is_raster_data_url(image_url) { + return None; + } + let (_, encoded) = image_url.split_once(',')?; + let bytes = BASE64_STANDARD.decode(encoded).ok()?; + image::load_from_memory(&bytes).ok() +} + +fn is_raster_data_url(image_url: &str) -> bool { + image_url.starts_with("data:image/png;base64,") + || image_url.starts_with("data:image/jpeg;base64,") + || image_url.starts_with("data:image/webp;base64,") +} + +fn is_gif_data_url(image_url: &str) -> bool { + image_url.starts_with("data:image/gif;base64,") +} + +#[cfg(test)] +mod tests { + use super::*; + use codex_app_server_protocol::DynamicToolCallStatus; + use image::DynamicImage; + + fn png_data_url(color: [u8; 4], width: u32, height: u32) -> String { + let image = RgbaImage::from_pixel(width, height, Rgba(color)); + let mut bytes = Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(image) + .write_to(&mut bytes, image::ImageFormat::Png) + .expect("encode png"); + format!( + "data:image/png;base64,{}", + BASE64_STANDARD.encode(bytes.into_inner()) + ) + } + + #[test] + fn builds_replay_from_remote_browser_tool_calls() { + let items = vec![ + ThreadItem::DynamicToolCall { + id: "tool-1".to_string(), + tool: "create_tab".to_string(), + arguments: JsonValue::Null, + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![ + DynamicToolCallOutputContentItem::InputText { + text: serde_json::json!({ + "result": { "tab_id": "tab_1" }, + "browser_state": { + "selectedTabId": "tab_1", + "tabs": [{ + "id": "tab_1", + "title": "First", + "url": "https://example.com/one", + "selected": true + }] + } + }) + .to_string(), + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: png_data_url([255, 0, 0, 255], 10, 10), + }, + ]), + success: Some(true), + duration_ms: Some(10), + }, + ThreadItem::DynamicToolCall { + id: "tool-2".to_string(), + tool: "tabs_content".to_string(), + arguments: JsonValue::Null, + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![ + DynamicToolCallOutputContentItem::InputText { + text: serde_json::json!({ + "result": { + "tabs": [{ + "title": "First", + "content": "headline one\nheadline two" + }] + }, + "browser_state": { + "selectedTabId": "tab_1", + "tabs": [{ + "id": "tab_1", + "title": "First", + "url": "https://example.com/one", + "selected": true + }] + } + }) + .to_string(), + }, + DynamicToolCallOutputContentItem::InputImage { + image_url: png_data_url([0, 0, 255, 255], 12, 8), + }, + ]), + success: Some(true), + duration_ms: Some(10), + }, + ]; + + let replay = build_browser_replay_item("turn-1", &items).expect("replay item"); + let ThreadItem::BrowserReplay { + id, + frames, + render_mode, + animation_image_url, + frame_duration_ms, + } = replay + else { + panic!("expected browser replay item"); + }; + + assert_eq!(id, "browser-replay-turn-1"); + assert_eq!(frames.len(), 2); + assert_eq!(render_mode, BrowserReplayRenderMode::Animation); + assert_eq!(frames[0].source_item_id, "tool-1"); + assert_eq!(frames[1].source_item_id, "tool-2"); + let second_snapshot = frames[1] + .text_snapshot + .as_deref() + .expect("second frame snapshot"); + assert!(second_snapshot.contains("\"title\": \"First\"")); + assert!(second_snapshot.contains("\"content\": \"headline one\\nheadline two\"")); + assert_eq!( + frames[1].text_snapshot_format, + Some(BrowserReplayTextSnapshotFormat::Json) + ); + assert!( + animation_image_url + .as_deref() + .is_some_and(|value| value.starts_with("data:image/gif;base64,")) + ); + assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS)); + } + + #[test] + fn builds_text_only_replay_when_no_images_are_available() { + let items = vec![ThreadItem::DynamicToolCall { + id: "tool-1".to_string(), + tool: "tabs_content".to_string(), + arguments: JsonValue::Null, + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::json!({ + "result": { + "tree": { + "role": "document", + "name": "Example page" + } + } + }) + .to_string(), + }]), + success: Some(true), + duration_ms: Some(5), + }]; + + let replay = build_browser_replay_item("turn-2", &items).expect("replay item"); + let ThreadItem::BrowserReplay { + frames, + render_mode, + animation_image_url, + frame_duration_ms, + .. + } = replay + else { + panic!("expected browser replay item"); + }; + + assert_eq!(frames.len(), 1); + assert_eq!(render_mode, BrowserReplayRenderMode::TextOnly); + assert_eq!( + frames[0].text_snapshot.as_deref(), + Some( + "{\n \"tree\": {\n \"role\": \"document\",\n \"name\": \"Example page\"\n }\n}" + ) + ); + assert_eq!( + frames[0].text_snapshot_format, + Some(BrowserReplayTextSnapshotFormat::Json) + ); + assert_eq!(animation_image_url, None); + assert_eq!(frame_duration_ms, None); + } + + #[test] + fn assigns_frame_duration_to_multi_frame_text_only_replays() { + let items = vec![ + ThreadItem::DynamicToolCall { + id: "tool-1".to_string(), + tool: "tabs_content".to_string(), + arguments: JsonValue::Null, + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::json!({ + "result": { "tree": { "role": "document", "name": "Step 1" } } + }) + .to_string(), + }]), + success: Some(true), + duration_ms: Some(5), + }, + ThreadItem::DynamicToolCall { + id: "tool-2".to_string(), + tool: "tabs_content".to_string(), + arguments: JsonValue::Null, + status: DynamicToolCallStatus::Completed, + content_items: Some(vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::json!({ + "result": { "tree": { "role": "document", "name": "Step 2" } } + }) + .to_string(), + }]), + success: Some(true), + duration_ms: Some(5), + }, + ]; + + let replay = build_browser_replay_item("turn-3", &items).expect("replay item"); + let ThreadItem::BrowserReplay { + frames, + render_mode, + animation_image_url, + frame_duration_ms, + .. + } = replay + else { + panic!("expected browser replay item"); + }; + + assert_eq!(frames.len(), 2); + assert_eq!(render_mode, BrowserReplayRenderMode::TextOnly); + assert_eq!(animation_image_url, None); + assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS)); + } +} diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index bcf9a542db..0c421c9f26 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -1,4 +1,6 @@ +use crate::atlas_command::merge_atlas_command_dynamic_tool; use crate::bespoke_event_handling::apply_bespoke_event_handling; +use crate::browser_replay::augment_turns_with_browser_replays; use crate::command_exec::CommandExecManager; use crate::command_exec::StartCommandExecParams; use crate::error_code::INPUT_TOO_LARGE_ERROR_CODE; @@ -1992,7 +1994,9 @@ impl CodexMessageProcessor { }; let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() { - merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default()) + let dynamic_tools = + merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default()); + merge_atlas_command_dynamic_tool(dynamic_tools) } else { dynamic_tools.unwrap_or_default() }; @@ -3299,11 +3303,28 @@ impl CodexMessageProcessor { self.attach_thread_name(thread_uuid, &mut thread).await; if include_turns && let Some(rollout_path) = rollout_path.as_ref() { - match read_rollout_items_from_rollout(rollout_path).await { - Ok(items) => { - thread.turns = build_turns_from_rollout_items(&items); - } - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + let active_turn = if loaded_thread.is_some() { + let state = self.thread_state_manager.thread_state(thread_uuid).await; + let state = state.lock().await; + state.active_turn_snapshot() + } else { + None + }; + + if let Err(message) = populate_thread_turns( + &mut thread, + ThreadTurnSource::RolloutPath(rollout_path), + active_turn.as_ref(), + ) + .await + { + let missing_rollout_message = format!( + "failed to load rollout `{}` for thread {thread_uuid}: ", + rollout_path.display() + ); + if message.contains(&missing_rollout_message) + && message.contains("No such file or directory") + { self.send_invalid_request_error( request_id, format!( @@ -3313,17 +3334,9 @@ impl CodexMessageProcessor { .await; return; } - Err(err) => { - self.send_internal_error( - request_id, - format!( - "failed to load rollout `{}` for thread {thread_uuid}: {err}", - rollout_path.display() - ), - ) - .await; - return; - } + + self.send_internal_error(request_id, message).await; + return; } } @@ -7473,6 +7486,7 @@ async fn populate_thread_turns( if let Some(active_turn) = active_turn { merge_turn_history_with_active_turn(&mut turns, active_turn.clone()); } + augment_turns_with_browser_replays(&mut turns); thread.turns = turns; Ok(()) } diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index b9bf5a8e5c..57bfc3aa04 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -57,7 +57,9 @@ use tracing_subscriber::registry::Registry; use tracing_subscriber::util::SubscriberInitExt; mod app_server_tracing; +mod atlas_command; mod bespoke_event_handling; +mod browser_replay; mod codex_message_processor; mod command_exec; mod config_api; diff --git a/codex-rs/app-server/src/message_processor/tracing_tests.rs b/codex-rs/app-server/src/message_processor/tracing_tests.rs index 4f7baf285e..1c9cbbd15d 100644 --- a/codex-rs/app-server/src/message_processor/tracing_tests.rs +++ b/codex-rs/app-server/src/message_processor/tracing_tests.rs @@ -7,9 +7,9 @@ use crate::transport::AppServerTransport; use anyhow::Result; use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::write_mock_responses_config_toml; +use codex_app_server_protocol::BrowserSessionArtifacts; use codex_app_server_protocol::BrowserSessionCommandParams; use codex_app_server_protocol::BrowserSessionCommandResponse; -use codex_app_server_protocol::BrowserSessionArtifacts; use codex_app_server_protocol::BrowserSessionState; use codex_app_server_protocol::BrowserSessionUpdatedNotification; use codex_app_server_protocol::BrowserTabState; diff --git a/codex-rs/app-server/src/remote_browser_api.rs b/codex-rs/app-server/src/remote_browser_api.rs index 68780e3d15..a31f3cac22 100644 --- a/codex-rs/app-server/src/remote_browser_api.rs +++ b/codex-rs/app-server/src/remote_browser_api.rs @@ -116,6 +116,10 @@ impl RemoteBrowserCommandOutcome { .map(|encoded| format!("data:image/png;base64,{encoded}")) } + pub(crate) fn screenshot_base64(&self) -> Option { + self.artifacts.screenshot_base64.clone() + } + pub(crate) fn replay_gif_data_url(&self) -> Option { self.artifacts .replay_gif_base64 diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 5c72d134b4..ec73a9a783 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -5,6 +5,8 @@ use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::to_response; +use codex_app_server_protocol::BrowserReplayRenderMode; +use codex_app_server_protocol::BrowserReplayTextSnapshotFormat; use codex_app_server_protocol::BrowserSessionArtifacts; use codex_app_server_protocol::BrowserSessionUpdatedNotification; use codex_app_server_protocol::DynamicToolCallOutputContentItem; @@ -19,6 +21,8 @@ use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadReadParams; +use codex_app_server_protocol::ThreadReadResponse; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::TurnStartParams; @@ -38,6 +42,7 @@ use tokio::time::timeout; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; +use wiremock::matchers::body_json; use wiremock::matchers::method; use wiremock::matchers::path; @@ -255,6 +260,655 @@ async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configu assert!(find_tool(body, "create_tab").is_some()); assert!(find_tool(body, "tabs_content").is_some()); assert!(find_tool(body, "playwright_screenshot").is_some()); + assert!(find_tool(body, "atlas_command").is_none()); + + Ok(()) +} + +#[tokio::test] +async fn atlas_command_dynamic_tool_calls_are_handled_internally() -> Result<()> { + let call_id = "dyn-atlas-1"; + let tool_name = "atlas_command"; + let tool_args = json!({ + "payload": { + "type": "create_tab" + } + }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Done")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let remote_browser_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/remote_browser")) + .and(body_json(json!({ + "browserSessionId": null, + "command": "create_tab", + "arguments": { + "select": true + } + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_atlas_123", + "result": { + "tab_id": "tab_123", + "selected_tab_id": "tab_123" + }, + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "New Tab", + "url": "about:blank", + "selected": true + }] + } + }))) + .expect(1) + .mount(&remote_browser_server) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + let thread_id = thread.id.clone(); + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread_id.clone(), + input: vec![V2UserInput::Text { + text: "use atlas".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); + + let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; + assert_eq!(started.thread_id, thread_id); + assert_eq!(started.turn_id, turn_id.clone()); + + assert!( + timeout( + Duration::from_millis(500), + mcp.read_stream_until_request_message() + ) + .await + .is_err(), + "atlas_command tool call should not be forwarded back to the client" + ); + + let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.browser_session_id, "sess_atlas_123"); + assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); + assert_eq!(browser_update.browser_state.tabs.len(), 1); + assert_eq!(browser_update.browser_state.tabs[0].title, "New Tab"); + assert_eq!(browser_update.artifacts, None); + + let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?; + let ThreadItem::DynamicToolCall { + status, + content_items, + success, + .. + } = completed.item + else { + panic!("expected dynamic tool call item"); + }; + assert_eq!(status, DynamicToolCallStatus::Completed); + assert_eq!(success, Some(true)); + assert_eq!( + content_items, + Some(vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::to_string(&json!({ + "type": "codex_repl", + "command_output": { + "id": "tab_123" + } + }))?, + }]) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let bodies = responses_bodies(&server).await?; + let payload = bodies + .iter() + .find_map(|body| function_call_output_payload(body, call_id)) + .context("expected function_call_output in follow-up request")?; + assert_eq!( + payload.body, + FunctionCallOutputBody::Text(serde_json::to_string(&json!({ + "type": "codex_repl", + "command_output": { + "id": "tab_123" + } + }))?) + ); + assert_eq!(payload.success, None); + + Ok(()) +} + +#[tokio::test] +async fn atlas_command_tabs_content_uses_background_tabs() -> Result<()> { + let call_id = "dyn-atlas-tabs-1"; + let tool_name = "atlas_command"; + let tool_args = json!({ + "payload": { + "type": "tabs_content", + "urls": ["https://news.ycombinator.com/"], + "content_type": "text" + } + }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Done")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let remote_browser_server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/remote_browser")) + .and(body_json(json!({ + "browserSessionId": null, + "command": "create_tab", + "arguments": { + "url": "https://news.ycombinator.com/", + "select": false + } + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_atlas_tabs_123", + "result": { + "tab_id": "tab_bg_123" + }, + "browserState": { + "selectedTabId": "tab_main", + "tabs": [ + { + "id": "tab_main", + "title": "Workspace", + "url": "http://localhost:3000/", + "selected": true + }, + { + "id": "tab_bg_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": false + } + ] + } + }))) + .expect(1) + .mount(&remote_browser_server) + .await; + + Mock::given(method("POST")) + .and(path("/remote_browser")) + .and(body_json(json!({ + "browserSessionId": "sess_atlas_tabs_123", + "command": "tabs_content", + "arguments": { + "tab_ids": ["tab_bg_123"] + } + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_atlas_tabs_123", + "result": { + "tabs": [{ + "url": "https://news.ycombinator.com/", + "title": "Hacker News", + "content": "Top stories" + }] + }, + "browserState": { + "selectedTabId": "tab_main", + "tabs": [ + { + "id": "tab_main", + "title": "Workspace", + "url": "http://localhost:3000/", + "selected": true + }, + { + "id": "tab_bg_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": false + } + ] + } + }))) + .expect(1) + .mount(&remote_browser_server) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "read the page".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _turn: TurnStartResponse = to_response::(turn_resp)?; + + let _started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; + let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.browser_session_id, "sess_atlas_tabs_123"); + assert_eq!(browser_update.browser_state.selected_tab_id, "tab_main"); + + let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?; + let ThreadItem::DynamicToolCall { + content_items, + success, + .. + } = completed.item + else { + panic!("expected dynamic tool call item"); + }; + assert_eq!(success, Some(true)); + assert_eq!( + content_items, + Some(vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::to_string(&json!({ + "type": "codex_repl", + "command_output": { + "results": [{ + "url": "https://news.ycombinator.com/", + "title": "Hacker News", + "content": "Top stories" + }] + } + }))?, + }]) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + Ok(()) +} + +#[tokio::test] +async fn atlas_command_locator_count_passes_through_to_remote_browser() -> Result<()> { + let call_id = "dyn-atlas-locator-count-1"; + let tool_name = "atlas_command"; + let tool_args = json!({ + "payload": { + "type": "playwright_locator_count", + "tab_id": "tab_123", + "selector": "span.titleline > a" + } + }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Done")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let remote_browser_server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/remote_browser")) + .and(body_json(json!({ + "browserSessionId": null, + "command": "playwright_locator_count", + "arguments": { + "tab_id": "tab_123", + "selector": "span.titleline > a" + } + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_atlas_count_123", + "result": { + "count": 5 + }, + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": true + }] + } + }))) + .expect(1) + .mount(&remote_browser_server) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + let thread_id = thread.id.clone(); + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread_id.clone(), + input: vec![V2UserInput::Text { + text: "count links".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); + + let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; + assert_eq!(started.thread_id, thread_id); + assert_eq!(started.turn_id, turn_id.clone()); + + let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.browser_session_id, "sess_atlas_count_123"); + assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); + + let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?; + let ThreadItem::DynamicToolCall { + content_items, + success, + .. + } = completed.item + else { + panic!("expected dynamic tool call item"); + }; + assert_eq!(success, Some(true)); + assert_eq!( + content_items, + Some(vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::to_string(&json!({ + "type": "codex_repl", + "command_output": { + "count": 5 + } + }))?, + }]) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let bodies = responses_bodies(&server).await?; + let payload = bodies + .iter() + .find_map(|body| function_call_output_payload(body, call_id)) + .context("expected function_call_output in follow-up request")?; + assert_eq!( + payload.body, + FunctionCallOutputBody::Text(serde_json::to_string(&json!({ + "type": "codex_repl", + "command_output": { + "count": 5 + } + }))?) + ); + assert_eq!(payload.success, None); + + Ok(()) +} + +#[tokio::test] +async fn atlas_command_locator_click_passes_through_to_remote_browser() -> Result<()> { + let call_id = "dyn-atlas-locator-click-1"; + let tool_name = "atlas_command"; + let tool_args = json!({ + "payload": { + "type": "playwright_locator_click", + "tab_id": "tab_123", + "selector": "[data-square=\"e2\"]", + "button": "left", + "timeout_ms": 1500 + } + }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Clicked")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let remote_browser_server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/remote_browser")) + .and(body_json(json!({ + "browserSessionId": null, + "command": "playwright_locator_click", + "arguments": { + "tab_id": "tab_123", + "selector": "[data-square=\"e2\"]", + "button": "left", + "timeout_ms": 1500 + } + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_atlas_click_123", + "result": {}, + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "3D Chess", + "url": "http://127.0.0.1:8123/", + "selected": true + }] + }, + "artifacts": { + "screenshotBase64": "AAA", + "replayGifBase64": "R0lGODlh", + "replayFrameCount": 3, + "replayFrameDurationMs": 700 + } + }))) + .expect(1) + .mount(&remote_browser_server) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + let thread_id = thread.id.clone(); + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread_id.clone(), + input: vec![V2UserInput::Text { + text: "click e2".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); + + let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?; + assert_eq!(started.thread_id, thread_id); + assert_eq!(started.turn_id, turn_id.clone()); + + let browser_update = wait_for_browser_session_updated(&mut mcp).await?; + assert_eq!(browser_update.browser_session_id, "sess_atlas_click_123"); + assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123"); + assert_eq!( + browser_update.artifacts, + Some(BrowserSessionArtifacts { + screenshot_image_url: Some("data:image/png;base64,AAA".to_string()), + replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()), + replay_frame_count: Some(3), + replay_frame_duration_ms: Some(700), + }) + ); + + let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?; + let ThreadItem::DynamicToolCall { + content_items, + success, + .. + } = completed.item + else { + panic!("expected dynamic tool call item"); + }; + assert_eq!(success, Some(true)); + assert_eq!( + content_items, + Some(vec![DynamicToolCallOutputContentItem::InputText { + text: serde_json::to_string(&json!({ + "type": "codex_repl", + "command_output": {} + }))?, + }]) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let bodies = responses_bodies(&server).await?; + let payload = bodies + .iter() + .find_map(|body| function_call_output_payload(body, call_id)) + .context("expected function_call_output in follow-up request")?; + assert_eq!( + payload.body, + FunctionCallOutputBody::Text(serde_json::to_string(&json!({ + "type": "codex_repl", + "command_output": {} + }))?) + ); + assert_eq!(payload.success, None); Ok(()) } @@ -477,6 +1131,269 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<() Ok(()) } +#[tokio::test] +async fn thread_read_derives_browser_replay_from_remote_browser_tool_history() -> Result<()> { + let call_id = "dyn-browser-read-1"; + let tool_name = "create_tab"; + let tool_args = json!({ "url": "https://news.ycombinator.com/" }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Done")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let remote_browser_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/remote_browser")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_123", + "result": { + "tab_id": "tab_123", + "selected_tab_id": "tab_123" + }, + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": true + }] + }, + "artifacts": { + "screenshotBase64": "AAA", + "replayGifBase64": "R0lGODlh", + "replayFrameCount": 2, + "replayFrameDurationMs": 700 + } + }))) + .mount(&remote_browser_server) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "open Hacker News".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _turn: TurnStartResponse = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let read_req = mcp + .send_thread_read_request(ThreadReadParams { + thread_id: thread.id.clone(), + include_turns: true, + }) + .await?; + let read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(read_req)), + ) + .await??; + let ThreadReadResponse { thread } = to_response::(read_resp)?; + + let turn = thread.turns.last().context("expected replay turn")?; + let replay = turn + .items + .iter() + .find_map(|item| match item { + ThreadItem::BrowserReplay { + frames, + animation_image_url, + frame_duration_ms, + .. + } => Some((frames, animation_image_url, frame_duration_ms)), + _ => None, + }) + .context("expected derived browser replay item")?; + + assert_eq!(replay.0.len(), 1); + assert_eq!(replay.0[0].source_item_id, call_id); + assert_eq!(replay.0[0].tool, tool_name); + assert_eq!( + replay.0[0].image_url.as_deref(), + Some("data:image/png;base64,AAA") + ); + assert_eq!(replay.1.as_deref(), Some("data:image/gif;base64,R0lGODlh")); + assert_eq!(*replay.2, Some(700)); + + Ok(()) +} + +#[tokio::test] +async fn thread_read_derives_text_only_browser_replay_from_remote_browser_tool_history() +-> Result<()> { + let call_id = "dyn-browser-read-text-1"; + let tool_name = "tabs_content"; + let tool_args = json!({ "tabId": "tab_123" }); + let tool_call_arguments = serde_json::to_string(&tool_args)?; + + let responses = vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + create_final_assistant_message_sse_response("Done")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let remote_browser_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/remote_browser")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "browserSessionId": "sess_123", + "result": { + "tree": { + "role": "document", + "name": "Hacker News" + } + }, + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Hacker News", + "url": "https://news.ycombinator.com/", + "selected": true + }] + } + }))) + .mount(&remote_browser_server) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri()); + let mut mcp = McpProcess::new_with_args( + codex_home.path(), + &["--remote-browser-endpoint", &remote_browser_endpoint], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "inspect the current page".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _turn: TurnStartResponse = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let read_req = mcp + .send_thread_read_request(ThreadReadParams { + thread_id: thread.id.clone(), + include_turns: true, + }) + .await?; + let read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(read_req)), + ) + .await??; + let ThreadReadResponse { thread } = to_response::(read_resp)?; + + let turn = thread.turns.last().context("expected replay turn")?; + let replay = turn + .items + .iter() + .find_map(|item| match item { + ThreadItem::BrowserReplay { + frames, + render_mode, + animation_image_url, + frame_duration_ms, + .. + } => Some((frames, render_mode, animation_image_url, frame_duration_ms)), + _ => None, + }) + .context("expected derived browser replay item")?; + + assert_eq!(replay.0.len(), 1); + assert_eq!(replay.0[0].source_item_id, call_id); + assert_eq!(replay.0[0].tool, tool_name); + assert_eq!(replay.1, &BrowserReplayRenderMode::TextOnly); + assert_eq!(replay.2, &None); + assert_eq!(replay.3, &None); + let text_snapshot = replay.0[0] + .text_snapshot + .as_deref() + .context("expected text snapshot")?; + assert!(text_snapshot.contains("\"role\": \"document\"")); + assert!(text_snapshot.contains("\"name\": \"Hacker News\"")); + assert_eq!( + replay.0[0].text_snapshot_format, + Some(BrowserReplayTextSnapshotFormat::Json) + ); + + Ok(()) +} + /// Exercises the full dynamic tool call path (server request, client response, model output). #[tokio::test] async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> { diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/core/src/rollout/policy.rs index 8b1f94dbd5..f24e036edf 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/core/src/rollout/policy.rs @@ -128,9 +128,9 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { | EventMsg::CollabAgentInteractionEnd(_) | EventMsg::CollabWaitingEnd(_) | EventMsg::CollabCloseEnd(_) - | EventMsg::CollabResumeEnd(_) - | EventMsg::DynamicToolCallRequest(_) - | EventMsg::DynamicToolCallResponse(_) => Some(EventPersistenceMode::Extended), + | EventMsg::CollabResumeEnd(_) => Some(EventPersistenceMode::Extended), + EventMsg::DynamicToolCallResponse(_) => Some(EventPersistenceMode::Limited), + EventMsg::DynamicToolCallRequest(_) => Some(EventPersistenceMode::Extended), EventMsg::Warning(_) | EventMsg::RealtimeConversationStarted(_) | EventMsg::RealtimeConversationRealtime(_) @@ -188,8 +188,11 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { mod tests { use super::EventPersistenceMode; use super::should_persist_event_msg; + use codex_protocol::dynamic_tools::DynamicToolCallRequest; + use codex_protocol::protocol::DynamicToolCallResponseEvent; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ImageGenerationEndEvent; + use std::time::Duration; #[test] fn persists_image_generation_end_events_in_limited_mode() { @@ -206,4 +209,42 @@ mod tests { EventPersistenceMode::Limited )); } + + #[test] + fn persists_dynamic_tool_responses_in_limited_mode() { + let event = EventMsg::DynamicToolCallResponse(DynamicToolCallResponseEvent { + call_id: "dyn_123".into(), + turn_id: "turn_123".into(), + tool: "create_tab".into(), + arguments: serde_json::json!({ "url": "https://example.com" }), + content_items: Vec::new(), + success: true, + error: None, + duration: Duration::from_millis(250), + }); + + assert!(should_persist_event_msg( + &event, + EventPersistenceMode::Limited + )); + } + + #[test] + fn does_not_persist_dynamic_tool_requests_in_limited_mode() { + let event = EventMsg::DynamicToolCallRequest(DynamicToolCallRequest { + call_id: "dyn_123".into(), + turn_id: "turn_123".into(), + tool: "create_tab".into(), + arguments: serde_json::json!({ "url": "https://example.com" }), + }); + + assert!(!should_persist_event_msg( + &event, + EventPersistenceMode::Limited + )); + assert!(should_persist_event_msg( + &event, + EventPersistenceMode::Extended + )); + } }