Wire Atlas browser bridge into app-server

Add the AgentLib js_repl bridge, persist browser replay items in thread history, and surface remote browser artifacts through turn/start.

Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Liang-Ting Jiang
2026-03-24 21:12:20 +00:00
co-authored by Codex
parent ba0d3b467b
commit ee865bcddc
41 changed files with 5135 additions and 48 deletions
+1
View File
@@ -1457,6 +1457,7 @@ dependencies = [
"codex-utils-pty",
"core_test_support",
"futures",
"image",
"opentelemetry",
"opentelemetry_sdk",
"owo-colors",
@@ -384,6 +384,111 @@
}
]
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionArtifacts": {
"properties": {
"replayFrameCount": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayFrameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayGifImageUrl": {
"type": [
"string",
"null"
]
},
"screenshotImageUrl": {
"type": [
"string",
"null"
]
}
},
"type": "object"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
@@ -404,6 +509,16 @@
},
"BrowserSessionUpdatedNotification": {
"properties": {
"artifacts": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionArtifacts"
},
{
"type": "null"
}
]
},
"browserSessionId": {
"type": "string"
},
@@ -2760,6 +2875,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -5515,6 +5515,111 @@
}
]
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/v2/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/v2/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/v2/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionArtifacts": {
"properties": {
"replayFrameCount": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayFrameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayGifImageUrl": {
"type": [
"string",
"null"
]
},
"screenshotImageUrl": {
"type": [
"string",
"null"
]
}
},
"type": "object"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
@@ -5536,6 +5641,16 @@
"BrowserSessionUpdatedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
"artifacts": {
"anyOf": [
{
"$ref": "#/definitions/v2/BrowserSessionArtifacts"
},
{
"type": "null"
}
]
},
"browserSessionId": {
"type": "string"
},
@@ -12543,6 +12658,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/v2/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/v2/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -715,6 +715,111 @@
}
]
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionArtifacts": {
"properties": {
"replayFrameCount": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayFrameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayGifImageUrl": {
"type": [
"string",
"null"
]
},
"screenshotImageUrl": {
"type": [
"string",
"null"
]
}
},
"type": "object"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
@@ -736,6 +841,16 @@
"BrowserSessionUpdatedNotification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"properties": {
"artifacts": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionArtifacts"
},
{
"type": "null"
}
]
},
"browserSessionId": {
"type": "string"
},
@@ -10303,6 +10418,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -1,6 +1,39 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BrowserSessionArtifacts": {
"properties": {
"replayFrameCount": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayFrameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"replayGifImageUrl": {
"type": [
"string",
"null"
]
},
"screenshotImageUrl": {
"type": [
"string",
"null"
]
}
},
"type": "object"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
@@ -44,6 +77,16 @@
}
},
"properties": {
"artifacts": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionArtifacts"
},
{
"type": "null"
}
]
},
"browserSessionId": {
"type": "string"
},
@@ -1,6 +1,119 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -869,6 +982,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -1,6 +1,119 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -869,6 +982,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -1,6 +1,119 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1012,6 +1125,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -66,6 +66,119 @@
}
]
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1519,6 +1632,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -4,6 +4,119 @@
"AgentPath": {
"type": "string"
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1277,6 +1390,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -4,6 +4,119 @@
"AgentPath": {
"type": "string"
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1277,6 +1390,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -4,6 +4,119 @@
"AgentPath": {
"type": "string"
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1277,6 +1390,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -66,6 +66,119 @@
}
]
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1519,6 +1632,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -4,6 +4,119 @@
"AgentPath": {
"type": "string"
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1277,6 +1390,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -66,6 +66,119 @@
}
]
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1519,6 +1632,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -4,6 +4,119 @@
"AgentPath": {
"type": "string"
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1277,6 +1390,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -4,6 +4,119 @@
"AgentPath": {
"type": "string"
},
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1277,6 +1390,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -1,6 +1,119 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1012,6 +1125,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -1,6 +1,119 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1012,6 +1125,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -1,6 +1,119 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"definitions": {
"BrowserReplayFrame": {
"properties": {
"browserState": {
"anyOf": [
{
"$ref": "#/definitions/BrowserSessionState"
},
{
"type": "null"
}
]
},
"imageUrl": {
"type": [
"string",
"null"
]
},
"sequenceNumber": {
"format": "uint32",
"minimum": 0.0,
"type": "integer"
},
"sourceItemId": {
"type": "string"
},
"status": {
"$ref": "#/definitions/DynamicToolCallStatus"
},
"textSnapshot": {
"type": [
"string",
"null"
]
},
"textSnapshotFormat": {
"anyOf": [
{
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
},
{
"type": "null"
}
]
},
"tool": {
"type": "string"
}
},
"required": [
"sequenceNumber",
"sourceItemId",
"status",
"tool"
],
"type": "object"
},
"BrowserReplayRenderMode": {
"enum": [
"animation",
"frames",
"textOnly"
],
"type": "string"
},
"BrowserReplayTextSnapshotFormat": {
"enum": [
"plain",
"json"
],
"type": "string"
},
"BrowserSessionState": {
"properties": {
"selectedTabId": {
"type": "string"
},
"tabs": {
"items": {
"$ref": "#/definitions/BrowserTabState"
},
"type": "array"
}
},
"required": [
"selectedTabId",
"tabs"
],
"type": "object"
},
"BrowserTabState": {
"properties": {
"id": {
"type": "string"
},
"selected": {
"type": "boolean"
},
"title": {
"type": "string"
},
"url": {
"type": "string"
}
},
"required": [
"id",
"selected",
"title",
"url"
],
"type": "object"
},
"ByteRange": {
"properties": {
"end": {
@@ -1012,6 +1125,51 @@
"title": "DynamicToolCallThreadItem",
"type": "object"
},
{
"properties": {
"animationImageUrl": {
"type": [
"string",
"null"
]
},
"frameDurationMs": {
"format": "uint32",
"minimum": 0.0,
"type": [
"integer",
"null"
]
},
"frames": {
"items": {
"$ref": "#/definitions/BrowserReplayFrame"
},
"type": "array"
},
"id": {
"type": "string"
},
"renderMode": {
"$ref": "#/definitions/BrowserReplayRenderMode"
},
"type": {
"enum": [
"browserReplay"
],
"title": "BrowserReplayThreadItemType",
"type": "string"
}
},
"required": [
"frames",
"id",
"renderMode",
"type"
],
"title": "BrowserReplayThreadItem",
"type": "object"
},
{
"properties": {
"agentsStates": {
@@ -0,0 +1,8 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { BrowserReplayTextSnapshotFormat } from "./BrowserReplayTextSnapshotFormat";
import type { BrowserSessionState } from "./BrowserSessionState";
import type { DynamicToolCallStatus } from "./DynamicToolCallStatus";
export type BrowserReplayFrame = { sequenceNumber: number, sourceItemId: string, tool: string, status: DynamicToolCallStatus, imageUrl?: string | null, textSnapshot?: string | null, textSnapshotFormat?: BrowserReplayTextSnapshotFormat | null, browserState?: BrowserSessionState | null, };
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type BrowserReplayRenderMode = "animation" | "frames" | "textOnly";
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type BrowserReplayTextSnapshotFormat = "plain" | "json";
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type BrowserSessionArtifacts = { screenshotImageUrl?: string | null, replayGifImageUrl?: string | null, replayFrameCount?: number | null, replayFrameDurationMs?: number | null, };
@@ -1,6 +1,7 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts";
import type { BrowserSessionState } from "./BrowserSessionState";
export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, };
export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, artifacts?: BrowserSessionArtifacts | null, };
@@ -4,6 +4,8 @@
import type { MessagePhase } from "../MessagePhase";
import type { ReasoningEffort } from "../ReasoningEffort";
import type { JsonValue } from "../serde_json/JsonValue";
import type { BrowserReplayFrame } from "./BrowserReplayFrame";
import type { BrowserReplayRenderMode } from "./BrowserReplayRenderMode";
import type { CollabAgentState } from "./CollabAgentState";
import type { CollabAgentTool } from "./CollabAgentTool";
import type { CollabAgentToolCallStatus } from "./CollabAgentToolCallStatus";
@@ -22,78 +24,78 @@ import type { PatchApplyStatus } from "./PatchApplyStatus";
import type { UserInput } from "./UserInput";
import type { WebSearchAction } from "./WebSearchAction";
export type ThreadItem = { "type": "userMessage", id: string, content: Array<UserInput>, } | { "type": "hookPrompt", id: string, fragments: Array<HookPromptFragment>, } | { "type": "agentMessage", id: string, text: string, phase: MessagePhase | null, memoryCitation: MemoryCitation | null, } | { "type": "plan", id: string, text: string, } | { "type": "reasoning", id: string, summary: Array<string>, content: Array<string>, } | { "type": "commandExecution", id: string,
export type ThreadItem = { "type": "userMessage", id: string, content: Array<UserInput>, } | { "type": "hookPrompt", id: string, fragments: Array<HookPromptFragment>, } | { "type": "agentMessage", id: string, text: string, phase: MessagePhase | null, memoryCitation: MemoryCitation | null, } | { "type": "plan", id: string, text: string, } | { "type": "reasoning", id: string, summary: Array<string>, content: Array<string>, } | { "type": "commandExecution", id: string,
/**
* The command to be executed.
*/
command: string,
command: string,
/**
* The command's working directory.
*/
cwd: string,
cwd: string,
/**
* Identifier for the underlying PTY process (when available).
*/
processId: string | null, source: CommandExecutionSource, status: CommandExecutionStatus,
processId: string | null, source: CommandExecutionSource, status: CommandExecutionStatus,
/**
* A best-effort parsing of the command to understand the action(s) it will perform.
* This returns a list of CommandAction objects because a single shell command may
* be composed of many commands piped together.
*/
commandActions: Array<CommandAction>,
commandActions: Array<CommandAction>,
/**
* The command's output, aggregated from stdout and stderr.
*/
aggregatedOutput: string | null,
aggregatedOutput: string | null,
/**
* The command's exit code.
*/
exitCode: number | null,
exitCode: number | null,
/**
* The duration of the command execution in milliseconds.
*/
durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array<FileUpdateChange>, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null,
durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array<FileUpdateChange>, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null,
/**
* The duration of the MCP tool call in milliseconds.
*/
durationMs: number | null, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array<DynamicToolCallOutputContentItem> | null, success: boolean | null,
durationMs: number | null, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array<DynamicToolCallOutputContentItem> | null, success: boolean | null,
/**
* The duration of the dynamic tool call in milliseconds.
*/
durationMs: number | null, } | { "type": "collabAgentToolCall",
durationMs: number | null, } | { "type": "browserReplay", id: string, frames: Array<BrowserReplayFrame>, renderMode: BrowserReplayRenderMode, animationImageUrl?: string | null, frameDurationMs?: number | null, } | { "type": "collabAgentToolCall",
/**
* Unique identifier for this collab tool call.
*/
id: string,
id: string,
/**
* Name of the collab tool that was invoked.
*/
tool: CollabAgentTool,
tool: CollabAgentTool,
/**
* Current status of the collab tool call.
*/
status: CollabAgentToolCallStatus,
status: CollabAgentToolCallStatus,
/**
* Thread ID of the agent issuing the collab request.
*/
senderThreadId: string,
senderThreadId: string,
/**
* Thread ID of the receiving agent, when applicable. In case of spawn operation,
* this corresponds to the newly spawned agent.
*/
receiverThreadIds: Array<string>,
receiverThreadIds: Array<string>,
/**
* Prompt text sent as part of the collab tool call, when available.
*/
prompt: string | null,
prompt: string | null,
/**
* Model requested for the spawned agent, when applicable.
*/
model: string | null,
model: string | null,
/**
* Reasoning effort requested for the spawned agent, when applicable.
*/
reasoningEffort: ReasoningEffort | null,
reasoningEffort: ReasoningEffort | null,
/**
* Last known status of the target agents, when available.
*/
@@ -25,6 +25,10 @@ export type { AppsDefaultConfig } from "./AppsDefaultConfig";
export type { AppsListParams } from "./AppsListParams";
export type { AppsListResponse } from "./AppsListResponse";
export type { AskForApproval } from "./AskForApproval";
export type { BrowserReplayFrame } from "./BrowserReplayFrame";
export type { BrowserReplayRenderMode } from "./BrowserReplayRenderMode";
export type { BrowserReplayTextSnapshotFormat } from "./BrowserReplayTextSnapshotFormat";
export type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts";
export type { BrowserSessionState } from "./BrowserSessionState";
export type { BrowserSessionUpdatedNotification } from "./BrowserSessionUpdatedNotification";
export type { BrowserTabState } from "./BrowserTabState";
@@ -179,6 +179,10 @@ pub fn generate_ts_with_options(
}
}
if !ts_files.is_empty() {
normalize_generated_ts_files(&ts_files)?;
}
Ok(())
}
@@ -1907,6 +1911,41 @@ fn prepend_header_if_missing(path: &Path) -> Result<()> {
Ok(())
}
pub(crate) fn normalize_typescript_text(text: &str) -> String {
let normalized = text.replace("\r\n", "\n").replace('\r', "\n");
let had_trailing_newline = normalized.ends_with('\n');
let mut result = normalized
.split('\n')
.map(|line| line.trim_end_matches([' ', '\t']))
.collect::<Vec<_>>()
.join("\n");
if had_trailing_newline && !result.ends_with('\n') {
result.push('\n');
}
result
}
fn normalize_generated_ts_files(paths: &[PathBuf]) -> Result<()> {
for path in paths {
if !path
.file_name()
.and_then(OsStr::to_str)
.is_some_and(|name| name == "ThreadItem.ts")
{
continue;
}
let content = fs::read_to_string(path)
.with_context(|| format!("Failed to read generated TypeScript {}", path.display()))?;
let normalized = normalize_typescript_text(&content);
if normalized == content {
continue;
}
fs::write(path, normalized)
.with_context(|| format!("Failed to normalize {}", path.display()))?;
}
Ok(())
}
fn ts_files_in(dir: &Path) -> Result<Vec<PathBuf>> {
let mut files = Vec::new();
for entry in
@@ -1921,6 +1921,57 @@ mod tests {
);
}
#[test]
fn reconstructs_dynamic_tool_items_from_response_events_only() {
let events = vec![
EventMsg::TurnStarted(TurnStartedEvent {
turn_id: "turn-1".into(),
model_context_window: None,
collaboration_mode_kind: Default::default(),
}),
EventMsg::UserMessage(UserMessageEvent {
message: "run dynamic tool".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::DynamicToolCallResponse(DynamicToolCallResponseEvent {
call_id: "dyn-1".into(),
turn_id: "turn-1".into(),
tool: "lookup_ticket".into(),
arguments: serde_json::json!({"id":"ABC-123"}),
content_items: vec![CoreDynamicToolCallOutputContentItem::InputText {
text: "Ticket is open".into(),
}],
success: true,
error: None,
duration: Duration::from_millis(42),
}),
];
let items = events
.into_iter()
.map(RolloutItem::EventMsg)
.collect::<Vec<_>>();
let turns = build_turns_from_rollout_items(&items);
assert_eq!(turns.len(), 1);
assert_eq!(turns[0].items.len(), 2);
assert_eq!(
turns[0].items[1],
ThreadItem::DynamicToolCall {
id: "dyn-1".into(),
tool: "lookup_ticket".into(),
arguments: serde_json::json!({"id":"ABC-123"}),
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
text: "Ticket is open".into(),
}]),
success: Some(true),
duration_ms: Some(42),
}
);
}
#[test]
fn reconstructs_declined_exec_and_patch_items() {
let events = vec![
@@ -4335,6 +4335,19 @@ pub enum ThreadItem {
},
#[serde(rename_all = "camelCase")]
#[ts(rename_all = "camelCase")]
BrowserReplay {
id: String,
frames: Vec<BrowserReplayFrame>,
render_mode: BrowserReplayRenderMode,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
animation_image_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
frame_duration_ms: Option<u32>,
},
#[serde(rename_all = "camelCase")]
#[ts(rename_all = "camelCase")]
CollabAgentToolCall {
/// Unique identifier for this collab tool call.
id: String,
@@ -4396,6 +4409,45 @@ pub struct HookPromptFragment {
pub hook_run_id: String,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct BrowserReplayFrame {
pub sequence_number: u32,
pub source_item_id: String,
pub tool: String,
pub status: DynamicToolCallStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub image_url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub text_snapshot: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub text_snapshot_format: Option<BrowserReplayTextSnapshotFormat>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[ts(optional = nullable)]
pub browser_state: Option<BrowserSessionState>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub enum BrowserReplayRenderMode {
Animation,
Frames,
TextOnly,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub enum BrowserReplayTextSnapshotFormat {
Plain,
Json,
}
impl ThreadItem {
pub fn id(&self) -> &str {
match self {
@@ -4408,6 +4460,7 @@ impl ThreadItem {
| ThreadItem::FileChange { id, .. }
| ThreadItem::McpToolCall { id, .. }
| ThreadItem::DynamicToolCall { id, .. }
| ThreadItem::BrowserReplay { id, .. }
| ThreadItem::CollabAgentToolCall { id, .. }
| ThreadItem::WebSearch { id, .. }
| ThreadItem::ImageView { id, .. }
@@ -8011,6 +8064,104 @@ mod tests {
);
}
#[test]
fn browser_replay_thread_item_serializes_animation_fields() {
let value = serde_json::to_value(ThreadItem::BrowserReplay {
id: "browser-replay-turn-1".to_string(),
frames: vec![BrowserReplayFrame {
sequence_number: 1,
source_item_id: "tool-call-1".to_string(),
tool: "create_tab".to_string(),
status: DynamicToolCallStatus::Completed,
image_url: Some("data:image/png;base64,AAA".to_string()),
text_snapshot: Some("Opened Hacker News".to_string()),
text_snapshot_format: Some(BrowserReplayTextSnapshotFormat::Plain),
browser_state: Some(BrowserSessionState {
selected_tab_id: "tab_123".to_string(),
tabs: vec![BrowserTabState {
id: "tab_123".to_string(),
title: "Hacker News".to_string(),
url: "https://news.ycombinator.com/".to_string(),
selected: true,
}],
}),
}],
render_mode: BrowserReplayRenderMode::Animation,
animation_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
frame_duration_ms: Some(700),
})
.unwrap();
assert_eq!(
value,
json!({
"type": "browserReplay",
"id": "browser-replay-turn-1",
"renderMode": "animation",
"frames": [{
"sequenceNumber": 1,
"sourceItemId": "tool-call-1",
"tool": "create_tab",
"status": "completed",
"imageUrl": "data:image/png;base64,AAA",
"textSnapshot": "Opened Hacker News",
"textSnapshotFormat": "plain",
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
}
}],
"animationImageUrl": "data:image/gif;base64,R0lGODlh",
"frameDurationMs": 700
})
);
}
#[test]
fn browser_replay_thread_item_serializes_text_only_fields() {
let value = serde_json::to_value(ThreadItem::BrowserReplay {
id: "browser-replay-turn-2".to_string(),
frames: vec![BrowserReplayFrame {
sequence_number: 1,
source_item_id: "tool-call-2".to_string(),
tool: "tabs_content".to_string(),
status: DynamicToolCallStatus::Completed,
image_url: None,
text_snapshot: Some(
"{\n \"tree\": {\n \"role\": \"document\"\n }\n}".to_string(),
),
text_snapshot_format: Some(BrowserReplayTextSnapshotFormat::Json),
browser_state: None,
}],
render_mode: BrowserReplayRenderMode::TextOnly,
animation_image_url: None,
frame_duration_ms: None,
})
.unwrap();
assert_eq!(
value,
json!({
"type": "browserReplay",
"id": "browser-replay-turn-2",
"renderMode": "textOnly",
"frames": [{
"sequenceNumber": 1,
"sourceItemId": "tool-call-2",
"tool": "tabs_content",
"status": "completed",
"textSnapshot": "{\n \"tree\": {\n \"role\": \"document\"\n }\n}",
"textSnapshotFormat": "json",
}],
})
);
}
#[test]
fn dynamic_tool_spec_deserializes_defer_loading() {
let value = json!({
@@ -5,6 +5,7 @@ use crate::ServerRequest;
use crate::export::GENERATED_TS_HEADER;
use crate::export::filter_experimental_ts_tree;
use crate::export::generate_index_ts_tree;
use crate::export::normalize_typescript_text;
use crate::protocol::common::visit_client_response_types;
use crate::protocol::common::visit_server_response_types;
use anyhow::Context;
@@ -129,11 +130,9 @@ fn read_file_bytes(path: &Path) -> Result<Vec<u8>> {
return Ok(normalized);
}
if path.extension().is_some_and(|ext| ext == "ts") {
// Windows checkouts (and some generators) may produce CRLF; normalize so the
// fixture test is platform-independent.
let text = String::from_utf8(bytes)
.with_context(|| format!("expected UTF-8 TypeScript in {}", path.display()))?;
let text = text.replace("\r\n", "\n").replace('\r', "\n");
let text = normalize_typescript_text(&text);
// Fixture comparisons care about schema content, not whether the generator
// re-prepended the standard banner to every TypeScript file.
let text = text
@@ -276,10 +275,7 @@ fn collect_typescript_fixture_file<T: TS + 'static + ?Sized>(
let contents = T::export_to_string().context("export TypeScript fixture content")?;
let output_path = normalize_relative_fixture_path(&output_path);
files.insert(
output_path,
contents.replace("\r\n", "\n").replace('\r', "\n"),
);
files.insert(output_path, normalize_typescript_text(&contents));
let mut visitor = TypeScriptFixtureCollector {
files,
+1
View File
@@ -52,6 +52,7 @@ codex-utils-json-to-toml = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true, features = ["derive"] }
futures = { workspace = true }
image = { workspace = true, features = ["gif", "jpeg", "png", "webp"] }
owo-colors = { workspace = true, features = ["supports-colors"] }
reqwest = { workspace = true, features = ["json", "rustls-tls"] }
serde = { workspace = true, features = ["derive"] }
+400
View File
@@ -0,0 +1,400 @@
use crate::remote_browser_api::RemoteBrowserApi;
use crate::remote_browser_api::RemoteBrowserCommandOutcome;
use codex_app_server_protocol::BrowserSessionArtifacts;
use codex_app_server_protocol::BrowserSessionCommandParams;
use codex_app_server_protocol::BrowserSessionState;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::DynamicToolCallResponse;
use codex_app_server_protocol::DynamicToolSpec;
use serde_json::Map;
use serde_json::Value as JsonValue;
use serde_json::json;
const ATLAS_COMMAND_TOOL: &str = "atlas_command";
const ATLAS_DISPLAY_TRUNCATE_MAX_CHARS: u32 = 4_000;
#[derive(Debug)]
pub(crate) struct AtlasCommandExecution {
pub(crate) browser_session_id: Option<String>,
pub(crate) browser_state: Option<BrowserSessionState>,
pub(crate) artifacts: Option<BrowserSessionArtifacts>,
pub(crate) response: DynamicToolCallResponse,
}
pub(crate) fn merge_atlas_command_dynamic_tool(
mut tools: Vec<DynamicToolSpec>,
) -> Vec<DynamicToolSpec> {
if !tools.iter().any(|tool| tool.name == ATLAS_COMMAND_TOOL) {
tools.push(DynamicToolSpec {
name: ATLAS_COMMAND_TOOL.to_string(),
description: "Hidden bridge for AgentLib inside js_repl. Call via codex.tool(...) only."
.to_string(),
input_schema: json!({
"type": "object",
"properties": {
"payload": {
"type": "object",
"additionalProperties": true
}
},
"required": ["payload"],
"additionalProperties": false
}),
defer_loading: true,
});
}
tools
}
pub(crate) fn is_atlas_command_dynamic_tool(name: &str) -> bool {
name == ATLAS_COMMAND_TOOL
}
pub(crate) async fn execute_atlas_command(
remote_browser_api: &RemoteBrowserApi,
browser_session_id: Option<String>,
arguments: JsonValue,
) -> AtlasCommandExecution {
match execute_atlas_command_impl(remote_browser_api, browser_session_id, arguments).await {
Ok(result) => result,
Err(message) => AtlasCommandExecution {
browser_session_id: None,
browser_state: None,
artifacts: None,
response: atlas_json_response(json!({
"type": "codex_repl",
"error": message,
})),
},
}
}
async fn execute_atlas_command_impl(
remote_browser_api: &RemoteBrowserApi,
browser_session_id: Option<String>,
arguments: JsonValue,
) -> Result<AtlasCommandExecution, String> {
let payload = arguments
.as_object()
.and_then(|args| args.get("payload"))
.cloned()
.ok_or_else(|| "atlas_command requires an object payload".to_string())?;
let payload_object = payload
.as_object()
.ok_or_else(|| "atlas_command payload must be an object".to_string())?;
let command_type = payload_object
.get("type")
.and_then(JsonValue::as_str)
.ok_or_else(|| "atlas_command payload.type is required".to_string())?;
let mut session_id = browser_session_id;
let mut last_browser_state = None;
let mut last_artifacts = None;
let command_output = match command_type {
"runtime_config" => json!({
"display_truncate_max_chars": ATLAS_DISPLAY_TRUNCATE_MAX_CHARS,
}),
"create_tab" => {
let outcome = call_remote(
remote_browser_api,
&mut session_id,
"create_tab",
json!({ "select": true }),
)
.await?;
let result = outcome.result.clone();
last_browser_state = outcome.browser_state.to_public_state();
last_artifacts = outcome.to_public_artifacts();
json!({
"id": result.get("tab_id").and_then(JsonValue::as_str).unwrap_or_default(),
})
}
"list_tabs" => {
let outcome =
call_remote(remote_browser_api, &mut session_id, "list_tabs", json!({})).await?;
let tabs = outcome
.result
.get("tabs")
.and_then(JsonValue::as_array)
.cloned()
.unwrap_or_default()
.into_iter()
.map(|tab| {
let tab_object = tab.as_object().cloned().unwrap_or_default();
json!({
"id": tab_object
.get("id")
.cloned()
.unwrap_or(JsonValue::String(String::new())),
"title": tab_object.get("title").cloned().unwrap_or(JsonValue::Null),
"url": tab_object.get("url").cloned().unwrap_or(JsonValue::Null),
})
})
.collect::<Vec<_>>();
last_browser_state = outcome.browser_state.to_public_state();
json!({ "tabs": tabs })
}
"selected_tab" => {
let outcome =
call_remote(remote_browser_api, &mut session_id, "selected_tab", json!({}))
.await?;
let selected_tab_id = outcome
.result
.get("selected_tab_id")
.and_then(JsonValue::as_str);
last_browser_state = outcome.browser_state.to_public_state();
json!({
"id": selected_tab_id,
})
}
"navigate_tab_url" => {
let remote_args = normalize_passthrough_payload(payload_object);
let outcome =
call_remote(remote_browser_api, &mut session_id, "navigate_tab_url", remote_args)
.await?;
last_browser_state = outcome.browser_state.to_public_state();
last_artifacts = outcome.to_public_artifacts();
json!({})
}
"playwright_wait_for_load_state" => {
let remote_args = normalize_passthrough_payload(payload_object);
let outcome = call_remote(
remote_browser_api,
&mut session_id,
"playwright_wait_for_load_state",
remote_args,
)
.await?;
last_browser_state = outcome.browser_state.to_public_state();
last_artifacts = outcome.to_public_artifacts();
json!({})
}
"playwright_locator_count"
| "playwright_locator_text_content"
| "playwright_locator_inner_text"
| "playwright_locator_get_attribute"
| "playwright_locator_is_visible"
| "playwright_locator_is_enabled" => {
let remote_args = normalize_passthrough_payload(payload_object);
let outcome = call_remote(
remote_browser_api,
&mut session_id,
command_type,
remote_args,
)
.await?;
last_browser_state = outcome.browser_state.to_public_state();
outcome.result.clone()
}
"playwright_locator_click" | "playwright_locator_dblclick" => {
let remote_args = normalize_passthrough_payload(payload_object);
let outcome = call_remote(
remote_browser_api,
&mut session_id,
command_type,
remote_args,
)
.await?;
last_browser_state = outcome.browser_state.to_public_state();
last_artifacts = outcome.to_public_artifacts();
json!({})
}
"playwright_locator_wait_for" => {
let remote_args = normalize_passthrough_payload(payload_object);
let outcome = call_remote(
remote_browser_api,
&mut session_id,
"playwright_locator_wait_for",
remote_args,
)
.await?;
last_browser_state = outcome.browser_state.to_public_state();
json!({})
}
"playwright_screenshot" => {
if has_any_keys(
payload_object,
&[
"cropX",
"cropY",
"cropWidth",
"cropHeight",
"crop_x",
"crop_y",
"crop_width",
"crop_height",
],
) {
return Err(
"atlas_command playwright_screenshot crop options are not supported by the remote browser bridge yet"
.to_string(),
);
}
let remote_args = normalize_passthrough_payload(payload_object);
let outcome = call_remote(
remote_browser_api,
&mut session_id,
"playwright_screenshot",
remote_args,
)
.await?;
last_browser_state = outcome.browser_state.to_public_state();
last_artifacts = outcome.to_public_artifacts();
json!({
"data": outcome
.screenshot_base64()
.ok_or_else(|| "remote browser did not return screenshot data".to_string())?,
})
}
"cua_get_visible_screenshot" => {
let tab_id = payload_object
.get("tab_id")
.cloned()
.ok_or_else(|| "cua_get_visible_screenshot requires tab_id".to_string())?;
let outcome = call_remote(
remote_browser_api,
&mut session_id,
"playwright_screenshot",
json!({
"tab_id": tab_id,
"full_page": false,
}),
)
.await?;
last_browser_state = outcome.browser_state.to_public_state();
last_artifacts = outcome.to_public_artifacts();
json!({
"data": outcome
.screenshot_base64()
.ok_or_else(|| "remote browser did not return screenshot data".to_string())?,
})
}
"tabs_content" => {
let urls = payload_object
.get("urls")
.and_then(JsonValue::as_array)
.ok_or_else(|| "tabs_content requires urls".to_string())?;
let content_type = payload_object
.get("content_type")
.and_then(JsonValue::as_str)
.ok_or_else(|| "tabs_content requires content_type".to_string())?;
if content_type != "text" {
return Err(format!(
"atlas_command tabs_content only supports content_type=\"text\" today; got {content_type:?}"
));
}
let timeout_ms = payload_object.get("timeout_ms").cloned();
let mut tab_ids = Vec::with_capacity(urls.len());
for url in urls {
let url = url
.as_str()
.ok_or_else(|| "tabs_content urls must be strings".to_string())?;
let mut create_args = Map::new();
create_args.insert("url".to_string(), JsonValue::String(url.to_string()));
create_args.insert("select".to_string(), JsonValue::Bool(false));
if let Some(timeout_ms) = timeout_ms.clone() {
create_args.insert("timeout_ms".to_string(), timeout_ms);
}
let outcome = call_remote(
remote_browser_api,
&mut session_id,
"create_tab",
JsonValue::Object(create_args),
)
.await?;
let tab_id = outcome
.result
.get("tab_id")
.and_then(JsonValue::as_str)
.ok_or_else(|| "remote browser create_tab did not return tab_id".to_string())?;
tab_ids.push(tab_id.to_string());
}
let outcome = call_remote(
remote_browser_api,
&mut session_id,
"tabs_content",
json!({ "tab_ids": tab_ids }),
)
.await?;
let tabs = outcome
.result
.get("tabs")
.and_then(JsonValue::as_array)
.cloned()
.unwrap_or_default()
.into_iter()
.map(|tab| {
let tab_object = tab.as_object().cloned().unwrap_or_default();
json!({
"url": tab_object
.get("url")
.cloned()
.unwrap_or(JsonValue::String(String::new())),
"title": tab_object.get("title").cloned().unwrap_or(JsonValue::Null),
"content": tab_object.get("content").cloned().unwrap_or(JsonValue::Null),
})
})
.collect::<Vec<_>>();
last_browser_state = outcome.browser_state.to_public_state();
json!({ "results": tabs })
}
other => {
return Err(format!(
"atlas_command does not support AgentLib command type {other:?} yet"
));
}
};
Ok(AtlasCommandExecution {
browser_session_id: session_id,
browser_state: last_browser_state,
artifacts: last_artifacts,
response: atlas_json_response(json!({
"type": "codex_repl",
"command_output": command_output,
})),
})
}
fn atlas_json_response(payload: JsonValue) -> DynamicToolCallResponse {
DynamicToolCallResponse {
content_items: vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::to_string(&payload).unwrap_or_else(|_| {
"{\"type\":\"codex_repl\",\"error\":\"serialization failure\"}".to_string()
}),
}],
success: true,
}
}
async fn call_remote(
remote_browser_api: &RemoteBrowserApi,
browser_session_id: &mut Option<String>,
command: &str,
arguments: JsonValue,
) -> Result<RemoteBrowserCommandOutcome, String> {
let outcome = remote_browser_api
.command_with_artifacts(BrowserSessionCommandParams {
browser_session_id: browser_session_id.clone(),
command: command.to_string(),
arguments: Some(arguments),
})
.await
.map_err(|err| err.message)?;
*browser_session_id = Some(outcome.browser_session_id.clone());
Ok(outcome)
}
fn normalize_passthrough_payload(payload: &Map<String, JsonValue>) -> JsonValue {
let mut args = payload.clone();
args.remove("type");
JsonValue::Object(args)
}
fn has_any_keys(payload: &Map<String, JsonValue>, keys: &[&str]) -> bool {
keys.iter().any(|key| payload.contains_key(*key))
}
@@ -1,3 +1,5 @@
use crate::atlas_command::execute_atlas_command;
use crate::atlas_command::is_atlas_command_dynamic_tool;
use crate::codex_message_processor::ApiVersion;
use crate::codex_message_processor::read_rollout_items_from_rollout;
use crate::codex_message_processor::read_summary_from_rollout;
@@ -286,6 +288,42 @@ async fn handle_remote_browser_dynamic_tool_request(
submit_dynamic_tool_response(conversation, call_id, response).await;
}
async fn handle_atlas_command_dynamic_tool_request(
call_id: String,
arguments: JsonValue,
conversation: Arc<CodexThread>,
thread_state: Arc<Mutex<ThreadState>>,
outgoing: ThreadScopedOutgoingMessageSender,
remote_browser_api: RemoteBrowserApi,
) {
let browser_session_id = { thread_state.lock().await.remote_browser_session_id.clone() };
let outcome = execute_atlas_command(&remote_browser_api, browser_session_id, arguments).await;
if let Some(browser_session_id) = outcome.browser_session_id.clone() {
let mut state = thread_state.lock().await;
state.remote_browser_session_id = Some(browser_session_id);
}
if let Some(browser_state) = outcome.browser_state {
outgoing
.send_server_notification(ServerNotification::BrowserSessionUpdated(
BrowserSessionUpdatedNotification {
browser_session_id: outcome.browser_session_id.unwrap_or_default(),
browser_state,
artifacts: outcome.artifacts,
},
))
.await;
}
submit_dynamic_tool_response(
conversation,
call_id,
to_core_dynamic_tool_response(outcome.response),
)
.await;
}
fn guardian_auto_approval_review_notification(
conversation_id: &ThreadId,
event_turn_id: &str,
@@ -1013,7 +1051,18 @@ pub(crate) async fn apply_bespoke_event_handling(
outgoing
.send_server_notification(ServerNotification::ItemStarted(notification))
.await;
if remote_browser_api.is_configured() && is_remote_browser_dynamic_tool(&tool) {
if remote_browser_api.is_configured() && is_atlas_command_dynamic_tool(&tool) {
tokio::spawn(handle_atlas_command_dynamic_tool_request(
call_id,
arguments,
conversation,
thread_state.clone(),
outgoing.clone(),
remote_browser_api.clone(),
));
} else if remote_browser_api.is_configured()
&& is_remote_browser_dynamic_tool(&tool)
{
tokio::spawn(handle_remote_browser_dynamic_tool_request(
call_id,
tool,
+498
View File
@@ -0,0 +1,498 @@
use crate::remote_browser_tools::is_remote_browser_dynamic_tool;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use codex_app_server_protocol::BrowserReplayFrame;
use codex_app_server_protocol::BrowserReplayRenderMode;
use codex_app_server_protocol::BrowserReplayTextSnapshotFormat;
use codex_app_server_protocol::BrowserSessionState;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::Turn;
use image::Delay;
use image::Frame;
use image::Rgba;
use image::RgbaImage;
use image::codecs::gif::GifEncoder;
use image::codecs::gif::Repeat;
use serde::Deserialize;
use serde_json::Value as JsonValue;
use std::io::Cursor;
const REPLAY_FRAME_DURATION_MS: u32 = 700;
const MAX_TEXT_SNAPSHOT_CHARS: usize = 2_000;
pub(crate) fn augment_turns_with_browser_replays(turns: &mut [Turn]) {
for turn in turns {
turn.items
.retain(|item| !matches!(item, ThreadItem::BrowserReplay { .. }));
if let Some(replay_item) = build_browser_replay_item(&turn.id, &turn.items) {
turn.items.push(replay_item);
}
}
}
fn build_browser_replay_item(turn_id: &str, items: &[ThreadItem]) -> Option<ThreadItem> {
let mut frames = Vec::new();
let mut static_image_urls = Vec::new();
let mut fallback_animation_url = None;
for item in items {
let ThreadItem::DynamicToolCall {
id,
tool,
status,
content_items: Some(content_items),
..
} = item
else {
continue;
};
if !is_remote_browser_dynamic_tool(tool) {
continue;
}
let frame_content = inspect_content_items(content_items);
if let Some(image_url) = frame_content.image_url.clone() {
static_image_urls.push(image_url);
}
if fallback_animation_url.is_none() {
fallback_animation_url = frame_content.animation_image_url.clone();
}
if frame_content.image_url.is_none()
&& frame_content.text_snapshot.is_none()
&& frame_content.browser_state.is_none()
{
continue;
}
frames.push(BrowserReplayFrame {
sequence_number: (frames.len() + 1) as u32,
source_item_id: id.clone(),
tool: tool.clone(),
status: status.clone(),
image_url: frame_content.image_url,
text_snapshot: frame_content.text_snapshot,
text_snapshot_format: frame_content.text_snapshot_format,
browser_state: frame_content.browser_state,
});
}
if frames.is_empty() {
return None;
}
let animation_image_url =
synthesize_animation_image_url(&static_image_urls).or(fallback_animation_url);
let render_mode = if animation_image_url.is_some() {
BrowserReplayRenderMode::Animation
} else if frames.iter().any(|frame| frame.image_url.is_some()) {
BrowserReplayRenderMode::Frames
} else {
BrowserReplayRenderMode::TextOnly
};
let frame_duration_ms =
(animation_image_url.is_some() || frames.len() > 1).then_some(REPLAY_FRAME_DURATION_MS);
Some(ThreadItem::BrowserReplay {
id: format!("browser-replay-{turn_id}"),
frames,
render_mode,
animation_image_url,
frame_duration_ms,
})
}
#[derive(Default)]
struct FrameContent {
image_url: Option<String>,
animation_image_url: Option<String>,
text_snapshot: Option<String>,
text_snapshot_format: Option<BrowserReplayTextSnapshotFormat>,
browser_state: Option<BrowserSessionState>,
}
#[derive(Debug, Deserialize)]
struct RemoteBrowserToolTextPayload {
#[serde(default)]
result: JsonValue,
#[serde(default)]
browser_state: Option<BrowserSessionState>,
}
fn inspect_content_items(items: &[DynamicToolCallOutputContentItem]) -> FrameContent {
let mut content = FrameContent::default();
for item in items {
match item {
DynamicToolCallOutputContentItem::InputText { text } => {
if content.text_snapshot.is_none() {
let snapshot = build_text_snapshot(text);
content.text_snapshot = snapshot.text;
content.text_snapshot_format = snapshot.format;
}
if content.browser_state.is_none() {
content.browser_state = parse_browser_state(text);
}
}
DynamicToolCallOutputContentItem::InputImage { image_url } => {
if is_gif_data_url(image_url) {
if content.animation_image_url.is_none() {
content.animation_image_url = Some(image_url.clone());
}
} else if is_raster_data_url(image_url) && content.image_url.is_none() {
content.image_url = Some(image_url.clone());
}
}
}
}
content
}
#[derive(Default)]
struct TextSnapshot {
text: Option<String>,
format: Option<BrowserReplayTextSnapshotFormat>,
}
fn build_text_snapshot(text: &str) -> TextSnapshot {
serde_json::from_str::<RemoteBrowserToolTextPayload>(text)
.ok()
.and_then(|payload| render_result_snapshot(&payload.result))
.or_else(|| {
truncate_snapshot(text).map(|value| TextSnapshot {
text: Some(value),
format: Some(BrowserReplayTextSnapshotFormat::Plain),
})
})
.unwrap_or_default()
}
fn parse_browser_state(text: &str) -> Option<BrowserSessionState> {
serde_json::from_str::<RemoteBrowserToolTextPayload>(text)
.ok()
.and_then(|payload| payload.browser_state)
}
fn render_result_snapshot(result: &JsonValue) -> Option<TextSnapshot> {
if result.is_null() {
return None;
}
match result {
JsonValue::String(text) => truncate_snapshot(text).map(|value| TextSnapshot {
text: Some(value),
format: Some(BrowserReplayTextSnapshotFormat::Plain),
}),
_ => serde_json::to_string_pretty(result)
.ok()
.and_then(|text| truncate_snapshot(&text))
.map(|value| TextSnapshot {
text: Some(value),
format: Some(BrowserReplayTextSnapshotFormat::Json),
}),
}
}
fn truncate_snapshot(text: &str) -> Option<String> {
let trimmed = text.trim();
if trimmed.is_empty() {
return None;
}
let char_count = trimmed.chars().count();
if char_count <= MAX_TEXT_SNAPSHOT_CHARS {
return Some(trimmed.to_string());
}
let truncated = trimmed
.chars()
.take(MAX_TEXT_SNAPSHOT_CHARS)
.collect::<String>();
Some(format!("{truncated}..."))
}
fn synthesize_animation_image_url(image_urls: &[String]) -> Option<String> {
if image_urls.len() < 2 {
return None;
}
let mut decoded_frames = Vec::new();
for image_url in image_urls {
let image = decode_raster_data_url(image_url)?;
decoded_frames.push(image.to_rgba8());
}
if decoded_frames.len() < 2 {
return None;
}
let max_width = decoded_frames.iter().map(RgbaImage::width).max()?;
let max_height = decoded_frames.iter().map(RgbaImage::height).max()?;
let delay = Delay::from_numer_denom_ms(REPLAY_FRAME_DURATION_MS, 1);
let frames = decoded_frames
.into_iter()
.map(|frame| Frame::from_parts(normalize_frame(frame, max_width, max_height), 0, 0, delay));
let mut bytes = Cursor::new(Vec::new());
{
let mut encoder = GifEncoder::new(&mut bytes);
encoder.set_repeat(Repeat::Infinite).ok()?;
encoder.encode_frames(frames).ok()?;
}
Some(format!(
"data:image/gif;base64,{}",
BASE64_STANDARD.encode(bytes.into_inner())
))
}
fn normalize_frame(frame: RgbaImage, width: u32, height: u32) -> RgbaImage {
if frame.width() == width && frame.height() == height {
return frame;
}
let mut canvas = RgbaImage::from_pixel(width, height, Rgba([255, 255, 255, 255]));
image::imageops::overlay(&mut canvas, &frame, 0, 0);
canvas
}
fn decode_raster_data_url(image_url: &str) -> Option<image::DynamicImage> {
if !is_raster_data_url(image_url) {
return None;
}
let (_, encoded) = image_url.split_once(',')?;
let bytes = BASE64_STANDARD.decode(encoded).ok()?;
image::load_from_memory(&bytes).ok()
}
fn is_raster_data_url(image_url: &str) -> bool {
image_url.starts_with("data:image/png;base64,")
|| image_url.starts_with("data:image/jpeg;base64,")
|| image_url.starts_with("data:image/webp;base64,")
}
fn is_gif_data_url(image_url: &str) -> bool {
image_url.starts_with("data:image/gif;base64,")
}
#[cfg(test)]
mod tests {
use super::*;
use codex_app_server_protocol::DynamicToolCallStatus;
use image::DynamicImage;
fn png_data_url(color: [u8; 4], width: u32, height: u32) -> String {
let image = RgbaImage::from_pixel(width, height, Rgba(color));
let mut bytes = Cursor::new(Vec::new());
DynamicImage::ImageRgba8(image)
.write_to(&mut bytes, image::ImageFormat::Png)
.expect("encode png");
format!(
"data:image/png;base64,{}",
BASE64_STANDARD.encode(bytes.into_inner())
)
}
#[test]
fn builds_replay_from_remote_browser_tool_calls() {
let items = vec![
ThreadItem::DynamicToolCall {
id: "tool-1".to_string(),
tool: "create_tab".to_string(),
arguments: JsonValue::Null,
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![
DynamicToolCallOutputContentItem::InputText {
text: serde_json::json!({
"result": { "tab_id": "tab_1" },
"browser_state": {
"selectedTabId": "tab_1",
"tabs": [{
"id": "tab_1",
"title": "First",
"url": "https://example.com/one",
"selected": true
}]
}
})
.to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: png_data_url([255, 0, 0, 255], 10, 10),
},
]),
success: Some(true),
duration_ms: Some(10),
},
ThreadItem::DynamicToolCall {
id: "tool-2".to_string(),
tool: "tabs_content".to_string(),
arguments: JsonValue::Null,
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![
DynamicToolCallOutputContentItem::InputText {
text: serde_json::json!({
"result": {
"tabs": [{
"title": "First",
"content": "headline one\nheadline two"
}]
},
"browser_state": {
"selectedTabId": "tab_1",
"tabs": [{
"id": "tab_1",
"title": "First",
"url": "https://example.com/one",
"selected": true
}]
}
})
.to_string(),
},
DynamicToolCallOutputContentItem::InputImage {
image_url: png_data_url([0, 0, 255, 255], 12, 8),
},
]),
success: Some(true),
duration_ms: Some(10),
},
];
let replay = build_browser_replay_item("turn-1", &items).expect("replay item");
let ThreadItem::BrowserReplay {
id,
frames,
render_mode,
animation_image_url,
frame_duration_ms,
} = replay
else {
panic!("expected browser replay item");
};
assert_eq!(id, "browser-replay-turn-1");
assert_eq!(frames.len(), 2);
assert_eq!(render_mode, BrowserReplayRenderMode::Animation);
assert_eq!(frames[0].source_item_id, "tool-1");
assert_eq!(frames[1].source_item_id, "tool-2");
let second_snapshot = frames[1]
.text_snapshot
.as_deref()
.expect("second frame snapshot");
assert!(second_snapshot.contains("\"title\": \"First\""));
assert!(second_snapshot.contains("\"content\": \"headline one\\nheadline two\""));
assert_eq!(
frames[1].text_snapshot_format,
Some(BrowserReplayTextSnapshotFormat::Json)
);
assert!(
animation_image_url
.as_deref()
.is_some_and(|value| value.starts_with("data:image/gif;base64,"))
);
assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS));
}
#[test]
fn builds_text_only_replay_when_no_images_are_available() {
let items = vec![ThreadItem::DynamicToolCall {
id: "tool-1".to_string(),
tool: "tabs_content".to_string(),
arguments: JsonValue::Null,
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::json!({
"result": {
"tree": {
"role": "document",
"name": "Example page"
}
}
})
.to_string(),
}]),
success: Some(true),
duration_ms: Some(5),
}];
let replay = build_browser_replay_item("turn-2", &items).expect("replay item");
let ThreadItem::BrowserReplay {
frames,
render_mode,
animation_image_url,
frame_duration_ms,
..
} = replay
else {
panic!("expected browser replay item");
};
assert_eq!(frames.len(), 1);
assert_eq!(render_mode, BrowserReplayRenderMode::TextOnly);
assert_eq!(
frames[0].text_snapshot.as_deref(),
Some(
"{\n \"tree\": {\n \"role\": \"document\",\n \"name\": \"Example page\"\n }\n}"
)
);
assert_eq!(
frames[0].text_snapshot_format,
Some(BrowserReplayTextSnapshotFormat::Json)
);
assert_eq!(animation_image_url, None);
assert_eq!(frame_duration_ms, None);
}
#[test]
fn assigns_frame_duration_to_multi_frame_text_only_replays() {
let items = vec![
ThreadItem::DynamicToolCall {
id: "tool-1".to_string(),
tool: "tabs_content".to_string(),
arguments: JsonValue::Null,
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::json!({
"result": { "tree": { "role": "document", "name": "Step 1" } }
})
.to_string(),
}]),
success: Some(true),
duration_ms: Some(5),
},
ThreadItem::DynamicToolCall {
id: "tool-2".to_string(),
tool: "tabs_content".to_string(),
arguments: JsonValue::Null,
status: DynamicToolCallStatus::Completed,
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::json!({
"result": { "tree": { "role": "document", "name": "Step 2" } }
})
.to_string(),
}]),
success: Some(true),
duration_ms: Some(5),
},
];
let replay = build_browser_replay_item("turn-3", &items).expect("replay item");
let ThreadItem::BrowserReplay {
frames,
render_mode,
animation_image_url,
frame_duration_ms,
..
} = replay
else {
panic!("expected browser replay item");
};
assert_eq!(frames.len(), 2);
assert_eq!(render_mode, BrowserReplayRenderMode::TextOnly);
assert_eq!(animation_image_url, None);
assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS));
}
}
@@ -1,4 +1,6 @@
use crate::atlas_command::merge_atlas_command_dynamic_tool;
use crate::bespoke_event_handling::apply_bespoke_event_handling;
use crate::browser_replay::augment_turns_with_browser_replays;
use crate::command_exec::CommandExecManager;
use crate::command_exec::StartCommandExecParams;
use crate::error_code::INPUT_TOO_LARGE_ERROR_CODE;
@@ -1992,7 +1994,9 @@ impl CodexMessageProcessor {
};
let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() {
merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default())
let dynamic_tools =
merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default());
merge_atlas_command_dynamic_tool(dynamic_tools)
} else {
dynamic_tools.unwrap_or_default()
};
@@ -3299,11 +3303,28 @@ impl CodexMessageProcessor {
self.attach_thread_name(thread_uuid, &mut thread).await;
if include_turns && let Some(rollout_path) = rollout_path.as_ref() {
match read_rollout_items_from_rollout(rollout_path).await {
Ok(items) => {
thread.turns = build_turns_from_rollout_items(&items);
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
let active_turn = if loaded_thread.is_some() {
let state = self.thread_state_manager.thread_state(thread_uuid).await;
let state = state.lock().await;
state.active_turn_snapshot()
} else {
None
};
if let Err(message) = populate_thread_turns(
&mut thread,
ThreadTurnSource::RolloutPath(rollout_path),
active_turn.as_ref(),
)
.await
{
let missing_rollout_message = format!(
"failed to load rollout `{}` for thread {thread_uuid}: ",
rollout_path.display()
);
if message.contains(&missing_rollout_message)
&& message.contains("No such file or directory")
{
self.send_invalid_request_error(
request_id,
format!(
@@ -3313,17 +3334,9 @@ impl CodexMessageProcessor {
.await;
return;
}
Err(err) => {
self.send_internal_error(
request_id,
format!(
"failed to load rollout `{}` for thread {thread_uuid}: {err}",
rollout_path.display()
),
)
.await;
return;
}
self.send_internal_error(request_id, message).await;
return;
}
}
@@ -7473,6 +7486,7 @@ async fn populate_thread_turns(
if let Some(active_turn) = active_turn {
merge_turn_history_with_active_turn(&mut turns, active_turn.clone());
}
augment_turns_with_browser_replays(&mut turns);
thread.turns = turns;
Ok(())
}
+2
View File
@@ -57,7 +57,9 @@ use tracing_subscriber::registry::Registry;
use tracing_subscriber::util::SubscriberInitExt;
mod app_server_tracing;
mod atlas_command;
mod bespoke_event_handling;
mod browser_replay;
mod codex_message_processor;
mod command_exec;
mod config_api;
@@ -7,9 +7,9 @@ use crate::transport::AppServerTransport;
use anyhow::Result;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::write_mock_responses_config_toml;
use codex_app_server_protocol::BrowserSessionArtifacts;
use codex_app_server_protocol::BrowserSessionCommandParams;
use codex_app_server_protocol::BrowserSessionCommandResponse;
use codex_app_server_protocol::BrowserSessionArtifacts;
use codex_app_server_protocol::BrowserSessionState;
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
use codex_app_server_protocol::BrowserTabState;
@@ -116,6 +116,10 @@ impl RemoteBrowserCommandOutcome {
.map(|encoded| format!("data:image/png;base64,{encoded}"))
}
pub(crate) fn screenshot_base64(&self) -> Option<String> {
self.artifacts.screenshot_base64.clone()
}
pub(crate) fn replay_gif_data_url(&self) -> Option<String> {
self.artifacts
.replay_gif_base64
@@ -5,6 +5,8 @@ use app_test_support::create_final_assistant_message_sse_response;
use app_test_support::create_mock_responses_server_repeating_assistant;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::to_response;
use codex_app_server_protocol::BrowserReplayRenderMode;
use codex_app_server_protocol::BrowserReplayTextSnapshotFormat;
use codex_app_server_protocol::BrowserSessionArtifacts;
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
@@ -19,6 +21,8 @@ use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerRequest;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadReadParams;
use codex_app_server_protocol::ThreadReadResponse;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::TurnStartParams;
@@ -38,6 +42,7 @@ use tokio::time::timeout;
use wiremock::Mock;
use wiremock::MockServer;
use wiremock::ResponseTemplate;
use wiremock::matchers::body_json;
use wiremock::matchers::method;
use wiremock::matchers::path;
@@ -255,6 +260,655 @@ async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configu
assert!(find_tool(body, "create_tab").is_some());
assert!(find_tool(body, "tabs_content").is_some());
assert!(find_tool(body, "playwright_screenshot").is_some());
assert!(find_tool(body, "atlas_command").is_none());
Ok(())
}
#[tokio::test]
async fn atlas_command_dynamic_tool_calls_are_handled_internally() -> Result<()> {
let call_id = "dyn-atlas-1";
let tool_name = "atlas_command";
let tool_args = json!({
"payload": {
"type": "create_tab"
}
});
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.and(body_json(json!({
"browserSessionId": null,
"command": "create_tab",
"arguments": {
"select": true
}
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_atlas_123",
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "New Tab",
"url": "about:blank",
"selected": true
}]
}
})))
.expect(1)
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let thread_id = thread.id.clone();
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread_id.clone(),
input: vec![V2UserInput::Text {
text: "use atlas".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
let turn_id = turn.id.clone();
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
assert_eq!(started.thread_id, thread_id);
assert_eq!(started.turn_id, turn_id.clone());
assert!(
timeout(
Duration::from_millis(500),
mcp.read_stream_until_request_message()
)
.await
.is_err(),
"atlas_command tool call should not be forwarded back to the client"
);
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
assert_eq!(browser_update.browser_session_id, "sess_atlas_123");
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
assert_eq!(browser_update.browser_state.tabs.len(), 1);
assert_eq!(browser_update.browser_state.tabs[0].title, "New Tab");
assert_eq!(browser_update.artifacts, None);
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
let ThreadItem::DynamicToolCall {
status,
content_items,
success,
..
} = completed.item
else {
panic!("expected dynamic tool call item");
};
assert_eq!(status, DynamicToolCallStatus::Completed);
assert_eq!(success, Some(true));
assert_eq!(
content_items,
Some(vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::to_string(&json!({
"type": "codex_repl",
"command_output": {
"id": "tab_123"
}
}))?,
}])
);
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let bodies = responses_bodies(&server).await?;
let payload = bodies
.iter()
.find_map(|body| function_call_output_payload(body, call_id))
.context("expected function_call_output in follow-up request")?;
assert_eq!(
payload.body,
FunctionCallOutputBody::Text(serde_json::to_string(&json!({
"type": "codex_repl",
"command_output": {
"id": "tab_123"
}
}))?)
);
assert_eq!(payload.success, None);
Ok(())
}
#[tokio::test]
async fn atlas_command_tabs_content_uses_background_tabs() -> Result<()> {
let call_id = "dyn-atlas-tabs-1";
let tool_name = "atlas_command";
let tool_args = json!({
"payload": {
"type": "tabs_content",
"urls": ["https://news.ycombinator.com/"],
"content_type": "text"
}
});
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.and(body_json(json!({
"browserSessionId": null,
"command": "create_tab",
"arguments": {
"url": "https://news.ycombinator.com/",
"select": false
}
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_atlas_tabs_123",
"result": {
"tab_id": "tab_bg_123"
},
"browserState": {
"selectedTabId": "tab_main",
"tabs": [
{
"id": "tab_main",
"title": "Workspace",
"url": "http://localhost:3000/",
"selected": true
},
{
"id": "tab_bg_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": false
}
]
}
})))
.expect(1)
.mount(&remote_browser_server)
.await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.and(body_json(json!({
"browserSessionId": "sess_atlas_tabs_123",
"command": "tabs_content",
"arguments": {
"tab_ids": ["tab_bg_123"]
}
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_atlas_tabs_123",
"result": {
"tabs": [{
"url": "https://news.ycombinator.com/",
"title": "Hacker News",
"content": "Top stories"
}]
},
"browserState": {
"selectedTabId": "tab_main",
"tabs": [
{
"id": "tab_main",
"title": "Workspace",
"url": "http://localhost:3000/",
"selected": true
},
{
"id": "tab_bg_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": false
}
]
}
})))
.expect(1)
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id,
input: vec![V2UserInput::Text {
text: "read the page".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
let _started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
assert_eq!(browser_update.browser_session_id, "sess_atlas_tabs_123");
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_main");
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
let ThreadItem::DynamicToolCall {
content_items,
success,
..
} = completed.item
else {
panic!("expected dynamic tool call item");
};
assert_eq!(success, Some(true));
assert_eq!(
content_items,
Some(vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::to_string(&json!({
"type": "codex_repl",
"command_output": {
"results": [{
"url": "https://news.ycombinator.com/",
"title": "Hacker News",
"content": "Top stories"
}]
}
}))?,
}])
);
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
Ok(())
}
#[tokio::test]
async fn atlas_command_locator_count_passes_through_to_remote_browser() -> Result<()> {
let call_id = "dyn-atlas-locator-count-1";
let tool_name = "atlas_command";
let tool_args = json!({
"payload": {
"type": "playwright_locator_count",
"tab_id": "tab_123",
"selector": "span.titleline > a"
}
});
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.and(body_json(json!({
"browserSessionId": null,
"command": "playwright_locator_count",
"arguments": {
"tab_id": "tab_123",
"selector": "span.titleline > a"
}
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_atlas_count_123",
"result": {
"count": 5
},
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
}
})))
.expect(1)
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let thread_id = thread.id.clone();
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread_id.clone(),
input: vec![V2UserInput::Text {
text: "count links".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
let turn_id = turn.id.clone();
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
assert_eq!(started.thread_id, thread_id);
assert_eq!(started.turn_id, turn_id.clone());
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
assert_eq!(browser_update.browser_session_id, "sess_atlas_count_123");
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
let ThreadItem::DynamicToolCall {
content_items,
success,
..
} = completed.item
else {
panic!("expected dynamic tool call item");
};
assert_eq!(success, Some(true));
assert_eq!(
content_items,
Some(vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::to_string(&json!({
"type": "codex_repl",
"command_output": {
"count": 5
}
}))?,
}])
);
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let bodies = responses_bodies(&server).await?;
let payload = bodies
.iter()
.find_map(|body| function_call_output_payload(body, call_id))
.context("expected function_call_output in follow-up request")?;
assert_eq!(
payload.body,
FunctionCallOutputBody::Text(serde_json::to_string(&json!({
"type": "codex_repl",
"command_output": {
"count": 5
}
}))?)
);
assert_eq!(payload.success, None);
Ok(())
}
#[tokio::test]
async fn atlas_command_locator_click_passes_through_to_remote_browser() -> Result<()> {
let call_id = "dyn-atlas-locator-click-1";
let tool_name = "atlas_command";
let tool_args = json!({
"payload": {
"type": "playwright_locator_click",
"tab_id": "tab_123",
"selector": "[data-square=\"e2\"]",
"button": "left",
"timeout_ms": 1500
}
});
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Clicked")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.and(body_json(json!({
"browserSessionId": null,
"command": "playwright_locator_click",
"arguments": {
"tab_id": "tab_123",
"selector": "[data-square=\"e2\"]",
"button": "left",
"timeout_ms": 1500
}
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_atlas_click_123",
"result": {},
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "3D Chess",
"url": "http://127.0.0.1:8123/",
"selected": true
}]
},
"artifacts": {
"screenshotBase64": "AAA",
"replayGifBase64": "R0lGODlh",
"replayFrameCount": 3,
"replayFrameDurationMs": 700
}
})))
.expect(1)
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let thread_id = thread.id.clone();
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread_id.clone(),
input: vec![V2UserInput::Text {
text: "click e2".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
let turn_id = turn.id.clone();
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
assert_eq!(started.thread_id, thread_id);
assert_eq!(started.turn_id, turn_id.clone());
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
assert_eq!(browser_update.browser_session_id, "sess_atlas_click_123");
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
assert_eq!(
browser_update.artifacts,
Some(BrowserSessionArtifacts {
screenshot_image_url: Some("data:image/png;base64,AAA".to_string()),
replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
replay_frame_count: Some(3),
replay_frame_duration_ms: Some(700),
})
);
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
let ThreadItem::DynamicToolCall {
content_items,
success,
..
} = completed.item
else {
panic!("expected dynamic tool call item");
};
assert_eq!(success, Some(true));
assert_eq!(
content_items,
Some(vec![DynamicToolCallOutputContentItem::InputText {
text: serde_json::to_string(&json!({
"type": "codex_repl",
"command_output": {}
}))?,
}])
);
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let bodies = responses_bodies(&server).await?;
let payload = bodies
.iter()
.find_map(|body| function_call_output_payload(body, call_id))
.context("expected function_call_output in follow-up request")?;
assert_eq!(
payload.body,
FunctionCallOutputBody::Text(serde_json::to_string(&json!({
"type": "codex_repl",
"command_output": {}
}))?)
);
assert_eq!(payload.success, None);
Ok(())
}
@@ -477,6 +1131,269 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
Ok(())
}
#[tokio::test]
async fn thread_read_derives_browser_replay_from_remote_browser_tool_history() -> Result<()> {
let call_id = "dyn-browser-read-1";
let tool_name = "create_tab";
let tool_args = json!({ "url": "https://news.ycombinator.com/" });
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_123",
"result": {
"tab_id": "tab_123",
"selected_tab_id": "tab_123"
},
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
},
"artifacts": {
"screenshotBase64": "AAA",
"replayGifBase64": "R0lGODlh",
"replayFrameCount": 2,
"replayFrameDurationMs": 700
}
})))
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "open Hacker News".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let read_req = mcp
.send_thread_read_request(ThreadReadParams {
thread_id: thread.id.clone(),
include_turns: true,
})
.await?;
let read_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(read_req)),
)
.await??;
let ThreadReadResponse { thread } = to_response::<ThreadReadResponse>(read_resp)?;
let turn = thread.turns.last().context("expected replay turn")?;
let replay = turn
.items
.iter()
.find_map(|item| match item {
ThreadItem::BrowserReplay {
frames,
animation_image_url,
frame_duration_ms,
..
} => Some((frames, animation_image_url, frame_duration_ms)),
_ => None,
})
.context("expected derived browser replay item")?;
assert_eq!(replay.0.len(), 1);
assert_eq!(replay.0[0].source_item_id, call_id);
assert_eq!(replay.0[0].tool, tool_name);
assert_eq!(
replay.0[0].image_url.as_deref(),
Some("data:image/png;base64,AAA")
);
assert_eq!(replay.1.as_deref(), Some("data:image/gif;base64,R0lGODlh"));
assert_eq!(*replay.2, Some(700));
Ok(())
}
#[tokio::test]
async fn thread_read_derives_text_only_browser_replay_from_remote_browser_tool_history()
-> Result<()> {
let call_id = "dyn-browser-read-text-1";
let tool_name = "tabs_content";
let tool_args = json!({ "tabId": "tab_123" });
let tool_call_arguments = serde_json::to_string(&tool_args)?;
let responses = vec![
responses::sse(vec![
responses::ev_response_created("resp-1"),
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
responses::ev_completed("resp-1"),
]),
create_final_assistant_message_sse_response("Done")?,
];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let remote_browser_server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/remote_browser"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"browserSessionId": "sess_123",
"result": {
"tree": {
"role": "document",
"name": "Hacker News"
}
},
"browserState": {
"selectedTabId": "tab_123",
"tabs": [{
"id": "tab_123",
"title": "Hacker News",
"url": "https://news.ycombinator.com/",
"selected": true
}]
}
})))
.mount(&remote_browser_server)
.await;
let codex_home = TempDir::new()?;
create_config_toml(codex_home.path(), &server.uri())?;
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
let mut mcp = McpProcess::new_with_args(
codex_home.path(),
&["--remote-browser-endpoint", &remote_browser_endpoint],
)
.await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams::default())
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "inspect the current page".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let read_req = mcp
.send_thread_read_request(ThreadReadParams {
thread_id: thread.id.clone(),
include_turns: true,
})
.await?;
let read_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(read_req)),
)
.await??;
let ThreadReadResponse { thread } = to_response::<ThreadReadResponse>(read_resp)?;
let turn = thread.turns.last().context("expected replay turn")?;
let replay = turn
.items
.iter()
.find_map(|item| match item {
ThreadItem::BrowserReplay {
frames,
render_mode,
animation_image_url,
frame_duration_ms,
..
} => Some((frames, render_mode, animation_image_url, frame_duration_ms)),
_ => None,
})
.context("expected derived browser replay item")?;
assert_eq!(replay.0.len(), 1);
assert_eq!(replay.0[0].source_item_id, call_id);
assert_eq!(replay.0[0].tool, tool_name);
assert_eq!(replay.1, &BrowserReplayRenderMode::TextOnly);
assert_eq!(replay.2, &None);
assert_eq!(replay.3, &None);
let text_snapshot = replay.0[0]
.text_snapshot
.as_deref()
.context("expected text snapshot")?;
assert!(text_snapshot.contains("\"role\": \"document\""));
assert!(text_snapshot.contains("\"name\": \"Hacker News\""));
assert_eq!(
replay.0[0].text_snapshot_format,
Some(BrowserReplayTextSnapshotFormat::Json)
);
Ok(())
}
/// Exercises the full dynamic tool call path (server request, client response, model output).
#[tokio::test]
async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> {
+44 -3
View File
@@ -128,9 +128,9 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option<EventPersistenceMode> {
| EventMsg::CollabAgentInteractionEnd(_)
| EventMsg::CollabWaitingEnd(_)
| EventMsg::CollabCloseEnd(_)
| EventMsg::CollabResumeEnd(_)
| EventMsg::DynamicToolCallRequest(_)
| EventMsg::DynamicToolCallResponse(_) => Some(EventPersistenceMode::Extended),
| EventMsg::CollabResumeEnd(_) => Some(EventPersistenceMode::Extended),
EventMsg::DynamicToolCallResponse(_) => Some(EventPersistenceMode::Limited),
EventMsg::DynamicToolCallRequest(_) => Some(EventPersistenceMode::Extended),
EventMsg::Warning(_)
| EventMsg::RealtimeConversationStarted(_)
| EventMsg::RealtimeConversationRealtime(_)
@@ -188,8 +188,11 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option<EventPersistenceMode> {
mod tests {
use super::EventPersistenceMode;
use super::should_persist_event_msg;
use codex_protocol::dynamic_tools::DynamicToolCallRequest;
use codex_protocol::protocol::DynamicToolCallResponseEvent;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ImageGenerationEndEvent;
use std::time::Duration;
#[test]
fn persists_image_generation_end_events_in_limited_mode() {
@@ -206,4 +209,42 @@ mod tests {
EventPersistenceMode::Limited
));
}
#[test]
fn persists_dynamic_tool_responses_in_limited_mode() {
let event = EventMsg::DynamicToolCallResponse(DynamicToolCallResponseEvent {
call_id: "dyn_123".into(),
turn_id: "turn_123".into(),
tool: "create_tab".into(),
arguments: serde_json::json!({ "url": "https://example.com" }),
content_items: Vec::new(),
success: true,
error: None,
duration: Duration::from_millis(250),
});
assert!(should_persist_event_msg(
&event,
EventPersistenceMode::Limited
));
}
#[test]
fn does_not_persist_dynamic_tool_requests_in_limited_mode() {
let event = EventMsg::DynamicToolCallRequest(DynamicToolCallRequest {
call_id: "dyn_123".into(),
turn_id: "turn_123".into(),
tool: "create_tab".into(),
arguments: serde_json::json!({ "url": "https://example.com" }),
});
assert!(!should_persist_event_msg(
&event,
EventPersistenceMode::Limited
));
assert!(should_persist_event_msg(
&event,
EventPersistenceMode::Extended
));
}
}