mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Wire Atlas browser bridge into app-server
Add the AgentLib js_repl bridge, persist browser replay items in thread history, and surface remote browser artifacts through turn/start. Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
parent
ba0d3b467b
commit
ee865bcddc
Generated
+1
@@ -1457,6 +1457,7 @@ dependencies = [
|
||||
"codex-utils-pty",
|
||||
"core_test_support",
|
||||
"futures",
|
||||
"image",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"owo-colors",
|
||||
|
||||
@@ -384,6 +384,111 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionArtifacts": {
|
||||
"properties": {
|
||||
"replayFrameCount": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayFrameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayGifImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"screenshotImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
@@ -404,6 +509,16 @@
|
||||
},
|
||||
"BrowserSessionUpdatedNotification": {
|
||||
"properties": {
|
||||
"artifacts": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionArtifacts"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"browserSessionId": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -2760,6 +2875,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -5515,6 +5515,111 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/v2/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/v2/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/v2/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionArtifacts": {
|
||||
"properties": {
|
||||
"replayFrameCount": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayFrameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayGifImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"screenshotImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
@@ -5536,6 +5641,16 @@
|
||||
"BrowserSessionUpdatedNotification": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"properties": {
|
||||
"artifacts": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/v2/BrowserSessionArtifacts"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"browserSessionId": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -12543,6 +12658,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/v2/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/v2/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -715,6 +715,111 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionArtifacts": {
|
||||
"properties": {
|
||||
"replayFrameCount": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayFrameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayGifImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"screenshotImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
@@ -736,6 +841,16 @@
|
||||
"BrowserSessionUpdatedNotification": {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"properties": {
|
||||
"artifacts": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionArtifacts"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"browserSessionId": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -10303,6 +10418,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -1,6 +1,39 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"BrowserSessionArtifacts": {
|
||||
"properties": {
|
||||
"replayFrameCount": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayFrameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"replayGifImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"screenshotImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
@@ -44,6 +77,16 @@
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"artifacts": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionArtifacts"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"browserSessionId": {
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
@@ -1,6 +1,119 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -869,6 +982,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -1,6 +1,119 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -869,6 +982,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -1,6 +1,119 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1012,6 +1125,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -66,6 +66,119 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1519,6 +1632,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -4,6 +4,119 @@
|
||||
"AgentPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1277,6 +1390,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -4,6 +4,119 @@
|
||||
"AgentPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1277,6 +1390,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -4,6 +4,119 @@
|
||||
"AgentPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1277,6 +1390,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -66,6 +66,119 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1519,6 +1632,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -4,6 +4,119 @@
|
||||
"AgentPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1277,6 +1390,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -66,6 +66,119 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1519,6 +1632,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -4,6 +4,119 @@
|
||||
"AgentPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1277,6 +1390,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -4,6 +4,119 @@
|
||||
"AgentPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1277,6 +1390,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -1,6 +1,119 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1012,6 +1125,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -1,6 +1,119 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1012,6 +1125,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -1,6 +1,119 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"BrowserReplayFrame": {
|
||||
"properties": {
|
||||
"browserState": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserSessionState"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"imageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sequenceNumber": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"sourceItemId": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/DynamicToolCallStatus"
|
||||
},
|
||||
"textSnapshot": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"textSnapshotFormat": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/BrowserReplayTextSnapshotFormat"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"sequenceNumber",
|
||||
"sourceItemId",
|
||||
"status",
|
||||
"tool"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserReplayRenderMode": {
|
||||
"enum": [
|
||||
"animation",
|
||||
"frames",
|
||||
"textOnly"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserReplayTextSnapshotFormat": {
|
||||
"enum": [
|
||||
"plain",
|
||||
"json"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"BrowserSessionState": {
|
||||
"properties": {
|
||||
"selectedTabId": {
|
||||
"type": "string"
|
||||
},
|
||||
"tabs": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserTabState"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"selectedTabId",
|
||||
"tabs"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"BrowserTabState": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"selected": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"selected",
|
||||
"title",
|
||||
"url"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ByteRange": {
|
||||
"properties": {
|
||||
"end": {
|
||||
@@ -1012,6 +1125,51 @@
|
||||
"title": "DynamicToolCallThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"animationImageUrl": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frameDurationMs": {
|
||||
"format": "uint32",
|
||||
"minimum": 0.0,
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frames": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/BrowserReplayFrame"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"renderMode": {
|
||||
"$ref": "#/definitions/BrowserReplayRenderMode"
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"browserReplay"
|
||||
],
|
||||
"title": "BrowserReplayThreadItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"frames",
|
||||
"id",
|
||||
"renderMode",
|
||||
"type"
|
||||
],
|
||||
"title": "BrowserReplayThreadItem",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"agentsStates": {
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { BrowserReplayTextSnapshotFormat } from "./BrowserReplayTextSnapshotFormat";
|
||||
import type { BrowserSessionState } from "./BrowserSessionState";
|
||||
import type { DynamicToolCallStatus } from "./DynamicToolCallStatus";
|
||||
|
||||
export type BrowserReplayFrame = { sequenceNumber: number, sourceItemId: string, tool: string, status: DynamicToolCallStatus, imageUrl?: string | null, textSnapshot?: string | null, textSnapshotFormat?: BrowserReplayTextSnapshotFormat | null, browserState?: BrowserSessionState | null, };
|
||||
@@ -0,0 +1,5 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type BrowserReplayRenderMode = "animation" | "frames" | "textOnly";
|
||||
@@ -0,0 +1,5 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type BrowserReplayTextSnapshotFormat = "plain" | "json";
|
||||
@@ -0,0 +1,5 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type BrowserSessionArtifacts = { screenshotImageUrl?: string | null, replayGifImageUrl?: string | null, replayFrameCount?: number | null, replayFrameDurationMs?: number | null, };
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts";
|
||||
import type { BrowserSessionState } from "./BrowserSessionState";
|
||||
|
||||
export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, };
|
||||
export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, artifacts?: BrowserSessionArtifacts | null, };
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
import type { MessagePhase } from "../MessagePhase";
|
||||
import type { ReasoningEffort } from "../ReasoningEffort";
|
||||
import type { JsonValue } from "../serde_json/JsonValue";
|
||||
import type { BrowserReplayFrame } from "./BrowserReplayFrame";
|
||||
import type { BrowserReplayRenderMode } from "./BrowserReplayRenderMode";
|
||||
import type { CollabAgentState } from "./CollabAgentState";
|
||||
import type { CollabAgentTool } from "./CollabAgentTool";
|
||||
import type { CollabAgentToolCallStatus } from "./CollabAgentToolCallStatus";
|
||||
@@ -22,78 +24,78 @@ import type { PatchApplyStatus } from "./PatchApplyStatus";
|
||||
import type { UserInput } from "./UserInput";
|
||||
import type { WebSearchAction } from "./WebSearchAction";
|
||||
|
||||
export type ThreadItem = { "type": "userMessage", id: string, content: Array<UserInput>, } | { "type": "hookPrompt", id: string, fragments: Array<HookPromptFragment>, } | { "type": "agentMessage", id: string, text: string, phase: MessagePhase | null, memoryCitation: MemoryCitation | null, } | { "type": "plan", id: string, text: string, } | { "type": "reasoning", id: string, summary: Array<string>, content: Array<string>, } | { "type": "commandExecution", id: string,
|
||||
export type ThreadItem = { "type": "userMessage", id: string, content: Array<UserInput>, } | { "type": "hookPrompt", id: string, fragments: Array<HookPromptFragment>, } | { "type": "agentMessage", id: string, text: string, phase: MessagePhase | null, memoryCitation: MemoryCitation | null, } | { "type": "plan", id: string, text: string, } | { "type": "reasoning", id: string, summary: Array<string>, content: Array<string>, } | { "type": "commandExecution", id: string,
|
||||
/**
|
||||
* The command to be executed.
|
||||
*/
|
||||
command: string,
|
||||
command: string,
|
||||
/**
|
||||
* The command's working directory.
|
||||
*/
|
||||
cwd: string,
|
||||
cwd: string,
|
||||
/**
|
||||
* Identifier for the underlying PTY process (when available).
|
||||
*/
|
||||
processId: string | null, source: CommandExecutionSource, status: CommandExecutionStatus,
|
||||
processId: string | null, source: CommandExecutionSource, status: CommandExecutionStatus,
|
||||
/**
|
||||
* A best-effort parsing of the command to understand the action(s) it will perform.
|
||||
* This returns a list of CommandAction objects because a single shell command may
|
||||
* be composed of many commands piped together.
|
||||
*/
|
||||
commandActions: Array<CommandAction>,
|
||||
commandActions: Array<CommandAction>,
|
||||
/**
|
||||
* The command's output, aggregated from stdout and stderr.
|
||||
*/
|
||||
aggregatedOutput: string | null,
|
||||
aggregatedOutput: string | null,
|
||||
/**
|
||||
* The command's exit code.
|
||||
*/
|
||||
exitCode: number | null,
|
||||
exitCode: number | null,
|
||||
/**
|
||||
* The duration of the command execution in milliseconds.
|
||||
*/
|
||||
durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array<FileUpdateChange>, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null,
|
||||
durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array<FileUpdateChange>, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null,
|
||||
/**
|
||||
* The duration of the MCP tool call in milliseconds.
|
||||
*/
|
||||
durationMs: number | null, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array<DynamicToolCallOutputContentItem> | null, success: boolean | null,
|
||||
durationMs: number | null, } | { "type": "dynamicToolCall", id: string, tool: string, arguments: JsonValue, status: DynamicToolCallStatus, contentItems: Array<DynamicToolCallOutputContentItem> | null, success: boolean | null,
|
||||
/**
|
||||
* The duration of the dynamic tool call in milliseconds.
|
||||
*/
|
||||
durationMs: number | null, } | { "type": "collabAgentToolCall",
|
||||
durationMs: number | null, } | { "type": "browserReplay", id: string, frames: Array<BrowserReplayFrame>, renderMode: BrowserReplayRenderMode, animationImageUrl?: string | null, frameDurationMs?: number | null, } | { "type": "collabAgentToolCall",
|
||||
/**
|
||||
* Unique identifier for this collab tool call.
|
||||
*/
|
||||
id: string,
|
||||
id: string,
|
||||
/**
|
||||
* Name of the collab tool that was invoked.
|
||||
*/
|
||||
tool: CollabAgentTool,
|
||||
tool: CollabAgentTool,
|
||||
/**
|
||||
* Current status of the collab tool call.
|
||||
*/
|
||||
status: CollabAgentToolCallStatus,
|
||||
status: CollabAgentToolCallStatus,
|
||||
/**
|
||||
* Thread ID of the agent issuing the collab request.
|
||||
*/
|
||||
senderThreadId: string,
|
||||
senderThreadId: string,
|
||||
/**
|
||||
* Thread ID of the receiving agent, when applicable. In case of spawn operation,
|
||||
* this corresponds to the newly spawned agent.
|
||||
*/
|
||||
receiverThreadIds: Array<string>,
|
||||
receiverThreadIds: Array<string>,
|
||||
/**
|
||||
* Prompt text sent as part of the collab tool call, when available.
|
||||
*/
|
||||
prompt: string | null,
|
||||
prompt: string | null,
|
||||
/**
|
||||
* Model requested for the spawned agent, when applicable.
|
||||
*/
|
||||
model: string | null,
|
||||
model: string | null,
|
||||
/**
|
||||
* Reasoning effort requested for the spawned agent, when applicable.
|
||||
*/
|
||||
reasoningEffort: ReasoningEffort | null,
|
||||
reasoningEffort: ReasoningEffort | null,
|
||||
/**
|
||||
* Last known status of the target agents, when available.
|
||||
*/
|
||||
|
||||
@@ -25,6 +25,10 @@ export type { AppsDefaultConfig } from "./AppsDefaultConfig";
|
||||
export type { AppsListParams } from "./AppsListParams";
|
||||
export type { AppsListResponse } from "./AppsListResponse";
|
||||
export type { AskForApproval } from "./AskForApproval";
|
||||
export type { BrowserReplayFrame } from "./BrowserReplayFrame";
|
||||
export type { BrowserReplayRenderMode } from "./BrowserReplayRenderMode";
|
||||
export type { BrowserReplayTextSnapshotFormat } from "./BrowserReplayTextSnapshotFormat";
|
||||
export type { BrowserSessionArtifacts } from "./BrowserSessionArtifacts";
|
||||
export type { BrowserSessionState } from "./BrowserSessionState";
|
||||
export type { BrowserSessionUpdatedNotification } from "./BrowserSessionUpdatedNotification";
|
||||
export type { BrowserTabState } from "./BrowserTabState";
|
||||
|
||||
@@ -179,6 +179,10 @@ pub fn generate_ts_with_options(
|
||||
}
|
||||
}
|
||||
|
||||
if !ts_files.is_empty() {
|
||||
normalize_generated_ts_files(&ts_files)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1907,6 +1911,41 @@ fn prepend_header_if_missing(path: &Path) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_typescript_text(text: &str) -> String {
|
||||
let normalized = text.replace("\r\n", "\n").replace('\r', "\n");
|
||||
let had_trailing_newline = normalized.ends_with('\n');
|
||||
let mut result = normalized
|
||||
.split('\n')
|
||||
.map(|line| line.trim_end_matches([' ', '\t']))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
if had_trailing_newline && !result.ends_with('\n') {
|
||||
result.push('\n');
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn normalize_generated_ts_files(paths: &[PathBuf]) -> Result<()> {
|
||||
for path in paths {
|
||||
if !path
|
||||
.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.is_some_and(|name| name == "ThreadItem.ts")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let content = fs::read_to_string(path)
|
||||
.with_context(|| format!("Failed to read generated TypeScript {}", path.display()))?;
|
||||
let normalized = normalize_typescript_text(&content);
|
||||
if normalized == content {
|
||||
continue;
|
||||
}
|
||||
fs::write(path, normalized)
|
||||
.with_context(|| format!("Failed to normalize {}", path.display()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ts_files_in(dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
let mut files = Vec::new();
|
||||
for entry in
|
||||
|
||||
@@ -1921,6 +1921,57 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstructs_dynamic_tool_items_from_response_events_only() {
|
||||
let events = vec![
|
||||
EventMsg::TurnStarted(TurnStartedEvent {
|
||||
turn_id: "turn-1".into(),
|
||||
model_context_window: None,
|
||||
collaboration_mode_kind: Default::default(),
|
||||
}),
|
||||
EventMsg::UserMessage(UserMessageEvent {
|
||||
message: "run dynamic tool".into(),
|
||||
images: None,
|
||||
text_elements: Vec::new(),
|
||||
local_images: Vec::new(),
|
||||
}),
|
||||
EventMsg::DynamicToolCallResponse(DynamicToolCallResponseEvent {
|
||||
call_id: "dyn-1".into(),
|
||||
turn_id: "turn-1".into(),
|
||||
tool: "lookup_ticket".into(),
|
||||
arguments: serde_json::json!({"id":"ABC-123"}),
|
||||
content_items: vec![CoreDynamicToolCallOutputContentItem::InputText {
|
||||
text: "Ticket is open".into(),
|
||||
}],
|
||||
success: true,
|
||||
error: None,
|
||||
duration: Duration::from_millis(42),
|
||||
}),
|
||||
];
|
||||
|
||||
let items = events
|
||||
.into_iter()
|
||||
.map(RolloutItem::EventMsg)
|
||||
.collect::<Vec<_>>();
|
||||
let turns = build_turns_from_rollout_items(&items);
|
||||
assert_eq!(turns.len(), 1);
|
||||
assert_eq!(turns[0].items.len(), 2);
|
||||
assert_eq!(
|
||||
turns[0].items[1],
|
||||
ThreadItem::DynamicToolCall {
|
||||
id: "dyn-1".into(),
|
||||
tool: "lookup_ticket".into(),
|
||||
arguments: serde_json::json!({"id":"ABC-123"}),
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: "Ticket is open".into(),
|
||||
}]),
|
||||
success: Some(true),
|
||||
duration_ms: Some(42),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstructs_declined_exec_and_patch_items() {
|
||||
let events = vec![
|
||||
|
||||
@@ -4335,6 +4335,19 @@ pub enum ThreadItem {
|
||||
},
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
BrowserReplay {
|
||||
id: String,
|
||||
frames: Vec<BrowserReplayFrame>,
|
||||
render_mode: BrowserReplayRenderMode,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional = nullable)]
|
||||
animation_image_url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional = nullable)]
|
||||
frame_duration_ms: Option<u32>,
|
||||
},
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase")]
|
||||
CollabAgentToolCall {
|
||||
/// Unique identifier for this collab tool call.
|
||||
id: String,
|
||||
@@ -4396,6 +4409,45 @@ pub struct HookPromptFragment {
|
||||
pub hook_run_id: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub struct BrowserReplayFrame {
|
||||
pub sequence_number: u32,
|
||||
pub source_item_id: String,
|
||||
pub tool: String,
|
||||
pub status: DynamicToolCallStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional = nullable)]
|
||||
pub image_url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional = nullable)]
|
||||
pub text_snapshot: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional = nullable)]
|
||||
pub text_snapshot_format: Option<BrowserReplayTextSnapshotFormat>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional = nullable)]
|
||||
pub browser_state: Option<BrowserSessionState>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub enum BrowserReplayRenderMode {
|
||||
Animation,
|
||||
Frames,
|
||||
TextOnly,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(export_to = "v2/")]
|
||||
pub enum BrowserReplayTextSnapshotFormat {
|
||||
Plain,
|
||||
Json,
|
||||
}
|
||||
|
||||
impl ThreadItem {
|
||||
pub fn id(&self) -> &str {
|
||||
match self {
|
||||
@@ -4408,6 +4460,7 @@ impl ThreadItem {
|
||||
| ThreadItem::FileChange { id, .. }
|
||||
| ThreadItem::McpToolCall { id, .. }
|
||||
| ThreadItem::DynamicToolCall { id, .. }
|
||||
| ThreadItem::BrowserReplay { id, .. }
|
||||
| ThreadItem::CollabAgentToolCall { id, .. }
|
||||
| ThreadItem::WebSearch { id, .. }
|
||||
| ThreadItem::ImageView { id, .. }
|
||||
@@ -8011,6 +8064,104 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_replay_thread_item_serializes_animation_fields() {
|
||||
let value = serde_json::to_value(ThreadItem::BrowserReplay {
|
||||
id: "browser-replay-turn-1".to_string(),
|
||||
frames: vec![BrowserReplayFrame {
|
||||
sequence_number: 1,
|
||||
source_item_id: "tool-call-1".to_string(),
|
||||
tool: "create_tab".to_string(),
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
image_url: Some("data:image/png;base64,AAA".to_string()),
|
||||
text_snapshot: Some("Opened Hacker News".to_string()),
|
||||
text_snapshot_format: Some(BrowserReplayTextSnapshotFormat::Plain),
|
||||
browser_state: Some(BrowserSessionState {
|
||||
selected_tab_id: "tab_123".to_string(),
|
||||
tabs: vec![BrowserTabState {
|
||||
id: "tab_123".to_string(),
|
||||
title: "Hacker News".to_string(),
|
||||
url: "https://news.ycombinator.com/".to_string(),
|
||||
selected: true,
|
||||
}],
|
||||
}),
|
||||
}],
|
||||
render_mode: BrowserReplayRenderMode::Animation,
|
||||
animation_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
|
||||
frame_duration_ms: Some(700),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"type": "browserReplay",
|
||||
"id": "browser-replay-turn-1",
|
||||
"renderMode": "animation",
|
||||
"frames": [{
|
||||
"sequenceNumber": 1,
|
||||
"sourceItemId": "tool-call-1",
|
||||
"tool": "create_tab",
|
||||
"status": "completed",
|
||||
"imageUrl": "data:image/png;base64,AAA",
|
||||
"textSnapshot": "Opened Hacker News",
|
||||
"textSnapshotFormat": "plain",
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
}],
|
||||
"animationImageUrl": "data:image/gif;base64,R0lGODlh",
|
||||
"frameDurationMs": 700
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_replay_thread_item_serializes_text_only_fields() {
|
||||
let value = serde_json::to_value(ThreadItem::BrowserReplay {
|
||||
id: "browser-replay-turn-2".to_string(),
|
||||
frames: vec![BrowserReplayFrame {
|
||||
sequence_number: 1,
|
||||
source_item_id: "tool-call-2".to_string(),
|
||||
tool: "tabs_content".to_string(),
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
image_url: None,
|
||||
text_snapshot: Some(
|
||||
"{\n \"tree\": {\n \"role\": \"document\"\n }\n}".to_string(),
|
||||
),
|
||||
text_snapshot_format: Some(BrowserReplayTextSnapshotFormat::Json),
|
||||
browser_state: None,
|
||||
}],
|
||||
render_mode: BrowserReplayRenderMode::TextOnly,
|
||||
animation_image_url: None,
|
||||
frame_duration_ms: None,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"type": "browserReplay",
|
||||
"id": "browser-replay-turn-2",
|
||||
"renderMode": "textOnly",
|
||||
"frames": [{
|
||||
"sequenceNumber": 1,
|
||||
"sourceItemId": "tool-call-2",
|
||||
"tool": "tabs_content",
|
||||
"status": "completed",
|
||||
"textSnapshot": "{\n \"tree\": {\n \"role\": \"document\"\n }\n}",
|
||||
"textSnapshotFormat": "json",
|
||||
}],
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_tool_spec_deserializes_defer_loading() {
|
||||
let value = json!({
|
||||
|
||||
@@ -5,6 +5,7 @@ use crate::ServerRequest;
|
||||
use crate::export::GENERATED_TS_HEADER;
|
||||
use crate::export::filter_experimental_ts_tree;
|
||||
use crate::export::generate_index_ts_tree;
|
||||
use crate::export::normalize_typescript_text;
|
||||
use crate::protocol::common::visit_client_response_types;
|
||||
use crate::protocol::common::visit_server_response_types;
|
||||
use anyhow::Context;
|
||||
@@ -129,11 +130,9 @@ fn read_file_bytes(path: &Path) -> Result<Vec<u8>> {
|
||||
return Ok(normalized);
|
||||
}
|
||||
if path.extension().is_some_and(|ext| ext == "ts") {
|
||||
// Windows checkouts (and some generators) may produce CRLF; normalize so the
|
||||
// fixture test is platform-independent.
|
||||
let text = String::from_utf8(bytes)
|
||||
.with_context(|| format!("expected UTF-8 TypeScript in {}", path.display()))?;
|
||||
let text = text.replace("\r\n", "\n").replace('\r', "\n");
|
||||
let text = normalize_typescript_text(&text);
|
||||
// Fixture comparisons care about schema content, not whether the generator
|
||||
// re-prepended the standard banner to every TypeScript file.
|
||||
let text = text
|
||||
@@ -276,10 +275,7 @@ fn collect_typescript_fixture_file<T: TS + 'static + ?Sized>(
|
||||
|
||||
let contents = T::export_to_string().context("export TypeScript fixture content")?;
|
||||
let output_path = normalize_relative_fixture_path(&output_path);
|
||||
files.insert(
|
||||
output_path,
|
||||
contents.replace("\r\n", "\n").replace('\r', "\n"),
|
||||
);
|
||||
files.insert(output_path, normalize_typescript_text(&contents));
|
||||
|
||||
let mut visitor = TypeScriptFixtureCollector {
|
||||
files,
|
||||
|
||||
@@ -52,6 +52,7 @@ codex-utils-json-to-toml = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
futures = { workspace = true }
|
||||
image = { workspace = true, features = ["gif", "jpeg", "png", "webp"] }
|
||||
owo-colors = { workspace = true, features = ["supports-colors"] }
|
||||
reqwest = { workspace = true, features = ["json", "rustls-tls"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
use crate::remote_browser_api::RemoteBrowserApi;
|
||||
use crate::remote_browser_api::RemoteBrowserCommandOutcome;
|
||||
use codex_app_server_protocol::BrowserSessionArtifacts;
|
||||
use codex_app_server_protocol::BrowserSessionCommandParams;
|
||||
use codex_app_server_protocol::BrowserSessionState;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
use codex_app_server_protocol::DynamicToolCallResponse;
|
||||
use codex_app_server_protocol::DynamicToolSpec;
|
||||
use serde_json::Map;
|
||||
use serde_json::Value as JsonValue;
|
||||
use serde_json::json;
|
||||
|
||||
const ATLAS_COMMAND_TOOL: &str = "atlas_command";
|
||||
const ATLAS_DISPLAY_TRUNCATE_MAX_CHARS: u32 = 4_000;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct AtlasCommandExecution {
|
||||
pub(crate) browser_session_id: Option<String>,
|
||||
pub(crate) browser_state: Option<BrowserSessionState>,
|
||||
pub(crate) artifacts: Option<BrowserSessionArtifacts>,
|
||||
pub(crate) response: DynamicToolCallResponse,
|
||||
}
|
||||
|
||||
pub(crate) fn merge_atlas_command_dynamic_tool(
|
||||
mut tools: Vec<DynamicToolSpec>,
|
||||
) -> Vec<DynamicToolSpec> {
|
||||
if !tools.iter().any(|tool| tool.name == ATLAS_COMMAND_TOOL) {
|
||||
tools.push(DynamicToolSpec {
|
||||
name: ATLAS_COMMAND_TOOL.to_string(),
|
||||
description: "Hidden bridge for AgentLib inside js_repl. Call via codex.tool(...) only."
|
||||
.to_string(),
|
||||
input_schema: json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"payload": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"required": ["payload"],
|
||||
"additionalProperties": false
|
||||
}),
|
||||
defer_loading: true,
|
||||
});
|
||||
}
|
||||
|
||||
tools
|
||||
}
|
||||
|
||||
pub(crate) fn is_atlas_command_dynamic_tool(name: &str) -> bool {
|
||||
name == ATLAS_COMMAND_TOOL
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_atlas_command(
|
||||
remote_browser_api: &RemoteBrowserApi,
|
||||
browser_session_id: Option<String>,
|
||||
arguments: JsonValue,
|
||||
) -> AtlasCommandExecution {
|
||||
match execute_atlas_command_impl(remote_browser_api, browser_session_id, arguments).await {
|
||||
Ok(result) => result,
|
||||
Err(message) => AtlasCommandExecution {
|
||||
browser_session_id: None,
|
||||
browser_state: None,
|
||||
artifacts: None,
|
||||
response: atlas_json_response(json!({
|
||||
"type": "codex_repl",
|
||||
"error": message,
|
||||
})),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_atlas_command_impl(
|
||||
remote_browser_api: &RemoteBrowserApi,
|
||||
browser_session_id: Option<String>,
|
||||
arguments: JsonValue,
|
||||
) -> Result<AtlasCommandExecution, String> {
|
||||
let payload = arguments
|
||||
.as_object()
|
||||
.and_then(|args| args.get("payload"))
|
||||
.cloned()
|
||||
.ok_or_else(|| "atlas_command requires an object payload".to_string())?;
|
||||
let payload_object = payload
|
||||
.as_object()
|
||||
.ok_or_else(|| "atlas_command payload must be an object".to_string())?;
|
||||
let command_type = payload_object
|
||||
.get("type")
|
||||
.and_then(JsonValue::as_str)
|
||||
.ok_or_else(|| "atlas_command payload.type is required".to_string())?;
|
||||
|
||||
let mut session_id = browser_session_id;
|
||||
let mut last_browser_state = None;
|
||||
let mut last_artifacts = None;
|
||||
|
||||
let command_output = match command_type {
|
||||
"runtime_config" => json!({
|
||||
"display_truncate_max_chars": ATLAS_DISPLAY_TRUNCATE_MAX_CHARS,
|
||||
}),
|
||||
"create_tab" => {
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
"create_tab",
|
||||
json!({ "select": true }),
|
||||
)
|
||||
.await?;
|
||||
let result = outcome.result.clone();
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
last_artifacts = outcome.to_public_artifacts();
|
||||
json!({
|
||||
"id": result.get("tab_id").and_then(JsonValue::as_str).unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
"list_tabs" => {
|
||||
let outcome =
|
||||
call_remote(remote_browser_api, &mut session_id, "list_tabs", json!({})).await?;
|
||||
let tabs = outcome
|
||||
.result
|
||||
.get("tabs")
|
||||
.and_then(JsonValue::as_array)
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|tab| {
|
||||
let tab_object = tab.as_object().cloned().unwrap_or_default();
|
||||
json!({
|
||||
"id": tab_object
|
||||
.get("id")
|
||||
.cloned()
|
||||
.unwrap_or(JsonValue::String(String::new())),
|
||||
"title": tab_object.get("title").cloned().unwrap_or(JsonValue::Null),
|
||||
"url": tab_object.get("url").cloned().unwrap_or(JsonValue::Null),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
json!({ "tabs": tabs })
|
||||
}
|
||||
"selected_tab" => {
|
||||
let outcome =
|
||||
call_remote(remote_browser_api, &mut session_id, "selected_tab", json!({}))
|
||||
.await?;
|
||||
let selected_tab_id = outcome
|
||||
.result
|
||||
.get("selected_tab_id")
|
||||
.and_then(JsonValue::as_str);
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
json!({
|
||||
"id": selected_tab_id,
|
||||
})
|
||||
}
|
||||
"navigate_tab_url" => {
|
||||
let remote_args = normalize_passthrough_payload(payload_object);
|
||||
let outcome =
|
||||
call_remote(remote_browser_api, &mut session_id, "navigate_tab_url", remote_args)
|
||||
.await?;
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
last_artifacts = outcome.to_public_artifacts();
|
||||
json!({})
|
||||
}
|
||||
"playwright_wait_for_load_state" => {
|
||||
let remote_args = normalize_passthrough_payload(payload_object);
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
"playwright_wait_for_load_state",
|
||||
remote_args,
|
||||
)
|
||||
.await?;
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
last_artifacts = outcome.to_public_artifacts();
|
||||
json!({})
|
||||
}
|
||||
"playwright_locator_count"
|
||||
| "playwright_locator_text_content"
|
||||
| "playwright_locator_inner_text"
|
||||
| "playwright_locator_get_attribute"
|
||||
| "playwright_locator_is_visible"
|
||||
| "playwright_locator_is_enabled" => {
|
||||
let remote_args = normalize_passthrough_payload(payload_object);
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
command_type,
|
||||
remote_args,
|
||||
)
|
||||
.await?;
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
outcome.result.clone()
|
||||
}
|
||||
"playwright_locator_click" | "playwright_locator_dblclick" => {
|
||||
let remote_args = normalize_passthrough_payload(payload_object);
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
command_type,
|
||||
remote_args,
|
||||
)
|
||||
.await?;
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
last_artifacts = outcome.to_public_artifacts();
|
||||
json!({})
|
||||
}
|
||||
"playwright_locator_wait_for" => {
|
||||
let remote_args = normalize_passthrough_payload(payload_object);
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
"playwright_locator_wait_for",
|
||||
remote_args,
|
||||
)
|
||||
.await?;
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
json!({})
|
||||
}
|
||||
"playwright_screenshot" => {
|
||||
if has_any_keys(
|
||||
payload_object,
|
||||
&[
|
||||
"cropX",
|
||||
"cropY",
|
||||
"cropWidth",
|
||||
"cropHeight",
|
||||
"crop_x",
|
||||
"crop_y",
|
||||
"crop_width",
|
||||
"crop_height",
|
||||
],
|
||||
) {
|
||||
return Err(
|
||||
"atlas_command playwright_screenshot crop options are not supported by the remote browser bridge yet"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
let remote_args = normalize_passthrough_payload(payload_object);
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
"playwright_screenshot",
|
||||
remote_args,
|
||||
)
|
||||
.await?;
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
last_artifacts = outcome.to_public_artifacts();
|
||||
json!({
|
||||
"data": outcome
|
||||
.screenshot_base64()
|
||||
.ok_or_else(|| "remote browser did not return screenshot data".to_string())?,
|
||||
})
|
||||
}
|
||||
"cua_get_visible_screenshot" => {
|
||||
let tab_id = payload_object
|
||||
.get("tab_id")
|
||||
.cloned()
|
||||
.ok_or_else(|| "cua_get_visible_screenshot requires tab_id".to_string())?;
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
"playwright_screenshot",
|
||||
json!({
|
||||
"tab_id": tab_id,
|
||||
"full_page": false,
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
last_artifacts = outcome.to_public_artifacts();
|
||||
json!({
|
||||
"data": outcome
|
||||
.screenshot_base64()
|
||||
.ok_or_else(|| "remote browser did not return screenshot data".to_string())?,
|
||||
})
|
||||
}
|
||||
"tabs_content" => {
|
||||
let urls = payload_object
|
||||
.get("urls")
|
||||
.and_then(JsonValue::as_array)
|
||||
.ok_or_else(|| "tabs_content requires urls".to_string())?;
|
||||
let content_type = payload_object
|
||||
.get("content_type")
|
||||
.and_then(JsonValue::as_str)
|
||||
.ok_or_else(|| "tabs_content requires content_type".to_string())?;
|
||||
if content_type != "text" {
|
||||
return Err(format!(
|
||||
"atlas_command tabs_content only supports content_type=\"text\" today; got {content_type:?}"
|
||||
));
|
||||
}
|
||||
|
||||
let timeout_ms = payload_object.get("timeout_ms").cloned();
|
||||
let mut tab_ids = Vec::with_capacity(urls.len());
|
||||
for url in urls {
|
||||
let url = url
|
||||
.as_str()
|
||||
.ok_or_else(|| "tabs_content urls must be strings".to_string())?;
|
||||
let mut create_args = Map::new();
|
||||
create_args.insert("url".to_string(), JsonValue::String(url.to_string()));
|
||||
create_args.insert("select".to_string(), JsonValue::Bool(false));
|
||||
if let Some(timeout_ms) = timeout_ms.clone() {
|
||||
create_args.insert("timeout_ms".to_string(), timeout_ms);
|
||||
}
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
"create_tab",
|
||||
JsonValue::Object(create_args),
|
||||
)
|
||||
.await?;
|
||||
let tab_id = outcome
|
||||
.result
|
||||
.get("tab_id")
|
||||
.and_then(JsonValue::as_str)
|
||||
.ok_or_else(|| "remote browser create_tab did not return tab_id".to_string())?;
|
||||
tab_ids.push(tab_id.to_string());
|
||||
}
|
||||
|
||||
let outcome = call_remote(
|
||||
remote_browser_api,
|
||||
&mut session_id,
|
||||
"tabs_content",
|
||||
json!({ "tab_ids": tab_ids }),
|
||||
)
|
||||
.await?;
|
||||
let tabs = outcome
|
||||
.result
|
||||
.get("tabs")
|
||||
.and_then(JsonValue::as_array)
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|tab| {
|
||||
let tab_object = tab.as_object().cloned().unwrap_or_default();
|
||||
json!({
|
||||
"url": tab_object
|
||||
.get("url")
|
||||
.cloned()
|
||||
.unwrap_or(JsonValue::String(String::new())),
|
||||
"title": tab_object.get("title").cloned().unwrap_or(JsonValue::Null),
|
||||
"content": tab_object.get("content").cloned().unwrap_or(JsonValue::Null),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
last_browser_state = outcome.browser_state.to_public_state();
|
||||
json!({ "results": tabs })
|
||||
}
|
||||
other => {
|
||||
return Err(format!(
|
||||
"atlas_command does not support AgentLib command type {other:?} yet"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(AtlasCommandExecution {
|
||||
browser_session_id: session_id,
|
||||
browser_state: last_browser_state,
|
||||
artifacts: last_artifacts,
|
||||
response: atlas_json_response(json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": command_output,
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
fn atlas_json_response(payload: JsonValue) -> DynamicToolCallResponse {
|
||||
DynamicToolCallResponse {
|
||||
content_items: vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(&payload).unwrap_or_else(|_| {
|
||||
"{\"type\":\"codex_repl\",\"error\":\"serialization failure\"}".to_string()
|
||||
}),
|
||||
}],
|
||||
success: true,
|
||||
}
|
||||
}
|
||||
|
||||
async fn call_remote(
|
||||
remote_browser_api: &RemoteBrowserApi,
|
||||
browser_session_id: &mut Option<String>,
|
||||
command: &str,
|
||||
arguments: JsonValue,
|
||||
) -> Result<RemoteBrowserCommandOutcome, String> {
|
||||
let outcome = remote_browser_api
|
||||
.command_with_artifacts(BrowserSessionCommandParams {
|
||||
browser_session_id: browser_session_id.clone(),
|
||||
command: command.to_string(),
|
||||
arguments: Some(arguments),
|
||||
})
|
||||
.await
|
||||
.map_err(|err| err.message)?;
|
||||
*browser_session_id = Some(outcome.browser_session_id.clone());
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
fn normalize_passthrough_payload(payload: &Map<String, JsonValue>) -> JsonValue {
|
||||
let mut args = payload.clone();
|
||||
args.remove("type");
|
||||
JsonValue::Object(args)
|
||||
}
|
||||
|
||||
fn has_any_keys(payload: &Map<String, JsonValue>, keys: &[&str]) -> bool {
|
||||
keys.iter().any(|key| payload.contains_key(*key))
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
use crate::atlas_command::execute_atlas_command;
|
||||
use crate::atlas_command::is_atlas_command_dynamic_tool;
|
||||
use crate::codex_message_processor::ApiVersion;
|
||||
use crate::codex_message_processor::read_rollout_items_from_rollout;
|
||||
use crate::codex_message_processor::read_summary_from_rollout;
|
||||
@@ -286,6 +288,42 @@ async fn handle_remote_browser_dynamic_tool_request(
|
||||
submit_dynamic_tool_response(conversation, call_id, response).await;
|
||||
}
|
||||
|
||||
async fn handle_atlas_command_dynamic_tool_request(
|
||||
call_id: String,
|
||||
arguments: JsonValue,
|
||||
conversation: Arc<CodexThread>,
|
||||
thread_state: Arc<Mutex<ThreadState>>,
|
||||
outgoing: ThreadScopedOutgoingMessageSender,
|
||||
remote_browser_api: RemoteBrowserApi,
|
||||
) {
|
||||
let browser_session_id = { thread_state.lock().await.remote_browser_session_id.clone() };
|
||||
let outcome = execute_atlas_command(&remote_browser_api, browser_session_id, arguments).await;
|
||||
|
||||
if let Some(browser_session_id) = outcome.browser_session_id.clone() {
|
||||
let mut state = thread_state.lock().await;
|
||||
state.remote_browser_session_id = Some(browser_session_id);
|
||||
}
|
||||
|
||||
if let Some(browser_state) = outcome.browser_state {
|
||||
outgoing
|
||||
.send_server_notification(ServerNotification::BrowserSessionUpdated(
|
||||
BrowserSessionUpdatedNotification {
|
||||
browser_session_id: outcome.browser_session_id.unwrap_or_default(),
|
||||
browser_state,
|
||||
artifacts: outcome.artifacts,
|
||||
},
|
||||
))
|
||||
.await;
|
||||
}
|
||||
|
||||
submit_dynamic_tool_response(
|
||||
conversation,
|
||||
call_id,
|
||||
to_core_dynamic_tool_response(outcome.response),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn guardian_auto_approval_review_notification(
|
||||
conversation_id: &ThreadId,
|
||||
event_turn_id: &str,
|
||||
@@ -1013,7 +1051,18 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
outgoing
|
||||
.send_server_notification(ServerNotification::ItemStarted(notification))
|
||||
.await;
|
||||
if remote_browser_api.is_configured() && is_remote_browser_dynamic_tool(&tool) {
|
||||
if remote_browser_api.is_configured() && is_atlas_command_dynamic_tool(&tool) {
|
||||
tokio::spawn(handle_atlas_command_dynamic_tool_request(
|
||||
call_id,
|
||||
arguments,
|
||||
conversation,
|
||||
thread_state.clone(),
|
||||
outgoing.clone(),
|
||||
remote_browser_api.clone(),
|
||||
));
|
||||
} else if remote_browser_api.is_configured()
|
||||
&& is_remote_browser_dynamic_tool(&tool)
|
||||
{
|
||||
tokio::spawn(handle_remote_browser_dynamic_tool_request(
|
||||
call_id,
|
||||
tool,
|
||||
|
||||
@@ -0,0 +1,498 @@
|
||||
use crate::remote_browser_tools::is_remote_browser_dynamic_tool;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
use codex_app_server_protocol::BrowserReplayFrame;
|
||||
use codex_app_server_protocol::BrowserReplayRenderMode;
|
||||
use codex_app_server_protocol::BrowserReplayTextSnapshotFormat;
|
||||
use codex_app_server_protocol::BrowserSessionState;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::Turn;
|
||||
use image::Delay;
|
||||
use image::Frame;
|
||||
use image::Rgba;
|
||||
use image::RgbaImage;
|
||||
use image::codecs::gif::GifEncoder;
|
||||
use image::codecs::gif::Repeat;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::io::Cursor;
|
||||
|
||||
const REPLAY_FRAME_DURATION_MS: u32 = 700;
|
||||
const MAX_TEXT_SNAPSHOT_CHARS: usize = 2_000;
|
||||
|
||||
pub(crate) fn augment_turns_with_browser_replays(turns: &mut [Turn]) {
|
||||
for turn in turns {
|
||||
turn.items
|
||||
.retain(|item| !matches!(item, ThreadItem::BrowserReplay { .. }));
|
||||
|
||||
if let Some(replay_item) = build_browser_replay_item(&turn.id, &turn.items) {
|
||||
turn.items.push(replay_item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_browser_replay_item(turn_id: &str, items: &[ThreadItem]) -> Option<ThreadItem> {
|
||||
let mut frames = Vec::new();
|
||||
let mut static_image_urls = Vec::new();
|
||||
let mut fallback_animation_url = None;
|
||||
|
||||
for item in items {
|
||||
let ThreadItem::DynamicToolCall {
|
||||
id,
|
||||
tool,
|
||||
status,
|
||||
content_items: Some(content_items),
|
||||
..
|
||||
} = item
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if !is_remote_browser_dynamic_tool(tool) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let frame_content = inspect_content_items(content_items);
|
||||
if let Some(image_url) = frame_content.image_url.clone() {
|
||||
static_image_urls.push(image_url);
|
||||
}
|
||||
if fallback_animation_url.is_none() {
|
||||
fallback_animation_url = frame_content.animation_image_url.clone();
|
||||
}
|
||||
|
||||
if frame_content.image_url.is_none()
|
||||
&& frame_content.text_snapshot.is_none()
|
||||
&& frame_content.browser_state.is_none()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
frames.push(BrowserReplayFrame {
|
||||
sequence_number: (frames.len() + 1) as u32,
|
||||
source_item_id: id.clone(),
|
||||
tool: tool.clone(),
|
||||
status: status.clone(),
|
||||
image_url: frame_content.image_url,
|
||||
text_snapshot: frame_content.text_snapshot,
|
||||
text_snapshot_format: frame_content.text_snapshot_format,
|
||||
browser_state: frame_content.browser_state,
|
||||
});
|
||||
}
|
||||
|
||||
if frames.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let animation_image_url =
|
||||
synthesize_animation_image_url(&static_image_urls).or(fallback_animation_url);
|
||||
let render_mode = if animation_image_url.is_some() {
|
||||
BrowserReplayRenderMode::Animation
|
||||
} else if frames.iter().any(|frame| frame.image_url.is_some()) {
|
||||
BrowserReplayRenderMode::Frames
|
||||
} else {
|
||||
BrowserReplayRenderMode::TextOnly
|
||||
};
|
||||
let frame_duration_ms =
|
||||
(animation_image_url.is_some() || frames.len() > 1).then_some(REPLAY_FRAME_DURATION_MS);
|
||||
|
||||
Some(ThreadItem::BrowserReplay {
|
||||
id: format!("browser-replay-{turn_id}"),
|
||||
frames,
|
||||
render_mode,
|
||||
animation_image_url,
|
||||
frame_duration_ms,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FrameContent {
|
||||
image_url: Option<String>,
|
||||
animation_image_url: Option<String>,
|
||||
text_snapshot: Option<String>,
|
||||
text_snapshot_format: Option<BrowserReplayTextSnapshotFormat>,
|
||||
browser_state: Option<BrowserSessionState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RemoteBrowserToolTextPayload {
|
||||
#[serde(default)]
|
||||
result: JsonValue,
|
||||
#[serde(default)]
|
||||
browser_state: Option<BrowserSessionState>,
|
||||
}
|
||||
|
||||
fn inspect_content_items(items: &[DynamicToolCallOutputContentItem]) -> FrameContent {
|
||||
let mut content = FrameContent::default();
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
DynamicToolCallOutputContentItem::InputText { text } => {
|
||||
if content.text_snapshot.is_none() {
|
||||
let snapshot = build_text_snapshot(text);
|
||||
content.text_snapshot = snapshot.text;
|
||||
content.text_snapshot_format = snapshot.format;
|
||||
}
|
||||
if content.browser_state.is_none() {
|
||||
content.browser_state = parse_browser_state(text);
|
||||
}
|
||||
}
|
||||
DynamicToolCallOutputContentItem::InputImage { image_url } => {
|
||||
if is_gif_data_url(image_url) {
|
||||
if content.animation_image_url.is_none() {
|
||||
content.animation_image_url = Some(image_url.clone());
|
||||
}
|
||||
} else if is_raster_data_url(image_url) && content.image_url.is_none() {
|
||||
content.image_url = Some(image_url.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
content
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TextSnapshot {
|
||||
text: Option<String>,
|
||||
format: Option<BrowserReplayTextSnapshotFormat>,
|
||||
}
|
||||
|
||||
fn build_text_snapshot(text: &str) -> TextSnapshot {
|
||||
serde_json::from_str::<RemoteBrowserToolTextPayload>(text)
|
||||
.ok()
|
||||
.and_then(|payload| render_result_snapshot(&payload.result))
|
||||
.or_else(|| {
|
||||
truncate_snapshot(text).map(|value| TextSnapshot {
|
||||
text: Some(value),
|
||||
format: Some(BrowserReplayTextSnapshotFormat::Plain),
|
||||
})
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn parse_browser_state(text: &str) -> Option<BrowserSessionState> {
|
||||
serde_json::from_str::<RemoteBrowserToolTextPayload>(text)
|
||||
.ok()
|
||||
.and_then(|payload| payload.browser_state)
|
||||
}
|
||||
|
||||
fn render_result_snapshot(result: &JsonValue) -> Option<TextSnapshot> {
|
||||
if result.is_null() {
|
||||
return None;
|
||||
}
|
||||
match result {
|
||||
JsonValue::String(text) => truncate_snapshot(text).map(|value| TextSnapshot {
|
||||
text: Some(value),
|
||||
format: Some(BrowserReplayTextSnapshotFormat::Plain),
|
||||
}),
|
||||
_ => serde_json::to_string_pretty(result)
|
||||
.ok()
|
||||
.and_then(|text| truncate_snapshot(&text))
|
||||
.map(|value| TextSnapshot {
|
||||
text: Some(value),
|
||||
format: Some(BrowserReplayTextSnapshotFormat::Json),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_snapshot(text: &str) -> Option<String> {
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let char_count = trimmed.chars().count();
|
||||
if char_count <= MAX_TEXT_SNAPSHOT_CHARS {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
let truncated = trimmed
|
||||
.chars()
|
||||
.take(MAX_TEXT_SNAPSHOT_CHARS)
|
||||
.collect::<String>();
|
||||
Some(format!("{truncated}..."))
|
||||
}
|
||||
|
||||
fn synthesize_animation_image_url(image_urls: &[String]) -> Option<String> {
|
||||
if image_urls.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut decoded_frames = Vec::new();
|
||||
for image_url in image_urls {
|
||||
let image = decode_raster_data_url(image_url)?;
|
||||
decoded_frames.push(image.to_rgba8());
|
||||
}
|
||||
|
||||
if decoded_frames.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let max_width = decoded_frames.iter().map(RgbaImage::width).max()?;
|
||||
let max_height = decoded_frames.iter().map(RgbaImage::height).max()?;
|
||||
let delay = Delay::from_numer_denom_ms(REPLAY_FRAME_DURATION_MS, 1);
|
||||
let frames = decoded_frames
|
||||
.into_iter()
|
||||
.map(|frame| Frame::from_parts(normalize_frame(frame, max_width, max_height), 0, 0, delay));
|
||||
|
||||
let mut bytes = Cursor::new(Vec::new());
|
||||
{
|
||||
let mut encoder = GifEncoder::new(&mut bytes);
|
||||
encoder.set_repeat(Repeat::Infinite).ok()?;
|
||||
encoder.encode_frames(frames).ok()?;
|
||||
}
|
||||
|
||||
Some(format!(
|
||||
"data:image/gif;base64,{}",
|
||||
BASE64_STANDARD.encode(bytes.into_inner())
|
||||
))
|
||||
}
|
||||
|
||||
fn normalize_frame(frame: RgbaImage, width: u32, height: u32) -> RgbaImage {
|
||||
if frame.width() == width && frame.height() == height {
|
||||
return frame;
|
||||
}
|
||||
|
||||
let mut canvas = RgbaImage::from_pixel(width, height, Rgba([255, 255, 255, 255]));
|
||||
image::imageops::overlay(&mut canvas, &frame, 0, 0);
|
||||
canvas
|
||||
}
|
||||
|
||||
fn decode_raster_data_url(image_url: &str) -> Option<image::DynamicImage> {
|
||||
if !is_raster_data_url(image_url) {
|
||||
return None;
|
||||
}
|
||||
let (_, encoded) = image_url.split_once(',')?;
|
||||
let bytes = BASE64_STANDARD.decode(encoded).ok()?;
|
||||
image::load_from_memory(&bytes).ok()
|
||||
}
|
||||
|
||||
fn is_raster_data_url(image_url: &str) -> bool {
|
||||
image_url.starts_with("data:image/png;base64,")
|
||||
|| image_url.starts_with("data:image/jpeg;base64,")
|
||||
|| image_url.starts_with("data:image/webp;base64,")
|
||||
}
|
||||
|
||||
fn is_gif_data_url(image_url: &str) -> bool {
|
||||
image_url.starts_with("data:image/gif;base64,")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use codex_app_server_protocol::DynamicToolCallStatus;
|
||||
use image::DynamicImage;
|
||||
|
||||
fn png_data_url(color: [u8; 4], width: u32, height: u32) -> String {
|
||||
let image = RgbaImage::from_pixel(width, height, Rgba(color));
|
||||
let mut bytes = Cursor::new(Vec::new());
|
||||
DynamicImage::ImageRgba8(image)
|
||||
.write_to(&mut bytes, image::ImageFormat::Png)
|
||||
.expect("encode png");
|
||||
format!(
|
||||
"data:image/png;base64,{}",
|
||||
BASE64_STANDARD.encode(bytes.into_inner())
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_replay_from_remote_browser_tool_calls() {
|
||||
let items = vec![
|
||||
ThreadItem::DynamicToolCall {
|
||||
id: "tool-1".to_string(),
|
||||
tool: "create_tab".to_string(),
|
||||
arguments: JsonValue::Null,
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: Some(vec![
|
||||
DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::json!({
|
||||
"result": { "tab_id": "tab_1" },
|
||||
"browser_state": {
|
||||
"selectedTabId": "tab_1",
|
||||
"tabs": [{
|
||||
"id": "tab_1",
|
||||
"title": "First",
|
||||
"url": "https://example.com/one",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
},
|
||||
DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: png_data_url([255, 0, 0, 255], 10, 10),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
duration_ms: Some(10),
|
||||
},
|
||||
ThreadItem::DynamicToolCall {
|
||||
id: "tool-2".to_string(),
|
||||
tool: "tabs_content".to_string(),
|
||||
arguments: JsonValue::Null,
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: Some(vec![
|
||||
DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::json!({
|
||||
"result": {
|
||||
"tabs": [{
|
||||
"title": "First",
|
||||
"content": "headline one\nheadline two"
|
||||
}]
|
||||
},
|
||||
"browser_state": {
|
||||
"selectedTabId": "tab_1",
|
||||
"tabs": [{
|
||||
"id": "tab_1",
|
||||
"title": "First",
|
||||
"url": "https://example.com/one",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
},
|
||||
DynamicToolCallOutputContentItem::InputImage {
|
||||
image_url: png_data_url([0, 0, 255, 255], 12, 8),
|
||||
},
|
||||
]),
|
||||
success: Some(true),
|
||||
duration_ms: Some(10),
|
||||
},
|
||||
];
|
||||
|
||||
let replay = build_browser_replay_item("turn-1", &items).expect("replay item");
|
||||
let ThreadItem::BrowserReplay {
|
||||
id,
|
||||
frames,
|
||||
render_mode,
|
||||
animation_image_url,
|
||||
frame_duration_ms,
|
||||
} = replay
|
||||
else {
|
||||
panic!("expected browser replay item");
|
||||
};
|
||||
|
||||
assert_eq!(id, "browser-replay-turn-1");
|
||||
assert_eq!(frames.len(), 2);
|
||||
assert_eq!(render_mode, BrowserReplayRenderMode::Animation);
|
||||
assert_eq!(frames[0].source_item_id, "tool-1");
|
||||
assert_eq!(frames[1].source_item_id, "tool-2");
|
||||
let second_snapshot = frames[1]
|
||||
.text_snapshot
|
||||
.as_deref()
|
||||
.expect("second frame snapshot");
|
||||
assert!(second_snapshot.contains("\"title\": \"First\""));
|
||||
assert!(second_snapshot.contains("\"content\": \"headline one\\nheadline two\""));
|
||||
assert_eq!(
|
||||
frames[1].text_snapshot_format,
|
||||
Some(BrowserReplayTextSnapshotFormat::Json)
|
||||
);
|
||||
assert!(
|
||||
animation_image_url
|
||||
.as_deref()
|
||||
.is_some_and(|value| value.starts_with("data:image/gif;base64,"))
|
||||
);
|
||||
assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_text_only_replay_when_no_images_are_available() {
|
||||
let items = vec![ThreadItem::DynamicToolCall {
|
||||
id: "tool-1".to_string(),
|
||||
tool: "tabs_content".to_string(),
|
||||
arguments: JsonValue::Null,
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::json!({
|
||||
"result": {
|
||||
"tree": {
|
||||
"role": "document",
|
||||
"name": "Example page"
|
||||
}
|
||||
}
|
||||
})
|
||||
.to_string(),
|
||||
}]),
|
||||
success: Some(true),
|
||||
duration_ms: Some(5),
|
||||
}];
|
||||
|
||||
let replay = build_browser_replay_item("turn-2", &items).expect("replay item");
|
||||
let ThreadItem::BrowserReplay {
|
||||
frames,
|
||||
render_mode,
|
||||
animation_image_url,
|
||||
frame_duration_ms,
|
||||
..
|
||||
} = replay
|
||||
else {
|
||||
panic!("expected browser replay item");
|
||||
};
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(render_mode, BrowserReplayRenderMode::TextOnly);
|
||||
assert_eq!(
|
||||
frames[0].text_snapshot.as_deref(),
|
||||
Some(
|
||||
"{\n \"tree\": {\n \"role\": \"document\",\n \"name\": \"Example page\"\n }\n}"
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
frames[0].text_snapshot_format,
|
||||
Some(BrowserReplayTextSnapshotFormat::Json)
|
||||
);
|
||||
assert_eq!(animation_image_url, None);
|
||||
assert_eq!(frame_duration_ms, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assigns_frame_duration_to_multi_frame_text_only_replays() {
|
||||
let items = vec![
|
||||
ThreadItem::DynamicToolCall {
|
||||
id: "tool-1".to_string(),
|
||||
tool: "tabs_content".to_string(),
|
||||
arguments: JsonValue::Null,
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::json!({
|
||||
"result": { "tree": { "role": "document", "name": "Step 1" } }
|
||||
})
|
||||
.to_string(),
|
||||
}]),
|
||||
success: Some(true),
|
||||
duration_ms: Some(5),
|
||||
},
|
||||
ThreadItem::DynamicToolCall {
|
||||
id: "tool-2".to_string(),
|
||||
tool: "tabs_content".to_string(),
|
||||
arguments: JsonValue::Null,
|
||||
status: DynamicToolCallStatus::Completed,
|
||||
content_items: Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::json!({
|
||||
"result": { "tree": { "role": "document", "name": "Step 2" } }
|
||||
})
|
||||
.to_string(),
|
||||
}]),
|
||||
success: Some(true),
|
||||
duration_ms: Some(5),
|
||||
},
|
||||
];
|
||||
|
||||
let replay = build_browser_replay_item("turn-3", &items).expect("replay item");
|
||||
let ThreadItem::BrowserReplay {
|
||||
frames,
|
||||
render_mode,
|
||||
animation_image_url,
|
||||
frame_duration_ms,
|
||||
..
|
||||
} = replay
|
||||
else {
|
||||
panic!("expected browser replay item");
|
||||
};
|
||||
|
||||
assert_eq!(frames.len(), 2);
|
||||
assert_eq!(render_mode, BrowserReplayRenderMode::TextOnly);
|
||||
assert_eq!(animation_image_url, None);
|
||||
assert_eq!(frame_duration_ms, Some(REPLAY_FRAME_DURATION_MS));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
use crate::atlas_command::merge_atlas_command_dynamic_tool;
|
||||
use crate::bespoke_event_handling::apply_bespoke_event_handling;
|
||||
use crate::browser_replay::augment_turns_with_browser_replays;
|
||||
use crate::command_exec::CommandExecManager;
|
||||
use crate::command_exec::StartCommandExecParams;
|
||||
use crate::error_code::INPUT_TOO_LARGE_ERROR_CODE;
|
||||
@@ -1992,7 +1994,9 @@ impl CodexMessageProcessor {
|
||||
};
|
||||
|
||||
let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() {
|
||||
merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default())
|
||||
let dynamic_tools =
|
||||
merge_remote_browser_dynamic_tools(dynamic_tools.unwrap_or_default());
|
||||
merge_atlas_command_dynamic_tool(dynamic_tools)
|
||||
} else {
|
||||
dynamic_tools.unwrap_or_default()
|
||||
};
|
||||
@@ -3299,11 +3303,28 @@ impl CodexMessageProcessor {
|
||||
self.attach_thread_name(thread_uuid, &mut thread).await;
|
||||
|
||||
if include_turns && let Some(rollout_path) = rollout_path.as_ref() {
|
||||
match read_rollout_items_from_rollout(rollout_path).await {
|
||||
Ok(items) => {
|
||||
thread.turns = build_turns_from_rollout_items(&items);
|
||||
}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
let active_turn = if loaded_thread.is_some() {
|
||||
let state = self.thread_state_manager.thread_state(thread_uuid).await;
|
||||
let state = state.lock().await;
|
||||
state.active_turn_snapshot()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if let Err(message) = populate_thread_turns(
|
||||
&mut thread,
|
||||
ThreadTurnSource::RolloutPath(rollout_path),
|
||||
active_turn.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let missing_rollout_message = format!(
|
||||
"failed to load rollout `{}` for thread {thread_uuid}: ",
|
||||
rollout_path.display()
|
||||
);
|
||||
if message.contains(&missing_rollout_message)
|
||||
&& message.contains("No such file or directory")
|
||||
{
|
||||
self.send_invalid_request_error(
|
||||
request_id,
|
||||
format!(
|
||||
@@ -3313,17 +3334,9 @@ impl CodexMessageProcessor {
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
self.send_internal_error(
|
||||
request_id,
|
||||
format!(
|
||||
"failed to load rollout `{}` for thread {thread_uuid}: {err}",
|
||||
rollout_path.display()
|
||||
),
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
self.send_internal_error(request_id, message).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7473,6 +7486,7 @@ async fn populate_thread_turns(
|
||||
if let Some(active_turn) = active_turn {
|
||||
merge_turn_history_with_active_turn(&mut turns, active_turn.clone());
|
||||
}
|
||||
augment_turns_with_browser_replays(&mut turns);
|
||||
thread.turns = turns;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -57,7 +57,9 @@ use tracing_subscriber::registry::Registry;
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
|
||||
mod app_server_tracing;
|
||||
mod atlas_command;
|
||||
mod bespoke_event_handling;
|
||||
mod browser_replay;
|
||||
mod codex_message_processor;
|
||||
mod command_exec;
|
||||
mod config_api;
|
||||
|
||||
@@ -7,9 +7,9 @@ use crate::transport::AppServerTransport;
|
||||
use anyhow::Result;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::write_mock_responses_config_toml;
|
||||
use codex_app_server_protocol::BrowserSessionArtifacts;
|
||||
use codex_app_server_protocol::BrowserSessionCommandParams;
|
||||
use codex_app_server_protocol::BrowserSessionCommandResponse;
|
||||
use codex_app_server_protocol::BrowserSessionArtifacts;
|
||||
use codex_app_server_protocol::BrowserSessionState;
|
||||
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
|
||||
use codex_app_server_protocol::BrowserTabState;
|
||||
|
||||
@@ -116,6 +116,10 @@ impl RemoteBrowserCommandOutcome {
|
||||
.map(|encoded| format!("data:image/png;base64,{encoded}"))
|
||||
}
|
||||
|
||||
pub(crate) fn screenshot_base64(&self) -> Option<String> {
|
||||
self.artifacts.screenshot_base64.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn replay_gif_data_url(&self) -> Option<String> {
|
||||
self.artifacts
|
||||
.replay_gif_base64
|
||||
|
||||
@@ -5,6 +5,8 @@ use app_test_support::create_final_assistant_message_sse_response;
|
||||
use app_test_support::create_mock_responses_server_repeating_assistant;
|
||||
use app_test_support::create_mock_responses_server_sequence_unchecked;
|
||||
use app_test_support::to_response;
|
||||
use codex_app_server_protocol::BrowserReplayRenderMode;
|
||||
use codex_app_server_protocol::BrowserReplayTextSnapshotFormat;
|
||||
use codex_app_server_protocol::BrowserSessionArtifacts;
|
||||
use codex_app_server_protocol::BrowserSessionUpdatedNotification;
|
||||
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
|
||||
@@ -19,6 +21,8 @@ use codex_app_server_protocol::JSONRPCResponse;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ServerRequest;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::ThreadReadParams;
|
||||
use codex_app_server_protocol::ThreadReadResponse;
|
||||
use codex_app_server_protocol::ThreadStartParams;
|
||||
use codex_app_server_protocol::ThreadStartResponse;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
@@ -38,6 +42,7 @@ use tokio::time::timeout;
|
||||
use wiremock::Mock;
|
||||
use wiremock::MockServer;
|
||||
use wiremock::ResponseTemplate;
|
||||
use wiremock::matchers::body_json;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::path;
|
||||
|
||||
@@ -255,6 +260,655 @@ async fn thread_start_auto_injects_remote_browser_tools_when_endpoint_is_configu
|
||||
assert!(find_tool(body, "create_tab").is_some());
|
||||
assert!(find_tool(body, "tabs_content").is_some());
|
||||
assert!(find_tool(body, "playwright_screenshot").is_some());
|
||||
assert!(find_tool(body, "atlas_command").is_none());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atlas_command_dynamic_tool_calls_are_handled_internally() -> Result<()> {
|
||||
let call_id = "dyn-atlas-1";
|
||||
let tool_name = "atlas_command";
|
||||
let tool_args = json!({
|
||||
"payload": {
|
||||
"type": "create_tab"
|
||||
}
|
||||
});
|
||||
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
||||
|
||||
let responses = vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
create_final_assistant_message_sse_response("Done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.and(body_json(json!({
|
||||
"browserSessionId": null,
|
||||
"command": "create_tab",
|
||||
"arguments": {
|
||||
"select": true
|
||||
}
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_atlas_123",
|
||||
"result": {
|
||||
"tab_id": "tab_123",
|
||||
"selected_tab_id": "tab_123"
|
||||
},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "New Tab",
|
||||
"url": "about:blank",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
let thread_id = thread.id.clone();
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread_id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "use atlas".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
let turn_id = turn.id.clone();
|
||||
|
||||
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
|
||||
assert_eq!(started.thread_id, thread_id);
|
||||
assert_eq!(started.turn_id, turn_id.clone());
|
||||
|
||||
assert!(
|
||||
timeout(
|
||||
Duration::from_millis(500),
|
||||
mcp.read_stream_until_request_message()
|
||||
)
|
||||
.await
|
||||
.is_err(),
|
||||
"atlas_command tool call should not be forwarded back to the client"
|
||||
);
|
||||
|
||||
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
|
||||
assert_eq!(browser_update.browser_session_id, "sess_atlas_123");
|
||||
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
|
||||
assert_eq!(browser_update.browser_state.tabs.len(), 1);
|
||||
assert_eq!(browser_update.browser_state.tabs[0].title, "New Tab");
|
||||
assert_eq!(browser_update.artifacts, None);
|
||||
|
||||
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
|
||||
let ThreadItem::DynamicToolCall {
|
||||
status,
|
||||
content_items,
|
||||
success,
|
||||
..
|
||||
} = completed.item
|
||||
else {
|
||||
panic!("expected dynamic tool call item");
|
||||
};
|
||||
assert_eq!(status, DynamicToolCallStatus::Completed);
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
content_items,
|
||||
Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(&json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": {
|
||||
"id": "tab_123"
|
||||
}
|
||||
}))?,
|
||||
}])
|
||||
);
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let bodies = responses_bodies(&server).await?;
|
||||
let payload = bodies
|
||||
.iter()
|
||||
.find_map(|body| function_call_output_payload(body, call_id))
|
||||
.context("expected function_call_output in follow-up request")?;
|
||||
assert_eq!(
|
||||
payload.body,
|
||||
FunctionCallOutputBody::Text(serde_json::to_string(&json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": {
|
||||
"id": "tab_123"
|
||||
}
|
||||
}))?)
|
||||
);
|
||||
assert_eq!(payload.success, None);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atlas_command_tabs_content_uses_background_tabs() -> Result<()> {
|
||||
let call_id = "dyn-atlas-tabs-1";
|
||||
let tool_name = "atlas_command";
|
||||
let tool_args = json!({
|
||||
"payload": {
|
||||
"type": "tabs_content",
|
||||
"urls": ["https://news.ycombinator.com/"],
|
||||
"content_type": "text"
|
||||
}
|
||||
});
|
||||
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
||||
|
||||
let responses = vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
create_final_assistant_message_sse_response("Done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.and(body_json(json!({
|
||||
"browserSessionId": null,
|
||||
"command": "create_tab",
|
||||
"arguments": {
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"select": false
|
||||
}
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_atlas_tabs_123",
|
||||
"result": {
|
||||
"tab_id": "tab_bg_123"
|
||||
},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_main",
|
||||
"tabs": [
|
||||
{
|
||||
"id": "tab_main",
|
||||
"title": "Workspace",
|
||||
"url": "http://localhost:3000/",
|
||||
"selected": true
|
||||
},
|
||||
{
|
||||
"id": "tab_bg_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": false
|
||||
}
|
||||
]
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.and(body_json(json!({
|
||||
"browserSessionId": "sess_atlas_tabs_123",
|
||||
"command": "tabs_content",
|
||||
"arguments": {
|
||||
"tab_ids": ["tab_bg_123"]
|
||||
}
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_atlas_tabs_123",
|
||||
"result": {
|
||||
"tabs": [{
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"title": "Hacker News",
|
||||
"content": "Top stories"
|
||||
}]
|
||||
},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_main",
|
||||
"tabs": [
|
||||
{
|
||||
"id": "tab_main",
|
||||
"title": "Workspace",
|
||||
"url": "http://localhost:3000/",
|
||||
"selected": true
|
||||
},
|
||||
{
|
||||
"id": "tab_bg_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": false
|
||||
}
|
||||
]
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread.id,
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "read the page".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
|
||||
let _started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
|
||||
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
|
||||
assert_eq!(browser_update.browser_session_id, "sess_atlas_tabs_123");
|
||||
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_main");
|
||||
|
||||
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
|
||||
let ThreadItem::DynamicToolCall {
|
||||
content_items,
|
||||
success,
|
||||
..
|
||||
} = completed.item
|
||||
else {
|
||||
panic!("expected dynamic tool call item");
|
||||
};
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
content_items,
|
||||
Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(&json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": {
|
||||
"results": [{
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"title": "Hacker News",
|
||||
"content": "Top stories"
|
||||
}]
|
||||
}
|
||||
}))?,
|
||||
}])
|
||||
);
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atlas_command_locator_count_passes_through_to_remote_browser() -> Result<()> {
|
||||
let call_id = "dyn-atlas-locator-count-1";
|
||||
let tool_name = "atlas_command";
|
||||
let tool_args = json!({
|
||||
"payload": {
|
||||
"type": "playwright_locator_count",
|
||||
"tab_id": "tab_123",
|
||||
"selector": "span.titleline > a"
|
||||
}
|
||||
});
|
||||
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
||||
|
||||
let responses = vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
create_final_assistant_message_sse_response("Done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.and(body_json(json!({
|
||||
"browserSessionId": null,
|
||||
"command": "playwright_locator_count",
|
||||
"arguments": {
|
||||
"tab_id": "tab_123",
|
||||
"selector": "span.titleline > a"
|
||||
}
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_atlas_count_123",
|
||||
"result": {
|
||||
"count": 5
|
||||
},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
let thread_id = thread.id.clone();
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread_id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "count links".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
let turn_id = turn.id.clone();
|
||||
|
||||
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
|
||||
assert_eq!(started.thread_id, thread_id);
|
||||
assert_eq!(started.turn_id, turn_id.clone());
|
||||
|
||||
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
|
||||
assert_eq!(browser_update.browser_session_id, "sess_atlas_count_123");
|
||||
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
|
||||
|
||||
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
|
||||
let ThreadItem::DynamicToolCall {
|
||||
content_items,
|
||||
success,
|
||||
..
|
||||
} = completed.item
|
||||
else {
|
||||
panic!("expected dynamic tool call item");
|
||||
};
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
content_items,
|
||||
Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(&json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": {
|
||||
"count": 5
|
||||
}
|
||||
}))?,
|
||||
}])
|
||||
);
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let bodies = responses_bodies(&server).await?;
|
||||
let payload = bodies
|
||||
.iter()
|
||||
.find_map(|body| function_call_output_payload(body, call_id))
|
||||
.context("expected function_call_output in follow-up request")?;
|
||||
assert_eq!(
|
||||
payload.body,
|
||||
FunctionCallOutputBody::Text(serde_json::to_string(&json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": {
|
||||
"count": 5
|
||||
}
|
||||
}))?)
|
||||
);
|
||||
assert_eq!(payload.success, None);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn atlas_command_locator_click_passes_through_to_remote_browser() -> Result<()> {
|
||||
let call_id = "dyn-atlas-locator-click-1";
|
||||
let tool_name = "atlas_command";
|
||||
let tool_args = json!({
|
||||
"payload": {
|
||||
"type": "playwright_locator_click",
|
||||
"tab_id": "tab_123",
|
||||
"selector": "[data-square=\"e2\"]",
|
||||
"button": "left",
|
||||
"timeout_ms": 1500
|
||||
}
|
||||
});
|
||||
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
||||
|
||||
let responses = vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
create_final_assistant_message_sse_response("Clicked")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.and(body_json(json!({
|
||||
"browserSessionId": null,
|
||||
"command": "playwright_locator_click",
|
||||
"arguments": {
|
||||
"tab_id": "tab_123",
|
||||
"selector": "[data-square=\"e2\"]",
|
||||
"button": "left",
|
||||
"timeout_ms": 1500
|
||||
}
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_atlas_click_123",
|
||||
"result": {},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "3D Chess",
|
||||
"url": "http://127.0.0.1:8123/",
|
||||
"selected": true
|
||||
}]
|
||||
},
|
||||
"artifacts": {
|
||||
"screenshotBase64": "AAA",
|
||||
"replayGifBase64": "R0lGODlh",
|
||||
"replayFrameCount": 3,
|
||||
"replayFrameDurationMs": 700
|
||||
}
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
let thread_id = thread.id.clone();
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread_id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "click e2".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
let turn_id = turn.id.clone();
|
||||
|
||||
let started = wait_for_dynamic_tool_started(&mut mcp, call_id).await?;
|
||||
assert_eq!(started.thread_id, thread_id);
|
||||
assert_eq!(started.turn_id, turn_id.clone());
|
||||
|
||||
let browser_update = wait_for_browser_session_updated(&mut mcp).await?;
|
||||
assert_eq!(browser_update.browser_session_id, "sess_atlas_click_123");
|
||||
assert_eq!(browser_update.browser_state.selected_tab_id, "tab_123");
|
||||
assert_eq!(
|
||||
browser_update.artifacts,
|
||||
Some(BrowserSessionArtifacts {
|
||||
screenshot_image_url: Some("data:image/png;base64,AAA".to_string()),
|
||||
replay_gif_image_url: Some("data:image/gif;base64,R0lGODlh".to_string()),
|
||||
replay_frame_count: Some(3),
|
||||
replay_frame_duration_ms: Some(700),
|
||||
})
|
||||
);
|
||||
|
||||
let completed = wait_for_dynamic_tool_completed(&mut mcp, call_id).await?;
|
||||
let ThreadItem::DynamicToolCall {
|
||||
content_items,
|
||||
success,
|
||||
..
|
||||
} = completed.item
|
||||
else {
|
||||
panic!("expected dynamic tool call item");
|
||||
};
|
||||
assert_eq!(success, Some(true));
|
||||
assert_eq!(
|
||||
content_items,
|
||||
Some(vec![DynamicToolCallOutputContentItem::InputText {
|
||||
text: serde_json::to_string(&json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": {}
|
||||
}))?,
|
||||
}])
|
||||
);
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let bodies = responses_bodies(&server).await?;
|
||||
let payload = bodies
|
||||
.iter()
|
||||
.find_map(|body| function_call_output_payload(body, call_id))
|
||||
.context("expected function_call_output in follow-up request")?;
|
||||
assert_eq!(
|
||||
payload.body,
|
||||
FunctionCallOutputBody::Text(serde_json::to_string(&json!({
|
||||
"type": "codex_repl",
|
||||
"command_output": {}
|
||||
}))?)
|
||||
);
|
||||
assert_eq!(payload.success, None);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -477,6 +1131,269 @@ async fn remote_browser_dynamic_tool_calls_are_handled_internally() -> Result<()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_read_derives_browser_replay_from_remote_browser_tool_history() -> Result<()> {
|
||||
let call_id = "dyn-browser-read-1";
|
||||
let tool_name = "create_tab";
|
||||
let tool_args = json!({ "url": "https://news.ycombinator.com/" });
|
||||
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
||||
|
||||
let responses = vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
create_final_assistant_message_sse_response("Done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_123",
|
||||
"result": {
|
||||
"tab_id": "tab_123",
|
||||
"selected_tab_id": "tab_123"
|
||||
},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": true
|
||||
}]
|
||||
},
|
||||
"artifacts": {
|
||||
"screenshotBase64": "AAA",
|
||||
"replayGifBase64": "R0lGODlh",
|
||||
"replayFrameCount": 2,
|
||||
"replayFrameDurationMs": 700
|
||||
}
|
||||
})))
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread.id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "open Hacker News".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let read_req = mcp
|
||||
.send_thread_read_request(ThreadReadParams {
|
||||
thread_id: thread.id.clone(),
|
||||
include_turns: true,
|
||||
})
|
||||
.await?;
|
||||
let read_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(read_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadReadResponse { thread } = to_response::<ThreadReadResponse>(read_resp)?;
|
||||
|
||||
let turn = thread.turns.last().context("expected replay turn")?;
|
||||
let replay = turn
|
||||
.items
|
||||
.iter()
|
||||
.find_map(|item| match item {
|
||||
ThreadItem::BrowserReplay {
|
||||
frames,
|
||||
animation_image_url,
|
||||
frame_duration_ms,
|
||||
..
|
||||
} => Some((frames, animation_image_url, frame_duration_ms)),
|
||||
_ => None,
|
||||
})
|
||||
.context("expected derived browser replay item")?;
|
||||
|
||||
assert_eq!(replay.0.len(), 1);
|
||||
assert_eq!(replay.0[0].source_item_id, call_id);
|
||||
assert_eq!(replay.0[0].tool, tool_name);
|
||||
assert_eq!(
|
||||
replay.0[0].image_url.as_deref(),
|
||||
Some("data:image/png;base64,AAA")
|
||||
);
|
||||
assert_eq!(replay.1.as_deref(), Some("data:image/gif;base64,R0lGODlh"));
|
||||
assert_eq!(*replay.2, Some(700));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_read_derives_text_only_browser_replay_from_remote_browser_tool_history()
|
||||
-> Result<()> {
|
||||
let call_id = "dyn-browser-read-text-1";
|
||||
let tool_name = "tabs_content";
|
||||
let tool_args = json!({ "tabId": "tab_123" });
|
||||
let tool_call_arguments = serde_json::to_string(&tool_args)?;
|
||||
|
||||
let responses = vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call(call_id, tool_name, &tool_call_arguments),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
create_final_assistant_message_sse_response("Done")?,
|
||||
];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
let remote_browser_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/remote_browser"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"browserSessionId": "sess_123",
|
||||
"result": {
|
||||
"tree": {
|
||||
"role": "document",
|
||||
"name": "Hacker News"
|
||||
}
|
||||
},
|
||||
"browserState": {
|
||||
"selectedTabId": "tab_123",
|
||||
"tabs": [{
|
||||
"id": "tab_123",
|
||||
"title": "Hacker News",
|
||||
"url": "https://news.ycombinator.com/",
|
||||
"selected": true
|
||||
}]
|
||||
}
|
||||
})))
|
||||
.mount(&remote_browser_server)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri())?;
|
||||
|
||||
let remote_browser_endpoint = format!("{}/remote_browser", remote_browser_server.uri());
|
||||
let mut mcp = McpProcess::new_with_args(
|
||||
codex_home.path(),
|
||||
&["--remote-browser-endpoint", &remote_browser_endpoint],
|
||||
)
|
||||
.await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams::default())
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread.id.clone(),
|
||||
input: vec![V2UserInput::Text {
|
||||
text: "inspect the current page".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let _turn: TurnStartResponse = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let read_req = mcp
|
||||
.send_thread_read_request(ThreadReadParams {
|
||||
thread_id: thread.id.clone(),
|
||||
include_turns: true,
|
||||
})
|
||||
.await?;
|
||||
let read_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(read_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadReadResponse { thread } = to_response::<ThreadReadResponse>(read_resp)?;
|
||||
|
||||
let turn = thread.turns.last().context("expected replay turn")?;
|
||||
let replay = turn
|
||||
.items
|
||||
.iter()
|
||||
.find_map(|item| match item {
|
||||
ThreadItem::BrowserReplay {
|
||||
frames,
|
||||
render_mode,
|
||||
animation_image_url,
|
||||
frame_duration_ms,
|
||||
..
|
||||
} => Some((frames, render_mode, animation_image_url, frame_duration_ms)),
|
||||
_ => None,
|
||||
})
|
||||
.context("expected derived browser replay item")?;
|
||||
|
||||
assert_eq!(replay.0.len(), 1);
|
||||
assert_eq!(replay.0[0].source_item_id, call_id);
|
||||
assert_eq!(replay.0[0].tool, tool_name);
|
||||
assert_eq!(replay.1, &BrowserReplayRenderMode::TextOnly);
|
||||
assert_eq!(replay.2, &None);
|
||||
assert_eq!(replay.3, &None);
|
||||
let text_snapshot = replay.0[0]
|
||||
.text_snapshot
|
||||
.as_deref()
|
||||
.context("expected text snapshot")?;
|
||||
assert!(text_snapshot.contains("\"role\": \"document\""));
|
||||
assert!(text_snapshot.contains("\"name\": \"Hacker News\""));
|
||||
assert_eq!(
|
||||
replay.0[0].text_snapshot_format,
|
||||
Some(BrowserReplayTextSnapshotFormat::Json)
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Exercises the full dynamic tool call path (server request, client response, model output).
|
||||
#[tokio::test]
|
||||
async fn dynamic_tool_call_round_trip_sends_text_content_items_to_model() -> Result<()> {
|
||||
|
||||
@@ -128,9 +128,9 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option<EventPersistenceMode> {
|
||||
| EventMsg::CollabAgentInteractionEnd(_)
|
||||
| EventMsg::CollabWaitingEnd(_)
|
||||
| EventMsg::CollabCloseEnd(_)
|
||||
| EventMsg::CollabResumeEnd(_)
|
||||
| EventMsg::DynamicToolCallRequest(_)
|
||||
| EventMsg::DynamicToolCallResponse(_) => Some(EventPersistenceMode::Extended),
|
||||
| EventMsg::CollabResumeEnd(_) => Some(EventPersistenceMode::Extended),
|
||||
EventMsg::DynamicToolCallResponse(_) => Some(EventPersistenceMode::Limited),
|
||||
EventMsg::DynamicToolCallRequest(_) => Some(EventPersistenceMode::Extended),
|
||||
EventMsg::Warning(_)
|
||||
| EventMsg::RealtimeConversationStarted(_)
|
||||
| EventMsg::RealtimeConversationRealtime(_)
|
||||
@@ -188,8 +188,11 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option<EventPersistenceMode> {
|
||||
mod tests {
|
||||
use super::EventPersistenceMode;
|
||||
use super::should_persist_event_msg;
|
||||
use codex_protocol::dynamic_tools::DynamicToolCallRequest;
|
||||
use codex_protocol::protocol::DynamicToolCallResponseEvent;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ImageGenerationEndEvent;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn persists_image_generation_end_events_in_limited_mode() {
|
||||
@@ -206,4 +209,42 @@ mod tests {
|
||||
EventPersistenceMode::Limited
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persists_dynamic_tool_responses_in_limited_mode() {
|
||||
let event = EventMsg::DynamicToolCallResponse(DynamicToolCallResponseEvent {
|
||||
call_id: "dyn_123".into(),
|
||||
turn_id: "turn_123".into(),
|
||||
tool: "create_tab".into(),
|
||||
arguments: serde_json::json!({ "url": "https://example.com" }),
|
||||
content_items: Vec::new(),
|
||||
success: true,
|
||||
error: None,
|
||||
duration: Duration::from_millis(250),
|
||||
});
|
||||
|
||||
assert!(should_persist_event_msg(
|
||||
&event,
|
||||
EventPersistenceMode::Limited
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_persist_dynamic_tool_requests_in_limited_mode() {
|
||||
let event = EventMsg::DynamicToolCallRequest(DynamicToolCallRequest {
|
||||
call_id: "dyn_123".into(),
|
||||
turn_id: "turn_123".into(),
|
||||
tool: "create_tab".into(),
|
||||
arguments: serde_json::json!({ "url": "https://example.com" }),
|
||||
});
|
||||
|
||||
assert!(!should_persist_event_msg(
|
||||
&event,
|
||||
EventPersistenceMode::Limited
|
||||
));
|
||||
assert!(should_persist_event_msg(
|
||||
&event,
|
||||
EventPersistenceMode::Extended
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user