diff --git a/codex-rs/.cargo/config.toml b/codex-rs/.cargo/config.toml index f5107cf73e..5d5eb8fd6f 100644 --- a/codex-rs/.cargo/config.toml +++ b/codex-rs/.cargo/config.toml @@ -1,5 +1,11 @@ [target.'cfg(all(windows, target_env = "msvc"))'] rustflags = ["-C", "link-arg=/STACK:8388608"] +# MSVC emits a warning about code that may trip "Cortex-A53 MPCore processor bug #843419" (see +# https://developer.arm.com/documentation/epm048406/latest) which is sometimes emitted by LLVM. +# Since Arm64 Windows 10+ isn't supported on that processor, it's safe to disable the warning. +[target.aarch64-pc-windows-msvc] +rustflags = ["-C", "link-arg=/STACK:8388608", "-C", "link-arg=/arm64hazardfree"] + [target.'cfg(all(windows, target_env = "gnu"))'] rustflags = ["-C", "link-arg=-Wl,--stack,8388608"] diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index c888f9368d..a68f99425c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1791,6 +1791,7 @@ dependencies = [ "eventsource-stream", "futures", "http 1.4.0", + "iana-time-zone", "image", "indexmap 2.13.0", "insta", @@ -2299,6 +2300,7 @@ dependencies = [ "anyhow", "async-trait", "clap", + "codex-protocol", "codex-utils-absolute-path", "libc", "pretty_assertions", @@ -2399,6 +2401,7 @@ dependencies = [ "codex-utils-pty", "codex-utils-sandbox-summary", "codex-utils-sleep-inhibitor", + "codex-utils-string", "codex-windows-sandbox", "color-eyre", "cpal", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index c72bb907f3..01836cde05 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -118,8 +118,8 @@ codex-shell-command = { path = "shell-command" } codex-shell-escalation = { path = "shell-escalation" } codex-skills = { path = "skills" } codex-state = { path = "state" } -codex-test-macros = { path = "test-macros" } codex-stdio-to-uds = { path = "stdio-to-uds" } +codex-test-macros = { path = "test-macros" } codex-tui = { path = "tui" } codex-utils-absolute-path = { path = "utils/absolute-path" } codex-utils-approval-presets = { path = "utils/approval-presets" } @@ -137,8 +137,8 @@ codex-utils-readiness = { path = "utils/readiness" } codex-utils-rustls-provider = { path = "utils/rustls-provider" } codex-utils-sandbox-summary = { path = "utils/sandbox-summary" } codex-utils-sleep-inhibitor = { path = "utils/sleep-inhibitor" } -codex-utils-string = { path = "utils/string" } codex-utils-stream-parser = { path = "utils/stream-parser" } +codex-utils-string = { path = "utils/string" } codex-windows-sandbox = { path = "windows-sandbox-rs" } core_test_support = { path = "core/tests/common" } mcp_test_support = { path = "mcp-server/tests/common" } @@ -165,8 +165,8 @@ clap = "4" clap_complete = "4" color-eyre = "0.6.3" crossbeam-channel = "0.5.15" -csv = "1.3.1" crossterm = "0.28.1" +csv = "1.3.1" ctor = "0.6.3" derive_more = "2" diffy = "0.4.2" @@ -178,14 +178,15 @@ env-flags = "0.1.1" env_logger = "0.11.9" eventsource-stream = "0.2.3" futures = { version = "0.3", default-features = false } -globset = "0.4" gethostname = "1.1.0" +globset = "0.4" http = "1.3.1" icu_decimal = "2.1" icu_locale_core = "2.1" icu_provider = { version = "2.1", features = ["sync"] } ignore = "0.4.23" image = { version = "^0.25.9", default-features = false } +iana-time-zone = "0.1.64" include_dir = "0.7.4" indexmap = "2.12.0" insta = "1.46.3" @@ -258,6 +259,7 @@ starlark = "0.13.0" strum = "0.27.2" strum_macros = "0.27.2" supports-color = "3.0.2" +syntect = "5" sys-locale = "0.3.2" tempfile = "3.23.0" test-log = "0.2.19" @@ -282,7 +284,6 @@ tracing-subscriber = "0.3.22" tracing-test = "0.2.5" tree-sitter = "0.25.10" tree-sitter-bash = "0.25" -syntect = "5" ts-rs = "11" tungstenite = { version = "0.27.0", features = ["deflate", "proxy"] } uds_windows = "1.1.0" @@ -352,6 +353,7 @@ ignored = [ [profile.release] lto = "fat" +split-debuginfo = "off" # Because we bundle some of these executables with the TypeScript CLI, we # remove everything to make the binary as small as possible. strip = "symbols" diff --git a/codex-rs/app-server-protocol/schema/json/ApplyPatchApprovalResponse.json b/codex-rs/app-server-protocol/schema/json/ApplyPatchApprovalResponse.json index 37380f5988..84842fd194 100644 --- a/codex-rs/app-server-protocol/schema/json/ApplyPatchApprovalResponse.json +++ b/codex-rs/app-server-protocol/schema/json/ApplyPatchApprovalResponse.json @@ -59,7 +59,7 @@ "type": "object" }, { - "description": "User has approved this command and wants to automatically approve any future identical instances (`command` and `cwd` match exactly) for the remainder of the session.", + "description": "User has approved this request and wants future prompts in the same session-scoped approval cache to be automatically approved for the remainder of the session.", "enum": [ "approved_for_session" ], diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 5ab197c84b..03e42bed45 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1340,7 +1340,7 @@ "type": "string" }, "output": { - "type": "string" + "$ref": "#/definitions/FunctionCallOutputPayload" }, "type": { "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalParams.json b/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalParams.json index 2122ad6521..4e2106bbbc 100644 --- a/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalParams.json +++ b/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalParams.json @@ -202,6 +202,85 @@ } ] }, + "CommandExecutionApprovalDecision": { + "oneOf": [ + { + "description": "User approved the command.", + "enum": [ + "accept" + ], + "type": "string" + }, + { + "description": "User approved the command and future prompts in the same session-scoped approval cache should run without prompting.", + "enum": [ + "acceptForSession" + ], + "type": "string" + }, + { + "additionalProperties": false, + "description": "User approved the command, and wants to apply the proposed execpolicy amendment so future matching commands can run without prompting.", + "properties": { + "acceptWithExecpolicyAmendment": { + "properties": { + "execpolicy_amendment": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "execpolicy_amendment" + ], + "type": "object" + } + }, + "required": [ + "acceptWithExecpolicyAmendment" + ], + "title": "AcceptWithExecpolicyAmendmentCommandExecutionApprovalDecision", + "type": "object" + }, + { + "additionalProperties": false, + "description": "User chose a persistent network policy rule (allow/deny) for this host.", + "properties": { + "applyNetworkPolicyAmendment": { + "properties": { + "network_policy_amendment": { + "$ref": "#/definitions/NetworkPolicyAmendment" + } + }, + "required": [ + "network_policy_amendment" + ], + "type": "object" + } + }, + "required": [ + "applyNetworkPolicyAmendment" + ], + "title": "ApplyNetworkPolicyAmendmentCommandExecutionApprovalDecision", + "type": "object" + }, + { + "description": "User denied the command. The agent will continue the turn.", + "enum": [ + "decline" + ], + "type": "string" + }, + { + "description": "User denied the command. The turn will also be immediately interrupted.", + "enum": [ + "cancel" + ], + "type": "string" + } + ] + }, "MacOsAutomationValue": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalResponse.json b/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalResponse.json index 04e914a390..0b7986fba9 100644 --- a/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalResponse.json +++ b/codex-rs/app-server-protocol/schema/json/CommandExecutionRequestApprovalResponse.json @@ -11,7 +11,7 @@ "type": "string" }, { - "description": "User approved the command and future identical commands should run without prompting.", + "description": "User approved the command and future prompts in the same session-scoped approval cache should run without prompting.", "enum": [ "acceptForSession" ], diff --git a/codex-rs/app-server-protocol/schema/json/EventMsg.json b/codex-rs/app-server-protocol/schema/json/EventMsg.json index 583a98c1d6..9f442ece9d 100644 --- a/codex-rs/app-server-protocol/schema/json/EventMsg.json +++ b/codex-rs/app-server-protocol/schema/json/EventMsg.json @@ -1675,6 +1675,16 @@ "null" ] }, + "available_decisions": { + "description": "Ordered list of decisions the client may present for this prompt.\n\nWhen absent, clients should derive the legacy default set from the other fields on this request.", + "items": { + "$ref": "#/definitions/ReviewDecision" + }, + "type": [ + "array", + "null" + ] + }, "call_id": { "description": "Identifier for the associated command execution item.", "type": "string" @@ -1884,30 +1894,6 @@ "title": "DynamicToolCallResponseEventMsg", "type": "object" }, - { - "properties": { - "item_id": { - "type": "string" - }, - "skill_name": { - "type": "string" - }, - "type": { - "enum": [ - "skill_request_approval" - ], - "title": "SkillRequestApprovalEventMsgType", - "type": "string" - } - }, - "required": [ - "item_id", - "skill_name", - "type" - ], - "title": "SkillRequestApprovalEventMsg", - "type": "object" - }, { "properties": { "id": { @@ -4836,7 +4822,7 @@ "type": "string" }, "output": { - "type": "string" + "$ref": "#/definitions/FunctionCallOutputPayload" }, "type": { "enum": [ @@ -4995,6 +4981,86 @@ ], "type": "object" }, + "ReviewDecision": { + "description": "User's decision in response to an ExecApprovalRequest.", + "oneOf": [ + { + "description": "User has approved this command and the agent should execute it.", + "enum": [ + "approved" + ], + "type": "string" + }, + { + "additionalProperties": false, + "description": "User has approved this command and wants to apply the proposed execpolicy amendment so future matching commands are permitted.", + "properties": { + "approved_execpolicy_amendment": { + "properties": { + "proposed_execpolicy_amendment": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "proposed_execpolicy_amendment" + ], + "type": "object" + } + }, + "required": [ + "approved_execpolicy_amendment" + ], + "title": "ApprovedExecpolicyAmendmentReviewDecision", + "type": "object" + }, + { + "description": "User has approved this request and wants future prompts in the same session-scoped approval cache to be automatically approved for the remainder of the session.", + "enum": [ + "approved_for_session" + ], + "type": "string" + }, + { + "additionalProperties": false, + "description": "User chose to persist a network policy rule (allow/deny) for future requests to the same host.", + "properties": { + "network_policy_amendment": { + "properties": { + "network_policy_amendment": { + "$ref": "#/definitions/NetworkPolicyAmendment" + } + }, + "required": [ + "network_policy_amendment" + ], + "type": "object" + } + }, + "required": [ + "network_policy_amendment" + ], + "title": "NetworkPolicyAmendmentReviewDecision", + "type": "object" + }, + { + "description": "User has denied this command and the agent should not execute it, but it should continue the session and try something else.", + "enum": [ + "denied" + ], + "type": "string" + }, + { + "description": "User has denied this command and the agent should not do anything until the user's next command.", + "enum": [ + "abort" + ], + "type": "string" + } + ] + }, "ReviewFinding": { "description": "A single review finding describing an observed issue or recommendation.", "properties": { @@ -7165,6 +7231,16 @@ "null" ] }, + "available_decisions": { + "description": "Ordered list of decisions the client may present for this prompt.\n\nWhen absent, clients should derive the legacy default set from the other fields on this request.", + "items": { + "$ref": "#/definitions/ReviewDecision" + }, + "type": [ + "array", + "null" + ] + }, "call_id": { "description": "Identifier for the associated command execution item.", "type": "string" @@ -7374,30 +7450,6 @@ "title": "DynamicToolCallResponseEventMsg", "type": "object" }, - { - "properties": { - "item_id": { - "type": "string" - }, - "skill_name": { - "type": "string" - }, - "type": { - "enum": [ - "skill_request_approval" - ], - "title": "SkillRequestApprovalEventMsgType", - "type": "string" - } - }, - "required": [ - "item_id", - "skill_name", - "type" - ], - "title": "SkillRequestApprovalEventMsg", - "type": "object" - }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/ExecCommandApprovalResponse.json b/codex-rs/app-server-protocol/schema/json/ExecCommandApprovalResponse.json index ac15f23658..baedafa403 100644 --- a/codex-rs/app-server-protocol/schema/json/ExecCommandApprovalResponse.json +++ b/codex-rs/app-server-protocol/schema/json/ExecCommandApprovalResponse.json @@ -59,7 +59,7 @@ "type": "object" }, { - "description": "User has approved this command and wants to automatically approve any future identical instances (`command` and `cwd` match exactly) for the remainder of the session.", + "description": "User has approved this request and wants future prompts in the same session-scoped approval cache to be automatically approved for the remainder of the session.", "enum": [ "approved_for_session" ], diff --git a/codex-rs/app-server-protocol/schema/json/ServerRequest.json b/codex-rs/app-server-protocol/schema/json/ServerRequest.json index ba94b889cf..ff810643d4 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ServerRequest.json @@ -268,6 +268,85 @@ } ] }, + "CommandExecutionApprovalDecision": { + "oneOf": [ + { + "description": "User approved the command.", + "enum": [ + "accept" + ], + "type": "string" + }, + { + "description": "User approved the command and future prompts in the same session-scoped approval cache should run without prompting.", + "enum": [ + "acceptForSession" + ], + "type": "string" + }, + { + "additionalProperties": false, + "description": "User approved the command, and wants to apply the proposed execpolicy amendment so future matching commands can run without prompting.", + "properties": { + "acceptWithExecpolicyAmendment": { + "properties": { + "execpolicy_amendment": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "execpolicy_amendment" + ], + "type": "object" + } + }, + "required": [ + "acceptWithExecpolicyAmendment" + ], + "title": "AcceptWithExecpolicyAmendmentCommandExecutionApprovalDecision", + "type": "object" + }, + { + "additionalProperties": false, + "description": "User chose a persistent network policy rule (allow/deny) for this host.", + "properties": { + "applyNetworkPolicyAmendment": { + "properties": { + "network_policy_amendment": { + "$ref": "#/definitions/NetworkPolicyAmendment" + } + }, + "required": [ + "network_policy_amendment" + ], + "type": "object" + } + }, + "required": [ + "applyNetworkPolicyAmendment" + ], + "title": "ApplyNetworkPolicyAmendmentCommandExecutionApprovalDecision", + "type": "object" + }, + { + "description": "User denied the command. The agent will continue the turn.", + "enum": [ + "decline" + ], + "type": "string" + }, + { + "description": "User denied the command. The turn will also be immediately interrupted.", + "enum": [ + "cancel" + ], + "type": "string" + } + ] + }, "CommandExecutionRequestApprovalParams": { "properties": { "approvalId": { @@ -722,21 +801,6 @@ } ] }, - "SkillRequestApprovalParams": { - "properties": { - "itemId": { - "type": "string" - }, - "skillName": { - "type": "string" - } - }, - "required": [ - "itemId", - "skillName" - ], - "type": "object" - }, "ThreadId": { "type": "string" }, @@ -898,30 +962,6 @@ "title": "Item/tool/requestUserInputRequest", "type": "object" }, - { - "properties": { - "id": { - "$ref": "#/definitions/RequestId" - }, - "method": { - "enum": [ - "skill/requestApproval" - ], - "title": "Skill/requestApprovalRequestMethod", - "type": "string" - }, - "params": { - "$ref": "#/definitions/SkillRequestApprovalParams" - } - }, - "required": [ - "id", - "method", - "params" - ], - "title": "Skill/requestApprovalRequest", - "type": "object" - }, { "description": "Execute a dynamic tool call on the client.", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/SkillRequestApprovalParams.json b/codex-rs/app-server-protocol/schema/json/SkillRequestApprovalParams.json deleted file mode 100644 index eebe6570b2..0000000000 --- a/codex-rs/app-server-protocol/schema/json/SkillRequestApprovalParams.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "itemId": { - "type": "string" - }, - "skillName": { - "type": "string" - } - }, - "required": [ - "itemId", - "skillName" - ], - "title": "SkillRequestApprovalParams", - "type": "object" -} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/SkillRequestApprovalResponse.json b/codex-rs/app-server-protocol/schema/json/SkillRequestApprovalResponse.json deleted file mode 100644 index 0bea163ccb..0000000000 --- a/codex-rs/app-server-protocol/schema/json/SkillRequestApprovalResponse.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "definitions": { - "SkillApprovalDecision": { - "enum": [ - "approve", - "decline" - ], - "type": "string" - } - }, - "properties": { - "decision": { - "$ref": "#/definitions/SkillApprovalDecision" - } - }, - "required": [ - "decision" - ], - "title": "SkillRequestApprovalResponse", - "type": "object" -} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 3da788229c..a107340464 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -1442,7 +1442,7 @@ "type": "string" }, { - "description": "User approved the command and future identical commands should run without prompting.", + "description": "User approved the command and future prompts in the same session-scoped approval cache should run without prompting.", "enum": [ "acceptForSession" ], @@ -2843,6 +2843,16 @@ "null" ] }, + "available_decisions": { + "description": "Ordered list of decisions the client may present for this prompt.\n\nWhen absent, clients should derive the legacy default set from the other fields on this request.", + "items": { + "$ref": "#/definitions/ReviewDecision" + }, + "type": [ + "array", + "null" + ] + }, "call_id": { "description": "Identifier for the associated command execution item.", "type": "string" @@ -3052,30 +3062,6 @@ "title": "DynamicToolCallResponseEventMsg", "type": "object" }, - { - "properties": { - "item_id": { - "type": "string" - }, - "skill_name": { - "type": "string" - }, - "type": { - "enum": [ - "skill_request_approval" - ], - "title": "SkillRequestApprovalEventMsgType", - "type": "string" - } - }, - "required": [ - "item_id", - "skill_name", - "type" - ], - "title": "SkillRequestApprovalEventMsg", - "type": "object" - }, { "properties": { "id": { @@ -5700,7 +5686,7 @@ "type": "object" }, { - "description": "User has approved this command and wants to automatically approve any future identical instances (`command` and `cwd` match exactly) for the remainder of the session.", + "description": "User has approved this request and wants future prompts in the same session-scoped approval cache to be automatically approved for the remainder of the session.", "enum": [ "approved_for_session" ], @@ -6732,30 +6718,6 @@ "title": "Item/tool/requestUserInputRequest", "type": "object" }, - { - "properties": { - "id": { - "$ref": "#/definitions/RequestId" - }, - "method": { - "enum": [ - "skill/requestApproval" - ], - "title": "Skill/requestApprovalRequestMethod", - "type": "string" - }, - "params": { - "$ref": "#/definitions/SkillRequestApprovalParams" - } - }, - "required": [ - "id", - "method", - "params" - ], - "title": "Skill/requestApprovalRequest", - "type": "object" - }, { "description": "Execute a dynamic tool call on the client.", "properties": { @@ -6877,43 +6839,6 @@ ], "type": "object" }, - "SkillApprovalDecision": { - "enum": [ - "approve", - "decline" - ], - "type": "string" - }, - "SkillRequestApprovalParams": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "itemId": { - "type": "string" - }, - "skillName": { - "type": "string" - } - }, - "required": [ - "itemId", - "skillName" - ], - "title": "SkillRequestApprovalParams", - "type": "object" - }, - "SkillRequestApprovalResponse": { - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "decision": { - "$ref": "#/definitions/SkillApprovalDecision" - } - }, - "required": [ - "decision" - ], - "title": "SkillRequestApprovalResponse", - "type": "object" - }, "StepStatus": { "enum": [ "pending", @@ -10300,6 +10225,16 @@ }, "Model": { "properties": { + "availabilityNux": { + "anyOf": [ + { + "$ref": "#/definitions/v2/ModelAvailabilityNux" + }, + { + "type": "null" + } + ] + }, "defaultReasoningEffort": { "$ref": "#/definitions/v2/ReasoningEffort" }, @@ -10346,6 +10281,16 @@ "string", "null" ] + }, + "upgradeInfo": { + "anyOf": [ + { + "$ref": "#/definitions/v2/ModelUpgradeInfo" + }, + { + "type": "null" + } + ] } }, "required": [ @@ -10360,6 +10305,17 @@ ], "type": "object" }, + "ModelAvailabilityNux": { + "properties": { + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, "ModelListParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { @@ -10448,6 +10404,35 @@ "title": "ModelReroutedNotification", "type": "object" }, + "ModelUpgradeInfo": { + "properties": { + "migrationMarkdown": { + "type": [ + "string", + "null" + ] + }, + "model": { + "type": "string" + }, + "modelLink": { + "type": [ + "string", + "null" + ] + }, + "upgradeCopy": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "model" + ], + "type": "object" + }, "NetworkAccess": { "enum": [ "restricted", @@ -11457,7 +11442,7 @@ "type": "string" }, "output": { - "type": "string" + "$ref": "#/definitions/v2/FunctionCallOutputPayload" }, "type": { "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json index 2afa018dc5..4023f76b51 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ModelListResponse.json @@ -22,6 +22,16 @@ }, "Model": { "properties": { + "availabilityNux": { + "anyOf": [ + { + "$ref": "#/definitions/ModelAvailabilityNux" + }, + { + "type": "null" + } + ] + }, "defaultReasoningEffort": { "$ref": "#/definitions/ReasoningEffort" }, @@ -68,6 +78,16 @@ "string", "null" ] + }, + "upgradeInfo": { + "anyOf": [ + { + "$ref": "#/definitions/ModelUpgradeInfo" + }, + { + "type": "null" + } + ] } }, "required": [ @@ -82,6 +102,46 @@ ], "type": "object" }, + "ModelAvailabilityNux": { + "properties": { + "message": { + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "ModelUpgradeInfo": { + "properties": { + "migrationMarkdown": { + "type": [ + "string", + "null" + ] + }, + "model": { + "type": "string" + }, + "modelLink": { + "type": [ + "string", + "null" + ] + }, + "upgradeCopy": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "model" + ], + "type": "object" + }, "ReasoningEffort": { "description": "See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning", "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index 748eeaab46..4717ff266b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -565,7 +565,7 @@ "type": "string" }, "output": { - "type": "string" + "$ref": "#/definitions/FunctionCallOutputPayload" }, "type": { "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index 29d6fbc6d7..ef7607d3ca 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -615,7 +615,7 @@ "type": "string" }, "output": { - "type": "string" + "$ref": "#/definitions/FunctionCallOutputPayload" }, "type": { "enum": [ diff --git a/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts b/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts index f1e528d7db..5ccbe1e697 100644 --- a/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts +++ b/codex-rs/app-server-protocol/schema/typescript/EventMsg.ts @@ -57,7 +57,6 @@ import type { RemoteSkillDownloadedEvent } from "./RemoteSkillDownloadedEvent"; import type { RequestUserInputEvent } from "./RequestUserInputEvent"; import type { ReviewRequest } from "./ReviewRequest"; import type { SessionConfiguredEvent } from "./SessionConfiguredEvent"; -import type { SkillRequestApprovalEvent } from "./SkillRequestApprovalEvent"; import type { StreamErrorEvent } from "./StreamErrorEvent"; import type { TerminalInteractionEvent } from "./TerminalInteractionEvent"; import type { ThreadNameUpdatedEvent } from "./ThreadNameUpdatedEvent"; @@ -80,4 +79,4 @@ import type { WebSearchEndEvent } from "./WebSearchEndEvent"; * Response event from the agent * NOTE: Make sure none of these values have optional types, as it will mess up the extension code-gen. */ -export type EventMsg = { "type": "error" } & ErrorEvent | { "type": "warning" } & WarningEvent | { "type": "realtime_conversation_started" } & RealtimeConversationStartedEvent | { "type": "realtime_conversation_realtime" } & RealtimeConversationRealtimeEvent | { "type": "realtime_conversation_closed" } & RealtimeConversationClosedEvent | { "type": "model_reroute" } & ModelRerouteEvent | { "type": "context_compacted" } & ContextCompactedEvent | { "type": "thread_rolled_back" } & ThreadRolledBackEvent | { "type": "task_started" } & TurnStartedEvent | { "type": "task_complete" } & TurnCompleteEvent | { "type": "token_count" } & TokenCountEvent | { "type": "agent_message" } & AgentMessageEvent | { "type": "user_message" } & UserMessageEvent | { "type": "agent_message_delta" } & AgentMessageDeltaEvent | { "type": "agent_reasoning" } & AgentReasoningEvent | { "type": "agent_reasoning_delta" } & AgentReasoningDeltaEvent | { "type": "agent_reasoning_raw_content" } & AgentReasoningRawContentEvent | { "type": "agent_reasoning_raw_content_delta" } & AgentReasoningRawContentDeltaEvent | { "type": "agent_reasoning_section_break" } & AgentReasoningSectionBreakEvent | { "type": "session_configured" } & SessionConfiguredEvent | { "type": "thread_name_updated" } & ThreadNameUpdatedEvent | { "type": "mcp_startup_update" } & McpStartupUpdateEvent | { "type": "mcp_startup_complete" } & McpStartupCompleteEvent | { "type": "mcp_tool_call_begin" } & McpToolCallBeginEvent | { "type": "mcp_tool_call_end" } & McpToolCallEndEvent | { "type": "web_search_begin" } & WebSearchBeginEvent | { "type": "web_search_end" } & WebSearchEndEvent | { "type": "exec_command_begin" } & ExecCommandBeginEvent | { "type": "exec_command_output_delta" } & ExecCommandOutputDeltaEvent | { "type": "terminal_interaction" } & TerminalInteractionEvent | { "type": "exec_command_end" } & ExecCommandEndEvent | { "type": "view_image_tool_call" } & ViewImageToolCallEvent | { "type": "exec_approval_request" } & ExecApprovalRequestEvent | { "type": "request_user_input" } & RequestUserInputEvent | { "type": "dynamic_tool_call_request" } & DynamicToolCallRequest | { "type": "dynamic_tool_call_response" } & DynamicToolCallResponseEvent | { "type": "skill_request_approval" } & SkillRequestApprovalEvent | { "type": "elicitation_request" } & ElicitationRequestEvent | { "type": "apply_patch_approval_request" } & ApplyPatchApprovalRequestEvent | { "type": "deprecation_notice" } & DeprecationNoticeEvent | { "type": "background_event" } & BackgroundEventEvent | { "type": "undo_started" } & UndoStartedEvent | { "type": "undo_completed" } & UndoCompletedEvent | { "type": "stream_error" } & StreamErrorEvent | { "type": "patch_apply_begin" } & PatchApplyBeginEvent | { "type": "patch_apply_end" } & PatchApplyEndEvent | { "type": "turn_diff" } & TurnDiffEvent | { "type": "get_history_entry_response" } & GetHistoryEntryResponseEvent | { "type": "mcp_list_tools_response" } & McpListToolsResponseEvent | { "type": "list_custom_prompts_response" } & ListCustomPromptsResponseEvent | { "type": "list_skills_response" } & ListSkillsResponseEvent | { "type": "list_remote_skills_response" } & ListRemoteSkillsResponseEvent | { "type": "remote_skill_downloaded" } & RemoteSkillDownloadedEvent | { "type": "skills_update_available" } | { "type": "plan_update" } & UpdatePlanArgs | { "type": "turn_aborted" } & TurnAbortedEvent | { "type": "shutdown_complete" } | { "type": "entered_review_mode" } & ReviewRequest | { "type": "exited_review_mode" } & ExitedReviewModeEvent | { "type": "raw_response_item" } & RawResponseItemEvent | { "type": "item_started" } & ItemStartedEvent | { "type": "item_completed" } & ItemCompletedEvent | { "type": "agent_message_content_delta" } & AgentMessageContentDeltaEvent | { "type": "plan_delta" } & PlanDeltaEvent | { "type": "reasoning_content_delta" } & ReasoningContentDeltaEvent | { "type": "reasoning_raw_content_delta" } & ReasoningRawContentDeltaEvent | { "type": "collab_agent_spawn_begin" } & CollabAgentSpawnBeginEvent | { "type": "collab_agent_spawn_end" } & CollabAgentSpawnEndEvent | { "type": "collab_agent_interaction_begin" } & CollabAgentInteractionBeginEvent | { "type": "collab_agent_interaction_end" } & CollabAgentInteractionEndEvent | { "type": "collab_waiting_begin" } & CollabWaitingBeginEvent | { "type": "collab_waiting_end" } & CollabWaitingEndEvent | { "type": "collab_close_begin" } & CollabCloseBeginEvent | { "type": "collab_close_end" } & CollabCloseEndEvent | { "type": "collab_resume_begin" } & CollabResumeBeginEvent | { "type": "collab_resume_end" } & CollabResumeEndEvent; +export type EventMsg = { "type": "error" } & ErrorEvent | { "type": "warning" } & WarningEvent | { "type": "realtime_conversation_started" } & RealtimeConversationStartedEvent | { "type": "realtime_conversation_realtime" } & RealtimeConversationRealtimeEvent | { "type": "realtime_conversation_closed" } & RealtimeConversationClosedEvent | { "type": "model_reroute" } & ModelRerouteEvent | { "type": "context_compacted" } & ContextCompactedEvent | { "type": "thread_rolled_back" } & ThreadRolledBackEvent | { "type": "task_started" } & TurnStartedEvent | { "type": "task_complete" } & TurnCompleteEvent | { "type": "token_count" } & TokenCountEvent | { "type": "agent_message" } & AgentMessageEvent | { "type": "user_message" } & UserMessageEvent | { "type": "agent_message_delta" } & AgentMessageDeltaEvent | { "type": "agent_reasoning" } & AgentReasoningEvent | { "type": "agent_reasoning_delta" } & AgentReasoningDeltaEvent | { "type": "agent_reasoning_raw_content" } & AgentReasoningRawContentEvent | { "type": "agent_reasoning_raw_content_delta" } & AgentReasoningRawContentDeltaEvent | { "type": "agent_reasoning_section_break" } & AgentReasoningSectionBreakEvent | { "type": "session_configured" } & SessionConfiguredEvent | { "type": "thread_name_updated" } & ThreadNameUpdatedEvent | { "type": "mcp_startup_update" } & McpStartupUpdateEvent | { "type": "mcp_startup_complete" } & McpStartupCompleteEvent | { "type": "mcp_tool_call_begin" } & McpToolCallBeginEvent | { "type": "mcp_tool_call_end" } & McpToolCallEndEvent | { "type": "web_search_begin" } & WebSearchBeginEvent | { "type": "web_search_end" } & WebSearchEndEvent | { "type": "exec_command_begin" } & ExecCommandBeginEvent | { "type": "exec_command_output_delta" } & ExecCommandOutputDeltaEvent | { "type": "terminal_interaction" } & TerminalInteractionEvent | { "type": "exec_command_end" } & ExecCommandEndEvent | { "type": "view_image_tool_call" } & ViewImageToolCallEvent | { "type": "exec_approval_request" } & ExecApprovalRequestEvent | { "type": "request_user_input" } & RequestUserInputEvent | { "type": "dynamic_tool_call_request" } & DynamicToolCallRequest | { "type": "dynamic_tool_call_response" } & DynamicToolCallResponseEvent | { "type": "elicitation_request" } & ElicitationRequestEvent | { "type": "apply_patch_approval_request" } & ApplyPatchApprovalRequestEvent | { "type": "deprecation_notice" } & DeprecationNoticeEvent | { "type": "background_event" } & BackgroundEventEvent | { "type": "undo_started" } & UndoStartedEvent | { "type": "undo_completed" } & UndoCompletedEvent | { "type": "stream_error" } & StreamErrorEvent | { "type": "patch_apply_begin" } & PatchApplyBeginEvent | { "type": "patch_apply_end" } & PatchApplyEndEvent | { "type": "turn_diff" } & TurnDiffEvent | { "type": "get_history_entry_response" } & GetHistoryEntryResponseEvent | { "type": "mcp_list_tools_response" } & McpListToolsResponseEvent | { "type": "list_custom_prompts_response" } & ListCustomPromptsResponseEvent | { "type": "list_skills_response" } & ListSkillsResponseEvent | { "type": "list_remote_skills_response" } & ListRemoteSkillsResponseEvent | { "type": "remote_skill_downloaded" } & RemoteSkillDownloadedEvent | { "type": "skills_update_available" } | { "type": "plan_update" } & UpdatePlanArgs | { "type": "turn_aborted" } & TurnAbortedEvent | { "type": "shutdown_complete" } | { "type": "entered_review_mode" } & ReviewRequest | { "type": "exited_review_mode" } & ExitedReviewModeEvent | { "type": "raw_response_item" } & RawResponseItemEvent | { "type": "item_started" } & ItemStartedEvent | { "type": "item_completed" } & ItemCompletedEvent | { "type": "agent_message_content_delta" } & AgentMessageContentDeltaEvent | { "type": "plan_delta" } & PlanDeltaEvent | { "type": "reasoning_content_delta" } & ReasoningContentDeltaEvent | { "type": "reasoning_raw_content_delta" } & ReasoningRawContentDeltaEvent | { "type": "collab_agent_spawn_begin" } & CollabAgentSpawnBeginEvent | { "type": "collab_agent_spawn_end" } & CollabAgentSpawnEndEvent | { "type": "collab_agent_interaction_begin" } & CollabAgentInteractionBeginEvent | { "type": "collab_agent_interaction_end" } & CollabAgentInteractionEndEvent | { "type": "collab_waiting_begin" } & CollabWaitingBeginEvent | { "type": "collab_waiting_end" } & CollabWaitingEndEvent | { "type": "collab_close_begin" } & CollabCloseBeginEvent | { "type": "collab_close_end" } & CollabCloseEndEvent | { "type": "collab_resume_begin" } & CollabResumeBeginEvent | { "type": "collab_resume_end" } & CollabResumeEndEvent; diff --git a/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts b/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts index 5144dab6e6..cc13400900 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ExecApprovalRequestEvent.ts @@ -6,6 +6,7 @@ import type { NetworkApprovalContext } from "./NetworkApprovalContext"; import type { NetworkPolicyAmendment } from "./NetworkPolicyAmendment"; import type { ParsedCommand } from "./ParsedCommand"; import type { PermissionProfile } from "./PermissionProfile"; +import type { ReviewDecision } from "./ReviewDecision"; export type ExecApprovalRequestEvent = { /** @@ -51,4 +52,11 @@ proposed_network_policy_amendments?: Array, /** * Optional additional filesystem permissions requested for this command. */ -additional_permissions?: PermissionProfile, parsed_cmd: Array, }; +additional_permissions?: PermissionProfile, +/** + * Ordered list of decisions the client may present for this prompt. + * + * When absent, clients should derive the legacy default set from the + * other fields on this request. + */ +available_decisions?: Array, parsed_cmd: Array, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts b/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts index 611c7fb22d..dd7621f01d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ResponseItem.ts @@ -15,4 +15,4 @@ export type ResponseItem = { "type": "message", role: string, content: Array | null, }; +proposedNetworkPolicyAmendments?: Array | null, +/** + * Ordered list of decisions the client may present for this prompt. + */ +availableDecisions?: Array | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts b/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts index de7b240173..cd99108190 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/Model.ts @@ -3,6 +3,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { InputModality } from "../InputModality"; import type { ReasoningEffort } from "../ReasoningEffort"; +import type { ModelAvailabilityNux } from "./ModelAvailabilityNux"; +import type { ModelUpgradeInfo } from "./ModelUpgradeInfo"; import type { ReasoningEffortOption } from "./ReasoningEffortOption"; -export type Model = { id: string, model: string, upgrade: string | null, displayName: string, description: string, hidden: boolean, supportedReasoningEfforts: Array, defaultReasoningEffort: ReasoningEffort, inputModalities: Array, supportsPersonality: boolean, isDefault: boolean, }; +export type Model = { id: string, model: string, upgrade: string | null, upgradeInfo: ModelUpgradeInfo | null, availabilityNux: ModelAvailabilityNux | null, displayName: string, description: string, hidden: boolean, supportedReasoningEfforts: Array, defaultReasoningEffort: ReasoningEffort, inputModalities: Array, supportsPersonality: boolean, isDefault: boolean, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/SkillApprovalDecision.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ModelAvailabilityNux.ts similarity index 72% rename from codex-rs/app-server-protocol/schema/typescript/v2/SkillApprovalDecision.ts rename to codex-rs/app-server-protocol/schema/typescript/v2/ModelAvailabilityNux.ts index 6773b0f1e0..7254aaec9b 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/SkillApprovalDecision.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ModelAvailabilityNux.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type SkillApprovalDecision = "approve" | "decline"; +export type ModelAvailabilityNux = { message: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/SkillRequestApprovalParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ModelUpgradeInfo.ts similarity index 52% rename from codex-rs/app-server-protocol/schema/typescript/v2/SkillRequestApprovalParams.ts rename to codex-rs/app-server-protocol/schema/typescript/v2/ModelUpgradeInfo.ts index 3f70fea595..82d73e9d06 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/SkillRequestApprovalParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ModelUpgradeInfo.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type SkillRequestApprovalParams = { itemId: string, skillName: string, }; +export type ModelUpgradeInfo = { model: string, upgradeCopy: string | null, modelLink: string | null, migrationMarkdown: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/SkillRequestApprovalResponse.ts b/codex-rs/app-server-protocol/schema/typescript/v2/SkillRequestApprovalResponse.ts deleted file mode 100644 index 9ce454ff4f..0000000000 --- a/codex-rs/app-server-protocol/schema/typescript/v2/SkillRequestApprovalResponse.ts +++ /dev/null @@ -1,6 +0,0 @@ -// GENERATED CODE! DO NOT MODIFY BY HAND! - -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { SkillApprovalDecision } from "./SkillApprovalDecision"; - -export type SkillRequestApprovalResponse = { decision: SkillApprovalDecision, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index 19594723db..4a881427d9 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -107,10 +107,12 @@ export type { McpToolCallResult } from "./McpToolCallResult"; export type { McpToolCallStatus } from "./McpToolCallStatus"; export type { MergeStrategy } from "./MergeStrategy"; export type { Model } from "./Model"; +export type { ModelAvailabilityNux } from "./ModelAvailabilityNux"; export type { ModelListParams } from "./ModelListParams"; export type { ModelListResponse } from "./ModelListResponse"; export type { ModelRerouteReason } from "./ModelRerouteReason"; export type { ModelReroutedNotification } from "./ModelReroutedNotification"; +export type { ModelUpgradeInfo } from "./ModelUpgradeInfo"; export type { NetworkAccess } from "./NetworkAccess"; export type { NetworkApprovalContext } from "./NetworkApprovalContext"; export type { NetworkApprovalProtocol } from "./NetworkApprovalProtocol"; @@ -141,13 +143,10 @@ export type { SandboxMode } from "./SandboxMode"; export type { SandboxPolicy } from "./SandboxPolicy"; export type { SandboxWorkspaceWrite } from "./SandboxWorkspaceWrite"; export type { SessionSource } from "./SessionSource"; -export type { SkillApprovalDecision } from "./SkillApprovalDecision"; export type { SkillDependencies } from "./SkillDependencies"; export type { SkillErrorInfo } from "./SkillErrorInfo"; export type { SkillInterface } from "./SkillInterface"; export type { SkillMetadata } from "./SkillMetadata"; -export type { SkillRequestApprovalParams } from "./SkillRequestApprovalParams"; -export type { SkillRequestApprovalResponse } from "./SkillRequestApprovalResponse"; export type { SkillScope } from "./SkillScope"; export type { SkillToolDependency } from "./SkillToolDependency"; export type { SkillsConfigWriteParams } from "./SkillsConfigWriteParams"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 357341cc4b..3af2976106 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -715,11 +715,6 @@ server_request_definitions! { response: v2::ToolRequestUserInputResponse, }, - SkillRequestApproval => "skill/requestApproval" { - params: v2::SkillRequestApprovalParams, - response: v2::SkillRequestApprovalResponse, - }, - /// Execute a dynamic tool call on the client. DynamicToolCall => "item/tool/call" { params: v2::DynamicToolCallParams, @@ -1545,6 +1540,7 @@ mod tests { }), proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, + available_decisions: None, }; let reason = crate::experimental_api::ExperimentalApi::experimental_reason(¶ms); assert_eq!( diff --git a/codex-rs/app-server-protocol/src/protocol/v1.rs b/codex-rs/app-server-protocol/src/protocol/v1.rs index 976f4e69d8..719268e9c5 100644 --- a/codex-rs/app-server-protocol/src/protocol/v1.rs +++ b/codex-rs/app-server-protocol/src/protocol/v1.rs @@ -531,6 +531,15 @@ impl From for CoreTextElement { } } +impl InputItem { + pub fn text_char_count(&self) -> usize { + match self { + InputItem::Text { text, .. } => text.chars().count(), + InputItem::Image { .. } | InputItem::LocalImage { .. } => 0, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] /// Deprecated in favor of AccountLoginCompletedNotification. diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index d563287fa8..32bc120f6d 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -31,6 +31,7 @@ use codex_protocol::models::MessagePhase; use codex_protocol::models::PermissionProfile as CorePermissionProfile; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::InputModality; +use codex_protocol::openai_models::ModelAvailabilityNux as CoreModelAvailabilityNux; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::openai_models::default_input_modalities; use codex_protocol::parse_command::ParsedCommand as CoreParsedCommand; @@ -49,6 +50,7 @@ use codex_protocol::protocol::RateLimitWindow as CoreRateLimitWindow; use codex_protocol::protocol::ReadOnlyAccess as CoreReadOnlyAccess; use codex_protocol::protocol::RealtimeAudioFrame as CoreRealtimeAudioFrame; use codex_protocol::protocol::RejectConfig as CoreRejectConfig; +use codex_protocol::protocol::ReviewDecision as CoreReviewDecision; use codex_protocol::protocol::SessionSource as CoreSessionSource; use codex_protocol::protocol::SkillDependencies as CoreSkillDependencies; use codex_protocol::protocol::SkillErrorInfo as CoreSkillErrorInfo; @@ -741,7 +743,8 @@ pub struct ConfigEdit { pub enum CommandExecutionApprovalDecision { /// User approved the command. Accept, - /// User approved the command and future identical commands should run without prompting. + /// User approved the command and future prompts in the same session-scoped + /// approval cache should run without prompting. AcceptForSession, /// User approved the command, and wants to apply the proposed execpolicy amendment so future /// matching commands can run without prompting. @@ -758,6 +761,27 @@ pub enum CommandExecutionApprovalDecision { Cancel, } +impl From for CommandExecutionApprovalDecision { + fn from(value: CoreReviewDecision) -> Self { + match value { + CoreReviewDecision::Approved => Self::Accept, + CoreReviewDecision::ApprovedExecpolicyAmendment { + proposed_execpolicy_amendment, + } => Self::AcceptWithExecpolicyAmendment { + execpolicy_amendment: proposed_execpolicy_amendment.into(), + }, + CoreReviewDecision::ApprovedForSession => Self::AcceptForSession, + CoreReviewDecision::NetworkPolicyAmendment { + network_policy_amendment, + } => Self::ApplyNetworkPolicyAmendment { + network_policy_amendment: network_policy_amendment.into(), + }, + CoreReviewDecision::Abort => Self::Cancel, + CoreReviewDecision::Denied => Self::Decline, + } + } +} + v2_enum_from_core! { pub enum NetworkApprovalProtocol from CoreNetworkApprovalProtocol { Http, @@ -1366,6 +1390,21 @@ pub struct ModelListParams { pub include_hidden: Option, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ModelAvailabilityNux { + pub message: String, +} + +impl From for ModelAvailabilityNux { + fn from(value: CoreModelAvailabilityNux) -> Self { + Self { + message: value.message, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -1373,6 +1412,8 @@ pub struct Model { pub id: String, pub model: String, pub upgrade: Option, + pub upgrade_info: Option, + pub availability_nux: Option, pub display_name: String, pub description: String, pub hidden: bool, @@ -1386,6 +1427,16 @@ pub struct Model { pub is_default: bool, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ModelUpgradeInfo { + pub model: String, + pub upgrade_copy: Option, + pub model_link: Option, + pub migration_markdown: Option, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -3029,6 +3080,18 @@ impl From for UserInput { } } +impl UserInput { + pub fn text_char_count(&self) -> usize { + match self { + UserInput::Text { text, .. } => text.chars().count(), + UserInput::Image { .. } + | UserInput::LocalImage { .. } + | UserInput::Skill { .. } + | UserInput::Mention { .. } => 0, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(tag = "type", rename_all = "camelCase")] #[ts(tag = "type")] @@ -3802,6 +3865,11 @@ pub struct CommandExecutionRequestApprovalParams { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional = nullable)] pub proposed_network_policy_amendments: Option>, + /// Ordered list of decisions the client may present for this prompt. + #[experimental("item/commandExecution/requestApproval.availableDecisions")] + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub available_decisions: Option>, } impl CommandExecutionRequestApprovalParams { @@ -3842,29 +3910,6 @@ pub struct FileChangeRequestApprovalResponse { pub decision: FileChangeApprovalDecision, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS, ExperimentalApi)] -#[serde(rename_all = "camelCase")] -#[ts(export_to = "v2/")] -pub struct SkillRequestApprovalParams { - pub item_id: String, - pub skill_name: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export_to = "v2/")] -pub enum SkillApprovalDecision { - Approve, - Decline, -} - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export_to = "v2/")] -pub struct SkillRequestApprovalResponse { - pub decision: SkillApprovalDecision, -} - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server-test-client/src/lib.rs b/codex-rs/app-server-test-client/src/lib.rs index a9225b0426..27203c6012 100644 --- a/codex-rs/app-server-test-client/src/lib.rs +++ b/codex-rs/app-server-test-client/src/lib.rs @@ -57,9 +57,6 @@ use codex_app_server_protocol::SendUserMessageParams; use codex_app_server_protocol::SendUserMessageResponse; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequest; -use codex_app_server_protocol::SkillApprovalDecision; -use codex_app_server_protocol::SkillRequestApprovalParams; -use codex_app_server_protocol::SkillRequestApprovalResponse; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; @@ -1511,9 +1508,6 @@ impl CodexClient { ServerRequest::FileChangeRequestApproval { request_id, params } => { self.approve_file_change_request(request_id, params)?; } - ServerRequest::SkillRequestApproval { request_id, params } => { - self.approve_skill_request(request_id, params)?; - } other => { bail!("received unsupported server request: {other:?}"); } @@ -1540,6 +1534,7 @@ impl CodexClient { additional_permissions, proposed_execpolicy_amendment, proposed_network_policy_amendments, + available_decisions, } = params; println!( @@ -1554,6 +1549,9 @@ impl CodexClient { if let Some(network_approval_context) = network_approval_context.as_ref() { println!("< network approval context: {network_approval_context:?}"); } + if let Some(available_decisions) = available_decisions.as_ref() { + println!("< available decisions: {available_decisions:?}"); + } if let Some(command) = command.as_deref() { println!("< command: {command}"); } @@ -1593,22 +1591,6 @@ impl CodexClient { Ok(()) } - fn approve_skill_request( - &mut self, - request_id: RequestId, - params: SkillRequestApprovalParams, - ) -> Result<()> { - println!( - "\n< skill approval requested for item {}, skill {}", - params.item_id, params.skill_name - ); - let response = SkillRequestApprovalResponse { - decision: SkillApprovalDecision::Approve, - }; - self.send_server_request_response(request_id, &response)?; - Ok(()) - } - fn approve_file_change_request( &mut self, request_id: RequestId, diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 505abb0457..60cf6d6107 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -142,7 +142,7 @@ Example with notification opt-out: - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). -- `model/list` — list available models (set `includeHidden: true` to include entries with `hidden: true`), with reasoning effort options and optional `upgrade` model ids. +- `model/list` — list available models (set `includeHidden: true` to include entries with `hidden: true`), with reasoning effort options, optional legacy `upgrade` model ids, optional `upgradeInfo` metadata (`model`, `upgradeCopy`, `modelLink`, `migrationMarkdown`), and optional `availabilityNux` metadata. - `experimentalFeature/list` — list feature flags with stage metadata (`beta`, `underDevelopment`, `stable`, etc.), enabled/default-enabled state, and cursor pagination. For non-beta flags, `displayName`/`description`/`announcement` are `null`. - `collaborationMode/list` — list available collaboration mode presets (experimental, no pagination). This response omits built-in developer instructions; clients should either pass `settings.developer_instructions: null` when setting a mode to use Codex's built-in instructions, or provide their own instructions explicitly. - `skills/list` — list skills for one or more `cwd` values (optional `forceReload`). @@ -710,7 +710,7 @@ Certain actions (shell commands or modifying files) may require explicit user ap Order of messages: 1. `item/started` — shows the pending `commandExecution` item with `command`, `cwd`, and other fields so you can render the proposed action. -2. `item/commandExecution/requestApproval` (request) — carries the same `itemId`, `threadId`, `turnId`, optionally `approvalId` (for subcommand callbacks), and `reason`. For normal command approvals, it also includes `command`, `cwd`, and `commandActions` for friendly display. When `initialize.params.capabilities.experimentalApi = true`, it may also include experimental `additionalPermissions` describing requested per-command sandbox access. For network-only approvals, those command fields may be omitted and `networkApprovalContext` is provided instead. Optional persistence hints may also be included via `proposedExecpolicyAmendment` and `proposedNetworkPolicyAmendments`. +2. `item/commandExecution/requestApproval` (request) — carries the same `itemId`, `threadId`, `turnId`, optionally `approvalId` (for subcommand callbacks), and `reason`. For normal command approvals, it also includes `command`, `cwd`, and `commandActions` for friendly display. When `initialize.params.capabilities.experimentalApi = true`, it may also include experimental `additionalPermissions` describing requested per-command sandbox access. For network-only approvals, those command fields may be omitted and `networkApprovalContext` is provided instead. Optional persistence hints may also be included via `proposedExecpolicyAmendment` and `proposedNetworkPolicyAmendments`. Clients can prefer `availableDecisions` when present to render the exact set of choices the server wants to expose, while still falling back to the older heuristics if it is omitted. 3. Client response — for example `{ "decision": "accept" }`, `{ "decision": "acceptForSession" }`, `{ "decision": { "acceptWithExecpolicyAmendment": { "execpolicy_amendment": [...] } } }`, `{ "decision": { "applyNetworkPolicyAmendment": { "network_policy_amendment": { "host": "example.com", "action": "allow" } } } }`, `{ "decision": "decline" }`, or `{ "decision": "cancel" }`. 4. `item/completed` — final `commandExecution` item with `status: "completed" | "failed" | "declined"` and execution output. Render this as the authoritative result. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 683fca361d..0bc2f1118f 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -58,9 +58,6 @@ use codex_app_server_protocol::ReasoningSummaryTextDeltaNotification; use codex_app_server_protocol::ReasoningTextDeltaNotification; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequestPayload; -use codex_app_server_protocol::SkillApprovalDecision as V2SkillApprovalDecision; -use codex_app_server_protocol::SkillRequestApprovalParams; -use codex_app_server_protocol::SkillRequestApprovalResponse; use codex_app_server_protocol::TerminalInteractionNotification; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadNameUpdatedNotification; @@ -111,7 +108,6 @@ use codex_protocol::protocol::TokenCountEvent; use codex_protocol::protocol::TurnDiffEvent; use codex_protocol::request_user_input::RequestUserInputAnswer as CoreRequestUserInputAnswer; use codex_protocol::request_user_input::RequestUserInputResponse as CoreRequestUserInputResponse; -use codex_protocol::skill_approval::SkillApprovalResponse as CoreSkillApprovalResponse; use codex_shell_command::parse_command::shlex_join; use std::collections::HashMap; use std::convert::TryFrom; @@ -336,6 +332,11 @@ pub(crate) async fn apply_bespoke_event_handling( .note_permission_requested(&conversation_id.to_string()) .await; let approval_id_for_op = ev.effective_approval_id(); + let available_decisions = ev + .effective_available_decisions() + .into_iter() + .map(CommandExecutionApprovalDecision::from) + .collect::>(); let ExecApprovalRequestEvent { call_id, approval_id, @@ -432,6 +433,7 @@ pub(crate) async fn apply_bespoke_event_handling( additional_permissions, proposed_execpolicy_amendment: proposed_execpolicy_amendment_v2, proposed_network_policy_amendments: proposed_network_policy_amendments_v2, + available_decisions: Some(available_decisions), }; let rx = outgoing .send_request(ServerRequestPayload::CommandExecutionRequestApproval( @@ -518,37 +520,6 @@ pub(crate) async fn apply_bespoke_event_handling( } } } - EventMsg::SkillRequestApproval(request) => { - if matches!(api_version, ApiVersion::V2) { - let item_id = request.item_id; - let skill_name = request.skill_name; - let params = SkillRequestApprovalParams { - item_id: item_id.clone(), - skill_name, - }; - let rx = outgoing - .send_request(ServerRequestPayload::SkillRequestApproval(params)) - .await; - tokio::spawn(async move { - let approved = match rx.await { - Ok(Ok(value)) => { - serde_json::from_value::(value) - .map(|response| { - matches!(response.decision, V2SkillApprovalDecision::Approve) - }) - .unwrap_or(false) - } - _ => false, - }; - let _ = conversation - .submit(Op::SkillApproval { - id: item_id, - response: CoreSkillApprovalResponse { approved }, - }) - .await; - }); - } - } EventMsg::DynamicToolCallRequest(request) => { if matches!(api_version, ApiVersion::V2) { let call_id = request.call_id; diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index ebebcceeca..9a1394fc2e 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -1,5 +1,7 @@ use crate::bespoke_event_handling::apply_bespoke_event_handling; +use crate::error_code::INPUT_TOO_LARGE_ERROR_CODE; use crate::error_code::INTERNAL_ERROR_CODE; +use crate::error_code::INVALID_PARAMS_ERROR_CODE; use crate::error_code::INVALID_REQUEST_ERROR_CODE; use crate::fuzzy_file_search::FuzzyFileSearchSession; use crate::fuzzy_file_search::run_fuzzy_file_search; @@ -267,6 +269,7 @@ use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionConfiguredEvent; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::USER_MESSAGE_BEGIN; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use codex_protocol::user_input::UserInput as CoreInputItem; use codex_rmcp_client::perform_oauth_login_return_url; use codex_utils_json_to_toml::json_to_toml; @@ -557,7 +560,12 @@ impl CodexMessageProcessor { Ok((review_request, hint)) } - pub async fn process_request(&mut self, connection_id: ConnectionId, request: ClientRequest) { + pub async fn process_request( + &mut self, + connection_id: ConnectionId, + request: ClientRequest, + app_server_client_name: Option, + ) { let to_connection_request_id = |request_id| ConnectionRequestId { connection_id, request_id, @@ -644,8 +652,12 @@ impl CodexMessageProcessor { .await; } ClientRequest::TurnStart { request_id, params } => { - self.turn_start(to_connection_request_id(request_id), params) - .await; + self.turn_start( + to_connection_request_id(request_id), + params, + app_server_client_name.clone(), + ) + .await; } ClientRequest::TurnSteer { request_id, params } => { self.turn_steer(to_connection_request_id(request_id), params) @@ -764,12 +776,20 @@ impl CodexMessageProcessor { .await; } ClientRequest::SendUserMessage { request_id, params } => { - self.send_user_message(to_connection_request_id(request_id), params) - .await; + self.send_user_message( + to_connection_request_id(request_id), + params, + app_server_client_name.clone(), + ) + .await; } ClientRequest::SendUserTurn { request_id, params } => { - self.send_user_turn(to_connection_request_id(request_id), params) - .await; + self.send_user_turn( + to_connection_request_id(request_id), + params, + app_server_client_name.clone(), + ) + .await; } ClientRequest::InterruptConversation { request_id, params } => { self.interrupt_conversation(to_connection_request_id(request_id), params) @@ -4146,6 +4166,7 @@ impl CodexMessageProcessor { http_headers, env_http_headers, scopes.as_deref().unwrap_or_default(), + server.oauth_resource.as_deref(), timeout_secs, config.mcp_oauth_callback_port, config.mcp_oauth_callback_url.as_deref(), @@ -4732,6 +4753,36 @@ impl CodexMessageProcessor { self.outgoing.send_error(request_id, error).await; } + fn input_too_large_error(actual_chars: usize) -> JSONRPCErrorError { + JSONRPCErrorError { + code: INVALID_PARAMS_ERROR_CODE, + message: format!( + "Input exceeds the maximum length of {MAX_USER_INPUT_TEXT_CHARS} characters." + ), + data: Some(serde_json::json!({ + "input_error_code": INPUT_TOO_LARGE_ERROR_CODE, + "max_chars": MAX_USER_INPUT_TEXT_CHARS, + "actual_chars": actual_chars, + })), + } + } + + fn validate_v1_input_limit(items: &[WireInputItem]) -> Result<(), JSONRPCErrorError> { + let actual_chars: usize = items.iter().map(WireInputItem::text_char_count).sum(); + if actual_chars > MAX_USER_INPUT_TEXT_CHARS { + return Err(Self::input_too_large_error(actual_chars)); + } + Ok(()) + } + + fn validate_v2_input_limit(items: &[V2UserInput]) -> Result<(), JSONRPCErrorError> { + let actual_chars: usize = items.iter().map(V2UserInput::text_char_count).sum(); + if actual_chars > MAX_USER_INPUT_TEXT_CHARS { + return Err(Self::input_too_large_error(actual_chars)); + } + Ok(()) + } + async fn send_internal_error(&self, request_id: ConnectionRequestId, message: String) { let error = JSONRPCErrorError { code: INTERNAL_ERROR_CODE, @@ -5026,11 +5077,16 @@ impl CodexMessageProcessor { &self, request_id: ConnectionRequestId, params: SendUserMessageParams, + app_server_client_name: Option, ) { let SendUserMessageParams { conversation_id, items, } = params; + if let Err(error) = Self::validate_v1_input_limit(&items) { + self.outgoing.send_error(request_id, error).await; + return; + } let Ok(conversation) = self.thread_manager.get_thread(conversation_id).await else { let error = JSONRPCErrorError { code: INVALID_REQUEST_ERROR_CODE, @@ -5040,6 +5096,12 @@ impl CodexMessageProcessor { self.outgoing.send_error(request_id, error).await; return; }; + if let Err(error) = + Self::set_app_server_client_name(conversation.as_ref(), app_server_client_name).await + { + self.outgoing.send_error(request_id, error).await; + return; + } let mapped_items: Vec = items .into_iter() @@ -5070,7 +5132,12 @@ impl CodexMessageProcessor { .await; } - async fn send_user_turn(&self, request_id: ConnectionRequestId, params: SendUserTurnParams) { + async fn send_user_turn( + &self, + request_id: ConnectionRequestId, + params: SendUserTurnParams, + app_server_client_name: Option, + ) { let SendUserTurnParams { conversation_id, items, @@ -5082,6 +5149,10 @@ impl CodexMessageProcessor { summary, output_schema, } = params; + if let Err(error) = Self::validate_v1_input_limit(&items) { + self.outgoing.send_error(request_id, error).await; + return; + } let Ok(conversation) = self.thread_manager.get_thread(conversation_id).await else { let error = JSONRPCErrorError { @@ -5092,6 +5163,12 @@ impl CodexMessageProcessor { self.outgoing.send_error(request_id, error).await; return; }; + if let Err(error) = + Self::set_app_server_client_name(conversation.as_ref(), app_server_client_name).await + { + self.outgoing.send_error(request_id, error).await; + return; + } let mapped_items: Vec = items .into_iter() @@ -5116,7 +5193,7 @@ impl CodexMessageProcessor { sandbox_policy, model, effort, - summary, + summary: Some(summary), final_output_json_schema: output_schema, collaboration_mode: None, personality: None, @@ -5205,6 +5282,7 @@ impl CodexMessageProcessor { connectors::list_cached_accessible_connectors_from_mcp_tools(&config), connectors::list_cached_all_connectors(&config) ); + let cached_all_connectors = all_connectors.clone(); let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); @@ -5231,6 +5309,19 @@ impl CodexMessageProcessor { let app_list_deadline = tokio::time::Instant::now() + APP_LIST_LOAD_TIMEOUT; let mut accessible_loaded = false; let mut all_loaded = false; + let mut last_notified_apps = None; + + if accessible_connectors.is_some() || all_connectors.is_some() { + let merged = connectors::with_app_enabled_state( + Self::merge_loaded_apps( + all_connectors.as_deref(), + accessible_connectors.as_deref(), + ), + &config, + ); + Self::send_app_list_updated_notification(&outgoing, merged.clone()).await; + last_notified_apps = Some(merged); + } loop { let result = match tokio::time::timeout_at(app_list_deadline, rx.recv()).await { @@ -5287,14 +5378,30 @@ impl CodexMessageProcessor { } } + let showing_interim_force_refetch = force_refetch && !(accessible_loaded && all_loaded); + let all_connectors_for_update = + if showing_interim_force_refetch && cached_all_connectors.is_some() { + cached_all_connectors.as_deref() + } else { + all_connectors.as_deref() + }; + let accessible_connectors_for_update = + if showing_interim_force_refetch && !accessible_loaded { + None + } else { + accessible_connectors.as_deref() + }; let merged = connectors::with_app_enabled_state( Self::merge_loaded_apps( - all_connectors.as_deref(), - accessible_connectors.as_deref(), + all_connectors_for_update, + accessible_connectors_for_update, ), &config, ); - Self::send_app_list_updated_notification(&outgoing, merged.clone()).await; + if last_notified_apps.as_ref() != Some(&merged) { + Self::send_app_list_updated_notification(&outgoing, merged.clone()).await; + last_notified_apps = Some(merged.clone()); + } if accessible_loaded && all_loaded { match Self::paginate_apps(merged.as_slice(), start, limit) { @@ -5563,7 +5670,16 @@ impl CodexMessageProcessor { let _ = conversation.submit(Op::Interrupt).await; } - async fn turn_start(&self, request_id: ConnectionRequestId, params: TurnStartParams) { + async fn turn_start( + &self, + request_id: ConnectionRequestId, + params: TurnStartParams, + app_server_client_name: Option, + ) { + if let Err(error) = Self::validate_v2_input_limit(¶ms.input) { + self.outgoing.send_error(request_id, error).await; + return; + } let (_, thread) = match self.load_thread(¶ms.thread_id).await { Ok(v) => v, Err(error) => { @@ -5571,6 +5687,12 @@ impl CodexMessageProcessor { return; } }; + if let Err(error) = + Self::set_app_server_client_name(thread.as_ref(), app_server_client_name).await + { + self.outgoing.send_error(request_id, error).await; + return; + } let collaboration_modes_config = CollaborationModesConfig { default_mode_request_user_input: thread.enabled(Feature::DefaultModeRequestUserInput), @@ -5652,6 +5774,20 @@ impl CodexMessageProcessor { } } + async fn set_app_server_client_name( + thread: &CodexThread, + app_server_client_name: Option, + ) -> Result<(), JSONRPCErrorError> { + thread + .set_app_server_client_name(app_server_client_name) + .await + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to set app server client name: {err}"), + data: None, + }) + } + async fn turn_steer(&self, request_id: ConnectionRequestId, params: TurnSteerParams) { let (_, thread) = match self.load_thread(¶ms.thread_id).await { Ok(v) => v, @@ -5669,6 +5805,10 @@ impl CodexMessageProcessor { .await; return; } + if let Err(error) = Self::validate_v2_input_limit(¶ms.input) { + self.outgoing.send_error(request_id, error).await; + return; + } let mapped_items: Vec = params .input @@ -7575,7 +7715,7 @@ mod tests { "role": "user", "content": [{ "type": "input_text", - "text": "\n\n".to_string(), + "text": "# AGENTS.md instructions for project\n\n\n\n".to_string(), }], }), json!({ diff --git a/codex-rs/app-server/src/error_code.rs b/codex-rs/app-server/src/error_code.rs index ca93b2f2d3..924a7086ae 100644 --- a/codex-rs/app-server/src/error_code.rs +++ b/codex-rs/app-server/src/error_code.rs @@ -1,3 +1,5 @@ pub(crate) const INVALID_REQUEST_ERROR_CODE: i64 = -32600; +pub const INVALID_PARAMS_ERROR_CODE: i64 = -32602; pub(crate) const INTERNAL_ERROR_CODE: i64 = -32603; pub(crate) const OVERLOADED_ERROR_CODE: i64 = -32001; +pub const INPUT_TOO_LARGE_ERROR_CODE: &str = "input_too_large"; diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index 206435592b..972f5af1e6 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -67,6 +67,8 @@ mod thread_state; mod thread_status; mod transport; +pub use crate::error_code::INPUT_TOO_LARGE_ERROR_CODE; +pub use crate::error_code::INVALID_PARAMS_ERROR_CODE; pub use crate::transport::AppServerTransport; const LOG_FORMAT_ENV_VAR: &str = "LOG_FORMAT"; diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index bd2b8b939e..3b01cd58a4 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -140,6 +140,7 @@ pub(crate) struct ConnectionSessionState { pub(crate) initialized: bool, pub(crate) experimental_api_enabled: bool, pub(crate) opted_out_notification_methods: HashSet, + pub(crate) app_server_client_name: Option, } pub(crate) struct MessageProcessorArgs { @@ -329,6 +330,7 @@ impl MessageProcessor { if let Ok(mut suffix) = USER_AGENT_SUFFIX.lock() { *suffix = Some(user_agent_suffix); } + session.app_server_client_name = Some(name.clone()); let user_agent = get_codex_user_agent(); let response = InitializeResponse { user_agent }; @@ -430,7 +432,7 @@ impl MessageProcessor { // inline the full `CodexMessageProcessor::process_request` future, which // can otherwise push worker-thread stack usage over the edge. self.codex_message_processor - .process_request(connection_id, other) + .process_request(connection_id, other, session.app_server_client_name.clone()) .boxed() .await; } diff --git a/codex-rs/app-server/src/models.rs b/codex-rs/app-server/src/models.rs index 1594c66229..6dbe77455f 100644 --- a/codex-rs/app-server/src/models.rs +++ b/codex-rs/app-server/src/models.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use codex_app_server_protocol::Model; +use codex_app_server_protocol::ModelUpgradeInfo; use codex_app_server_protocol::ReasoningEffortOption; use codex_core::ThreadManager; use codex_core::models_manager::manager::RefreshStrategy; @@ -24,7 +25,14 @@ fn model_from_preset(preset: ModelPreset) -> Model { Model { id: preset.id.to_string(), model: preset.model.to_string(), - upgrade: preset.upgrade.map(|upgrade| upgrade.id), + upgrade: preset.upgrade.as_ref().map(|upgrade| upgrade.id.clone()), + upgrade_info: preset.upgrade.as_ref().map(|upgrade| ModelUpgradeInfo { + model: upgrade.id.clone(), + upgrade_copy: upgrade.upgrade_copy.clone(), + model_link: upgrade.model_link.clone(), + migration_markdown: upgrade.migration_markdown.clone(), + }), + availability_nux: preset.availability_nux.map(Into::into), display_name: preset.display_name.to_string(), description: preset.description.to_string(), hidden: !preset.show_in_picker, diff --git a/codex-rs/app-server/src/transport.rs b/codex-rs/app-server/src/transport.rs index 8c36df9822..6ac3e0d466 100644 --- a/codex-rs/app-server/src/transport.rs +++ b/codex-rs/app-server/src/transport.rs @@ -986,6 +986,7 @@ mod tests { ), proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, + available_decisions: None, }, }), }, @@ -1047,6 +1048,7 @@ mod tests { ), proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, + available_decisions: None, }, }), }, diff --git a/codex-rs/app-server/tests/common/models_cache.rs b/codex-rs/app-server/tests/common/models_cache.rs index 218a1d0e40..0de8fda5f2 100644 --- a/codex-rs/app-server/tests/common/models_cache.rs +++ b/codex-rs/app-server/tests/common/models_cache.rs @@ -1,6 +1,7 @@ use chrono::DateTime; use chrono::Utc; use codex_core::test_support::all_model_presets; +use codex_protocol::config_types::ReasoningSummary; use codex_protocol::openai_models::ConfigShellToolType; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelPreset; @@ -30,8 +31,10 @@ fn preset_to_info(preset: &ModelPreset, priority: i32) -> ModelInfo { base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, diff --git a/codex-rs/app-server/tests/suite/output_schema.rs b/codex-rs/app-server/tests/suite/output_schema.rs index 3d83bec7dd..db006e3284 100644 --- a/codex-rs/app-server/tests/suite/output_schema.rs +++ b/codex-rs/app-server/tests/suite/output_schema.rs @@ -1,8 +1,11 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::to_response; +use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; +use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::AddConversationListenerParams; use codex_app_server_protocol::InputItem; +use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::NewConversationParams; use codex_app_server_protocol::NewConversationResponse; @@ -13,6 +16,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use core_test_support::responses; use core_test_support::skip_if_no_network; use pretty_assertions::assert_eq; @@ -124,6 +128,85 @@ async fn send_user_turn_accepts_output_schema_v1() -> Result<()> { Ok(()) } +#[tokio::test] +async fn send_user_turn_rejects_oversized_input_v1() -> Result<()> { + let server = responses::start_mock_server().await; + let body = responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-1"), + ]); + let _response_mock = responses::mount_sse_once(&server, body).await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let new_conv_id = mcp + .send_new_conversation_request(NewConversationParams { + ..Default::default() + }) + .await?; + let new_conv_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(new_conv_id)), + ) + .await??; + let NewConversationResponse { + conversation_id, .. + } = to_response::(new_conv_resp)?; + + let listener_id = mcp + .send_add_conversation_listener_request(AddConversationListenerParams { + conversation_id, + experimental_raw_events: false, + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(listener_id)), + ) + .await??; + + let oversized_input = "x".repeat(MAX_USER_INPUT_TEXT_CHARS + 1); + let send_turn_id = mcp + .send_send_user_turn_request(SendUserTurnParams { + conversation_id, + items: vec![InputItem::Text { + text: oversized_input.clone(), + text_elements: Vec::new(), + }], + cwd: codex_home.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: "mock-model".to_string(), + effort: Some(ReasoningEffort::Low), + summary: ReasoningSummary::Auto, + output_schema: None, + }) + .await?; + + let err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(send_turn_id)), + ) + .await??; + + assert_eq!(err.error.code, INVALID_PARAMS_ERROR_CODE); + assert_eq!( + err.error.message, + format!("Input exceeds the maximum length of {MAX_USER_INPUT_TEXT_CHARS} characters.") + ); + let data = err.error.data.expect("expected structured error data"); + assert_eq!(data["input_error_code"], INPUT_TOO_LARGE_ERROR_CODE); + assert_eq!(data["max_chars"], MAX_USER_INPUT_TEXT_CHARS); + assert_eq!(data["actual_chars"], oversized_input.chars().count()); + + Ok(()) +} + #[tokio::test] async fn send_user_turn_output_schema_is_per_turn_v1() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/app-server/tests/suite/send_message.rs b/codex-rs/app-server/tests/suite/send_message.rs index 0bf6e98cfb..e755a11b77 100644 --- a/codex-rs/app-server/tests/suite/send_message.rs +++ b/codex-rs/app-server/tests/suite/send_message.rs @@ -3,9 +3,12 @@ use app_test_support::McpProcess; use app_test_support::create_fake_rollout; use app_test_support::rollout_path; use app_test_support::to_response; +use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; +use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::AddConversationListenerParams; use codex_app_server_protocol::AddConversationSubscriptionResponse; use codex_app_server_protocol::InputItem; +use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::NewConversationParams; @@ -27,6 +30,7 @@ use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::TurnContextItem; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use core_test_support::responses; use pretty_assertions::assert_eq; use std::io::Write; @@ -210,17 +214,13 @@ async fn test_send_message_raw_notifications_opt_in() -> Result<()> { }) .await?; - let permissions = read_raw_response_item(&mut mcp, conversation_id).await; - assert_permissions_message(&permissions); - let developer = read_raw_response_item(&mut mcp, conversation_id).await; + assert_permissions_message(&developer); assert_developer_message(&developer, "Use the test harness tools."); - let instructions = read_raw_response_item(&mut mcp, conversation_id).await; - assert_instructions_message(&instructions); - - let environment = read_raw_response_item(&mut mcp, conversation_id).await; - assert_environment_message(&environment); + let contextual_user = read_raw_response_item(&mut mcp, conversation_id).await; + assert_instructions_message(&contextual_user); + assert_environment_message(&contextual_user); let response: JSONRPCResponse = timeout( DEFAULT_READ_TIMEOUT, @@ -272,6 +272,66 @@ async fn test_send_message_session_not_found() -> Result<()> { Ok(()) } +#[tokio::test] +async fn test_send_message_rejects_oversized_input() -> Result<()> { + let server = responses::start_mock_server().await; + let body = responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-1"), + ]); + let _response_mock = responses::mount_sse_once(&server, body).await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let new_conv_id = mcp + .send_new_conversation_request(NewConversationParams { + ..Default::default() + }) + .await?; + let new_conv_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(new_conv_id)), + ) + .await??; + let NewConversationResponse { + conversation_id, .. + } = to_response::<_>(new_conv_resp)?; + + let oversized_input = "x".repeat(MAX_USER_INPUT_TEXT_CHARS + 1); + let req_id = mcp + .send_send_user_message_request(SendUserMessageParams { + conversation_id, + items: vec![InputItem::Text { + text: oversized_input.clone(), + text_elements: Vec::new(), + }], + }) + .await?; + + let err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(req_id)), + ) + .await??; + + assert_eq!(err.error.code, INVALID_PARAMS_ERROR_CODE); + assert_eq!( + err.error.message, + format!("Input exceeds the maximum length of {MAX_USER_INPUT_TEXT_CHARS} characters.") + ); + let data = err.error.data.expect("expected structured error data"); + assert_eq!(data["input_error_code"], INPUT_TOO_LARGE_ERROR_CODE); + assert_eq!(data["max_chars"], MAX_USER_INPUT_TEXT_CHARS); + assert_eq!(data["actual_chars"], oversized_input.chars().count()); + + Ok(()) +} + #[tokio::test] async fn resume_with_model_mismatch_appends_model_switch_once() -> Result<()> { let server = responses::start_mock_server().await; @@ -481,9 +541,8 @@ fn assert_permissions_message(item: &ResponseItem) { false, ) .into_text(); - assert_eq!( - texts, - vec![expected.as_str()], + assert!( + texts.iter().any(|text| *text == expected), "expected permissions developer message, got {texts:?}" ); } @@ -496,9 +555,8 @@ fn assert_developer_message(item: &ResponseItem, expected_text: &str) { ResponseItem::Message { role, content, .. } => { assert_eq!(role, "developer"); let texts = content_texts(content); - assert_eq!( - texts, - vec![expected_text], + assert!( + texts.contains(&expected_text), "expected developer instructions message, got {texts:?}" ); } @@ -562,6 +620,8 @@ fn append_rollout_turn_context(path: &Path, timestamp: &str, model: &str) -> std item: RolloutItem::TurnContext(TurnContextItem { turn_id: None, cwd: PathBuf::from("/"), + current_date: None, + timezone: None, approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, network: None, diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index a4956f77ab..b54d771caf 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -1027,6 +1027,41 @@ async fn list_apps_force_refetch_patches_updates_from_cached_snapshots() -> Resu let first_update = read_app_list_updated_notification(&mut mcp).await?; assert_eq!( first_update.data, + vec![ + AppInfo { + id: "beta".to_string(), + name: "Beta App".to_string(), + description: Some("Beta v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some("https://chatgpt.com/apps/beta-app/beta".to_string()), + is_accessible: true, + is_enabled: true, + }, + AppInfo { + id: "alpha".to_string(), + name: "Alpha".to_string(), + description: Some("Alpha v1".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some("https://chatgpt.com/apps/alpha/alpha".to_string()), + is_accessible: false, + is_enabled: true, + }, + ] + ); + + let second_update = read_app_list_updated_notification(&mut mcp).await?; + assert_eq!( + second_update.data, vec![ AppInfo { id: "alpha".to_string(), @@ -1073,8 +1108,8 @@ async fn list_apps_force_refetch_patches_updates_from_cached_snapshots() -> Resu is_accessible: false, is_enabled: true, }]; - let second_update = read_app_list_updated_notification(&mut mcp).await?; - assert_eq!(second_update.data, expected_final); + let third_update = read_app_list_updated_notification(&mut mcp).await?; + assert_eq!(third_update.data, expected_final); let refetch_response: JSONRPCResponse = timeout( DEFAULT_TIMEOUT, diff --git a/codex-rs/app-server/tests/suite/v2/initialize.rs b/codex-rs/app-server/tests/suite/v2/initialize.rs index 2edc83a7f4..de437c5bf5 100644 --- a/codex-rs/app-server/tests/suite/v2/initialize.rs +++ b/codex-rs/app-server/tests/suite/v2/initialize.rs @@ -1,16 +1,24 @@ use anyhow::Result; use app_test_support::McpProcess; +use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::to_response; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeResponse; use codex_app_server_protocol::JSONRPCMessage; +use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::UserInput as V2UserInput; +use core_test_support::fs_wait; use pretty_assertions::assert_eq; +use serde_json::Value; use std::path::Path; +use std::time::Duration; use tempfile::TempDir; use tokio::time::timeout; @@ -178,11 +186,103 @@ async fn initialize_opt_out_notification_methods_filters_notifications() -> Resu Ok(()) } +#[tokio::test] +async fn turn_start_notify_payload_includes_initialize_client_name() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + let codex_home = TempDir::new()?; + let notify_script = codex_home.path().join("notify.py"); + std::fs::write( + ¬ify_script, + r#"from pathlib import Path +import sys + +payload_path = Path(__file__).with_name("notify.json") +tmp_path = payload_path.with_suffix(".json.tmp") +tmp_path.write_text(sys.argv[-1], encoding="utf-8") +tmp_path.replace(payload_path) +"#, + )?; + let notify_file = codex_home.path().join("notify.json"); + let notify_script = notify_script + .to_str() + .expect("notify script path should be valid UTF-8"); + create_config_toml_with_extra( + codex_home.path(), + &server.uri(), + "never", + &format!( + "notify = [\"python3\", {}]", + toml_basic_string(notify_script) + ), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.initialize_with_client_info(ClientInfo { + name: "xcode".to_string(), + title: Some("Xcode".to_string()), + version: "1.0.0".to_string(), + }), + ) + .await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "Hello".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _: TurnStartResponse = to_response(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + fs_wait::wait_for_path_exists(¬ify_file, Duration::from_secs(5)).await?; + let payload_raw = tokio::fs::read_to_string(¬ify_file).await?; + let payload: Value = serde_json::from_str(&payload_raw)?; + assert_eq!(payload["client"], "xcode"); + + Ok(()) +} + // Helper to create a config.toml pointing at the mock model server. fn create_config_toml( codex_home: &Path, server_uri: &str, approval_policy: &str, +) -> std::io::Result<()> { + create_config_toml_with_extra(codex_home, server_uri, approval_policy, "") +} + +fn create_config_toml_with_extra( + codex_home: &Path, + server_uri: &str, + approval_policy: &str, + extra: &str, ) -> std::io::Result<()> { let config_toml = codex_home.join("config.toml"); std::fs::write( @@ -195,6 +295,8 @@ sandbox_mode = "read-only" model_provider = "mock_provider" +{extra} + [model_providers.mock_provider] name = "Mock provider for test" base_url = "{server_uri}/v1" @@ -205,3 +307,7 @@ stream_max_retries = 0 ), ) } + +fn toml_basic_string(value: &str) -> String { + format!("\"{}\"", value.replace('\\', "\\\\").replace('"', "\\\"")) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 2444130978..cb3acb633d 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -19,7 +19,6 @@ mod realtime_conversation; mod request_user_input; mod review; mod safety_check_downgrade; -mod skill_approval; mod skills_list; mod thread_archive; mod thread_fork; diff --git a/codex-rs/app-server/tests/suite/v2/model_list.rs b/codex-rs/app-server/tests/suite/v2/model_list.rs index a71a8a3377..eba8905a77 100644 --- a/codex-rs/app-server/tests/suite/v2/model_list.rs +++ b/codex-rs/app-server/tests/suite/v2/model_list.rs @@ -9,6 +9,7 @@ use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::Model; use codex_app_server_protocol::ModelListParams; use codex_app_server_protocol::ModelListResponse; +use codex_app_server_protocol::ModelUpgradeInfo; use codex_app_server_protocol::ReasoningEffortOption; use codex_app_server_protocol::RequestId; use codex_protocol::openai_models::ModelPreset; @@ -24,6 +25,13 @@ fn model_from_preset(preset: &ModelPreset) -> Model { id: preset.id.clone(), model: preset.model.clone(), upgrade: preset.upgrade.as_ref().map(|upgrade| upgrade.id.clone()), + upgrade_info: preset.upgrade.as_ref().map(|upgrade| ModelUpgradeInfo { + model: upgrade.id.clone(), + upgrade_copy: upgrade.upgrade_copy.clone(), + model_link: upgrade.model_link.clone(), + migration_markdown: upgrade.migration_markdown.clone(), + }), + availability_nux: preset.availability_nux.clone().map(Into::into), display_name: preset.display_name.clone(), description: preset.description.clone(), hidden: !preset.show_in_picker, diff --git a/codex-rs/app-server/tests/suite/v2/skill_approval.rs b/codex-rs/app-server/tests/suite/v2/skill_approval.rs deleted file mode 100644 index 945a545a8d..0000000000 --- a/codex-rs/app-server/tests/suite/v2/skill_approval.rs +++ /dev/null @@ -1,109 +0,0 @@ -use anyhow::Result; -use app_test_support::McpProcess; -use app_test_support::create_final_assistant_message_sse_response; -use app_test_support::create_mock_responses_server_sequence; -use app_test_support::to_response; -use codex_app_server_protocol::JSONRPCResponse; -use codex_app_server_protocol::RequestId; -use codex_app_server_protocol::ServerRequest; -use codex_app_server_protocol::ThreadStartParams; -use codex_app_server_protocol::ThreadStartResponse; -use codex_app_server_protocol::TurnStartParams; -use codex_app_server_protocol::TurnStartResponse; -use codex_app_server_protocol::UserInput as V2UserInput; -use tokio::time::timeout; - -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn skill_request_approval_round_trip() -> Result<()> { - let codex_home = tempfile::TempDir::new()?; - let server = - create_mock_responses_server_sequence(vec![create_final_assistant_message_sse_response( - "done", - )?]) - .await; - create_config_toml(codex_home.path(), &server.uri())?; - - let mut mcp = McpProcess::new(codex_home.path()).await?; - timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; - - let thread_start_id = mcp - .send_thread_start_request(ThreadStartParams { - model: Some("mock-model".to_string()), - ..Default::default() - }) - .await?; - let thread_start_resp: JSONRPCResponse = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_response_message(RequestId::Integer(thread_start_id)), - ) - .await??; - let ThreadStartResponse { thread, .. } = to_response(thread_start_resp)?; - - let turn_start_id = mcp - .send_turn_start_request(TurnStartParams { - thread_id: thread.id.clone(), - input: vec![V2UserInput::Text { - text: "ask something".to_string(), - text_elements: Vec::new(), - }], - model: Some("mock-model".to_string()), - ..Default::default() - }) - .await?; - let turn_start_resp: JSONRPCResponse = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_response_message(RequestId::Integer(turn_start_id)), - ) - .await??; - let TurnStartResponse { turn, .. } = to_response(turn_start_resp)?; - - let server_req = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_request_message(), - ) - .await??; - let ServerRequest::SkillRequestApproval { request_id, params } = server_req else { - panic!("expected SkillRequestApproval request, got: {server_req:?}"); - }; - - assert_eq!(params.item_id, turn.id); - assert_eq!(params.skill_name, "test-skill"); - - mcp.send_response(request_id, serde_json::json!({ "decision": "approve" })) - .await?; - - timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("turn/completed"), - ) - .await??; - - Ok(()) -} - -fn create_config_toml(codex_home: &std::path::Path, server_uri: &str) -> std::io::Result<()> { - let config_toml = codex_home.join("config.toml"); - std::fs::write( - config_toml, - format!( - r#" -model = "mock-model" -approval_policy = "never" -sandbox_mode = "read-only" - -model_provider = "mock_provider" - -[features] -skill_approval = true - -[model_providers.mock_provider] -name = "Mock provider for test" -base_url = "{server_uri}/v1" -request_max_retries = 0 -stream_max_retries = 0 -"# - ), - ) -} diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index afd116be17..f396aa8c44 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -9,6 +9,8 @@ use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::format_with_current_shell_display; use app_test_support::to_response; +use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; +use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::ByteRange; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::CommandExecutionApprovalDecision; @@ -19,6 +21,7 @@ use codex_app_server_protocol::FileChangeOutputDeltaNotification; use codex_app_server_protocol::FileChangeRequestApprovalResponse; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; +use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::PatchApplyStatus; @@ -45,6 +48,7 @@ use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::Settings; use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use core_test_support::responses; use core_test_support::skip_if_no_network; use pretty_assertions::assert_eq; @@ -223,6 +227,143 @@ async fn turn_start_emits_user_message_item_with_text_elements() -> Result<()> { Ok(()) } +#[tokio::test] +async fn turn_start_accepts_text_at_limit_with_mention_item() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &server.uri(), + "never", + &BTreeMap::from([(Feature::Personality, true)]), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![ + V2UserInput::Text { + text: "x".repeat(MAX_USER_INPUT_TEXT_CHARS), + text_elements: Vec::new(), + }, + V2UserInput::Mention { + name: "Demo App".to_string(), + path: "app://demo-app".to_string(), + }, + ], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + assert_eq!(turn.status, TurnStatus::InProgress); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + Ok(()) +} + +#[tokio::test] +async fn turn_start_rejects_combined_oversized_text_input() -> Result<()> { + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + "http://localhost/unused", + "never", + &BTreeMap::from([(Feature::Personality, true)]), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let first = "x".repeat(MAX_USER_INPUT_TEXT_CHARS / 2); + let second = "y".repeat(MAX_USER_INPUT_TEXT_CHARS / 2 + 1); + let actual_chars = first.chars().count() + second.chars().count(); + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![ + V2UserInput::Text { + text: first, + text_elements: Vec::new(), + }, + V2UserInput::Text { + text: second, + text_elements: Vec::new(), + }, + ], + ..Default::default() + }) + .await?; + let err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(turn_req)), + ) + .await??; + + assert_eq!(err.error.code, INVALID_PARAMS_ERROR_CODE); + assert_eq!( + err.error.message, + format!("Input exceeds the maximum length of {MAX_USER_INPUT_TEXT_CHARS} characters.") + ); + let data = err.error.data.expect("expected structured error data"); + assert_eq!(data["input_error_code"], INPUT_TOO_LARGE_ERROR_CODE); + assert_eq!(data["max_chars"], MAX_USER_INPUT_TEXT_CHARS); + assert_eq!(data["actual_chars"], actual_chars); + + let turn_started = tokio::time::timeout( + std::time::Duration::from_millis(250), + mcp.read_stream_until_notification_message("turn/started"), + ) + .await; + assert!( + turn_started.is_err(), + "did not expect a turn/started notification for rejected input" + ); + + Ok(()) +} + #[tokio::test] async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<()> { // Provide a mock server and config so model wiring is valid. diff --git a/codex-rs/app-server/tests/suite/v2/turn_steer.rs b/codex-rs/app-server/tests/suite/v2/turn_steer.rs index 779e77577b..61a8a9794c 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_steer.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_steer.rs @@ -6,6 +6,8 @@ use app_test_support::create_mock_responses_server_sequence; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; +use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; +use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; @@ -17,6 +19,7 @@ use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnSteerParams; use codex_app_server_protocol::TurnSteerResponse; use codex_app_server_protocol::UserInput as V2UserInput; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use tempfile::TempDir; use tokio::time::timeout; @@ -67,6 +70,109 @@ async fn turn_steer_requires_active_turn() -> Result<()> { Ok(()) } +#[tokio::test] +async fn turn_steer_rejects_oversized_text_input() -> Result<()> { + #[cfg(target_os = "windows")] + let shell_command = vec![ + "powershell".to_string(), + "-Command".to_string(), + "Start-Sleep -Seconds 10".to_string(), + ]; + #[cfg(not(target_os = "windows"))] + let shell_command = vec!["sleep".to_string(), "10".to_string()]; + + let tmp = TempDir::new()?; + let codex_home = tmp.path().join("codex_home"); + std::fs::create_dir(&codex_home)?; + let working_directory = tmp.path().join("workdir"); + std::fs::create_dir(&working_directory)?; + + let server = + create_mock_responses_server_sequence_unchecked(vec![create_shell_command_sse_response( + shell_command.clone(), + Some(&working_directory), + Some(10_000), + "call_sleep", + )?]) + .await; + create_config_toml(&codex_home, &server.uri())?; + + let mut mcp = McpProcess::new(&codex_home).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "run sleep".to_string(), + text_elements: Vec::new(), + }], + cwd: Some(working_directory.clone()), + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + let _task_started: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("codex/event/task_started"), + ) + .await??; + + let oversized_input = "x".repeat(MAX_USER_INPUT_TEXT_CHARS + 1); + let steer_req = mcp + .send_turn_steer_request(TurnSteerParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: oversized_input.clone(), + text_elements: Vec::new(), + }], + expected_turn_id: turn.id.clone(), + }) + .await?; + let steer_err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(steer_req)), + ) + .await??; + + assert_eq!(steer_err.error.code, INVALID_PARAMS_ERROR_CODE); + assert_eq!( + steer_err.error.message, + format!("Input exceeds the maximum length of {MAX_USER_INPUT_TEXT_CHARS} characters.") + ); + let data = steer_err + .error + .data + .expect("expected structured error data"); + assert_eq!(data["input_error_code"], INPUT_TOO_LARGE_ERROR_CODE); + assert_eq!(data["max_chars"], MAX_USER_INPUT_TEXT_CHARS); + assert_eq!(data["actual_chars"], oversized_input.chars().count()); + + mcp.interrupt_turn_and_wait_for_aborted(thread.id, turn.id, DEFAULT_READ_TIMEOUT) + .await?; + + Ok(()) +} + #[tokio::test] async fn turn_steer_returns_active_turn_id() -> Result<()> { #[cfg(target_os = "windows")] diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index a3470ff1b9..620a93872e 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -298,12 +298,7 @@ fn merge_directory_app(existing: &mut DirectoryApp, incoming: DirectoryApp) { .as_deref() .map(|value| !value.trim().is_empty()) .unwrap_or(false); - let existing_description_present = existing - .description - .as_deref() - .map(|value| !value.trim().is_empty()) - .unwrap_or(false); - if !existing_description_present && incoming_description_present { + if incoming_description_present { existing.description = description; } diff --git a/codex-rs/cli/src/login.rs b/codex-rs/cli/src/login.rs index 42622f25d3..f2fd5a3bdc 100644 --- a/codex-rs/cli/src/login.rs +++ b/codex-rs/cli/src/login.rs @@ -22,7 +22,7 @@ const LOGIN_SUCCESS_MESSAGE: &str = "Successfully logged in"; fn print_login_server_start(actual_port: u16, auth_url: &str) { eprintln!( - "Starting local login server on http://localhost:{actual_port}.\nIf your browser did not open, navigate to this URL to authenticate:\n\n{auth_url}" + "Starting local login server on http://localhost:{actual_port}.\nIf your browser did not open, navigate to this URL to authenticate:\n\n{auth_url}\n\nOn a remote or headless machine? Use `codex login --device-auth` instead." ); } diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 42ded11c06..c271fe0b9e 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -808,6 +808,7 @@ async fn cli_main(arg0_paths: Arg0DispatchPaths) -> anyhow::Result<()> { stage_width = stage_width.max(stage.len()); rows.push((name, stage, enabled)); } + rows.sort_unstable_by_key(|(name, _, _)| *name); for (name, stage, enabled) in rows { println!("{name: Re enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }; servers.insert(name.clone(), new_entry); @@ -272,6 +273,7 @@ async fn run_add(config_overrides: &CliConfigOverrides, add_args: AddArgs) -> Re oauth_config.http_headers, oauth_config.env_http_headers, &Vec::new(), + None, config.mcp_oauth_callback_port, config.mcp_oauth_callback_url.as_deref(), ) @@ -356,6 +358,7 @@ async fn run_login(config_overrides: &CliConfigOverrides, login_args: LoginArgs) http_headers, env_http_headers, &scopes, + server.oauth_resource.as_deref(), config.mcp_oauth_callback_port, config.mcp_oauth_callback_url.as_deref(), ) diff --git a/codex-rs/cli/tests/features.rs b/codex-rs/cli/tests/features.rs index 6b2ed72f8a..17a7eff679 100644 --- a/codex-rs/cli/tests/features.rs +++ b/codex-rs/cli/tests/features.rs @@ -2,6 +2,7 @@ use std::path::Path; use anyhow::Result; use predicates::str::contains; +use pretty_assertions::assert_eq; use tempfile::TempDir; fn codex_command(codex_home: &Path) -> Result { @@ -58,3 +59,33 @@ async fn features_enable_under_development_feature_prints_warning() -> Result<() Ok(()) } + +#[tokio::test] +async fn features_list_is_sorted_alphabetically_by_feature_name() -> Result<()> { + let codex_home = TempDir::new()?; + + let mut cmd = codex_command(codex_home.path())?; + let output = cmd + .args(["features", "list"]) + .assert() + .success() + .get_output() + .stdout + .clone(); + let stdout = String::from_utf8(output)?; + + let actual_names = stdout + .lines() + .map(|line| { + line.split_once(" ") + .map(|(name, _)| name.trim_end().to_string()) + .expect("feature list output should contain aligned columns") + }) + .collect::>(); + let mut expected_names = actual_names.clone(); + expected_names.sort(); + + assert_eq!(actual_names, expected_names); + + Ok(()) +} diff --git a/codex-rs/cloud-requirements/src/lib.rs b/codex-rs/cloud-requirements/src/lib.rs index 6f6bf3b6de..d89ff53524 100644 --- a/codex-rs/cloud-requirements/src/lib.rs +++ b/codex-rs/cloud-requirements/src/lib.rs @@ -28,17 +28,21 @@ use serde::Serialize; use sha2::Sha256; use std::path::PathBuf; use std::sync::Arc; +use std::sync::Mutex; +use std::sync::OnceLock; use std::time::Duration; use std::time::Instant; use thiserror::Error; use tokio::fs; +use tokio::task::JoinHandle; use tokio::time::sleep; use tokio::time::timeout; const CLOUD_REQUIREMENTS_TIMEOUT: Duration = Duration::from_secs(15); const CLOUD_REQUIREMENTS_MAX_ATTEMPTS: usize = 5; const CLOUD_REQUIREMENTS_CACHE_FILENAME: &str = "cloud-requirements-cache.json"; -const CLOUD_REQUIREMENTS_CACHE_TTL: Duration = Duration::from_secs(60 * 60); +const CLOUD_REQUIREMENTS_CACHE_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); +const CLOUD_REQUIREMENTS_CACHE_TTL: Duration = Duration::from_secs(30 * 60); const CLOUD_REQUIREMENTS_CACHE_WRITE_HMAC_KEY: &[u8] = b"codex-cloud-requirements-cache-v3-064f8542-75b4-494c-a294-97d3ce597271"; const CLOUD_REQUIREMENTS_CACHE_READ_HMAC_KEYS: &[&[u8]] = @@ -46,6 +50,11 @@ const CLOUD_REQUIREMENTS_CACHE_READ_HMAC_KEYS: &[&[u8]] = type HmacSha256 = Hmac; +fn refresher_task_slot() -> &'static Mutex>> { + static REFRESHER_TASK: OnceLock>>> = OnceLock::new(); + REFRESHER_TASK.get_or_init(|| Mutex::new(None)) +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum FetchCloudRequirementsStatus { BackendClientInit, @@ -188,6 +197,7 @@ impl RequirementsFetcher for BackendRequirementsFetcher { } } +#[derive(Clone)] struct CloudRequirementsService { auth_manager: Arc, fetcher: Arc, @@ -325,6 +335,54 @@ impl CloudRequirementsService { None } + async fn refresh_cache_in_background(&self) { + loop { + sleep(CLOUD_REQUIREMENTS_CACHE_REFRESH_INTERVAL).await; + match timeout(self.timeout, self.refresh_cache()).await { + Ok(true) => {} + Ok(false) => break, + Err(_) => { + tracing::warn!( + "Timed out refreshing cloud requirements cache from remote; keeping existing cache" + ); + } + } + } + } + + async fn refresh_cache(&self) -> bool { + let Some(auth) = self.auth_manager.auth().await else { + return false; + }; + if !auth.is_chatgpt_auth() + || !matches!( + auth.account_plan_type(), + Some(PlanType::Business | PlanType::Enterprise) + ) + { + return false; + } + + let token_data = auth.get_token_data().ok(); + let chatgpt_user_id = token_data + .as_ref() + .and_then(|token_data| token_data.id_token.chatgpt_user_id.as_deref()); + let account_id = auth.get_account_id(); + let account_id = account_id.as_deref(); + + if self + .fetch_with_retries(&auth, chatgpt_user_id, account_id) + .await + .is_none() + { + tracing::warn!( + path = %self.cache_path.display(), + "Failed to refresh cloud requirements cache from remote" + ); + } + true + } + async fn load_cache( &self, chatgpt_user_id: Option<&str>, @@ -452,7 +510,17 @@ pub fn cloud_requirements_loader( codex_home, CLOUD_REQUIREMENTS_TIMEOUT, ); + let refresh_service = service.clone(); let task = tokio::spawn(async move { service.fetch_with_timeout().await }); + let refresh_task = + tokio::spawn(async move { refresh_service.refresh_cache_in_background().await }); + let mut refresher_guard = refresher_task_slot().lock().unwrap_or_else(|err| { + tracing::warn!("cloud requirements refresher task slot was poisoned"); + err.into_inner() + }); + if let Some(existing_task) = refresher_guard.replace(refresh_task) { + existing_task.abort(); + } CloudRequirementsLoader::new(async move { task.await .inspect_err(|err| tracing::warn!(error = %err, "Cloud requirements task failed")) @@ -1052,7 +1120,11 @@ mod tests { let cache_file: CloudRequirementsCacheFile = serde_json::from_str(&std::fs::read_to_string(path).expect("read cache")) .expect("parse cache"); - assert!(cache_file.signed_payload.expires_at > Utc::now()); + assert!( + cache_file.signed_payload.expires_at + <= cache_file.signed_payload.cached_at + ChronoDuration::minutes(30) + ); + assert!(cache_file.signed_payload.expires_at > cache_file.signed_payload.cached_at); assert!(cache_file.signed_payload.cached_at <= Utc::now()); assert_eq!( cache_file.signed_payload.chatgpt_user_id, @@ -1130,4 +1202,57 @@ mod tests { CLOUD_REQUIREMENTS_MAX_ATTEMPTS ); } + + #[tokio::test] + async fn refresh_from_remote_updates_cached_cloud_requirements() { + let codex_home = tempdir().expect("tempdir"); + let fetcher = Arc::new(SequenceFetcher::new(vec![ + Ok(Some("allowed_approval_policies = [\"never\"]".to_string())), + Ok(Some( + "allowed_approval_policies = [\"on-request\"]".to_string(), + )), + ])); + let service = CloudRequirementsService::new( + auth_manager_with_plan("business"), + fetcher, + codex_home.path().to_path_buf(), + CLOUD_REQUIREMENTS_TIMEOUT, + ); + + assert_eq!( + service.fetch().await, + Some(ConfigRequirementsToml { + allowed_approval_policies: Some(vec![AskForApproval::Never]), + allowed_sandbox_modes: None, + allowed_web_search_modes: None, + mcp_servers: None, + rules: None, + enforce_residency: None, + network: None, + }) + ); + + service.refresh_cache().await; + + let path = codex_home.path().join(CLOUD_REQUIREMENTS_CACHE_FILENAME); + let cache_file: CloudRequirementsCacheFile = + serde_json::from_str(&std::fs::read_to_string(path).expect("read cache")) + .expect("parse cache"); + assert_eq!( + cache_file + .signed_payload + .contents + .as_deref() + .and_then(|contents| parse_cloud_requirements(contents).ok().flatten()), + Some(ConfigRequirementsToml { + allowed_approval_policies: Some(vec![AskForApproval::OnRequest]), + allowed_sandbox_modes: None, + allowed_web_search_modes: None, + mcp_servers: None, + rules: None, + enforce_residency: None, + network: None, + }) + ); + } } diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index 48cc1d7002..97d0b9e320 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -25,6 +25,8 @@ use tokio_tungstenite::WebSocketStream; use tokio_tungstenite::tungstenite::Error as WsError; use tokio_tungstenite::tungstenite::Message; use tokio_tungstenite::tungstenite::client::IntoClientRequest; +use tracing::debug; +use tracing::error; use tracing::info; use tracing::trace; use tungstenite::protocol::WebSocketConfig; @@ -62,15 +64,23 @@ impl WsStream { }; match command { WsCommand::Send { message, tx_result } => { + debug!("realtime websocket sending message"); let result = inner.send(message).await; let should_break = result.is_err(); + if let Err(err) = &result { + error!("realtime websocket send failed: {err}"); + } let _ = tx_result.send(result); if should_break { break; } } WsCommand::Close { tx_result } => { + info!("realtime websocket sending close"); let result = inner.close(None).await; + if let Err(err) = &result { + error!("realtime websocket close failed: {err}"); + } let _ = tx_result.send(result); break; } @@ -82,7 +92,9 @@ impl WsStream { }; match message { Ok(Message::Ping(payload)) => { + trace!(payload_len = payload.len(), "realtime websocket received ping"); if let Err(err) = inner.send(Message::Pong(payload)).await { + error!("realtime websocket failed to send pong: {err}"); let _ = tx_message.send(Err(err)); break; } @@ -93,6 +105,24 @@ impl WsStream { | Message::Close(_) | Message::Frame(_))) => { let is_close = matches!(message, Message::Close(_)); + match &message { + Message::Text(_) => trace!("realtime websocket received text frame"), + Message::Binary(binary) => { + error!( + payload_len = binary.len(), + "realtime websocket received unexpected binary frame" + ); + } + Message::Close(frame) => info!( + "realtime websocket received close frame: code={:?} reason={:?}", + frame.as_ref().map(|frame| frame.code), + frame.as_ref().map(|frame| frame.reason.as_str()) + ), + Message::Frame(_) => { + trace!("realtime websocket received raw frame"); + } + Message::Ping(_) | Message::Pong(_) => {} + } if tx_message.send(Ok(message)).is_err() { break; } @@ -101,6 +131,7 @@ impl WsStream { } } Err(err) => { + error!("realtime websocket receive failed: {err}"); let _ = tx_message.send(Err(err)); break; } @@ -108,6 +139,7 @@ impl WsStream { } } } + info!("realtime websocket pump exiting"); }); ( @@ -298,7 +330,7 @@ impl RealtimeWebsocketWriter { async fn send_json(&self, message: RealtimeOutboundMessage) -> Result<(), ApiError> { let payload = serde_json::to_string(&message) .map_err(|err| ApiError::Stream(format!("failed to encode realtime request: {err}")))?; - trace!("realtime websocket request: {payload}"); + debug!(?message, "realtime websocket request"); if self.is_closed.load(Ordering::SeqCst) { return Err(ApiError::Stream( @@ -325,12 +357,14 @@ impl RealtimeWebsocketEvents { Some(Ok(msg)) => msg, Some(Err(err)) => { self.is_closed.store(true, Ordering::SeqCst); + error!("realtime websocket read failed: {err}"); return Err(ApiError::Stream(format!( "failed to read websocket message: {err}" ))); } None => { self.is_closed.store(true, Ordering::SeqCst); + info!("realtime websocket event stream ended"); return Ok(None); } }; @@ -338,11 +372,18 @@ impl RealtimeWebsocketEvents { match msg { Message::Text(text) => { if let Some(event) = parse_realtime_event(&text) { + debug!(?event, "realtime websocket parsed event"); return Ok(Some(event)); } + debug!("realtime websocket ignored unsupported text frame"); } - Message::Close(_) => { + Message::Close(frame) => { self.is_closed.store(true, Ordering::SeqCst); + info!( + "realtime websocket closed: code={:?} reason={:?}", + frame.as_ref().map(|frame| frame.code), + frame.as_ref().map(|frame| frame.reason.as_str()) + ); return Ok(None); } Message::Binary(_) => { @@ -383,15 +424,24 @@ impl RealtimeWebsocketClient { request.headers_mut().extend(headers); info!("connecting realtime websocket: {ws_url}"); - let (stream, _) = + let (stream, response) = tokio_tungstenite::connect_async_with_config(request, Some(websocket_config()), false) .await .map_err(|err| { ApiError::Stream(format!("failed to connect realtime websocket: {err}")) })?; + info!( + ws_url = %ws_url, + status = %response.status(), + "realtime websocket connected" + ); let (stream, rx_message) = WsStream::new(stream); let connection = RealtimeWebsocketConnection::new(stream, rx_message); + debug!( + conversation_id = config.session_id.as_deref().unwrap_or(""), + "realtime websocket sending session.create" + ); connection .send_session_create(config.prompt, config.session_id) .await?; diff --git a/codex-rs/codex-api/tests/models_integration.rs b/codex-rs/codex-api/tests/models_integration.rs index 9f7e380665..2b61f0de60 100644 --- a/codex-rs/codex-api/tests/models_integration.rs +++ b/codex-rs/codex-api/tests/models_integration.rs @@ -3,6 +3,7 @@ use codex_api::ModelsClient; use codex_api::provider::Provider; use codex_api::provider::RetryConfig; use codex_client::ReqwestTransport; +use codex_protocol::config_types::ReasoningSummary; use codex_protocol::openai_models::ConfigShellToolType; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelVisibility; @@ -78,8 +79,10 @@ async fn models_client_hits_models_endpoint() { base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 47731291a4..87f18eb800 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -61,6 +61,7 @@ env-flags = { workspace = true } eventsource-stream = { workspace = true } futures = { workspace = true } http = { workspace = true } +iana-time-zone = { workspace = true } indexmap = { workspace = true } keyring = { workspace = true, features = ["crypto-rust"] } libc = { workspace = true } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 6c4c78fdf1..59140d9d16 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -409,9 +409,6 @@ "shell_zsh_fork": { "type": "boolean" }, - "skill_approval": { - "type": "boolean" - }, "skill_env_var_dependency_prompt": { "type": "boolean" }, @@ -616,6 +613,10 @@ "additionalProperties": false, "description": "Memories settings loaded from config.toml.", "properties": { + "generate_memories": { + "description": "When `false`, newly created threads are stored with `memory_mode = \"disabled\"` in the state DB.", + "type": "boolean" + }, "max_raw_memories_for_global": { "description": "Maximum number of recent raw memories retained for global consolidation.", "format": "uint", @@ -633,6 +634,11 @@ "minimum": 0.0, "type": "integer" }, + "max_unused_days": { + "description": "Maximum number of days since a memory was last used before it becomes ineligible for phase 2 selection.", + "format": "int64", + "type": "integer" + }, "min_rollout_idle_hours": { "description": "Minimum idle time between last thread activity and memory creation (hours). > 12h recommended.", "format": "int64", @@ -645,6 +651,10 @@ "phase_2_model": { "description": "Model used for memory consolidation.", "type": "string" + }, + "use_memories": { + "description": "When `false`, skip injecting memory usage instructions into developer prompts.", + "type": "boolean" } }, "type": "object" @@ -1144,6 +1154,10 @@ }, "type": "object" }, + "oauth_resource": { + "default": null, + "type": "string" + }, "required": { "default": null, "type": "boolean" @@ -1177,6 +1191,18 @@ }, "type": "object" }, + "RealtimeAudioToml": { + "additionalProperties": false, + "properties": { + "microphone": { + "type": "string" + }, + "speaker": { + "type": "string" + } + }, + "type": "object" + }, "ReasoningEffort": { "description": "See https://platform.openai.com/docs/guides/reasoning?api-mode=responses#get-started-with-reasoning", "enum": [ @@ -1538,6 +1564,15 @@ "default": null, "description": "Settings for app-specific controls." }, + "audio": { + "allOf": [ + { + "$ref": "#/definitions/RealtimeAudioToml" + } + ], + "default": null, + "description": "Machine-local realtime audio device preferences used by realtime voice." + }, "background_terminal_max_timeout": { "description": "Maximum poll window for background terminal output (`write_stdin`), in milliseconds. Default: `300000` (5 minutes).", "format": "uint64", @@ -1705,9 +1740,6 @@ "shell_zsh_fork": { "type": "boolean" }, - "skill_approval": { - "type": "boolean" - }, "skill_env_var_dependency_prompt": { "type": "boolean" }, diff --git a/codex-rs/core/src/agent/agent_names.txt b/codex-rs/core/src/agent/agent_names.txt index e5a712ff5c..7574e69a80 100644 --- a/codex-rs/core/src/agent/agent_names.txt +++ b/codex-rs/core/src/agent/agent_names.txt @@ -1,87 +1,100 @@ -Ash -Elm -Yew -Fir -Oak -Pine -Spruce -Cedar -Birch -Maple -Beech -Alder -Willow -Poplar -Aspen -Larch -Juniper -Cypress -Hemlock -Sequoia -Baobab -Acacia -Banyan -Teak -Olive -Cherry -Apple -Pear -Plum -Peach -Mango -Guava -Fig -Date -Palm -Holly -Ivy -Fern -Moss -Reed -Cane -Rush -Sage -Mint -Thyme -Basil -Dill -Fennel -Parsley -Laurel -Myrtle -Lotus -Tulip -Rose -Daisy -Lily -Iris -Poppy -Aster -Peony -Camellia -Azalea -Begonia -Petunia -Verbena -Marigold -Daffodil -Orchid -Violet -Clover -Bamboo -Eucalypt -Oleander -Yucca -Agave -Aloe -Cactus -Sedum -Boxwood -Hawthorn -Redwood -Ironwood -Sandal -Kapok -Mahonia -Spirea -Wisteria \ No newline at end of file +Euclid +Archimedes +Ptolemy +Hypatia +Avicenna +Averroes +Aquinas +Copernicus +Kepler +Galileo +Bacon +Descartes +Pascal +Fermat +Huygens +Leibniz +Newton +Halley +Euler +Lagrange +Laplace +Volta +Gauss +Ampere +Faraday +Darwin +Lovelace +Boole +Pasteur +Maxwell +Mendel +Curie +Planck +Tesla +Poincare +Noether +Hilbert +Einstein +Raman +Bohr +Turing +Hubble +Feynman +Franklin +McClintock +Meitner +Herschel +Linnaeus +Wegener +Chandrasekhar +Sagan +Goodall +Carson +Carver +Socrates +Plato +Aristotle +Epicurus +Cicero +Confucius +Mencius +Zeno +Locke +Hume +Kant +Hegel +Kierkegaard +Mill +Nietzsche +Peirce +James +Dewey +Russell +Popper +Sartre +Beauvoir +Arendt +Rawls +Singer +Anscombe +Parfit +Kuhn +Boyle +Hooke +Harvey +Dalton +Ohm +Helmholtz +Gibbs +Lorentz +Schrodinger +Heisenberg +Pauli +Dirac +Bernoulli +Godel +Nash +Banach +Ramanujan +Erdos \ No newline at end of file diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index e94ac79669..8ea2362283 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -4,11 +4,17 @@ use crate::agent::status::is_final; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::find_thread_path_by_id_str; +use crate::rollout::RolloutRecorder; +use crate::session_prefix::format_subagent_context_line; use crate::session_prefix::format_subagent_notification_message; use crate::state_db; use crate::thread_manager::ThreadManagerState; use codex_protocol::ThreadId; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::Op; +use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::TokenUsage; @@ -18,6 +24,12 @@ use std::sync::Weak; use tokio::sync::watch; const AGENT_NAMES: &str = include_str!("agent_names.txt"); +const FORKED_SPAWN_AGENT_OUTPUT_MESSAGE: &str = "You are the newly spawned agent. The prior conversation history was forked from your parent agent. Treat the next user message as your new task, and use the forked history only as background context."; + +#[derive(Clone, Debug, Default)] +pub(crate) struct SpawnAgentOptions { + pub(crate) fork_parent_spawn_call_id: Option, +} fn agent_nickname_list() -> Vec<&'static str> { AGENT_NAMES @@ -57,6 +69,17 @@ impl AgentControl { config: crate::config::Config, items: Vec, session_source: Option, + ) -> CodexResult { + self.spawn_agent_with_options(config, items, session_source, SpawnAgentOptions::default()) + .await + } + + pub(crate) async fn spawn_agent_with_options( + &self, + config: crate::config::Config, + items: Vec, + session_source: Option, + options: SpawnAgentOptions, ) -> CodexResult { let state = self.upgrade()?; let mut reservation = self.state.reserve_spawn_slot(config.agent_max_threads)?; @@ -82,9 +105,75 @@ impl AgentControl { // The same `AgentControl` is sent to spawn the thread. let new_thread = match session_source { Some(session_source) => { - state - .spawn_new_thread_with_source(config, self.clone(), session_source, false, None) - .await? + if let Some(call_id) = options.fork_parent_spawn_call_id.as_ref() { + let SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + .. + }) = session_source.clone() + else { + return Err(CodexErr::Fatal( + "spawn_agent fork requires a thread-spawn session source".to_string(), + )); + }; + let parent_thread = state.get_thread(parent_thread_id).await.ok(); + if let Some(parent_thread) = parent_thread.as_ref() { + // `record_conversation_items` only queues rollout writes asynchronously. + // Flush/materialize the live parent before snapshotting JSONL for a fork. + parent_thread + .codex + .session + .ensure_rollout_materialized() + .await; + parent_thread.codex.session.flush_rollout().await; + } + let rollout_path = parent_thread + .as_ref() + .and_then(|parent_thread| parent_thread.rollout_path()) + .or(find_thread_path_by_id_str( + config.codex_home.as_path(), + &parent_thread_id.to_string(), + ) + .await?) + .ok_or_else(|| { + CodexErr::Fatal(format!( + "parent thread rollout unavailable for fork: {parent_thread_id}" + )) + })?; + let mut forked_rollout_items = + RolloutRecorder::get_rollout_history(&rollout_path) + .await? + .get_rollout_items(); + let mut output = FunctionCallOutputPayload::from_text( + FORKED_SPAWN_AGENT_OUTPUT_MESSAGE.to_string(), + ); + output.success = Some(true); + forked_rollout_items.push(RolloutItem::ResponseItem( + ResponseItem::FunctionCallOutput { + call_id: call_id.clone(), + output, + }, + )); + let initial_history = InitialHistory::Forked(forked_rollout_items); + state + .fork_thread_with_source( + config, + initial_history, + self.clone(), + session_source, + false, + ) + .await? + } else { + state + .spawn_new_thread_with_source( + config, + self.clone(), + session_source, + false, + None, + ) + .await? + } } None => state.spawn_new_thread(config, self.clone()).await?, }; @@ -255,6 +344,40 @@ impl AgentControl { thread.total_token_usage().await } + pub(crate) async fn format_environment_context_subagents( + &self, + parent_thread_id: ThreadId, + ) -> String { + let Ok(state) = self.upgrade() else { + return String::new(); + }; + + let mut agents = Vec::new(); + for thread_id in state.list_thread_ids().await { + let Ok(thread) = state.get_thread(thread_id).await else { + continue; + }; + let snapshot = thread.config_snapshot().await; + let SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id: agent_parent_thread_id, + agent_nickname, + .. + }) = snapshot.session_source + else { + continue; + }; + if agent_parent_thread_id != parent_thread_id { + continue; + } + agents.push(format_subagent_context_line( + &thread_id.to_string(), + agent_nickname.as_deref(), + )); + } + agents.sort(); + agents.join("\n") + } + /// Starts a detached watcher for sub-agents spawned from another thread. /// /// This is only enabled for `SubAgentSource::ThreadSpawn`, where a parent thread exists and @@ -319,8 +442,8 @@ mod tests { use crate::config::Config; use crate::config::ConfigBuilder; use crate::config_loader::LoaderOverrides; + use crate::contextual_user_message::SUBAGENT_NOTIFICATION_OPEN_TAG; use crate::features::Feature; - use crate::session_prefix::SUBAGENT_NOTIFICATION_OPEN_TAG; use assert_matches::assert_matches; use codex_protocol::config_types::ModeKind; use codex_protocol::models::ContentItem; @@ -421,6 +544,21 @@ mod tests { }) } + /// Returns true when any message item contains `needle` in a text span. + fn history_contains_text(history_items: &[ResponseItem], needle: &str) -> bool { + history_items.iter().any(|item| { + let ResponseItem::Message { content, .. } = item else { + return false; + }; + content.iter().any(|content_item| match content_item { + ContentItem::InputText { text } | ContentItem::OutputText { text } => { + text.contains(needle) + } + ContentItem::InputImage { .. } => false, + }) + }) + } + async fn wait_for_subagent_notification(parent_thread: &Arc) -> bool { let wait = async { loop { @@ -673,6 +811,242 @@ mod tests { assert_eq!(captured, Some(expected)); } + #[tokio::test] + async fn spawn_agent_can_fork_parent_thread_history() { + let harness = AgentControlHarness::new().await; + let (parent_thread_id, parent_thread) = harness.start_thread().await; + parent_thread + .inject_user_message_without_turn("parent seed context".to_string()) + .await; + let turn_context = parent_thread.codex.session.new_default_turn().await; + let parent_spawn_call_id = "spawn-call-history".to_string(); + let parent_spawn_call = ResponseItem::FunctionCall { + id: None, + name: "spawn_agent".to_string(), + arguments: "{}".to_string(), + call_id: parent_spawn_call_id.clone(), + }; + parent_thread + .codex + .session + .record_conversation_items(turn_context.as_ref(), &[parent_spawn_call]) + .await; + parent_thread + .codex + .session + .ensure_rollout_materialized() + .await; + parent_thread.codex.session.flush_rollout().await; + + let child_thread_id = harness + .control + .spawn_agent_with_options( + harness.config.clone(), + text_input("child task"), + Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: 1, + agent_nickname: None, + agent_role: None, + })), + SpawnAgentOptions { + fork_parent_spawn_call_id: Some(parent_spawn_call_id), + }, + ) + .await + .expect("forked spawn should succeed"); + + let child_thread = harness + .manager + .get_thread(child_thread_id) + .await + .expect("child thread should be registered"); + assert_ne!(child_thread_id, parent_thread_id); + let history = child_thread.codex.session.clone_history().await; + assert!(history_contains_text( + history.raw_items(), + "parent seed context" + )); + + let expected = ( + child_thread_id, + Op::UserInput { + items: vec![UserInput::Text { + text: "child task".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }, + ); + let captured = harness + .manager + .captured_ops() + .into_iter() + .find(|entry| *entry == expected); + assert_eq!(captured, Some(expected)); + + let _ = harness + .control + .shutdown_agent(child_thread_id) + .await + .expect("child shutdown should submit"); + let _ = parent_thread + .submit(Op::Shutdown {}) + .await + .expect("parent shutdown should submit"); + } + + #[tokio::test] + async fn spawn_agent_fork_injects_output_for_parent_spawn_call() { + let harness = AgentControlHarness::new().await; + let (parent_thread_id, parent_thread) = harness.start_thread().await; + let turn_context = parent_thread.codex.session.new_default_turn().await; + let parent_spawn_call_id = "spawn-call-1".to_string(); + let parent_spawn_call = ResponseItem::FunctionCall { + id: None, + name: "spawn_agent".to_string(), + arguments: "{}".to_string(), + call_id: parent_spawn_call_id.clone(), + }; + parent_thread + .codex + .session + .record_conversation_items(turn_context.as_ref(), &[parent_spawn_call]) + .await; + parent_thread + .codex + .session + .ensure_rollout_materialized() + .await; + parent_thread.codex.session.flush_rollout().await; + + let child_thread_id = harness + .control + .spawn_agent_with_options( + harness.config.clone(), + text_input("child task"), + Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: 1, + agent_nickname: None, + agent_role: None, + })), + SpawnAgentOptions { + fork_parent_spawn_call_id: Some(parent_spawn_call_id.clone()), + }, + ) + .await + .expect("forked spawn should succeed"); + + let child_thread = harness + .manager + .get_thread(child_thread_id) + .await + .expect("child thread should be registered"); + let history = child_thread.codex.session.clone_history().await; + let injected_output = history.raw_items().iter().find_map(|item| match item { + ResponseItem::FunctionCallOutput { call_id, output } + if call_id == &parent_spawn_call_id => + { + Some(output) + } + _ => None, + }); + let injected_output = + injected_output.expect("forked child should contain synthetic tool output"); + assert_eq!( + injected_output.text_content(), + Some(FORKED_SPAWN_AGENT_OUTPUT_MESSAGE) + ); + assert_eq!(injected_output.success, Some(true)); + + let _ = harness + .control + .shutdown_agent(child_thread_id) + .await + .expect("child shutdown should submit"); + let _ = parent_thread + .submit(Op::Shutdown {}) + .await + .expect("parent shutdown should submit"); + } + + #[tokio::test] + async fn spawn_agent_fork_flushes_parent_rollout_before_loading_history() { + let harness = AgentControlHarness::new().await; + let (parent_thread_id, parent_thread) = harness.start_thread().await; + let turn_context = parent_thread.codex.session.new_default_turn().await; + let parent_spawn_call_id = "spawn-call-unflushed".to_string(); + let parent_spawn_call = ResponseItem::FunctionCall { + id: None, + name: "spawn_agent".to_string(), + arguments: "{}".to_string(), + call_id: parent_spawn_call_id.clone(), + }; + parent_thread + .codex + .session + .record_conversation_items(turn_context.as_ref(), &[parent_spawn_call]) + .await; + + let child_thread_id = harness + .control + .spawn_agent_with_options( + harness.config.clone(), + text_input("child task"), + Some(SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: 1, + agent_nickname: None, + agent_role: None, + })), + SpawnAgentOptions { + fork_parent_spawn_call_id: Some(parent_spawn_call_id.clone()), + }, + ) + .await + .expect("forked spawn should flush parent rollout before loading history"); + + let child_thread = harness + .manager + .get_thread(child_thread_id) + .await + .expect("child thread should be registered"); + let history = child_thread.codex.session.clone_history().await; + + let mut parent_call_index = None; + let mut injected_output_index = None; + for (idx, item) in history.raw_items().iter().enumerate() { + match item { + ResponseItem::FunctionCall { call_id, .. } if call_id == &parent_spawn_call_id => { + parent_call_index = Some(idx); + } + ResponseItem::FunctionCallOutput { call_id, .. } + if call_id == &parent_spawn_call_id => + { + injected_output_index = Some(idx); + } + _ => {} + } + } + + let parent_call_index = + parent_call_index.expect("forked child should include the parent spawn_agent call"); + let injected_output_index = injected_output_index + .expect("forked child should include synthetic output for the parent spawn_agent call"); + assert!(parent_call_index < injected_output_index); + + let _ = harness + .control + .shutdown_agent(child_thread_id) + .await + .expect("child shutdown should submit"); + let _ = parent_thread + .submit(Op::Shutdown {}) + .await + .expect("parent shutdown should submit"); + } + #[tokio::test] async fn spawn_agent_respects_max_threads_limit() { let max_threads = 1usize; diff --git a/codex-rs/core/src/agent/role.rs b/codex-rs/core/src/agent/role.rs index ba9bb51a0d..3b34d3a71e 100644 --- a/codex-rs/core/src/agent/role.rs +++ b/codex-rs/core/src/agent/role.rs @@ -187,19 +187,24 @@ Rules: config_file: None, } ), - ( - "awaiter".to_string(), - AgentRoleConfig { - description: Some(r#"Use an `awaiter` agent EVERY TIME you must run a command that might take some time. -This includes, but not only: -* testing -* monitoring of a long running process -* explicit ask to wait for something - -When YOU wait for the `awaiter` agent to be done, use the largest possible timeout."#.to_string()), - config_file: Some("awaiter.toml".to_string().parse().unwrap_or_default()), - } - ) + // Awaiter is temp removed +// ( +// "awaiter".to_string(), +// AgentRoleConfig { +// description: Some(r#"Use an `awaiter` agent EVERY TIME you must run a command that will take some very long time. +// This includes, but not only: +// * testing +// * monitoring of a long running process +// * explicit ask to wait for something +// +// Rules: +// - When an awaiter is running, you can work on something else. If you need to wait for its completion, use the largest possible timeout. +// - Be patient with the `awaiter`. +// - Do not use an awaiter for every compilation/test if it won't take time. Only use if for long running commands. +// - Close the awaiter when you're done with it."#.to_string()), +// config_file: Some("awaiter.toml".to_string().parse().unwrap_or_default()), +// } +// ) ]) }); &CONFIG diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 33fc438727..3fea6eed56 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -84,19 +84,13 @@ fn reserialize_shell_outputs(items: &mut [ResponseItem]) { shell_call_ids.insert(call_id.clone()); } } - ResponseItem::CustomToolCallOutput { call_id, output } => { - if shell_call_ids.remove(call_id) - && let Some(structured) = parse_structured_shell_output(output) - { - *output = structured - } - } ResponseItem::FunctionCall { name, call_id, .. } if is_shell_tool_name(name) || name == "apply_patch" => { shell_call_ids.insert(call_id.clone()); } - ResponseItem::FunctionCallOutput { call_id, output } => { + ResponseItem::FunctionCallOutput { call_id, output } + | ResponseItem::CustomToolCallOutput { call_id, output } => { if shell_call_ids.remove(call_id) && let Some(structured) = output .text_content() @@ -240,6 +234,7 @@ mod tests { use codex_api::common::OpenAiVerbosity; use codex_api::common::TextControls; use codex_api::create_text_param_for_request; + use codex_protocol::models::FunctionCallOutputPayload; use pretty_assertions::assert_eq; use super::*; @@ -343,4 +338,62 @@ mod tests { let v = serde_json::to_value(&req).expect("json"); assert!(v.get("text").is_none()); } + + #[test] + fn reserializes_shell_outputs_for_function_and_custom_tool_calls() { + let raw_output = r#"{"output":"hello","metadata":{"exit_code":0,"duration_seconds":0.5}}"#; + let expected_output = "Exit code: 0\nWall time: 0.5 seconds\nOutput:\nhello"; + let mut items = vec![ + ResponseItem::FunctionCall { + id: None, + name: "shell".to_string(), + arguments: "{}".to_string(), + call_id: "call-1".to_string(), + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_text(raw_output.to_string()), + }, + ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "call-2".to_string(), + name: "apply_patch".to_string(), + input: "*** Begin Patch".to_string(), + }, + ResponseItem::CustomToolCallOutput { + call_id: "call-2".to_string(), + output: FunctionCallOutputPayload::from_text(raw_output.to_string()), + }, + ]; + + reserialize_shell_outputs(&mut items); + + assert_eq!( + items, + vec![ + ResponseItem::FunctionCall { + id: None, + name: "shell".to_string(), + arguments: "{}".to_string(), + call_id: "call-1".to_string(), + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_text(expected_output.to_string()), + }, + ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "call-2".to_string(), + name: "apply_patch".to_string(), + input: "*** Begin Patch".to_string(), + }, + ResponseItem::CustomToolCallOutput { + call_id: "call-2".to_string(), + output: FunctionCallOutputPayload::from_text(expected_output.to_string()), + }, + ] + ); + } } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index a63aa24148..91b24a0f25 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -53,6 +53,8 @@ use crate::util::error_or_panic; use crate::ws_version_from_features; use async_channel::Receiver; use async_channel::Sender; +use chrono::Local; +use chrono::Utc; use codex_hooks::HookEvent; use codex_hooks::HookEventAfterAgent; use codex_hooks::HookPayload; @@ -94,7 +96,6 @@ use codex_protocol::protocol::TurnContextNetworkItem; use codex_protocol::protocol::TurnStartedEvent; use codex_protocol::request_user_input::RequestUserInputArgs; use codex_protocol::request_user_input::RequestUserInputResponse; -use codex_protocol::skill_approval::SkillApprovalResponse; use codex_rmcp_client::ElicitationResponse; use codex_rmcp_client::OAuthCredentialsStoreMode; use codex_utils_stream_parser::AssistantTextChunk; @@ -252,6 +253,7 @@ use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; use crate::tools::handlers::SEARCH_TOOL_BM25_TOOL_NAME; use crate::tools::js_repl::JsReplHandle; +use crate::tools::js_repl::resolve_compatible_node; use crate::tools::network_approval::NetworkApprovalService; use crate::tools::network_approval::build_blocked_request_observer; use crate::tools::network_approval::build_network_policy_decider; @@ -342,6 +344,18 @@ impl Codex { config.features.disable(Feature::Collab); } + if config.features.enabled(Feature::JsRepl) + && let Err(err) = resolve_compatible_node(config.js_repl_node_path.as_deref()).await + { + let message = format!( + "Disabled `js_repl` for this session because the configured Node runtime is unavailable or incompatible. {err}" + ); + warn!("{message}"); + config.features.disable(Feature::JsRepl); + config.features.disable(Feature::JsReplToolsOnly); + config.startup_warnings.push(message); + } + let allowed_skills_for_implicit_invocation = loaded_skills.allowed_skills_for_implicit_invocation(); let user_instructions = @@ -429,6 +443,7 @@ impl Codex { thread_name: None, original_config_do_not_use: Arc::clone(&config), metrics_service_name, + app_server_client_name: None, session_source, dynamic_tools, persist_extended_history, @@ -516,6 +531,18 @@ impl Codex { self.session.steer_input(input, expected_turn_id).await } + pub(crate) async fn set_app_server_client_name( + &self, + app_server_client_name: Option, + ) -> ConstraintResult<()> { + self.session + .update_settings(SessionSettingsUpdate { + app_server_client_name, + ..Default::default() + }) + .await + } + pub(crate) async fn agent_status(&self) -> AgentStatus { self.agent_status.borrow().clone() } @@ -583,6 +610,9 @@ pub(crate) struct TurnContext { /// the model as well as sandbox policies are resolved against this path /// instead of `std::env::current_dir()`. pub(crate) cwd: PathBuf, + pub(crate) current_date: Option, + pub(crate) timezone: Option, + pub(crate) app_server_client_name: Option, pub(crate) developer_instructions: Option, pub(crate) compact_prompt: Option, pub(crate) user_instructions: Option, @@ -667,6 +697,9 @@ impl TurnContext { reasoning_summary: self.reasoning_summary, session_source: self.session_source.clone(), cwd: self.cwd.clone(), + current_date: self.current_date.clone(), + timezone: self.timezone.clone(), + app_server_client_name: self.app_server_client_name.clone(), developer_instructions: self.developer_instructions.clone(), compact_prompt: self.compact_prompt.clone(), user_instructions: self.user_instructions.clone(), @@ -707,6 +740,8 @@ impl TurnContext { TurnContextItem { turn_id: Some(self.sub_id.clone()), cwd: self.cwd.clone(), + current_date: self.current_date.clone(), + timezone: self.timezone.clone(), approval_policy: self.approval_policy.value(), sandbox_policy: self.sandbox_policy.get().clone(), network: self.turn_context_network_item(), @@ -736,13 +771,23 @@ impl TurnContext { } } +fn local_time_context() -> (String, String) { + match iana_time_zone::get_timezone() { + Ok(timezone) => (Local::now().format("%Y-%m-%d").to_string(), timezone), + Err(_) => ( + Utc::now().format("%Y-%m-%d").to_string(), + "Etc/UTC".to_string(), + ), + } +} + #[derive(Clone)] pub(crate) struct SessionConfiguration { /// Provider identifier ("openai", "openrouter", ...). provider: ModelProviderInfo, collaboration_mode: CollaborationMode, - model_reasoning_summary: ReasoningSummaryConfig, + model_reasoning_summary: Option, /// Developer instructions that supplement the base instructions. developer_instructions: Option, @@ -782,6 +827,7 @@ pub(crate) struct SessionConfiguration { original_config_do_not_use: Arc, /// Optional service name tag for session metrics. metrics_service_name: Option, + app_server_client_name: Option, /// Source of the session (cli, vscode, exec, mcp, ...) session_source: SessionSource, dynamic_tools: Vec, @@ -812,7 +858,7 @@ impl SessionConfiguration { next_configuration.collaboration_mode = collaboration_mode; } if let Some(summary) = updates.reasoning_summary { - next_configuration.model_reasoning_summary = summary; + next_configuration.model_reasoning_summary = Some(summary); } if let Some(personality) = updates.personality { next_configuration.personality = Some(personality); @@ -829,6 +875,9 @@ impl SessionConfiguration { if let Some(cwd) = updates.cwd.clone() { next_configuration.cwd = cwd; } + if let Some(app_server_client_name) = updates.app_server_client_name.clone() { + next_configuration.app_server_client_name = Some(app_server_client_name); + } Ok(next_configuration) } } @@ -843,6 +892,7 @@ pub(crate) struct SessionSettingsUpdate { pub(crate) reasoning_summary: Option, pub(crate) final_output_json_schema: Option>, pub(crate) personality: Option, + pub(crate) app_server_client_name: Option, } impl Session { @@ -973,7 +1023,9 @@ impl Session { skills_outcome: Arc, ) -> TurnContext { let reasoning_effort = session_configuration.collaboration_mode.reasoning_effort(); - let reasoning_summary = session_configuration.model_reasoning_summary; + let reasoning_summary = session_configuration + .model_reasoning_summary + .unwrap_or(model_info.default_reasoning_summary); let otel_manager = otel_manager.clone().with_model( session_configuration.collaboration_mode.model(), model_info.slug.as_str(), @@ -1003,6 +1055,7 @@ impl Session { .features .enabled(Feature::UseLinuxSandboxBwrap), )); + let (current_date, timezone) = local_time_context(); TurnContext { sub_id, config: per_turn_config.clone(), @@ -1014,6 +1067,9 @@ impl Session { reasoning_summary, session_source, cwd, + current_date: Some(current_date), + timezone: Some(timezone), + app_server_client_name: session_configuration.app_server_client_name.clone(), developer_instructions: session_configuration.developer_instructions.clone(), compact_prompt: session_configuration.compact_prompt.clone(), user_instructions: session_configuration.user_instructions.clone(), @@ -1112,7 +1168,7 @@ impl Session { // // - initialize RolloutRecorder with new or resumed session info // - perform default shell discovery - // - load history metadata + // - load history metadata (skipped for subagents) let rollout_fut = async { if config.ephemeral { Ok::<_, anyhow::Error>((None, None)) @@ -1129,7 +1185,16 @@ impl Session { } }; - let history_meta_fut = crate::message_history::history_metadata(&config); + let history_meta_fut = async { + if matches!( + session_configuration.session_source, + SessionSource::SubAgent(_) + ) { + (0, 0) + } else { + crate::message_history::history_metadata(&config).await + } + }; let auth_manager_clone = Arc::clone(&auth_manager); let config_for_mcp = Arc::clone(&config); let auth_and_mcp_fut = async move { @@ -1250,7 +1315,9 @@ impl Session { otel_manager.conversation_starts( config.model_provider.name.as_str(), session_configuration.collaboration_mode.reasoning_effort(), - config.model_reasoning_summary, + config + .model_reasoning_summary + .unwrap_or(ReasoningSummaryConfig::Auto), config.model_context_window, config.model_auto_compact_token_limit, config.permissions.approval_policy.value(), @@ -1657,7 +1724,10 @@ impl Session { match conversation_history { InitialHistory::New => { // Build and record initial items (user instructions + environment context) - let items = self.build_initial_context(&turn_context).await; + // TODO(ccunningham): Defer initial context insertion until the first real turn + // starts so it reflects the actual first-turn settings (permissions, etc.) and + // we do not emit model-visible "diff" updates before the first user message. + let items = self.build_initial_context(&turn_context, None).await; self.record_conversation_items(&turn_context, &items).await; { let mut state = self.state.lock().await; @@ -1761,7 +1831,7 @@ impl Session { } // Append the current session's initial context after the reconstructed history. - let initial_context = self.build_initial_context(&turn_context).await; + let initial_context = self.build_initial_context(&turn_context, None).await; self.record_conversation_items(&turn_context, &initial_context) .await; { @@ -2590,9 +2660,13 @@ impl Session { /// Emit an exec approval request event and await the user's decision. /// - /// The request is keyed by `call_id` + `approval_id` so matching responses are delivered - /// to the correct in-flight turn. If the task is aborted, this returns the - /// default `ReviewDecision` (`Denied`). + /// The request is keyed by `call_id` + `approval_id` so matching responses + /// are delivered to the correct in-flight turn. If the task is aborted, + /// this returns the default `ReviewDecision` (`Denied`). + /// + /// Note that if `available_decisions` is `None`, then the other fields will + /// be used to derive the available decisions via + /// [ExecApprovalRequestEvent::default_available_decisions]. #[allow(clippy::too_many_arguments)] pub async fn request_command_approval( &self, @@ -2605,6 +2679,7 @@ impl Session { network_approval_context: Option, proposed_execpolicy_amendment: Option, additional_permissions: Option, + available_decisions: Option>, ) -> ReviewDecision { // command-level approvals use `call_id`. // `approval_id` is only present for subcommand callbacks (execve intercept) @@ -2638,6 +2713,14 @@ impl Session { }, ] }); + let available_decisions = available_decisions.unwrap_or_else(|| { + ExecApprovalRequestEvent::default_available_decisions( + network_approval_context.as_ref(), + proposed_execpolicy_amendment.as_ref(), + proposed_network_policy_amendments.as_deref(), + additional_permissions.as_ref(), + ) + }); let event = EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { call_id, approval_id, @@ -2649,6 +2732,7 @@ impl Session { proposed_execpolicy_amendment, proposed_network_policy_amendments, additional_permissions, + available_decisions: Some(available_decisions), parsed_cmd, }); self.send_event(turn_context, event).await; @@ -2723,40 +2807,6 @@ impl Session { rx_response.await.ok() } - pub async fn request_skill_approval( - &self, - turn_context: &TurnContext, - item_id: String, - skill_name: String, - ) -> Option { - let (tx_response, rx_response) = oneshot::channel(); - let prev_entry = { - let mut active = self.active_turn.lock().await; - match active.as_mut() { - Some(at) => { - let mut ts = at.turn_state.lock().await; - ts.insert_pending_skill_approval(item_id.clone(), tx_response) - } - None => None, - } - }; - if prev_entry.is_some() { - warn!("Overwriting existing pending skill approval for item_id: {item_id}"); - } - - self.send_event( - turn_context, - EventMsg::SkillRequestApproval( - codex_protocol::skill_approval::SkillRequestApprovalEvent { - item_id, - skill_name, - }, - ), - ) - .await; - rx_response.await.ok() - } - pub async fn notify_user_input_response( &self, sub_id: &str, @@ -2803,31 +2853,6 @@ impl Session { } } - pub async fn notify_skill_approval_response( - &self, - item_id: &str, - response: SkillApprovalResponse, - ) { - let entry = { - let mut active = self.active_turn.lock().await; - match active.as_mut() { - Some(at) => { - let mut ts = at.turn_state.lock().await; - ts.remove_pending_skill_approval(item_id) - } - None => None, - } - }; - match entry { - Some(tx_response) => { - tx_response.send(response).ok(); - } - None => { - warn!("No pending skill approval found for item_id: {item_id}"); - } - } - } - pub async fn notify_approval(&self, approval_id: &str, decision: ReviewDecision) { let entry = { let mut active = self.active_turn.lock().await; @@ -2895,7 +2920,7 @@ impl Session { } else { let user_messages = collect_user_messages(history.raw_items()); let rebuilt = compact::build_compacted_history( - self.build_initial_context(turn_context).await, + self.build_initial_context(turn_context, None).await, &user_messages, &compacted.message, ); @@ -3023,10 +3048,20 @@ impl Session { pub(crate) async fn build_initial_context( &self, turn_context: &TurnContext, + previous_user_turn_model: Option<&str>, ) -> Vec { - let mut items = Vec::::with_capacity(4); + let mut developer_sections = Vec::::with_capacity(8); + let mut contextual_user_sections = Vec::::with_capacity(2); let shell = self.user_shell(); - items.push( + if let Some(model_switch_message) = + crate::context_manager::updates::build_model_instructions_update_item( + previous_user_turn_model, + turn_context, + ) + { + developer_sections.push(model_switch_message.into_text()); + } + developer_sections.push( DeveloperInstructions::from_policy( turn_context.sandbox_policy.get(), turn_context.approval_policy.value(), @@ -3034,17 +3069,18 @@ impl Session { &turn_context.cwd, turn_context.features.enabled(Feature::RequestPermissions), ) - .into(), + .into_text(), ); if let Some(developer_instructions) = turn_context.developer_instructions.as_deref() { - items.push(DeveloperInstructions::new(developer_instructions.to_string()).into()); + developer_sections.push(developer_instructions.to_string()); } // Add developer instructions for memories. - if let Some(memory_prompt) = - build_memory_tool_developer_instructions(&turn_context.config.codex_home).await - && turn_context.features.enabled(Feature::MemoryTool) + if turn_context.features.enabled(Feature::MemoryTool) + && turn_context.config.memories.use_memories + && let Some(memory_prompt) = + build_memory_tool_developer_instructions(&turn_context.config.codex_home).await { - items.push(DeveloperInstructions::new(memory_prompt).into()); + developer_sections.push(memory_prompt); } // Add developer instructions from collaboration_mode if they exist and are non-empty let (collaboration_mode, base_instructions) = { @@ -3057,7 +3093,7 @@ impl Session { if let Some(collab_instructions) = DeveloperInstructions::from_collaboration_mode(&collaboration_mode) { - items.push(collab_instructions.into()); + developer_sections.push(collab_instructions.into_text()); } if self.features.enabled(Feature::Personality) && let Some(personality) = turn_context.personality @@ -3072,34 +3108,53 @@ impl Session { personality, ) { - items.push( - DeveloperInstructions::personality_spec_message(personality_message).into(), + developer_sections.push( + DeveloperInstructions::personality_spec_message(personality_message) + .into_text(), ); } } if turn_context.features.enabled(Feature::Apps) { - items.push(DeveloperInstructions::new(render_apps_section()).into()); + developer_sections.push(render_apps_section()); } if turn_context.features.enabled(Feature::CodexGitCommit) && let Some(commit_message_instruction) = commit_message_trailer_instruction( turn_context.config.commit_attribution.as_deref(), ) { - items.push(DeveloperInstructions::new(commit_message_instruction).into()); + developer_sections.push(commit_message_instruction); } if let Some(user_instructions) = turn_context.user_instructions.as_deref() { - items.push( + contextual_user_sections.push( UserInstructions { text: user_instructions.to_string(), directory: turn_context.cwd.to_string_lossy().into_owned(), } - .into(), + .serialize_to_text(), ); } - items.push(ResponseItem::from(EnvironmentContext::from_turn_context( - turn_context, - shell.as_ref(), - ))); + let subagents = self + .services + .agent_control + .format_environment_context_subagents(self.conversation_id) + .await; + contextual_user_sections.push( + EnvironmentContext::from_turn_context(turn_context, shell.as_ref()) + .with_subagents(subagents) + .serialize_to_xml(), + ); + + let mut items = Vec::with_capacity(2); + if let Some(developer_message) = + crate::context_manager::updates::build_developer_update_item(developer_sections) + { + items.push(developer_message); + } + if let Some(contextual_user_message) = + crate::context_manager::updates::build_contextual_user_message(contextual_user_sections) + { + items.push(contextual_user_message); + } items } @@ -3144,22 +3199,8 @@ impl Session { let reference_context_item = self.reference_context_item().await; let should_inject_full_context = reference_context_item.is_none(); let context_items = if should_inject_full_context { - let mut initial_context = self.build_initial_context(turn_context).await; - // Full reinjection bypasses the settings-diff path, so add the model-switch - // instruction explicitly when needed. Keep it before the rest of full context so - // model-specific guidance is read first. - if let Some(model_switch_item) = - crate::context_manager::updates::build_model_instructions_update_item( - previous_user_turn_model, - turn_context, - ) - { - // TODO(ccunningham): When a model switch changes the effective personality - // instructions, inject the updated personality spec alongside - // here so resume/model-switch paths can avoid forcing full reinjection. - initial_context.insert(0, model_switch_item); - } - initial_context + self.build_initial_context(turn_context, previous_user_turn_model) + .await } else { // Steady-state path: append only context diffs to minimize token overhead. self.build_settings_update_items( @@ -3549,9 +3590,9 @@ impl Session { use_linux_sandbox_bwrap: turn_context.features.enabled(Feature::UseLinuxSandboxBwrap), }; { - let mut cancel_guard = self.services.mcp_startup_cancellation_token.lock().await; - cancel_guard.cancel(); - *cancel_guard = CancellationToken::new(); + let mut guard = self.services.mcp_startup_cancellation_token.lock().await; + guard.cancel(); + *guard = CancellationToken::new(); } let (refreshed_manager, cancel_token) = McpConnectionManager::new( &mcp_servers, @@ -3565,11 +3606,11 @@ impl Session { ) .await; { - let mut cancel_guard = self.services.mcp_startup_cancellation_token.lock().await; - if cancel_guard.is_cancelled() { + let mut guard = self.services.mcp_startup_cancellation_token.lock().await; + if guard.is_cancelled() { cancel_token.cancel(); } - *cancel_guard = cancel_token; + *guard = cancel_token; } let mut manager = self.services.mcp_connection_manager.write().await; @@ -3731,9 +3772,6 @@ async fn submission_loop(sess: Arc, config: Arc, rx_sub: Receiv Op::DynamicToolResponse { id, response } => { handlers::dynamic_tool_response(&sess, id, response).await; } - Op::SkillApproval { id, response } => { - handlers::skill_approval_response(&sess, id, response).await; - } Op::AddToHistory { text } => { handlers::add_to_history(&sess, &config, text).await; } @@ -3857,7 +3895,6 @@ mod handlers { use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::WarningEvent; use codex_protocol::request_user_input::RequestUserInputResponse; - use codex_protocol::skill_approval::SkillApprovalResponse; use crate::context_manager::is_user_turn_boundary; use codex_protocol::config_types::CollaborationMode; @@ -3930,9 +3967,10 @@ mod handlers { sandbox_policy: Some(sandbox_policy), windows_sandbox_level: None, collaboration_mode, - reasoning_summary: Some(summary), + reasoning_summary: summary, final_output_json_schema: Some(final_output_json_schema), personality, + app_server_client_name: None, }, ) } @@ -4098,14 +4136,6 @@ mod handlers { sess.notify_dynamic_tool_response(&id, response).await; } - pub async fn skill_approval_response( - sess: &Arc, - id: String, - response: SkillApprovalResponse, - ) { - sess.notify_skill_approval_response(&id, response).await; - } - pub async fn add_to_history(sess: &Arc, config: &Arc, text: String) { let id = sess.conversation_id; let config = Arc::clone(config); @@ -4660,7 +4690,9 @@ async fn spawn_review_thread( let provider_for_context = provider.clone(); let otel_manager_for_context = otel_manager.clone(); let reasoning_effort = per_turn_config.model_reasoning_effort; - let reasoning_summary = per_turn_config.model_reasoning_summary; + let reasoning_summary = per_turn_config + .model_reasoning_summary + .unwrap_or(model_info.default_reasoning_summary); let session_source = parent_turn_context.session_source.clone(); let per_turn_config = Arc::new(per_turn_config); @@ -4688,6 +4720,9 @@ async fn spawn_review_thread( tools_config, features: parent_turn_context.features.clone(), ghost_snapshot: parent_turn_context.ghost_snapshot.clone(), + current_date: parent_turn_context.current_date.clone(), + timezone: parent_turn_context.timezone.clone(), + app_server_client_name: parent_turn_context.app_server_client_name.clone(), developer_instructions: None, user_instructions: None, compact_prompt: parent_turn_context.compact_prompt.clone(), @@ -4819,15 +4854,6 @@ pub(crate) async fn run_turn( collaboration_mode_kind: turn_context.collaboration_mode.mode, }); sess.send_event(&turn_context, event).await; - if turn_context.config.features.enabled(Feature::SkillApproval) { - let _ = sess - .request_skill_approval( - turn_context.as_ref(), - turn_context.sub_id.clone(), - "test-skill".to_string(), - ) - .await; - } // TODO(ccunningham): Pre-turn compaction runs before context updates and the // new user message are recorded. Estimate pending incoming items (context // diffs/full reinjection + user input) and trigger compaction preemptively @@ -5065,6 +5091,7 @@ pub(crate) async fn run_turn( &sess, &turn_context, InitialContextInjection::BeforeLastUserMessage, + previous_model.as_deref(), ) .await .is_err() @@ -5081,6 +5108,7 @@ pub(crate) async fn run_turn( .dispatch(HookPayload { session_id: sess.conversation_id, cwd: turn_context.cwd.clone(), + client: turn_context.app_server_client_name.clone(), triggered_at: chrono::Utc::now(), hook_event: HookEvent::AfterAgent { event: HookEventAfterAgent { @@ -5188,7 +5216,13 @@ async fn run_pre_sampling_compact( .unwrap_or(i64::MAX); // Compact if the total usage tokens are greater than the auto compact limit if total_usage_tokens >= auto_compact_limit { - run_auto_compact(sess, turn_context, InitialContextInjection::DoNotInject).await?; + run_auto_compact( + sess, + turn_context, + InitialContextInjection::DoNotInject, + None, + ) + .await?; } Ok(()) } @@ -5231,6 +5265,7 @@ async fn maybe_run_previous_model_inline_compact( sess, &previous_model_turn_context, InitialContextInjection::DoNotInject, + None, ) .await?; return Ok(true); @@ -5242,12 +5277,14 @@ async fn run_auto_compact( sess: &Arc, turn_context: &Arc, initial_context_injection: InitialContextInjection, + previous_user_turn_model: Option<&str>, ) -> CodexResult<()> { if should_use_remote_compact_task(&turn_context.provider) { run_inline_remote_auto_compact_task( Arc::clone(sess), Arc::clone(turn_context), initial_context_injection, + previous_user_turn_model, ) .await?; } else { @@ -5255,6 +5292,7 @@ async fn run_auto_compact( Arc::clone(sess), Arc::clone(turn_context), initial_context_injection, + previous_user_turn_model, ) .await?; } @@ -5856,7 +5894,6 @@ fn realtime_text_for_event(msg: &EventMsg) -> Option { | EventMsg::RequestUserInput(_) | EventMsg::DynamicToolCallRequest(_) | EventMsg::DynamicToolCallResponse(_) - | EventMsg::SkillRequestApproval(_) | EventMsg::ElicitationRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) | EventMsg::DeprecationNotice(_) @@ -7270,6 +7307,8 @@ mod tests { let previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -7307,6 +7346,8 @@ mod tests { let mut previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -7493,7 +7534,7 @@ mod tests { session .record_context_updates_and_set_reference_context_item(&turn_context, None) .await; - expected.extend(session.build_initial_context(&turn_context).await); + expected.extend(session.build_initial_context(&turn_context, None).await); let history_after_seed = session.clone_history().await; assert_eq!(expected, history_after_seed.raw_items()); @@ -7501,7 +7542,10 @@ mod tests { .record_context_updates_and_set_reference_context_item(&turn_context, None) .await; let history_after_second_seed = session.clone_history().await; - assert_eq!(expected, history_after_second_seed.raw_items()); + assert_eq!( + history_after_seed.raw_items(), + history_after_second_seed.raw_items() + ); } #[tokio::test] @@ -7655,7 +7699,7 @@ mod tests { let reconstruction_turn = session.new_default_turn().await; expected.extend( session - .build_initial_context(reconstruction_turn.as_ref()) + .build_initial_context(reconstruction_turn.as_ref(), None) .await, ); let history = session.state.lock().await.clone_history(); @@ -7669,6 +7713,8 @@ mod tests { let previous_context_item = TurnContextItem { turn_id: Some(turn_context.sub_id.clone()), cwd: turn_context.cwd.clone(), + current_date: turn_context.current_date.clone(), + timezone: turn_context.timezone.clone(), approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, @@ -7698,7 +7744,7 @@ mod tests { async fn thread_rollback_drops_last_turn_from_history() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let initial_context = sess.build_initial_context(tc.as_ref()).await; + let initial_context = sess.build_initial_context(tc.as_ref(), None).await; sess.record_into_history(&initial_context, tc.as_ref()) .await; @@ -7769,7 +7815,7 @@ mod tests { async fn thread_rollback_clears_history_when_num_turns_exceeds_existing_turns() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let initial_context = sess.build_initial_context(tc.as_ref()).await; + let initial_context = sess.build_initial_context(tc.as_ref(), None).await; sess.record_into_history(&initial_context, tc.as_ref()) .await; @@ -7797,7 +7843,7 @@ mod tests { async fn thread_rollback_fails_when_turn_in_progress() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let initial_context = sess.build_initial_context(tc.as_ref()).await; + let initial_context = sess.build_initial_context(tc.as_ref(), None).await; sess.record_into_history(&initial_context, tc.as_ref()) .await; @@ -7818,7 +7864,7 @@ mod tests { async fn thread_rollback_fails_when_num_turns_is_zero() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let initial_context = sess.build_initial_context(tc.as_ref()).await; + let initial_context = sess.build_initial_context(tc.as_ref(), None).await; sess.record_into_history(&initial_context, tc.as_ref()) .await; @@ -7872,6 +7918,7 @@ mod tests { thread_name: None, original_config_do_not_use: Arc::clone(&config), metrics_service_name: None, + app_server_client_name: None, session_source: SessionSource::Exec, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -7964,6 +8011,7 @@ mod tests { thread_name: None, original_config_do_not_use: Arc::clone(&config), metrics_service_name: None, + app_server_client_name: None, session_source: SessionSource::Exec, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -8275,6 +8323,7 @@ mod tests { thread_name: None, original_config_do_not_use: Arc::clone(&config), metrics_service_name: None, + app_server_client_name: None, session_source: SessionSource::Exec, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -8328,6 +8377,7 @@ mod tests { thread_name: None, original_config_do_not_use: Arc::clone(&config), metrics_service_name: None, + app_server_client_name: None, session_source: SessionSource::Exec, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -8409,6 +8459,7 @@ mod tests { thread_name: None, original_config_do_not_use: Arc::clone(&config), metrics_service_name: None, + app_server_client_name: None, session_source: SessionSource::Exec, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -8568,6 +8619,7 @@ mod tests { thread_name: None, original_config_do_not_use: Arc::clone(&config), metrics_service_name: None, + app_server_client_name: None, session_source: SessionSource::Exec, dynamic_tools, persist_extended_history: false, @@ -8834,6 +8886,42 @@ mod tests { assert!(environment_update.contains("blocked.example.com")); } + #[tokio::test] + async fn build_settings_update_items_emits_environment_item_for_time_changes() { + let (session, previous_context) = make_session_and_context().await; + let previous_context = Arc::new(previous_context); + let mut current_context = previous_context + .with_model( + previous_context.model_info.slug.clone(), + &session.services.models_manager, + ) + .await; + current_context.current_date = Some("2026-02-27".to_string()); + current_context.timezone = Some("Europe/Berlin".to_string()); + + let reference_context_item = previous_context.to_turn_context_item(); + let update_items = session.build_settings_update_items( + Some(&reference_context_item), + None, + ¤t_context, + ); + + let environment_update = update_items + .iter() + .find_map(|item| match item { + ResponseItem::Message { role, content, .. } if role == "user" => { + let [ContentItem::InputText { text }] = content.as_slice() else { + return None; + }; + text.contains("").then_some(text) + } + _ => None, + }) + .expect("environment update item should be emitted"); + assert!(environment_update.contains("2026-02-27")); + assert!(environment_update.contains("Europe/Berlin")); + } + #[tokio::test] async fn record_context_updates_and_set_reference_context_item_injects_full_context_when_baseline_missing() { @@ -8842,7 +8930,7 @@ mod tests { .record_context_updates_and_set_reference_context_item(&turn_context, None) .await; let history = session.clone_history().await; - let initial_context = session.build_initial_context(&turn_context).await; + let initial_context = session.build_initial_context(&turn_context, None).await; assert_eq!(history.raw_items().to_vec(), initial_context); let current_context = session.reference_context_item().await; @@ -8886,10 +8974,28 @@ mod tests { let history = session.clone_history().await; let mut expected_history = vec![compacted_summary]; - expected_history.extend(session.build_initial_context(&turn_context).await); + expected_history.extend(session.build_initial_context(&turn_context, None).await); assert_eq!(history.raw_items().to_vec(), expected_history); } + #[tokio::test] + async fn build_initial_context_prepends_model_switch_message() { + let (session, turn_context) = make_session_and_context().await; + + let initial_context = session + .build_initial_context(&turn_context, Some("previous-regular-model")) + .await; + + let ResponseItem::Message { role, content, .. } = &initial_context[0] else { + panic!("expected developer message"); + }; + assert_eq!(role, "developer"); + let [ContentItem::InputText { text }, ..] = content.as_slice() else { + panic!("expected developer text"); + }; + assert!(text.contains("")); + } + #[tokio::test] async fn run_user_shell_command_does_not_set_reference_context_item() { let (session, _turn_context, rx) = make_session_and_context_with_rx().await; @@ -9211,7 +9317,7 @@ mod tests { let ContentItem::InputText { text } = content_item else { return false; }; - text.contains(crate::session_prefix::TURN_ABORTED_OPEN_TAG) + text.contains(crate::contextual_user_message::TURN_ABORTED_OPEN_TAG) }) }), "expected a model-visible turn aborted marker in history after interrupt" @@ -9285,7 +9391,7 @@ mod tests { // personality_spec) matches reconstruction. let reconstruction_turn = session.new_default_turn().await; let mut initial_context = session - .build_initial_context(reconstruction_turn.as_ref()) + .build_initial_context(reconstruction_turn.as_ref(), None) .await; // Ensure personality_spec is present when Personality is enabled, so expected matches // what reconstruction produces (build_initial_context may omit it when baked into model). diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index c1fb31ebd1..0777f8e3b2 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -322,6 +322,7 @@ async fn handle_exec_approval( network_approval_context, proposed_execpolicy_amendment, additional_permissions, + available_decisions, .. } = event; // Race approval with cancellation and timeout to avoid hangs. @@ -335,6 +336,7 @@ async fn handle_exec_approval( network_approval_context, proposed_execpolicy_amendment, additional_permissions, + available_decisions, ); let decision = await_approval_with_cancel( approval_fut, diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index b493075d4a..19a8214ee0 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -1,6 +1,7 @@ use crate::agent::AgentStatus; use crate::codex::Codex; use crate::codex::SteerInputError; +use crate::config::ConstraintResult; use crate::error::Result as CodexResult; use crate::features::Feature; use crate::file_watcher::WatchRegistration; @@ -67,6 +68,15 @@ impl CodexThread { self.codex.steer_input(input, expected_turn_id).await } + pub async fn set_app_server_client_name( + &self, + app_server_client_name: Option, + ) -> ConstraintResult<()> { + self.codex + .set_app_server_client_name(app_server_client_name) + .await + } + /// Use sparingly: this is intended to be removed soon. pub async fn submit_with_id(&self, sub: Submission) -> CodexResult<()> { self.codex.submit_with_id(sub).await diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 43ee08fd4f..7470a14895 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -7,7 +7,6 @@ use crate::client_common::ResponseEvent; use crate::codex::Session; use crate::codex::TurnContext; use crate::codex::get_last_assistant_message_from_turn; -use crate::context_manager::ContextManager; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::protocol::CompactedItem; @@ -51,47 +50,11 @@ pub(crate) fn should_use_remote_compact_task(provider: &ModelProviderInfo) -> bo provider.is_openai() } -fn is_model_switch_developer_message(item: &ResponseItem) -> bool { - match item { - ResponseItem::Message { role, content, .. } if role == "developer" => { - matches!( - content.as_slice(), - [ContentItem::InputText { text }] if text.starts_with("\n") - ) - } - _ => false, - } -} - -pub(crate) fn extract_trailing_model_switch_update_for_compaction_request( - history: &mut ContextManager, -) -> Option { - let history_items = history.raw_items(); - let last_user_turn_boundary_index = history_items - .iter() - .rposition(crate::context_manager::is_user_turn_boundary); - let model_switch_index = history_items - .iter() - .enumerate() - .rev() - .find_map(|(i, item)| { - let is_trailing = last_user_turn_boundary_index.is_none_or(|boundary| i > boundary); - if is_trailing && is_model_switch_developer_message(item) { - Some(i) - } else { - None - } - })?; - let mut replacement = history_items.to_vec(); - let model_switch_item = replacement.remove(model_switch_index); - history.replace(replacement); - Some(model_switch_item) -} - pub(crate) async fn run_inline_auto_compact_task( sess: Arc, turn_context: Arc, initial_context_injection: InitialContextInjection, + previous_user_turn_model: Option<&str>, ) -> CodexResult<()> { let prompt = turn_context.compact_prompt().to_string(); let input = vec![UserInput::Text { @@ -100,7 +63,14 @@ pub(crate) async fn run_inline_auto_compact_task( text_elements: Vec::new(), }]; - run_compact_task_inner(sess, turn_context, input, initial_context_injection).await?; + run_compact_task_inner( + sess, + turn_context, + input, + initial_context_injection, + previous_user_turn_model, + ) + .await?; Ok(()) } @@ -120,6 +90,7 @@ pub(crate) async fn run_compact_task( turn_context, input, InitialContextInjection::DoNotInject, + None, ) .await } @@ -129,6 +100,7 @@ async fn run_compact_task_inner( turn_context: Arc, input: Vec, initial_context_injection: InitialContextInjection, + previous_user_turn_model: Option<&str>, ) -> CodexResult<()> { let compaction_item = TurnItem::ContextCompaction(ContextCompactionItem::new()); sess.emit_turn_item_started(&turn_context, &compaction_item) @@ -136,10 +108,6 @@ async fn run_compact_task_inner( let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); let mut history = sess.clone_history().await; - // Keep compaction prompts in-distribution: if a model-switch update was injected at the - // tail of history (between turns), exclude it from the compaction request payload. - let stripped_model_switch_item = - extract_trailing_model_switch_update_for_compaction_request(&mut history); history.record_items( &[initial_input_for_turn.into()], turn_context.truncation_policy, @@ -240,15 +208,12 @@ async fn run_compact_task_inner( initial_context_injection, InitialContextInjection::BeforeLastUserMessage ) { - let initial_context = sess.build_initial_context(turn_context.as_ref()).await; + let initial_context = sess + .build_initial_context(turn_context.as_ref(), previous_user_turn_model) + .await; new_history = insert_initial_context_before_last_real_user_or_summary(new_history, initial_context); } - // Reattach the stripped model-switch update only after successful compaction so the model - // still sees the switch instructions on the next real sampling request. - if let Some(model_switch_item) = stripped_model_switch_item { - new_history.push(model_switch_item); - } let ghost_snapshots: Vec = history_items .iter() .filter(|item| matches!(item, ResponseItem::GhostSnapshot { .. })) @@ -491,14 +456,18 @@ mod tests { async fn process_compacted_history_with_test_session( compacted_history: Vec, + previous_user_turn_model: Option<&str>, ) -> (Vec, Vec) { let (session, turn_context) = crate::codex::make_session_and_context().await; - let initial_context = session.build_initial_context(&turn_context).await; + let initial_context = session + .build_initial_context(&turn_context, previous_user_turn_model) + .await; let refreshed = crate::compact_remote::process_compacted_history( &session, &turn_context, compacted_history, InitialContextInjection::BeforeLastUserMessage, + previous_user_turn_model, ) .await; (refreshed, initial_context) @@ -534,107 +503,6 @@ mod tests { assert_eq!(None, joined); } - #[test] - fn extract_trailing_model_switch_update_for_compaction_request_removes_trailing_item() { - let mut history = ContextManager::new(); - history.replace(vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "USER_MESSAGE".to_string(), - }], - end_turn: None, - phase: None, - }, - ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: "ASSISTANT_REPLY".to_string(), - }], - end_turn: None, - phase: None, - }, - ResponseItem::Message { - id: None, - role: "developer".to_string(), - content: vec![ContentItem::InputText { - text: "\nNEW_MODEL_INSTRUCTIONS".to_string(), - }], - end_turn: None, - phase: None, - }, - ]); - - let model_switch_item = - extract_trailing_model_switch_update_for_compaction_request(&mut history); - - assert_eq!(history.raw_items().len(), 2); - assert!(model_switch_item.is_some()); - assert!( - history - .raw_items() - .iter() - .all(|item| !is_model_switch_developer_message(item)) - ); - } - - #[test] - fn extract_trailing_model_switch_update_for_compaction_request_keeps_historical_item() { - let mut history = ContextManager::new(); - history.replace(vec![ - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "FIRST_USER_MESSAGE".to_string(), - }], - end_turn: None, - phase: None, - }, - ResponseItem::Message { - id: None, - role: "developer".to_string(), - content: vec![ContentItem::InputText { - text: "\nOLDER_MODEL_INSTRUCTIONS".to_string(), - }], - end_turn: None, - phase: None, - }, - ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: "ASSISTANT_REPLY".to_string(), - }], - end_turn: None, - phase: None, - }, - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: "SECOND_USER_MESSAGE".to_string(), - }], - end_turn: None, - phase: None, - }, - ]); - - let model_switch_item = - extract_trailing_model_switch_update_for_compaction_request(&mut history); - - assert_eq!(history.raw_items().len(), 4); - assert!(model_switch_item.is_none()); - assert!( - history - .raw_items() - .iter() - .any(is_model_switch_developer_message) - ); - } - #[test] fn collect_user_messages_extracts_user_text_only() { let items = vec![ @@ -802,7 +670,7 @@ do things }, ]; let (refreshed, mut expected) = - process_compacted_history_with_test_session(compacted_history).await; + process_compacted_history_with_test_session(compacted_history, None).await; expected.push(ResponseItem::Message { id: None, role: "user".to_string(), @@ -827,7 +695,7 @@ do things phase: None, }]; let (refreshed, mut expected) = - process_compacted_history_with_test_session(compacted_history).await; + process_compacted_history_with_test_session(compacted_history, None).await; expected.push(ResponseItem::Message { id: None, role: "user".to_string(), @@ -903,7 +771,7 @@ keep me updated }, ]; let (refreshed, mut expected) = - process_compacted_history_with_test_session(compacted_history).await; + process_compacted_history_with_test_session(compacted_history, None).await; expected.push(ResponseItem::Message { id: None, role: "user".to_string(), @@ -949,7 +817,7 @@ keep me updated ]; let (refreshed, initial_context) = - process_compacted_history_with_test_session(compacted_history).await; + process_compacted_history_with_test_session(compacted_history, None).await; let mut expected = vec![ ResponseItem::Message { id: None, @@ -983,6 +851,46 @@ keep me updated assert_eq!(refreshed, expected); } + #[tokio::test] + async fn process_compacted_history_reinjects_model_switch_message() { + let compacted_history = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "summary".to_string(), + }], + end_turn: None, + phase: None, + }]; + + let (refreshed, initial_context) = process_compacted_history_with_test_session( + compacted_history, + Some("previous-regular-model"), + ) + .await; + + let ResponseItem::Message { role, content, .. } = &initial_context[0] else { + panic!("expected developer message"); + }; + assert_eq!(role, "developer"); + let [ContentItem::InputText { text }, ..] = content.as_slice() else { + panic!("expected developer text"); + }; + assert!(text.contains("")); + + let mut expected = initial_context; + expected.push(ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "summary".to_string(), + }], + end_turn: None, + phase: None, + }); + assert_eq!(refreshed, expected); + } + #[test] fn insert_initial_context_before_last_real_user_or_summary_keeps_summary_last() { let compacted_history = vec![ diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 2a9106c722..cc5f5164c3 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -4,7 +4,6 @@ use crate::Prompt; use crate::codex::Session; use crate::codex::TurnContext; use crate::compact::InitialContextInjection; -use crate::compact::extract_trailing_model_switch_update_for_compaction_request; use crate::compact::insert_initial_context_before_last_real_user_or_summary; use crate::context_manager::ContextManager; use crate::context_manager::TotalTokenUsageBreakdown; @@ -28,8 +27,15 @@ pub(crate) async fn run_inline_remote_auto_compact_task( sess: Arc, turn_context: Arc, initial_context_injection: InitialContextInjection, + previous_user_turn_model: Option<&str>, ) -> CodexResult<()> { - run_remote_compact_task_inner(&sess, &turn_context, initial_context_injection).await?; + run_remote_compact_task_inner( + &sess, + &turn_context, + initial_context_injection, + previous_user_turn_model, + ) + .await?; Ok(()) } @@ -44,16 +50,28 @@ pub(crate) async fn run_remote_compact_task( }); sess.send_event(&turn_context, start_event).await; - run_remote_compact_task_inner(&sess, &turn_context, InitialContextInjection::DoNotInject).await + run_remote_compact_task_inner( + &sess, + &turn_context, + InitialContextInjection::DoNotInject, + None, + ) + .await } async fn run_remote_compact_task_inner( sess: &Arc, turn_context: &Arc, initial_context_injection: InitialContextInjection, + previous_user_turn_model: Option<&str>, ) -> CodexResult<()> { - if let Err(err) = - run_remote_compact_task_inner_impl(sess, turn_context, initial_context_injection).await + if let Err(err) = run_remote_compact_task_inner_impl( + sess, + turn_context, + initial_context_injection, + previous_user_turn_model, + ) + .await { let event = EventMsg::Error( err.to_error_event(Some("Error running remote compact task".to_string())), @@ -68,15 +86,12 @@ async fn run_remote_compact_task_inner_impl( sess: &Arc, turn_context: &Arc, initial_context_injection: InitialContextInjection, + previous_user_turn_model: Option<&str>, ) -> CodexResult<()> { let compaction_item = TurnItem::ContextCompaction(ContextCompactionItem::new()); sess.emit_turn_item_started(turn_context, &compaction_item) .await; let mut history = sess.clone_history().await; - // Keep compaction prompts in-distribution: if a model-switch update was injected at the - // tail of history (between turns), exclude it from the compaction request payload. - let stripped_model_switch_item = - extract_trailing_model_switch_update_for_compaction_request(&mut history); let base_instructions = sess.get_base_instructions().await; let deleted_items = trim_function_call_history_to_fit_context_window( &mut history, @@ -90,7 +105,6 @@ async fn run_remote_compact_task_inner_impl( "trimmed history items before remote compaction" ); } - // Required to keep `/undo` available after compaction let ghost_snapshots: Vec = history .raw_items() @@ -134,13 +148,9 @@ async fn run_remote_compact_task_inner_impl( turn_context.as_ref(), new_history, initial_context_injection, + previous_user_turn_model, ) .await; - // Reattach the stripped model-switch update only after successful compaction so the model - // still sees the switch instructions on the next real sampling request. - if let Some(model_switch_item) = stripped_model_switch_item { - new_history.push(model_switch_item); - } if !ghost_snapshots.is_empty() { new_history.extend(ghost_snapshots); @@ -170,6 +180,7 @@ pub(crate) async fn process_compacted_history( turn_context: &TurnContext, mut compacted_history: Vec, initial_context_injection: InitialContextInjection, + previous_user_turn_model: Option<&str>, ) -> Vec { // Mid-turn compaction is the only path that must inject initial context above the last user // message in the replacement history. Pre-turn compaction instead injects context after the @@ -178,7 +189,8 @@ pub(crate) async fn process_compacted_history( initial_context_injection, InitialContextInjection::BeforeLastUserMessage ) { - sess.build_initial_context(turn_context).await + sess.build_initial_context(turn_context, previous_user_turn_model) + .await } else { Vec::new() }; @@ -205,10 +217,12 @@ pub(crate) async fn process_compacted_history( fn should_keep_compacted_history_item(item: &ResponseItem) -> bool { match item { ResponseItem::Message { role, .. } if role == "developer" => false, - ResponseItem::Message { role, .. } if role == "user" => matches!( - crate::event_mapping::parse_turn_item(item), - Some(TurnItem::UserMessage(_)) - ), + ResponseItem::Message { role, .. } if role == "user" => { + matches!( + crate::event_mapping::parse_turn_item(item), + Some(TurnItem::UserMessage(_)) + ) + } ResponseItem::Message { role, .. } if role == "assistant" => true, ResponseItem::Message { .. } => false, ResponseItem::Compaction { .. } => true, diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index fceb965998..592f50d907 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -195,6 +195,11 @@ mod document_helpers { { entry["scopes"] = array_from_iter(scopes.iter().cloned()); } + if let Some(resource) = &config.oauth_resource + && !resource.is_empty() + { + entry["oauth_resource"] = value(resource.clone()); + } entry } @@ -839,6 +844,30 @@ impl ConfigEditsBuilder { self } + pub fn set_realtime_microphone(mut self, microphone: Option<&str>) -> Self { + let segments = vec!["audio".to_string(), "microphone".to_string()]; + match microphone { + Some(microphone) => self.edits.push(ConfigEdit::SetPath { + segments, + value: value(microphone), + }), + None => self.edits.push(ConfigEdit::ClearPath { segments }), + } + self + } + + pub fn set_realtime_speaker(mut self, speaker: Option<&str>) -> Self { + let segments = vec!["audio".to_string(), "speaker".to_string()]; + match speaker { + Some(speaker) => self.edits.push(ConfigEdit::SetPath { + segments, + value: value(speaker), + }), + None => self.edits.push(ConfigEdit::ClearPath { segments }), + } + self + } + pub fn clear_legacy_windows_sandbox_keys(mut self) -> Self { for key in [ "experimental_windows_sandbox", @@ -1441,6 +1470,7 @@ gpt-5 = "gpt-5.1" enabled_tools: Some(vec!["one".to_string(), "two".to_string()]), disabled_tools: None, scopes: None, + oauth_resource: None, }, ); @@ -1465,6 +1495,7 @@ gpt-5 = "gpt-5.1" enabled_tools: None, disabled_tools: Some(vec!["forbidden".to_string()]), scopes: None, + oauth_resource: Some("https://resource.example.com".to_string()), }, ); @@ -1483,6 +1514,7 @@ bearer_token_env_var = \"TOKEN\" enabled = false startup_timeout_sec = 5.0 disabled_tools = [\"forbidden\"] +oauth_resource = \"https://resource.example.com\" [mcp_servers.http.http_headers] Z-Header = \"z\" @@ -1532,6 +1564,7 @@ foo = { command = "cmd" } enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); @@ -1578,6 +1611,7 @@ foo = { command = "cmd" } # keep me enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); @@ -1623,6 +1657,7 @@ foo = { command = "cmd", args = ["--flag"] } # keep me enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); @@ -1669,6 +1704,7 @@ foo = { command = "cmd" } enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); @@ -1804,6 +1840,50 @@ model_reasoning_effort = "high" assert_eq!(notice, Some(true)); } + #[test] + fn blocking_builder_set_realtime_audio_persists_and_clears() { + let tmp = tempdir().expect("tmpdir"); + let codex_home = tmp.path(); + + ConfigEditsBuilder::new(codex_home) + .set_realtime_microphone(Some("USB Mic")) + .set_realtime_speaker(Some("Desk Speakers")) + .apply_blocking() + .expect("persist realtime audio"); + + let raw = std::fs::read_to_string(codex_home.join(CONFIG_TOML_FILE)).expect("read config"); + let config: TomlValue = toml::from_str(&raw).expect("parse config"); + let realtime_audio = config + .get("audio") + .and_then(TomlValue::as_table) + .expect("audio table should exist"); + assert_eq!( + realtime_audio.get("microphone").and_then(TomlValue::as_str), + Some("USB Mic") + ); + assert_eq!( + realtime_audio.get("speaker").and_then(TomlValue::as_str), + Some("Desk Speakers") + ); + + ConfigEditsBuilder::new(codex_home) + .set_realtime_microphone(None) + .apply_blocking() + .expect("clear realtime microphone"); + + let raw = std::fs::read_to_string(codex_home.join(CONFIG_TOML_FILE)).expect("read config"); + let config: TomlValue = toml::from_str(&raw).expect("parse config"); + let realtime_audio = config + .get("audio") + .and_then(TomlValue::as_table) + .expect("audio table should exist"); + assert_eq!(realtime_audio.get("microphone"), None); + assert_eq!( + realtime_audio.get("speaker").and_then(TomlValue::as_str), + Some("Desk Speakers") + ); + } + #[test] fn replace_mcp_servers_blocking_clears_table_when_empty() { let tmp = tempdir().expect("tmpdir"); diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 306ba49c10..5eb1ad4c30 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -49,8 +49,6 @@ use crate::project_doc::LOCAL_PROJECT_DOC_FILENAME; use crate::protocol::AskForApproval; use crate::protocol::ReadOnlyAccess; use crate::protocol::SandboxPolicy; -#[cfg(target_os = "macos")] -use crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions; use crate::unified_exec::DEFAULT_MAX_BACKGROUND_TERMINAL_TIMEOUT_MS; use crate::unified_exec::MIN_EMPTY_YIELD_TIME_MS; use crate::windows_sandbox::WindowsSandboxLevelExt; @@ -66,6 +64,7 @@ use codex_protocol::config_types::TrustLevel; use codex_protocol::config_types::Verbosity; use codex_protocol::config_types::WebSearchMode; use codex_protocol::config_types::WindowsSandboxLevel; +use codex_protocol::models::MacOsSeatbeltProfileExtensions; use codex_protocol::openai_models::ModelsResponse; use codex_protocol::openai_models::ReasoningEffort; use codex_rmcp_client::OAuthCredentialsStoreMode; @@ -82,8 +81,6 @@ use std::path::Path; use std::path::PathBuf; #[cfg(test)] use tempfile::tempdir; -#[cfg(not(target_os = "macos"))] -type MacOsSeatbeltProfileExtensions = (); use crate::config::permissions::network_proxy_config_from_permissions; use crate::config::profile::ConfigProfile; @@ -412,9 +409,9 @@ pub struct Config { /// global default"). pub plan_mode_reasoning_effort: Option, - /// If not "none", the value to use for `reasoning.summary` when making a - /// request using the Responses API. - pub model_reasoning_summary: ReasoningSummary, + /// Optional value to use for `reasoning.summary` when making a request + /// using the Responses API. When unset, the model catalog default is used. + pub model_reasoning_summary: Option, /// Optional override to force-enable reasoning summaries for the configured model. pub model_supports_reasoning_summaries: Option, @@ -429,6 +426,9 @@ pub struct Config { /// Base URL for requests to ChatGPT (as opposed to the OpenAI API). pub chatgpt_base_url: String, + /// Machine-local realtime audio device preferences used by realtime voice. + pub realtime_audio: RealtimeAudioConfig, + /// Experimental / do not use. Overrides only the realtime conversation /// websocket transport base URL (the `Op::RealtimeConversation` `/ws` /// connection) without changing normal provider HTTP requests. @@ -1178,6 +1178,10 @@ pub struct ConfigToml { /// Base URL for requests to ChatGPT (as opposed to the OpenAI API). pub chatgpt_base_url: Option, + /// Machine-local realtime audio device preferences used by realtime voice. + #[serde(default)] + pub audio: Option, + /// Experimental / do not use. Overrides only the realtime conversation /// websocket transport base URL (the `Op::RealtimeConversation` `/ws` /// connection) without changing normal provider HTTP requests. @@ -1309,6 +1313,19 @@ impl ProjectConfig { } } +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RealtimeAudioConfig { + pub microphone: Option, + pub speaker: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct RealtimeAudioToml { + pub microphone: Option, + pub speaker: Option, +} + #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, JsonSchema)] #[schemars(deny_unknown_fields)] pub struct ToolsToml { @@ -2141,8 +2158,7 @@ impl Config { .or(cfg.plan_mode_reasoning_effort), model_reasoning_summary: config_profile .model_reasoning_summary - .or(cfg.model_reasoning_summary) - .unwrap_or_default(), + .or(cfg.model_reasoning_summary), model_supports_reasoning_summaries: cfg.model_supports_reasoning_summaries, model_catalog, model_verbosity: config_profile.model_verbosity.or(cfg.model_verbosity), @@ -2150,6 +2166,12 @@ impl Config { .chatgpt_base_url .or(cfg.chatgpt_base_url) .unwrap_or("https://chatgpt.com/backend-api/".to_string()), + realtime_audio: cfg + .audio + .map_or_else(RealtimeAudioConfig::default, |audio| RealtimeAudioConfig { + microphone: audio.microphone, + speaker: audio.speaker, + }), experimental_realtime_ws_base_url: cfg.experimental_realtime_ws_base_url, experimental_realtime_ws_backend_prompt: cfg.experimental_realtime_ws_backend_prompt, forced_chatgpt_workspace_id, @@ -2411,6 +2433,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, } } @@ -2430,6 +2453,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, } } @@ -2466,7 +2490,10 @@ persistence = "none" let memories = r#" [memories] +generate_memories = false +use_memories = false max_raw_memories_for_global = 512 +max_unused_days = 21 max_rollout_age_days = 42 max_rollouts_per_startup = 9 min_rollout_idle_hours = 24 @@ -2477,7 +2504,10 @@ phase_2_model = "gpt-5" toml::from_str::(memories).expect("TOML deserialization should succeed"); assert_eq!( Some(MemoriesToml { + generate_memories: Some(false), + use_memories: Some(false), max_raw_memories_for_global: Some(512), + max_unused_days: Some(21), max_rollout_age_days: Some(42), max_rollouts_per_startup: Some(9), min_rollout_idle_hours: Some(24), @@ -2496,7 +2526,10 @@ phase_2_model = "gpt-5" assert_eq!( config.memories, MemoriesConfig { + generate_memories: false, + use_memories: false, max_raw_memories_for_global: 512, + max_unused_days: 21, max_rollout_age_days: 42, max_rollouts_per_startup: 9, min_rollout_idle_hours: 24, @@ -3464,6 +3497,7 @@ profile = "project" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); @@ -3621,6 +3655,7 @@ bearer_token = "secret" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -3692,6 +3727,7 @@ ZIG_VAR = "3" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -3743,6 +3779,7 @@ ZIG_VAR = "3" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -3792,6 +3829,7 @@ ZIG_VAR = "3" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -3857,6 +3895,7 @@ startup_timeout_sec = 2.0 enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); apply_blocking( @@ -3934,6 +3973,7 @@ X-Auth = "DOCS_AUTH" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -3964,6 +4004,7 @@ X-Auth = "DOCS_AUTH" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); apply_blocking( @@ -4032,6 +4073,7 @@ url = "https://example.com/mcp" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ), ( @@ -4052,6 +4094,7 @@ url = "https://example.com/mcp" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ), ]); @@ -4135,6 +4178,7 @@ url = "https://example.com/mcp" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -4180,6 +4224,7 @@ url = "https://example.com/mcp" enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -4225,6 +4270,7 @@ url = "https://example.com/mcp" enabled_tools: Some(vec!["allowed".to_string()]), disabled_tools: Some(vec!["blocked".to_string()]), scopes: None, + oauth_resource: None, }, )]); @@ -4253,6 +4299,51 @@ url = "https://example.com/mcp" Ok(()) } + #[tokio::test] + async fn replace_mcp_servers_streamable_http_serializes_oauth_resource() -> anyhow::Result<()> { + let codex_home = TempDir::new()?; + + let servers = BTreeMap::from([( + "docs".to_string(), + McpServerConfig { + transport: McpServerTransportConfig::StreamableHttp { + url: "https://example.com/mcp".to_string(), + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, + }, + enabled: true, + required: false, + disabled_reason: None, + startup_timeout_sec: None, + tool_timeout_sec: None, + enabled_tools: None, + disabled_tools: None, + scopes: None, + oauth_resource: Some("https://resource.example.com".to_string()), + }, + )]); + + apply_blocking( + codex_home.path(), + None, + &[ConfigEdit::ReplaceMcpServers(servers.clone())], + )?; + + let config_path = codex_home.path().join(CONFIG_TOML_FILE); + let serialized = std::fs::read_to_string(&config_path)?; + assert!(serialized.contains(r#"oauth_resource = "https://resource.example.com""#)); + + let loaded = load_global_mcp_servers(codex_home.path()).await?; + let docs = loaded.get("docs").expect("docs entry"); + assert_eq!( + docs.oauth_resource.as_deref(), + Some("https://resource.example.com") + ); + + Ok(()) + } + #[tokio::test] async fn set_model_updates_defaults() -> anyhow::Result<()> { let codex_home = TempDir::new()?; @@ -4761,12 +4852,13 @@ model_verbosity = "high" show_raw_agent_reasoning: false, model_reasoning_effort: Some(ReasoningEffort::High), plan_mode_reasoning_effort: None, - model_reasoning_summary: ReasoningSummary::Detailed, + model_reasoning_summary: Some(ReasoningSummary::Detailed), model_supports_reasoning_summaries: None, model_catalog: None, model_verbosity: None, personality: Some(Personality::Pragmatic), chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), + realtime_audio: RealtimeAudioConfig::default(), experimental_realtime_ws_base_url: None, experimental_realtime_ws_backend_prompt: None, base_instructions: None, @@ -4887,12 +4979,13 @@ model_verbosity = "high" show_raw_agent_reasoning: false, model_reasoning_effort: None, plan_mode_reasoning_effort: None, - model_reasoning_summary: ReasoningSummary::default(), + model_reasoning_summary: None, model_supports_reasoning_summaries: None, model_catalog: None, model_verbosity: None, personality: Some(Personality::Pragmatic), chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), + realtime_audio: RealtimeAudioConfig::default(), experimental_realtime_ws_base_url: None, experimental_realtime_ws_backend_prompt: None, base_instructions: None, @@ -5011,12 +5104,13 @@ model_verbosity = "high" show_raw_agent_reasoning: false, model_reasoning_effort: None, plan_mode_reasoning_effort: None, - model_reasoning_summary: ReasoningSummary::default(), + model_reasoning_summary: None, model_supports_reasoning_summaries: None, model_catalog: None, model_verbosity: None, personality: Some(Personality::Pragmatic), chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), + realtime_audio: RealtimeAudioConfig::default(), experimental_realtime_ws_base_url: None, experimental_realtime_ws_backend_prompt: None, base_instructions: None, @@ -5121,12 +5215,13 @@ model_verbosity = "high" show_raw_agent_reasoning: false, model_reasoning_effort: Some(ReasoningEffort::High), plan_mode_reasoning_effort: None, - model_reasoning_summary: ReasoningSummary::Detailed, + model_reasoning_summary: Some(ReasoningSummary::Detailed), model_supports_reasoning_summaries: None, model_catalog: None, model_verbosity: Some(Verbosity::High), personality: Some(Personality::Pragmatic), chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), + realtime_audio: RealtimeAudioConfig::default(), experimental_realtime_ws_base_url: None, experimental_realtime_ws_backend_prompt: None, base_instructions: None, @@ -5971,6 +6066,39 @@ experimental_realtime_ws_backend_prompt = "prompt from config" ); Ok(()) } + + #[test] + fn realtime_audio_loads_from_config_toml() -> std::io::Result<()> { + let cfg: ConfigToml = toml::from_str( + r#" +[audio] +microphone = "USB Mic" +speaker = "Desk Speakers" +"#, + ) + .expect("TOML deserialization should succeed"); + + let realtime_audio = cfg + .audio + .as_ref() + .expect("realtime audio config should be present"); + assert_eq!(realtime_audio.microphone.as_deref(), Some("USB Mic")); + assert_eq!(realtime_audio.speaker.as_deref(), Some("Desk Speakers")); + + let codex_home = TempDir::new()?; + let config = Config::load_from_base_config_with_overrides( + cfg, + ConfigOverrides::default(), + codex_home.path().to_path_buf(), + )?; + + assert_eq!(config.realtime_audio.microphone.as_deref(), Some("USB Mic")); + assert_eq!( + config.realtime_audio.speaker.as_deref(), + Some("Desk Speakers") + ); + Ok(()) + } } #[cfg(test)] diff --git a/codex-rs/core/src/config/types.rs b/codex-rs/core/src/config/types.rs index 678766e807..3ee85226e1 100644 --- a/codex-rs/core/src/config/types.rs +++ b/codex-rs/core/src/config/types.rs @@ -27,6 +27,7 @@ pub const DEFAULT_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 16; pub const DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS: i64 = 30; pub const DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS: i64 = 6; pub const DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL: usize = 1_024; +pub const DEFAULT_MEMORIES_MAX_UNUSED_DAYS: i64 = 30; #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "kebab-case")] @@ -98,6 +99,10 @@ pub struct McpServerConfig { /// Optional OAuth scopes to request during MCP login. #[serde(default, skip_serializing_if = "Option::is_none")] pub scopes: Option>, + + /// Optional OAuth resource parameter to include during MCP login (RFC 8707). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oauth_resource: Option, } // Raw MCP config shape used for deserialization and JSON Schema generation. @@ -142,6 +147,8 @@ pub(crate) struct RawMcpServerConfig { pub disabled_tools: Option>, #[serde(default)] pub scopes: Option>, + #[serde(default)] + pub oauth_resource: Option, } impl<'de> Deserialize<'de> for McpServerConfig { @@ -165,6 +172,7 @@ impl<'de> Deserialize<'de> for McpServerConfig { let enabled_tools = raw.enabled_tools.clone(); let disabled_tools = raw.disabled_tools.clone(); let scopes = raw.scopes.clone(); + let oauth_resource = raw.oauth_resource.clone(); fn throw_if_set(transport: &str, field: &str, value: Option<&T>) -> Result<(), E> where @@ -188,6 +196,7 @@ impl<'de> Deserialize<'de> for McpServerConfig { throw_if_set("stdio", "bearer_token", raw.bearer_token.as_ref())?; throw_if_set("stdio", "http_headers", raw.http_headers.as_ref())?; throw_if_set("stdio", "env_http_headers", raw.env_http_headers.as_ref())?; + throw_if_set("stdio", "oauth_resource", raw.oauth_resource.as_ref())?; McpServerTransportConfig::Stdio { command, args: raw.args.clone().unwrap_or_default(), @@ -221,6 +230,7 @@ impl<'de> Deserialize<'de> for McpServerConfig { enabled_tools, disabled_tools, scopes, + oauth_resource, }) } } @@ -361,8 +371,14 @@ pub struct FeedbackConfigToml { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, JsonSchema)] #[schemars(deny_unknown_fields)] pub struct MemoriesToml { + /// When `false`, newly created threads are stored with `memory_mode = "disabled"` in the state DB. + pub generate_memories: Option, + /// When `false`, skip injecting memory usage instructions into developer prompts. + pub use_memories: Option, /// Maximum number of recent raw memories retained for global consolidation. pub max_raw_memories_for_global: Option, + /// Maximum number of days since a memory was last used before it becomes ineligible for phase 2 selection. + pub max_unused_days: Option, /// Maximum age of the threads used for memories. pub max_rollout_age_days: Option, /// Maximum number of rollout candidates processed per pass. @@ -378,7 +394,10 @@ pub struct MemoriesToml { /// Effective memories settings after defaults are applied. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MemoriesConfig { + pub generate_memories: bool, + pub use_memories: bool, pub max_raw_memories_for_global: usize, + pub max_unused_days: i64, pub max_rollout_age_days: i64, pub max_rollouts_per_startup: usize, pub min_rollout_idle_hours: i64, @@ -389,7 +408,10 @@ pub struct MemoriesConfig { impl Default for MemoriesConfig { fn default() -> Self { Self { + generate_memories: true, + use_memories: true, max_raw_memories_for_global: DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_GLOBAL, + max_unused_days: DEFAULT_MEMORIES_MAX_UNUSED_DAYS, max_rollout_age_days: DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS, max_rollouts_per_startup: DEFAULT_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, min_rollout_idle_hours: DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS, @@ -403,10 +425,16 @@ impl From for MemoriesConfig { fn from(toml: MemoriesToml) -> Self { let defaults = Self::default(); Self { + generate_memories: toml.generate_memories.unwrap_or(defaults.generate_memories), + use_memories: toml.use_memories.unwrap_or(defaults.use_memories), max_raw_memories_for_global: toml .max_raw_memories_for_global .unwrap_or(defaults.max_raw_memories_for_global) .min(4096), + max_unused_days: toml + .max_unused_days + .unwrap_or(defaults.max_unused_days) + .clamp(0, 365), max_rollout_age_days: toml .max_rollout_age_days .unwrap_or(defaults.max_rollout_age_days) @@ -1084,6 +1112,22 @@ mod tests { ); } + #[test] + fn deserialize_streamable_http_server_config_with_oauth_resource() { + let cfg: McpServerConfig = toml::from_str( + r#" + url = "https://example.com/mcp" + oauth_resource = "https://api.example.com" + "#, + ) + .expect("should deserialize http config with oauth_resource"); + + assert_eq!( + cfg.oauth_resource, + Some("https://api.example.com".to_string()) + ); + } + #[test] fn deserialize_server_config_with_tool_filters() { let cfg: McpServerConfig = toml::from_str( @@ -1138,6 +1182,20 @@ mod tests { "#, ) .expect_err("should reject env_http_headers for stdio transport"); + + let err = toml::from_str::( + r#" + command = "echo" + oauth_resource = "https://api.example.com" + "#, + ) + .expect_err("should reject oauth_resource for stdio transport"); + + assert!( + err.to_string() + .contains("oauth_resource is not supported for stdio"), + "unexpected error: {err}" + ); } #[test] diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index f6f7198f6e..e4b7755abf 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,14 +1,11 @@ use crate::codex::TurnContext; use crate::context_manager::normalize; -use crate::instructions::SkillInstructions; -use crate::instructions::UserInstructions; -use crate::session_prefix::is_session_prefix; +use crate::event_mapping::is_contextual_user_message_content; use crate::truncate::TruncationPolicy; use crate::truncate::approx_token_count; use crate::truncate::approx_tokens_from_byte_count_i64; use crate::truncate::truncate_function_output_items_with_policy; use crate::truncate::truncate_text; -use crate::user_shell_command::is_user_shell_command_text; use codex_protocol::models::BaseInstructions; use codex_protocol::models::ContentItem; use codex_protocol::models::FunctionCallOutputBody; @@ -347,32 +344,21 @@ impl ContextManager { let policy_with_serialization_budget = policy * 1.2; match item { ResponseItem::FunctionCallOutput { call_id, output } => { - let body = match &output.body { - FunctionCallOutputBody::Text(content) => FunctionCallOutputBody::Text( - truncate_text(content, policy_with_serialization_budget), - ), - FunctionCallOutputBody::ContentItems(items) => { - FunctionCallOutputBody::ContentItems( - truncate_function_output_items_with_policy( - items, - policy_with_serialization_budget, - ), - ) - } - }; ResponseItem::FunctionCallOutput { call_id: call_id.clone(), - output: FunctionCallOutputPayload { - body, - success: output.success, - }, + output: truncate_function_output_payload( + output, + policy_with_serialization_budget, + ), } } ResponseItem::CustomToolCallOutput { call_id, output } => { - let truncated = truncate_text(output, policy_with_serialization_budget); ResponseItem::CustomToolCallOutput { call_id: call_id.clone(), - output: truncated, + output: truncate_function_output_payload( + output, + policy_with_serialization_budget, + ), } } ResponseItem::Message { .. } @@ -388,6 +374,25 @@ impl ContextManager { } } +fn truncate_function_output_payload( + output: &FunctionCallOutputPayload, + policy: TruncationPolicy, +) -> FunctionCallOutputPayload { + let body = match &output.body { + FunctionCallOutputBody::Text(content) => { + FunctionCallOutputBody::Text(truncate_text(content, policy)) + } + FunctionCallOutputBody::ContentItems(items) => FunctionCallOutputBody::ContentItems( + truncate_function_output_items_with_policy(items, policy), + ), + }; + + FunctionCallOutputPayload { + body, + success: output.success, + } +} + /// API messages include every non-system item (user/assistant messages, reasoning, /// tool calls, tool outputs, shell calls, and web-search calls). fn is_api_message(message: &ResponseItem) -> bool { @@ -511,7 +516,8 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { } } } - ResponseItem::FunctionCallOutput { output, .. } => { + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { if let FunctionCallOutputBody::ContentItems(items) = &output.body { for content_item in items { if let FunctionCallOutputContentItem::InputImage { image_url } = content_item { @@ -554,33 +560,7 @@ pub(crate) fn is_user_turn_boundary(item: &ResponseItem) -> bool { return false; }; - if role != "user" { - return false; - } - - if UserInstructions::is_user_instructions(content) - || SkillInstructions::is_skill_instructions(content) - { - return false; - } - - for content_item in content { - match content_item { - ContentItem::InputText { text } => { - if is_session_prefix(text) || is_user_shell_command_text(text) { - return false; - } - } - ContentItem::OutputText { text } => { - if is_session_prefix(text) { - return false; - } - } - ContentItem::InputImage { .. } => {} - } - } - - true + role == "user" && !is_contextual_user_message_content(content) } fn user_message_positions(items: &[ResponseItem]) -> Vec { diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 3e11dae40a..798abc7675 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -67,7 +67,7 @@ fn user_input_text_msg(text: &str) -> ResponseItem { fn custom_tool_call_output(call_id: &str, output: &str) -> ResponseItem { ResponseItem::CustomToolCallOutput { call_id: call_id.to_string(), - output: output.to_string(), + output: FunctionCallOutputPayload::from_text(output.to_string()), } } @@ -279,6 +279,24 @@ fn for_prompt_strips_images_when_model_does_not_support_images() { }, ]), }, + ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "tool-1".to_string(), + name: "js_repl".to_string(), + input: "view_image".to_string(), + }, + ResponseItem::CustomToolCallOutput { + call_id: "tool-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputText { + text: "js repl result".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "https://example.com/js-repl-result.png".to_string(), + }, + ]), + }, ]; let history = create_history_with_items(items); let text_only_modalities = vec![InputModality::Text]; @@ -321,6 +339,25 @@ fn for_prompt_strips_images_when_model_does_not_support_images() { }, ]), }, + ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "tool-1".to_string(), + name: "js_repl".to_string(), + input: "view_image".to_string(), + }, + ResponseItem::CustomToolCallOutput { + call_id: "tool-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputText { + text: "js repl result".to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: "image content omitted because you do not support image input" + .to_string(), + }, + ]), + }, ]; assert_eq!(stripped, expected); @@ -563,7 +600,6 @@ fn drop_last_n_user_turns_preserves_prefix() { fn drop_last_n_user_turns_ignores_session_prefix_user_messages() { let items = vec![ user_input_text_msg("ctx"), - user_input_text_msg("do the thing"), user_input_text_msg( "# AGENTS.md instructions for test_directory\n\n\ntest_text\n", ), @@ -586,7 +622,6 @@ fn drop_last_n_user_turns_ignores_session_prefix_user_messages() { let expected_prefix_and_first_turn = vec![ user_input_text_msg("ctx"), - user_input_text_msg("do the thing"), user_input_text_msg( "# AGENTS.md instructions for test_directory\n\n\ntest_text\n", ), @@ -608,7 +643,6 @@ fn drop_last_n_user_turns_ignores_session_prefix_user_messages() { let expected_prefix_only = vec![ user_input_text_msg("ctx"), - user_input_text_msg("do the thing"), user_input_text_msg( "# AGENTS.md instructions for test_directory\n\n\ntest_text\n", ), @@ -623,7 +657,6 @@ fn drop_last_n_user_turns_ignores_session_prefix_user_messages() { let mut history = create_history_with_items(vec![ user_input_text_msg("ctx"), - user_input_text_msg("do the thing"), user_input_text_msg( "# AGENTS.md instructions for test_directory\n\n\ntest_text\n", ), @@ -644,7 +677,6 @@ fn drop_last_n_user_turns_ignores_session_prefix_user_messages() { let mut history = create_history_with_items(vec![ user_input_text_msg("ctx"), - user_input_text_msg("do the thing"), user_input_text_msg( "# AGENTS.md instructions for test_directory\n\n\ntest_text\n", ), @@ -676,7 +708,7 @@ fn remove_first_item_handles_custom_tool_pair() { }, ResponseItem::CustomToolCallOutput { call_id: "tool-1".to_string(), - output: "ok".to_string(), + output: FunctionCallOutputPayload::from_text("ok".to_string()), }, ]; let mut h = create_history_with_items(items); @@ -755,7 +787,7 @@ fn record_items_truncates_custom_tool_call_output_content() { let long_output = line.repeat(2_500); let item = ResponseItem::CustomToolCallOutput { call_id: "tool-200".to_string(), - output: long_output.clone(), + output: FunctionCallOutputPayload::from_text(long_output.clone()), }; history.record_items([&item], policy); @@ -763,7 +795,8 @@ fn record_items_truncates_custom_tool_call_output_content() { assert_eq!(history.items.len(), 1); match &history.items[0] { ResponseItem::CustomToolCallOutput { output, .. } => { - assert_ne!(output, &long_output); + let output = output.text_content().unwrap_or_default(); + assert_ne!(output, long_output); assert!( output.contains("tokens truncated"), "expected token-based truncation marker, got {output}" @@ -954,7 +987,7 @@ fn normalize_adds_missing_output_for_custom_tool_call() { }, ResponseItem::CustomToolCallOutput { call_id: "tool-x".to_string(), - output: "aborted".to_string(), + output: FunctionCallOutputPayload::from_text("aborted".to_string()), }, ] ); @@ -1021,7 +1054,7 @@ fn normalize_removes_orphan_function_call_output() { fn normalize_removes_orphan_custom_tool_call_output() { let items = vec![ResponseItem::CustomToolCallOutput { call_id: "orphan-2".to_string(), - output: "ok".to_string(), + output: FunctionCallOutputPayload::from_text("ok".to_string()), }]; let mut h = create_history_with_items(items); @@ -1094,7 +1127,7 @@ fn normalize_mixed_inserts_and_removals() { }, ResponseItem::CustomToolCallOutput { call_id: "t1".to_string(), - output: "aborted".to_string(), + output: FunctionCallOutputPayload::from_text("aborted".to_string()), }, ResponseItem::LocalShellCall { id: None, @@ -1196,7 +1229,7 @@ fn normalize_removes_orphan_function_call_output_panics_in_debug() { fn normalize_removes_orphan_custom_tool_call_output_panics_in_debug() { let items = vec![ResponseItem::CustomToolCallOutput { call_id: "orphan-2".to_string(), - output: "ok".to_string(), + output: FunctionCallOutputPayload::from_text("ok".to_string()), }]; let mut h = create_history_with_items(items); h.normalize_history(&default_input_modalities()); @@ -1299,6 +1332,28 @@ fn image_data_url_payload_does_not_dominate_function_call_output_estimate() { assert!(estimated < raw_len); } +#[test] +fn image_data_url_payload_does_not_dominate_custom_tool_call_output_estimate() { + let payload = "C".repeat(50_000); + let image_url = format!("data:image/png;base64,{payload}"); + let item = ResponseItem::CustomToolCallOutput { + call_id: "call-js-repl".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputText { + text: "Screenshot captured".to_string(), + }, + FunctionCallOutputContentItem::InputImage { image_url }, + ]), + }; + + let raw_len = serde_json::to_string(&item).unwrap().len() as i64; + let estimated = estimate_response_item_model_visible_bytes(&item); + let expected = raw_len - payload.len() as i64 + IMAGE_BYTES_ESTIMATE; + + assert_eq!(estimated, expected); + assert!(estimated < raw_len); +} + #[test] fn non_base64_image_urls_are_unchanged() { let message_item = ResponseItem::Message { diff --git a/codex-rs/core/src/context_manager/normalize.rs b/codex-rs/core/src/context_manager/normalize.rs index a4fe9e64fd..572ac51fc8 100644 --- a/codex-rs/core/src/context_manager/normalize.rs +++ b/codex-rs/core/src/context_manager/normalize.rs @@ -1,7 +1,6 @@ use std::collections::HashSet; use codex_protocol::models::ContentItem; -use codex_protocol::models::FunctionCallOutputBody; use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; @@ -35,10 +34,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { idx, ResponseItem::FunctionCallOutput { call_id: call_id.clone(), - output: FunctionCallOutputPayload { - body: FunctionCallOutputBody::Text("aborted".to_string()), - ..Default::default() - }, + output: FunctionCallOutputPayload::from_text("aborted".to_string()), }, )); } @@ -59,7 +55,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { idx, ResponseItem::CustomToolCallOutput { call_id: call_id.clone(), - output: "aborted".to_string(), + output: FunctionCallOutputPayload::from_text("aborted".to_string()), }, )); } @@ -82,10 +78,7 @@ pub(crate) fn ensure_call_outputs_present(items: &mut Vec) { idx, ResponseItem::FunctionCallOutput { call_id: call_id.clone(), - output: FunctionCallOutputPayload { - body: FunctionCallOutputBody::Text("aborted".to_string()), - ..Default::default() - }, + output: FunctionCallOutputPayload::from_text("aborted".to_string()), }, )); } @@ -245,7 +238,8 @@ pub(crate) fn strip_images_when_unsupported( } *content = normalized_content; } - ResponseItem::FunctionCallOutput { output, .. } => { + ResponseItem::FunctionCallOutput { output, .. } + | ResponseItem::CustomToolCallOutput { output, .. } => { if let Some(content_items) = output.content_items_mut() { let mut normalized_content_items = Vec::with_capacity(content_items.len()); for content_item in content_items.iter() { diff --git a/codex-rs/core/src/context_manager/updates.rs b/codex-rs/core/src/context_manager/updates.rs index 54c5667d4d..b9fc537ec7 100644 --- a/codex-rs/core/src/context_manager/updates.rs +++ b/codex-rs/core/src/context_manager/updates.rs @@ -4,6 +4,7 @@ use crate::features::Feature; use crate::shell::Shell; use codex_execpolicy::Policy; use codex_protocol::config_types::Personality; +use codex_protocol::models::ContentItem; use codex_protocol::models::DeveloperInstructions; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ModelInfo; @@ -30,7 +31,7 @@ fn build_permissions_update_item( previous: Option<&TurnContextItem>, next: &TurnContext, exec_policy: &Policy, -) -> Option { +) -> Option { let prev = previous?; if prev.sandbox_policy == *next.sandbox_policy.get() && prev.approval_policy == next.approval_policy.value() @@ -38,27 +39,26 @@ fn build_permissions_update_item( return None; } - Some( - DeveloperInstructions::from_policy( - next.sandbox_policy.get(), - next.approval_policy.value(), - exec_policy, - &next.cwd, - next.features.enabled(Feature::RequestPermissions), - ) - .into(), - ) + Some(DeveloperInstructions::from_policy( + next.sandbox_policy.get(), + next.approval_policy.value(), + exec_policy, + &next.cwd, + next.features.enabled(Feature::RequestPermissions), + )) } fn build_collaboration_mode_update_item( previous: Option<&TurnContextItem>, next: &TurnContext, -) -> Option { +) -> Option { let prev = previous?; if prev.collaboration_mode.as_ref() != Some(&next.collaboration_mode) { // If the next mode has empty developer instructions, this returns None and we emit no // update, so prior collaboration instructions remain in the prompt history. - Some(DeveloperInstructions::from_collaboration_mode(&next.collaboration_mode)?.into()) + Some(DeveloperInstructions::from_collaboration_mode( + &next.collaboration_mode, + )?) } else { None } @@ -68,7 +68,7 @@ fn build_personality_update_item( previous: Option<&TurnContextItem>, next: &TurnContext, personality_feature_enabled: bool, -) -> Option { +) -> Option { if !personality_feature_enabled { return None; } @@ -82,8 +82,7 @@ fn build_personality_update_item( { let model_info = &next.model_info; let personality_message = personality_message_for(model_info, personality); - personality_message - .map(|message| DeveloperInstructions::personality_spec_message(message).into()) + personality_message.map(DeveloperInstructions::personality_spec_message) } else { None } @@ -103,7 +102,7 @@ pub(crate) fn personality_message_for( pub(crate) fn build_model_instructions_update_item( previous_user_turn_model: Option<&str>, next: &TurnContext, -) -> Option { +) -> Option { let previous_model = previous_user_turn_model?; if previous_model == next.model_info.slug { return None; @@ -114,7 +113,36 @@ pub(crate) fn build_model_instructions_update_item( return None; } - Some(DeveloperInstructions::model_switch_message(model_instructions).into()) + Some(DeveloperInstructions::model_switch_message( + model_instructions, + )) +} + +pub(crate) fn build_developer_update_item(text_sections: Vec) -> Option { + build_text_message("developer", text_sections) +} + +pub(crate) fn build_contextual_user_message(text_sections: Vec) -> Option { + build_text_message("user", text_sections) +} + +fn build_text_message(role: &str, text_sections: Vec) -> Option { + if text_sections.is_empty() { + return None; + } + + let content = text_sections + .into_iter() + .map(|text| ContentItem::InputText { text }) + .collect(); + + Some(ResponseItem::Message { + id: None, + role: role.to_string(), + content, + end_turn: None, + phase: None, + }) } pub(crate) fn build_settings_update_items( @@ -125,29 +153,26 @@ pub(crate) fn build_settings_update_items( exec_policy: &Policy, personality_feature_enabled: bool, ) -> Vec { - let mut update_items = Vec::new(); + let contextual_user_message = build_environment_update_item(previous, next, shell); + let developer_update_sections = [ + // Keep model-switch instructions first so model-specific guidance is read before + // any other context diffs on this turn. + build_model_instructions_update_item(previous_user_turn_model, next), + build_permissions_update_item(previous, next, exec_policy), + build_collaboration_mode_update_item(previous, next), + build_personality_update_item(previous, next, personality_feature_enabled), + ] + .into_iter() + .flatten() + .map(DeveloperInstructions::into_text) + .collect(); - // Keep model-switch instructions first so model-specific guidance is read before - // any other context diffs on this turn. - if let Some(model_instructions_item) = - build_model_instructions_update_item(previous_user_turn_model, next) - { - update_items.push(model_instructions_item); + let mut items = Vec::with_capacity(2); + if let Some(developer_message) = build_developer_update_item(developer_update_sections) { + items.push(developer_message); } - if let Some(env_item) = build_environment_update_item(previous, next, shell) { - update_items.push(env_item); + if let Some(contextual_user_message) = contextual_user_message { + items.push(contextual_user_message); } - if let Some(permissions_item) = build_permissions_update_item(previous, next, exec_policy) { - update_items.push(permissions_item); - } - if let Some(collaboration_mode_item) = build_collaboration_mode_update_item(previous, next) { - update_items.push(collaboration_mode_item); - } - if let Some(personality_item) = - build_personality_update_item(previous, next, personality_feature_enabled) - { - update_items.push(personality_item); - } - - update_items + items } diff --git a/codex-rs/core/src/contextual_user_message.rs b/codex-rs/core/src/contextual_user_message.rs new file mode 100644 index 0000000000..51a2d23ea9 --- /dev/null +++ b/codex-rs/core/src/contextual_user_message.rs @@ -0,0 +1,139 @@ +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::ENVIRONMENT_CONTEXT_CLOSE_TAG; +use codex_protocol::protocol::ENVIRONMENT_CONTEXT_OPEN_TAG; + +pub(crate) const AGENTS_MD_START_MARKER: &str = "# AGENTS.md instructions for "; +pub(crate) const AGENTS_MD_END_MARKER: &str = ""; +pub(crate) const SKILL_OPEN_TAG: &str = ""; +pub(crate) const SKILL_CLOSE_TAG: &str = ""; +pub(crate) const USER_SHELL_COMMAND_OPEN_TAG: &str = ""; +pub(crate) const USER_SHELL_COMMAND_CLOSE_TAG: &str = ""; +pub(crate) const TURN_ABORTED_OPEN_TAG: &str = ""; +pub(crate) const TURN_ABORTED_CLOSE_TAG: &str = ""; +pub(crate) const SUBAGENT_NOTIFICATION_OPEN_TAG: &str = ""; +pub(crate) const SUBAGENT_NOTIFICATION_CLOSE_TAG: &str = ""; + +#[derive(Clone, Copy)] +pub(crate) struct ContextualUserFragmentDefinition { + start_marker: &'static str, + end_marker: &'static str, +} + +impl ContextualUserFragmentDefinition { + pub(crate) const fn new(start_marker: &'static str, end_marker: &'static str) -> Self { + Self { + start_marker, + end_marker, + } + } + + pub(crate) fn matches_text(&self, text: &str) -> bool { + let trimmed = text.trim_start(); + let starts_with_marker = trimmed + .get(..self.start_marker.len()) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(self.start_marker)); + let trimmed = trimmed.trim_end(); + let ends_with_marker = trimmed + .get(trimmed.len().saturating_sub(self.end_marker.len())..) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(self.end_marker)); + starts_with_marker && ends_with_marker + } + + pub(crate) const fn start_marker(&self) -> &'static str { + self.start_marker + } + + pub(crate) const fn end_marker(&self) -> &'static str { + self.end_marker + } + + pub(crate) fn wrap(&self, body: String) -> String { + format!("{}\n{}\n{}", self.start_marker, body, self.end_marker) + } + + pub(crate) fn into_message(self, text: String) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { text }], + end_turn: None, + phase: None, + } + } +} + +pub(crate) const AGENTS_MD_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new(AGENTS_MD_START_MARKER, AGENTS_MD_END_MARKER); +pub(crate) const ENVIRONMENT_CONTEXT_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new( + ENVIRONMENT_CONTEXT_OPEN_TAG, + ENVIRONMENT_CONTEXT_CLOSE_TAG, + ); +pub(crate) const SKILL_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new(SKILL_OPEN_TAG, SKILL_CLOSE_TAG); +pub(crate) const USER_SHELL_COMMAND_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new( + USER_SHELL_COMMAND_OPEN_TAG, + USER_SHELL_COMMAND_CLOSE_TAG, + ); +pub(crate) const TURN_ABORTED_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new(TURN_ABORTED_OPEN_TAG, TURN_ABORTED_CLOSE_TAG); +pub(crate) const SUBAGENT_NOTIFICATION_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new( + SUBAGENT_NOTIFICATION_OPEN_TAG, + SUBAGENT_NOTIFICATION_CLOSE_TAG, + ); + +const CONTEXTUAL_USER_FRAGMENTS: &[ContextualUserFragmentDefinition] = &[ + AGENTS_MD_FRAGMENT, + ENVIRONMENT_CONTEXT_FRAGMENT, + SKILL_FRAGMENT, + USER_SHELL_COMMAND_FRAGMENT, + TURN_ABORTED_FRAGMENT, + SUBAGENT_NOTIFICATION_FRAGMENT, +]; + +pub(crate) fn is_contextual_user_fragment(content_item: &ContentItem) -> bool { + let ContentItem::InputText { text } = content_item else { + return false; + }; + CONTEXTUAL_USER_FRAGMENTS + .iter() + .any(|definition| definition.matches_text(text)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_environment_context_fragment() { + assert!(is_contextual_user_fragment(&ContentItem::InputText { + text: "\n/tmp\n".to_string(), + })); + } + + #[test] + fn detects_agents_instructions_fragment() { + assert!(is_contextual_user_fragment(&ContentItem::InputText { + text: "# AGENTS.md instructions for /tmp\n\n\nbody\n" + .to_string(), + })); + } + + #[test] + fn detects_subagent_notification_fragment_case_insensitively() { + assert!( + SUBAGENT_NOTIFICATION_FRAGMENT + .matches_text("{}") + ); + } + + #[test] + fn ignores_regular_user_text() { + assert!(!is_contextual_user_fragment(&ContentItem::InputText { + text: "hello".to_string(), + })); + } +} diff --git a/codex-rs/core/src/environment_context.rs b/codex-rs/core/src/environment_context.rs index 3c6df40adb..3e9ed871e3 100644 --- a/codex-rs/core/src/environment_context.rs +++ b/codex-rs/core/src/environment_context.rs @@ -1,9 +1,7 @@ use crate::codex::TurnContext; +use crate::contextual_user_message::ENVIRONMENT_CONTEXT_FRAGMENT; use crate::shell::Shell; -use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::ENVIRONMENT_CONTEXT_CLOSE_TAG; -use codex_protocol::protocol::ENVIRONMENT_CONTEXT_OPEN_TAG; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::TurnContextNetworkItem; use serde::Deserialize; @@ -15,7 +13,10 @@ use std::path::PathBuf; pub(crate) struct EnvironmentContext { pub cwd: Option, pub shell: Shell, + pub current_date: Option, + pub timezone: Option, pub network: Option, + pub subagents: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] @@ -25,11 +26,21 @@ pub(crate) struct NetworkContext { } impl EnvironmentContext { - pub fn new(cwd: Option, shell: Shell, network: Option) -> Self { + pub fn new( + cwd: Option, + shell: Shell, + current_date: Option, + timezone: Option, + network: Option, + subagents: Option, + ) -> Self { Self { cwd, shell, + current_date, + timezone, network, + subagents, } } @@ -39,11 +50,17 @@ impl EnvironmentContext { pub fn equals_except_shell(&self, other: &EnvironmentContext) -> bool { let EnvironmentContext { cwd, + current_date, + timezone, network, - // should compare all fields except shell + subagents, shell: _, } = other; - self.cwd == *cwd && self.network == *network + self.cwd == *cwd + && self.current_date == *current_date + && self.timezone == *timezone + && self.network == *network + && self.subagents == *subagents } pub fn diff_from_turn_context_item( @@ -58,19 +75,24 @@ impl EnvironmentContext { } else { None }; + let current_date = after.current_date.clone(); + let timezone = after.timezone.clone(); let network = if before_network != after_network { after_network } else { before_network }; - EnvironmentContext::new(cwd, shell.clone(), network) + EnvironmentContext::new(cwd, shell.clone(), current_date, timezone, network, None) } pub fn from_turn_context(turn_context: &TurnContext, shell: &Shell) -> Self { Self::new( Some(turn_context.cwd.clone()), shell.clone(), + turn_context.current_date.clone(), + turn_context.timezone.clone(), Self::network_from_turn_context(turn_context), + None, ) } @@ -78,10 +100,20 @@ impl EnvironmentContext { Self::new( Some(turn_context_item.cwd.clone()), shell.clone(), + turn_context_item.current_date.clone(), + turn_context_item.timezone.clone(), Self::network_from_turn_context_item(turn_context_item), + None, ) } + pub fn with_subagents(mut self, subagents: String) -> Self { + if !subagents.is_empty() { + self.subagents = Some(subagents); + } + self + } + fn network_from_turn_context(turn_context: &TurnContext) -> Option { let network = turn_context .config @@ -122,13 +154,19 @@ impl EnvironmentContext { /// /// ``` pub fn serialize_to_xml(self) -> String { - let mut lines = vec![ENVIRONMENT_CONTEXT_OPEN_TAG.to_string()]; + let mut lines = Vec::new(); if let Some(cwd) = self.cwd { lines.push(format!(" {}", cwd.to_string_lossy())); } let shell_name = self.shell.name(); lines.push(format!(" {shell_name}")); + if let Some(current_date) = self.current_date { + lines.push(format!(" {current_date}")); + } + if let Some(timezone) = self.timezone { + lines.push(format!(" {timezone}")); + } match self.network { Some(ref network) => { lines.push(" ".to_string()); @@ -145,22 +183,18 @@ impl EnvironmentContext { // lines.push(" ".to_string()); } } - lines.push(ENVIRONMENT_CONTEXT_CLOSE_TAG.to_string()); - lines.join("\n") + if let Some(subagents) = self.subagents { + lines.push(" ".to_string()); + lines.extend(subagents.lines().map(|line| format!(" {line}"))); + lines.push(" ".to_string()); + } + ENVIRONMENT_CONTEXT_FRAGMENT.wrap(lines.join("\n")) } } impl From for ResponseItem { fn from(ec: EnvironmentContext) -> Self { - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: ec.serialize_to_xml(), - }], - end_turn: None, - phase: None, - } + ENVIRONMENT_CONTEXT_FRAGMENT.into_message(ec.serialize_to_xml()) } } @@ -183,12 +217,21 @@ mod tests { #[test] fn serialize_workspace_write_environment_context() { let cwd = test_path_buf("/repo"); - let context = EnvironmentContext::new(Some(cwd.clone()), fake_shell(), None); + let context = EnvironmentContext::new( + Some(cwd.clone()), + fake_shell(), + Some("2026-02-26".to_string()), + Some("America/Los_Angeles".to_string()), + None, + None, + ); let expected = format!( r#" {cwd} bash + 2026-02-26 + America/Los_Angeles "#, cwd = cwd.display(), ); @@ -202,13 +245,21 @@ mod tests { allowed_domains: vec!["api.example.com".to_string(), "*.openai.com".to_string()], denied_domains: vec!["blocked.example.com".to_string()], }; - let context = - EnvironmentContext::new(Some(test_path_buf("/repo")), fake_shell(), Some(network)); + let context = EnvironmentContext::new( + Some(test_path_buf("/repo")), + fake_shell(), + Some("2026-02-26".to_string()), + Some("America/Los_Angeles".to_string()), + Some(network), + None, + ); let expected = format!( r#" {} bash + 2026-02-26 + America/Los_Angeles api.example.com *.openai.com @@ -223,10 +274,19 @@ mod tests { #[test] fn serialize_read_only_environment_context() { - let context = EnvironmentContext::new(None, fake_shell(), None); + let context = EnvironmentContext::new( + None, + fake_shell(), + Some("2026-02-26".to_string()), + Some("America/Los_Angeles".to_string()), + None, + None, + ); let expected = r#" bash + 2026-02-26 + America/Los_Angeles "#; assert_eq!(context.serialize_to_xml(), expected); @@ -234,10 +294,19 @@ mod tests { #[test] fn serialize_external_sandbox_environment_context() { - let context = EnvironmentContext::new(None, fake_shell(), None); + let context = EnvironmentContext::new( + None, + fake_shell(), + Some("2026-02-26".to_string()), + Some("America/Los_Angeles".to_string()), + None, + None, + ); let expected = r#" bash + 2026-02-26 + America/Los_Angeles "#; assert_eq!(context.serialize_to_xml(), expected); @@ -245,10 +314,19 @@ mod tests { #[test] fn serialize_external_sandbox_with_restricted_network_environment_context() { - let context = EnvironmentContext::new(None, fake_shell(), None); + let context = EnvironmentContext::new( + None, + fake_shell(), + Some("2026-02-26".to_string()), + Some("America/Los_Angeles".to_string()), + None, + None, + ); let expected = r#" bash + 2026-02-26 + America/Los_Angeles "#; assert_eq!(context.serialize_to_xml(), expected); @@ -256,10 +334,19 @@ mod tests { #[test] fn serialize_full_access_environment_context() { - let context = EnvironmentContext::new(None, fake_shell(), None); + let context = EnvironmentContext::new( + None, + fake_shell(), + Some("2026-02-26".to_string()), + Some("America/Los_Angeles".to_string()), + None, + None, + ); let expected = r#" bash + 2026-02-26 + America/Los_Angeles "#; assert_eq!(context.serialize_to_xml(), expected); @@ -267,23 +354,65 @@ mod tests { #[test] fn equals_except_shell_compares_cwd() { - let context1 = EnvironmentContext::new(Some(PathBuf::from("/repo")), fake_shell(), None); - let context2 = EnvironmentContext::new(Some(PathBuf::from("/repo")), fake_shell(), None); + let context1 = EnvironmentContext::new( + Some(PathBuf::from("/repo")), + fake_shell(), + None, + None, + None, + None, + ); + let context2 = EnvironmentContext::new( + Some(PathBuf::from("/repo")), + fake_shell(), + None, + None, + None, + None, + ); assert!(context1.equals_except_shell(&context2)); } #[test] fn equals_except_shell_ignores_sandbox_policy() { - let context1 = EnvironmentContext::new(Some(PathBuf::from("/repo")), fake_shell(), None); - let context2 = EnvironmentContext::new(Some(PathBuf::from("/repo")), fake_shell(), None); + let context1 = EnvironmentContext::new( + Some(PathBuf::from("/repo")), + fake_shell(), + None, + None, + None, + None, + ); + let context2 = EnvironmentContext::new( + Some(PathBuf::from("/repo")), + fake_shell(), + None, + None, + None, + None, + ); assert!(context1.equals_except_shell(&context2)); } #[test] fn equals_except_shell_compares_cwd_differences() { - let context1 = EnvironmentContext::new(Some(PathBuf::from("/repo1")), fake_shell(), None); - let context2 = EnvironmentContext::new(Some(PathBuf::from("/repo2")), fake_shell(), None); + let context1 = EnvironmentContext::new( + Some(PathBuf::from("/repo1")), + fake_shell(), + None, + None, + None, + None, + ); + let context2 = EnvironmentContext::new( + Some(PathBuf::from("/repo2")), + fake_shell(), + None, + None, + None, + None, + ); assert!(!context1.equals_except_shell(&context2)); } @@ -298,6 +427,9 @@ mod tests { shell_snapshot: crate::shell::empty_shell_snapshot_receiver(), }, None, + None, + None, + None, ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), @@ -307,8 +439,39 @@ mod tests { shell_snapshot: crate::shell::empty_shell_snapshot_receiver(), }, None, + None, + None, + None, ); assert!(context1.equals_except_shell(&context2)); } + + #[test] + fn serialize_environment_context_with_subagents() { + let context = EnvironmentContext::new( + Some(test_path_buf("/repo")), + fake_shell(), + Some("2026-02-26".to_string()), + Some("America/Los_Angeles".to_string()), + None, + Some("- agent-1: atlas\n- agent-2".to_string()), + ); + + let expected = format!( + r#" + {} + bash + 2026-02-26 + America/Los_Angeles + + - agent-1: atlas + - agent-2 + +"#, + test_path_buf("/repo").display() + ); + + assert_eq!(context.serialize_to_xml(), expected); + } } diff --git a/codex-rs/core/src/event_mapping.rs b/codex-rs/core/src/event_mapping.rs index 16f7e1c47a..afffdb0f1d 100644 --- a/codex-rs/core/src/event_mapping.rs +++ b/codex-rs/core/src/event_mapping.rs @@ -18,16 +18,15 @@ use codex_protocol::user_input::UserInput; use tracing::warn; use uuid::Uuid; -use crate::instructions::SkillInstructions; -use crate::instructions::UserInstructions; -use crate::session_prefix::is_session_prefix; -use crate::user_shell_command::is_user_shell_command_text; +use crate::contextual_user_message::is_contextual_user_fragment; use crate::web_search::web_search_action_detail; +pub(crate) fn is_contextual_user_message_content(message: &[ContentItem]) -> bool { + message.iter().any(is_contextual_user_fragment) +} + fn parse_user_message(message: &[ContentItem]) -> Option { - if UserInstructions::is_user_instructions(message) - || SkillInstructions::is_skill_instructions(message) - { + if is_contextual_user_message_content(message) { return None; } @@ -44,9 +43,6 @@ fn parse_user_message(message: &[ContentItem]) -> Option { { continue; } - if is_session_prefix(text) || is_user_shell_command_text(text) { - return None; - } content.push(UserInput::Text { text: text.clone(), // Model input content does not carry UI element ranges. @@ -59,9 +55,6 @@ fn parse_user_message(message: &[ContentItem]) -> Option { }); } ContentItem::OutputText { text } => { - if is_session_prefix(text) { - return None; - } warn!("Output text in user message: {}", text); } } @@ -299,7 +292,7 @@ mod tests { id: None, role: "user".to_string(), content: vec![ContentItem::InputText { - text: "test_text".to_string(), + text: "# AGENTS.md instructions for test_directory\n\n\ntest_text\n".to_string(), }], end_turn: None, phase: None, @@ -341,6 +334,22 @@ mod tests { end_turn: None, phase: None, }, + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "ctx".to_string(), + }, + ContentItem::InputText { + text: + "# AGENTS.md instructions for dir\n\n\nbody\n" + .to_string(), + }, + ], + end_turn: None, + phase: None, + }, ]; for item in items { diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 288ee7f212..5a805a3b8b 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -219,6 +219,8 @@ pub async fn process_exec_tool_call( enforce_managed_network, network: network.as_ref(), sandbox_policy_cwd: sandbox_cwd, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: codex_linux_sandbox_exe.as_ref(), use_linux_sandbox_bwrap, windows_sandbox_level, diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index cc65ad572f..8f8178d201 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -133,8 +133,6 @@ pub enum Feature { SkillMcpDependencyInstall, /// Prompt for missing skill env var dependencies. SkillEnvVarDependencyPrompt, - /// Emit skill approval test prompts/events. - SkillApproval, /// Steer feature flag - when enabled, Enter submits immediately instead of queuing. /// Kept for config backward compatibility; behavior is always steer-enabled. Steer, @@ -465,7 +463,7 @@ pub const FEATURES: &[FeatureSpec] = &[ key: "js_repl", stage: Stage::Experimental { name: "JavaScript REPL", - menu_description: "Enable a persistent Node-backed JavaScript REPL for interactive website debugging and other inline JavaScript execution capabilities. Requires Node >= v24.13.1 installed.", + menu_description: "Enable a persistent Node-backed JavaScript REPL for interactive website debugging and other inline JavaScript execution capabilities. Requires Node >= v22.22.0 installed.", announcement: "NEW: JavaScript REPL is now available in /experimental. Enable it, then start a new chat or restart Codex to use it.", }, default_enabled: false, @@ -630,12 +628,6 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::UnderDevelopment, default_enabled: false, }, - FeatureSpec { - id: Feature::SkillApproval, - key: "skill_approval", - stage: Stage::UnderDevelopment, - default_enabled: false, - }, FeatureSpec { id: Feature::Steer, key: "steer", diff --git a/codex-rs/core/src/instructions/mod.rs b/codex-rs/core/src/instructions/mod.rs index 9da925ac74..9f1d95d2f6 100644 --- a/codex-rs/core/src/instructions/mod.rs +++ b/codex-rs/core/src/instructions/mod.rs @@ -1,6 +1,5 @@ mod user_instructions; pub(crate) use user_instructions::SkillInstructions; -pub use user_instructions::USER_INSTRUCTIONS_OPEN_TAG_LEGACY; pub use user_instructions::USER_INSTRUCTIONS_PREFIX; pub(crate) use user_instructions::UserInstructions; diff --git a/codex-rs/core/src/instructions/user_instructions.rs b/codex-rs/core/src/instructions/user_instructions.rs index 525834847e..09e6e4c2f7 100644 --- a/codex-rs/core/src/instructions/user_instructions.rs +++ b/codex-rs/core/src/instructions/user_instructions.rs @@ -1,12 +1,12 @@ use serde::Deserialize; use serde::Serialize; -use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; -pub const USER_INSTRUCTIONS_OPEN_TAG_LEGACY: &str = ""; +use crate::contextual_user_message::AGENTS_MD_FRAGMENT; +use crate::contextual_user_message::SKILL_FRAGMENT; + pub const USER_INSTRUCTIONS_PREFIX: &str = "# AGENTS.md instructions for "; -pub const SKILL_INSTRUCTIONS_PREFIX: &str = " bool { - if let [ContentItem::InputText { text }] = message { - text.starts_with(USER_INSTRUCTIONS_PREFIX) - || text.starts_with(USER_INSTRUCTIONS_OPEN_TAG_LEGACY) - } else { - false - } + pub(crate) fn serialize_to_text(&self) -> String { + format!( + "{prefix}{directory}\n\n\n{contents}\n{suffix}", + prefix = AGENTS_MD_FRAGMENT.start_marker(), + directory = self.directory, + contents = self.text, + suffix = AGENTS_MD_FRAGMENT.end_marker(), + ) } } impl From for ResponseItem { fn from(ui: UserInstructions) -> Self { - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: format!( - "{USER_INSTRUCTIONS_PREFIX}{directory}\n\n\n{contents}\n", - directory = ui.directory, - contents = ui.text - ), - }], - end_turn: None, - phase: None, - } + AGENTS_MD_FRAGMENT.into_message(ui.serialize_to_text()) } } @@ -52,36 +41,21 @@ pub(crate) struct SkillInstructions { pub contents: String, } -impl SkillInstructions { - pub fn is_skill_instructions(message: &[ContentItem]) -> bool { - if let [ContentItem::InputText { text }] = message { - text.starts_with(SKILL_INSTRUCTIONS_PREFIX) - } else { - false - } - } -} +impl SkillInstructions {} impl From for ResponseItem { fn from(si: SkillInstructions) -> Self { - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: format!( - "\n{}\n{}\n{}\n", - si.name, si.path, si.contents - ), - }], - end_turn: None, - phase: None, - } + SKILL_FRAGMENT.into_message(SKILL_FRAGMENT.wrap(format!( + "{}\n{}\n{}", + si.name, si.path, si.contents + ))) } } #[cfg(test)] mod tests { use super::*; + use codex_protocol::models::ContentItem; use pretty_assertions::assert_eq; #[test] @@ -110,21 +84,10 @@ mod tests { #[test] fn test_is_user_instructions() { - assert!(UserInstructions::is_user_instructions( - &[ContentItem::InputText { - text: "# AGENTS.md instructions for test_directory\n\n\ntest_text\n".to_string(), - }] + assert!(AGENTS_MD_FRAGMENT.matches_text( + "# AGENTS.md instructions for test_directory\n\n\ntest_text\n" )); - assert!(UserInstructions::is_user_instructions(&[ - ContentItem::InputText { - text: "test_text".to_string(), - } - ])); - assert!(!UserInstructions::is_user_instructions(&[ - ContentItem::InputText { - text: "test_text".to_string(), - } - ])); + assert!(!AGENTS_MD_FRAGMENT.matches_text("test_text")); } #[test] @@ -154,16 +117,9 @@ mod tests { #[test] fn test_is_skill_instructions() { - assert!(SkillInstructions::is_skill_instructions(&[ - ContentItem::InputText { - text: "\ndemo-skill\nskills/demo/SKILL.md\nbody\n" - .to_string(), - } - ])); - assert!(!SkillInstructions::is_skill_instructions(&[ - ContentItem::InputText { - text: "regular text".to_string(), - } - ])); + assert!(SKILL_FRAGMENT.matches_text( + "\ndemo-skill\nskills/demo/SKILL.md\nbody\n" + )); + assert!(!SKILL_FRAGMENT.matches_text("regular text")); } } diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index c34f33f394..7edb94b793 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -27,6 +27,7 @@ pub mod config; pub mod config_loader; pub mod connectors; mod context_manager; +mod contextual_user_message; pub mod custom_prompts; pub mod env; mod environment_context; diff --git a/codex-rs/core/src/mcp/mod.rs b/codex-rs/core/src/mcp/mod.rs index 3abe4e9233..d744010469 100644 --- a/codex-rs/core/src/mcp/mod.rs +++ b/codex-rs/core/src/mcp/mod.rs @@ -139,6 +139,7 @@ fn codex_apps_mcp_server_config(config: &Config, auth: Option<&CodexAuth>) -> Mc enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, } } diff --git a/codex-rs/core/src/mcp/skill_dependencies.rs b/codex-rs/core/src/mcp/skill_dependencies.rs index 1969bc6322..8d5fc3c1a3 100644 --- a/codex-rs/core/src/mcp/skill_dependencies.rs +++ b/codex-rs/core/src/mcp/skill_dependencies.rs @@ -241,6 +241,7 @@ pub(crate) async fn maybe_install_mcp_dependencies( oauth_config.http_headers, oauth_config.env_http_headers, &[], + server_config.oauth_resource.as_deref(), config.mcp_oauth_callback_port, config.mcp_oauth_callback_url.as_deref(), ) @@ -387,6 +388,7 @@ fn mcp_dependency_to_server_config( enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }); } @@ -411,6 +413,7 @@ fn mcp_dependency_to_server_config( enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }); } @@ -468,6 +471,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); @@ -516,6 +520,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, )]); diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index 12e3d325d3..2d22351d1f 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -2115,6 +2115,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, auth_status: McpAuthStatus::Unsupported, }; @@ -2162,6 +2163,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, auth_status: McpAuthStatus::Unsupported, }; diff --git a/codex-rs/core/src/memories/README.md b/codex-rs/core/src/memories/README.md index c19a0c6806..0a49b58316 100644 --- a/codex-rs/core/src/memories/README.md +++ b/codex-rs/core/src/memories/README.md @@ -59,7 +59,14 @@ Phase 2 consolidates the latest stage-1 outputs into the filesystem memory artif What it does: - claims a single global phase-2 job (so only one consolidation runs at a time) -- loads a bounded set of the most recent stage-1 outputs from the state DB (the per-rollout memories produced by Phase 1, used as the consolidation input set) +- loads a bounded set of stage-1 outputs from the state DB using phase-2 + selection rules: + - ignores memories whose `last_usage` falls outside the configured + `max_unused_days` window + - for memories with no `last_usage`, falls back to `generated_at` so fresh + never-used memories can still be selected + - ranks eligible memories by `usage_count` first, then by the most recent + `last_usage` / `generated_at` - computes a completion watermark from the claimed watermark + newest input timestamps - syncs local memory artifacts under the memories root: - `raw_memories.md` (merged raw memories, latest first) @@ -70,11 +77,31 @@ What it does: If there is input, it then: - spawns an internal consolidation sub-agent +- builds the Phase 2 prompt with a diff of the current Phase 1 input + selection versus the last successful Phase 2 selection (`added`, + `retained`, `removed`) - runs it with no approvals, no network, and local write access only - disables collab for that agent (to prevent recursive delegation) - watches the agent status and heartbeats the global job lease while it runs - marks the phase-2 job success/failure in the state DB when the agent finishes +Selection diff behavior: + +- successful Phase 2 runs mark the exact stage-1 snapshots they consumed with + `selected_for_phase2 = 1` and persist the matching + `selected_for_phase2_source_updated_at` +- Phase 1 upserts preserve the previous `selected_for_phase2` baseline until + the next successful Phase 2 run rewrites it +- the next Phase 2 run compares the current top-N stage-1 inputs against that + prior snapshot selection to label inputs as `added` or `retained`; a + refreshed thread stays `added` until Phase 2 successfully selects its newer + snapshot +- rows that were previously selected but still exist outside the current top-N + selection are surfaced as `removed` +- before the agent starts, local `rollout_summaries/` and `raw_memories.md` + keep the union of the current selection and the previous successful + selection, so removed-thread evidence stays available during forgetting + Watermark behavior: - The global phase-2 job claim includes an input watermark representing the latest input timestamp known when the job was claimed. diff --git a/codex-rs/core/src/memories/phase2.rs b/codex-rs/core/src/memories/phase2.rs index 6794a4574f..fb6e99d2df 100644 --- a/codex-rs/core/src/memories/phase2.rs +++ b/codex-rs/core/src/memories/phase2.rs @@ -8,6 +8,7 @@ use crate::memories::metrics; use crate::memories::phase_two; use crate::memories::prompts::build_consolidation_prompt; use crate::memories::storage::rebuild_raw_memories_file_from_memories; +use crate::memories::storage::rollout_summary_file_stem; use crate::memories::storage::sync_rollout_summaries_from_memories; use codex_config::Constrained; use codex_protocol::ThreadId; @@ -17,8 +18,10 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::TokenUsage; use codex_protocol::user_input::UserInput; +use codex_state::Stage1Output; use codex_state::StateRuntime; use codex_utils_absolute_path::AbsolutePathBuf; +use std::collections::HashSet; use std::sync::Arc; use std::time::Duration; use tokio::sync::watch; @@ -50,6 +53,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { }; let root = memory_root(&config.codex_home); let max_raw_memories = config.memories.max_raw_memories_for_global; + let max_unused_days = config.memories.max_unused_days; // 1. Claim the job. let claim = match job::claim(session, db).await { @@ -73,21 +77,27 @@ pub(super) async fn run(session: &Arc, config: Arc) { }; // 3. Query the memories - let raw_memories = match db.list_stage1_outputs_for_global(max_raw_memories).await { - Ok(memories) => memories, + let selection = match db + .get_phase2_input_selection(max_raw_memories, max_unused_days) + .await + { + Ok(selection) => selection, Err(err) => { tracing::error!("failed to list stage1 outputs from global: {}", err); job::failed(session, db, &claim, "failed_load_stage1_outputs").await; return; } }; + let raw_memories = selection.selected.to_vec(); + let artifact_memories = artifact_memories_for_phase2(&selection); let new_watermark = get_watermark(claim.watermark, &raw_memories); // 4. Update the file system by syncing the raw memories with the one extracted from DB at // step 3 // [`rollout_summaries/`] if let Err(err) = - sync_rollout_summaries_from_memories(&root, &raw_memories, max_raw_memories).await + sync_rollout_summaries_from_memories(&root, &artifact_memories, artifact_memories.len()) + .await { tracing::error!("failed syncing local memory artifacts for global consolidation: {err}"); job::failed(session, db, &claim, "failed_sync_artifacts").await; @@ -95,7 +105,8 @@ pub(super) async fn run(session: &Arc, config: Arc) { } // [`raw_memories.md`] if let Err(err) = - rebuild_raw_memories_file_from_memories(&root, &raw_memories, max_raw_memories).await + rebuild_raw_memories_file_from_memories(&root, &artifact_memories, artifact_memories.len()) + .await { tracing::error!("failed syncing local memory artifacts for global consolidation: {err}"); job::failed(session, db, &claim, "failed_rebuild_raw_memories").await; @@ -103,12 +114,20 @@ pub(super) async fn run(session: &Arc, config: Arc) { } if raw_memories.is_empty() { // We check only after sync of the file system. - job::succeed(session, db, &claim, new_watermark, "succeeded_no_input").await; + job::succeed( + session, + db, + &claim, + new_watermark, + &[], + "succeeded_no_input", + ) + .await; return; } // 5. Spawn the agent - let prompt = agent::get_prompt(config); + let prompt = agent::get_prompt(config, &selection); let source = SessionSource::SubAgent(SubAgentSource::MemoryConsolidation); let thread_id = match session .services @@ -129,6 +148,7 @@ pub(super) async fn run(session: &Arc, config: Arc) { session, claim, new_watermark, + raw_memories.clone(), thread_id, phase_two_e2e_timer, ); @@ -140,6 +160,22 @@ pub(super) async fn run(session: &Arc, config: Arc) { emit_metrics(session, counters); } +fn artifact_memories_for_phase2( + selection: &codex_state::Phase2InputSelection, +) -> Vec { + let mut seen = HashSet::new(); + let mut memories = selection.selected.clone(); + for memory in &selection.selected { + seen.insert(rollout_summary_file_stem(memory)); + } + for memory in &selection.previous_selected { + if seen.insert(rollout_summary_file_stem(memory)) { + memories.push(memory.clone()); + } + } + memories +} + mod job { use super::*; @@ -205,6 +241,7 @@ mod job { db: &StateRuntime, claim: &Claim, completion_watermark: i64, + selected_outputs: &[codex_state::Stage1Output], reason: &'static str, ) { session.services.otel_manager.counter( @@ -213,7 +250,7 @@ mod job { &[("status", reason)], ); let _ = db - .mark_global_phase2_job_succeeded(&claim.token, completion_watermark) + .mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) .await; } } @@ -266,9 +303,12 @@ mod agent { Some(agent_config) } - pub(super) fn get_prompt(config: Arc) -> Vec { + pub(super) fn get_prompt( + config: Arc, + selection: &codex_state::Phase2InputSelection, + ) -> Vec { let root = memory_root(&config.codex_home); - let prompt = build_consolidation_prompt(&root); + let prompt = build_consolidation_prompt(&root, selection); vec![UserInput::Text { text: prompt, text_elements: vec![], @@ -280,6 +320,7 @@ mod agent { session: &Arc, claim: Claim, new_watermark: i64, + selected_outputs: Vec, thread_id: ThreadId, phase_two_e2e_timer: Option, ) { @@ -316,7 +357,15 @@ mod agent { if let Some(token_usage) = agent_control.get_total_token_usage(thread_id).await { emit_token_usage_metrics(&session, &token_usage); } - job::succeed(&session, &db, &claim, new_watermark, "succeeded").await; + job::succeed( + &session, + &db, + &claim, + new_watermark, + &selected_outputs, + "succeeded", + ) + .await; } else { job::failed(&session, &db, &claim, "failed_agent").await; } diff --git a/codex-rs/core/src/memories/prompts.rs b/codex-rs/core/src/memories/prompts.rs index 9b341c1ffe..35cfe1edf0 100644 --- a/codex-rs/core/src/memories/prompts.rs +++ b/codex-rs/core/src/memories/prompts.rs @@ -1,9 +1,13 @@ use crate::memories::memory_root; use crate::memories::phase_one; +use crate::memories::storage::rollout_summary_file_stem_from_parts; use crate::truncate::TruncationPolicy; use crate::truncate::truncate_text; use askama::Template; use codex_protocol::openai_models::ModelInfo; +use codex_state::Phase2InputSelection; +use codex_state::Stage1Output; +use codex_state::Stage1OutputRef; use std::path::Path; use tokio::fs; use tracing::warn; @@ -12,6 +16,7 @@ use tracing::warn; #[template(path = "memories/consolidation.md", escape = "none")] struct ConsolidationPromptTemplate<'a> { memory_root: &'a str, + phase2_input_selection: &'a str, } #[derive(Template)] @@ -30,17 +35,91 @@ struct MemoryToolDeveloperInstructionsTemplate<'a> { } /// Builds the consolidation subagent prompt for a specific memory root. -pub(super) fn build_consolidation_prompt(memory_root: &Path) -> String { +pub(super) fn build_consolidation_prompt( + memory_root: &Path, + selection: &Phase2InputSelection, +) -> String { let memory_root = memory_root.display().to_string(); + let phase2_input_selection = render_phase2_input_selection(selection); let template = ConsolidationPromptTemplate { memory_root: &memory_root, + phase2_input_selection: &phase2_input_selection, }; template.render().unwrap_or_else(|err| { warn!("failed to render memories consolidation prompt template: {err}"); - format!("## Memory Phase 2 (Consolidation)\nConsolidate Codex memories in: {memory_root}") + format!( + "## Memory Phase 2 (Consolidation)\nConsolidate Codex memories in: {memory_root}\n\n{phase2_input_selection}" + ) }) } +fn render_phase2_input_selection(selection: &Phase2InputSelection) -> String { + let retained = selection.retained_thread_ids.len(); + let added = selection.selected.len().saturating_sub(retained); + let selected = if selection.selected.is_empty() { + "- none".to_string() + } else { + selection + .selected + .iter() + .map(|item| { + render_selected_input_line( + item, + selection.retained_thread_ids.contains(&item.thread_id), + ) + }) + .collect::>() + .join("\n") + }; + let removed = if selection.removed.is_empty() { + "- none".to_string() + } else { + selection + .removed + .iter() + .map(render_removed_input_line) + .collect::>() + .join("\n") + }; + + format!( + "- selected inputs this run: {}\n- newly added since the last successful Phase 2 run: {added}\n- retained from the last successful Phase 2 run: {retained}\n- removed from the last successful Phase 2 run: {}\n\nCurrent selected Phase 1 inputs:\n{selected}\n\nRemoved from the last successful Phase 2 selection:\n{removed}\n", + selection.selected.len(), + selection.removed.len(), + ) +} + +fn render_selected_input_line(item: &Stage1Output, retained: bool) -> String { + let status = if retained { "retained" } else { "added" }; + let rollout_summary_file = format!( + "rollout_summaries/{}.md", + rollout_summary_file_stem_from_parts( + item.thread_id, + item.source_updated_at, + item.rollout_slug.as_deref(), + ) + ); + format!( + "- [{status}] thread_id={}, rollout_summary_file={rollout_summary_file}", + item.thread_id + ) +} + +fn render_removed_input_line(item: &Stage1OutputRef) -> String { + let rollout_summary_file = format!( + "rollout_summaries/{}.md", + rollout_summary_file_stem_from_parts( + item.thread_id, + item.source_updated_at, + item.rollout_slug.as_deref(), + ) + ); + format!( + "- thread_id={}, rollout_summary_file={rollout_summary_file}", + item.thread_id + ) +} + /// Builds the stage-1 user message containing rollout metadata and content. /// /// Large rollout payloads are truncated to 70% of the active model's effective diff --git a/codex-rs/core/src/memories/storage.rs b/codex-rs/core/src/memories/storage.rs index 1a07dbe9b8..1410c4e73a 100644 --- a/codex-rs/core/src/memories/storage.rs +++ b/codex-rs/core/src/memories/storage.rs @@ -82,6 +82,8 @@ async fn rebuild_raw_memories_file( ) .map_err(raw_memories_format_error)?; writeln!(body, "cwd: {}", memory.cwd.display()).map_err(raw_memories_format_error)?; + writeln!(body, "rollout_path: {}", memory.rollout_path.display()) + .map_err(raw_memories_format_error)?; let rollout_summary_file = format!("{}.md", rollout_summary_file_stem(memory)); writeln!(body, "rollout_summary_file: {rollout_summary_file}") .map_err(raw_memories_format_error)?; @@ -138,7 +140,12 @@ async fn write_rollout_summary_for_thread( memory.source_updated_at.to_rfc3339() ) .map_err(rollout_summary_format_error)?; + writeln!(body, "rollout_path: {}", memory.rollout_path.display()) + .map_err(rollout_summary_format_error)?; writeln!(body, "cwd: {}", memory.cwd.display()).map_err(rollout_summary_format_error)?; + if let Some(git_branch) = memory.git_branch.as_deref() { + writeln!(body, "git_branch: {git_branch}").map_err(rollout_summary_format_error)?; + } writeln!(body).map_err(rollout_summary_format_error)?; body.push_str(&memory.rollout_summary); body.push('\n'); @@ -267,7 +274,9 @@ mod tests { raw_memory: "raw memory".to_string(), rollout_summary: "summary".to_string(), rollout_slug: rollout_slug.map(ToString::to_string), + rollout_path: PathBuf::from("/tmp/rollout.jsonl"), cwd: PathBuf::from("/tmp/workspace"), + git_branch: None, generated_at: Utc.timestamp_opt(124, 0).single().expect("timestamp"), } } diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index b47e7f8aab..62cef7eae3 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -86,7 +86,9 @@ async fn sync_rollout_summaries_and_raw_memories_file_keeps_latest_memories_only raw_memory: "raw memory".to_string(), rollout_summary: "short summary".to_string(), rollout_slug: None, + rollout_path: PathBuf::from("/tmp/rollout-100.jsonl"), cwd: PathBuf::from("/tmp/workspace"), + git_branch: None, generated_at: Utc.timestamp_opt(101, 0).single().expect("timestamp"), }]; @@ -135,6 +137,7 @@ async fn sync_rollout_summaries_and_raw_memories_file_keeps_latest_memories_only assert!(raw_memories.contains("raw memory")); assert!(raw_memories.contains(&keep_id)); assert!(raw_memories.contains("cwd: /tmp/workspace")); + assert!(raw_memories.contains("rollout_path: /tmp/rollout-100.jsonl")); assert!(raw_memories.contains(&format!( "rollout_summary_file: {canonical_rollout_summary_file}" ))); @@ -150,6 +153,10 @@ async fn sync_rollout_summaries_and_raw_memories_file_keeps_latest_memories_only .find("cwd: /tmp/workspace") .map(|offset| thread_pos + offset) .expect("cwd should exist after thread header"); + let rollout_path_pos = raw_memories[thread_pos..] + .find("rollout_path: /tmp/rollout-100.jsonl") + .map(|offset| thread_pos + offset) + .expect("rollout_path should exist after thread header"); let file_pos = raw_memories[thread_pos..] .find(&format!( "rollout_summary_file: {canonical_rollout_summary_file}" @@ -158,7 +165,8 @@ async fn sync_rollout_summaries_and_raw_memories_file_keeps_latest_memories_only .expect("rollout_summary_file should exist after thread header"); assert!(thread_pos < updated_pos); assert!(updated_pos < cwd_pos); - assert!(cwd_pos < file_pos); + assert!(cwd_pos < rollout_path_pos); + assert!(rollout_path_pos < file_pos); } #[tokio::test] @@ -184,7 +192,9 @@ async fn sync_rollout_summaries_uses_timestamp_hash_and_sanitized_slug_filename( raw_memory: "raw memory".to_string(), rollout_summary: "short summary".to_string(), rollout_slug: Some("Unsafe Slug/With Spaces & Symbols + EXTRA_LONG_12345".to_string()), + rollout_path: PathBuf::from("/tmp/rollout-200.jsonl"), cwd: PathBuf::from("/tmp/workspace"), + git_branch: Some("feature/memory-branch".to_string()), generated_at: Utc.timestamp_opt(201, 0).single().expect("timestamp"), }]; @@ -239,6 +249,8 @@ async fn sync_rollout_summaries_uses_timestamp_hash_and_sanitized_slug_filename( .await .expect("read rollout summary"); assert!(summary.contains(&format!("thread_id: {thread_id}"))); + assert!(summary.contains("rollout_path: /tmp/rollout-200.jsonl")); + assert!(summary.contains("git_branch: feature/memory-branch")); assert!( !tokio::fs::try_exists(&stale_unslugged_path) .await @@ -283,7 +295,9 @@ task_outcome: success .to_string(), rollout_summary: "short summary".to_string(), rollout_slug: Some("Unsafe Slug/With Spaces & Symbols + EXTRA_LONG_12345".to_string()), + rollout_path: PathBuf::from("/tmp/rollout-200.jsonl"), cwd: PathBuf::from("/tmp/workspace"), + git_branch: None, generated_at: Utc.timestamp_opt(201, 0).single().expect("timestamp"), }]; @@ -316,6 +330,12 @@ task_outcome: success let raw_memories = tokio::fs::read_to_string(raw_memories_file(&root)) .await .expect("read raw memories"); + let summary = tokio::fs::read_to_string( + rollout_summaries_dir(&root).join(canonical_rollout_summary_file), + ) + .await + .expect("read rollout summary"); + assert!(summary.contains("rollout_path: /tmp/rollout-200.jsonl")); assert!(raw_memories.contains(&format!( "rollout_summary_file: {canonical_rollout_summary_file}" ))); @@ -360,7 +380,9 @@ mod phase2 { raw_memory: "raw memory".to_string(), rollout_summary: "rollout summary".to_string(), rollout_slug: None, + rollout_path: PathBuf::from("/tmp/rollout-summary.jsonl"), cwd: PathBuf::from("/tmp/workspace"), + git_branch: None, generated_at: chrono::DateTime::::from_timestamp(source_updated_at + 1, 0) .expect("valid generated_at timestamp"), } @@ -542,7 +564,7 @@ mod phase2 { #[tokio::test] async fn dispatch_reclaims_stale_global_lock_and_starts_consolidation() { let harness = DispatchHarness::new().await; - harness.seed_stage1_output(100).await; + harness.seed_stage1_output(Utc::now().timestamp()).await; let stale_claim = harness .state_db @@ -556,12 +578,18 @@ mod phase2 { phase2::run(&harness.session, Arc::clone(&harness.config)).await; - let running_claim = harness + let post_dispatch_claim = harness .state_db .try_claim_global_phase2_job(ThreadId::new(), 3_600) .await - .expect("claim while running"); - pretty_assertions::assert_eq!(running_claim, Phase2JobClaimOutcome::SkippedRunning); + .expect("claim after stale lock dispatch"); + assert!( + matches!( + post_dispatch_claim, + Phase2JobClaimOutcome::SkippedRunning | Phase2JobClaimOutcome::SkippedNotDirty + ), + "stale-lock dispatch should either keep the reclaimed job running or finish it before re-claim" + ); let user_input_ops = harness.user_input_ops_count(); pretty_assertions::assert_eq!(user_input_ops, 1); diff --git a/codex-rs/core/src/models_manager/model_info.rs b/codex-rs/core/src/models_manager/model_info.rs index 19a945f951..4824e4cd11 100644 --- a/codex-rs/core/src/models_manager/model_info.rs +++ b/codex-rs/core/src/models_manager/model_info.rs @@ -1,3 +1,4 @@ +use codex_protocol::config_types::ReasoningSummary; use codex_protocol::openai_models::ConfigShellToolType; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelInstructionsVariables; @@ -68,10 +69,12 @@ pub(crate) fn model_info_from_slug(slug: &str) -> ModelInfo { visibility: ModelVisibility::None, supported_in_api: true, priority: 99, + availability_nux: None, upgrade: None, base_instructions: BASE_INSTRUCTIONS.to_string(), model_messages: local_personality_messages_for_slug(slug), supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, apply_patch_tool_type: None, diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 1b858aedcf..d643d74682 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -31,6 +31,7 @@ use tokio::sync::Mutex; use tokio::task::JoinHandle; use tracing::debug; use tracing::error; +use tracing::info; use tracing::warn; const AUDIO_IN_QUEUE_CAPACITY: usize = 256; @@ -184,6 +185,7 @@ pub(crate) async fn handle_start( let requested_session_id = params .session_id .or_else(|| Some(sess.conversation_id.to_string())); + info!("starting realtime conversation"); let events_rx = match sess .conversation .start(api_provider, None, prompt, requested_session_id.clone()) @@ -191,11 +193,14 @@ pub(crate) async fn handle_start( { Ok(events_rx) => events_rx, Err(err) => { + error!("failed to start realtime conversation: {err}"); send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::Other).await; return Ok(()); } }; + info!("realtime conversation started"); + sess.send_event_raw(Event { id: sub_id.clone(), msg: EventMsg::RealtimeConversationStarted(RealtimeConversationStartedEvent { @@ -211,6 +216,7 @@ pub(crate) async fn handle_start( msg, }; while let Ok(event) = events_rx.recv().await { + debug!(conversation_id = %sess_clone.conversation_id, "received realtime conversation event"); let maybe_routed_text = match &event { RealtimeEvent::ConversationItemAdded(item) => { realtime_text_from_conversation_item(item) @@ -231,6 +237,7 @@ pub(crate) async fn handle_start( .await; } if let Some(()) = sess_clone.conversation.running_state().await { + info!("realtime conversation transport closed"); sess_clone .send_event_raw(ev(EventMsg::RealtimeConversationClosed( RealtimeConversationClosedEvent { @@ -250,6 +257,7 @@ pub(crate) async fn handle_audio( params: ConversationAudioParams, ) { if let Err(err) = sess.conversation.audio_in(params.frame).await { + error!("failed to append realtime audio: {err}"); send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest).await; } } @@ -284,6 +292,7 @@ pub(crate) async fn handle_text( debug!(text = %params.text, "[realtime-text] appending realtime conversation text input"); if let Err(err) = sess.conversation.text_in(params.text).await { + error!("failed to append realtime text: {err}"); send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest).await; } } diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/core/src/rollout/policy.rs index a1e77ba4fc..4c76bc54f6 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/core/src/rollout/policy.rs @@ -141,7 +141,6 @@ fn event_msg_persistence_mode(ev: &EventMsg) -> Option { | EventMsg::TerminalInteraction(_) | EventMsg::ExecCommandOutputDelta(_) | EventMsg::ExecApprovalRequest(_) - | EventMsg::SkillRequestApproval(_) | EventMsg::RequestUserInput(_) | EventMsg::ElicitationRequest(_) | EventMsg::ApplyPatchApprovalRequest(_) diff --git a/codex-rs/core/src/rollout/recorder.rs b/codex-rs/core/src/rollout/recorder.rs index d612bd14e5..d7404fd2c6 100644 --- a/codex-rs/core/src/rollout/recorder.rs +++ b/codex-rs/core/src/rollout/recorder.rs @@ -326,7 +326,9 @@ impl RolloutRecorder { else { break; }; - if let Some(path) = select_resume_path_from_db_page(&db_page, filter_cwd) { + if let Some(path) = + select_resume_path_from_db_page(&db_page, filter_cwd, default_provider).await + { return Ok(Some(path)); } db_cursor = db_page.next_anchor.map(Into::into); @@ -348,7 +350,7 @@ impl RolloutRecorder { default_provider, ) .await?; - if let Some(path) = select_resume_path(&page, filter_cwd) { + if let Some(path) = select_resume_path(&page, filter_cwd, default_provider).await { return Ok(Some(path)); } cursor = page.next_cursor; @@ -458,6 +460,7 @@ impl RolloutRecorder { state_db_ctx.clone(), state_builder, config.model_provider_id.clone(), + config.memories.generate_memories, )); Ok(Self { @@ -709,6 +712,7 @@ async fn rollout_writer( state_db_ctx: Option, mut state_builder: Option, default_provider: String, + generate_memories: bool, ) -> std::io::Result<()> { let mut writer = file.map(|file| JsonlWriter { file }); let mut buffered_items = Vec::::new(); @@ -729,6 +733,7 @@ async fn rollout_writer( state_db_ctx.as_deref(), &mut state_builder, default_provider.as_str(), + generate_memories, ) .await?; } @@ -782,6 +787,7 @@ async fn rollout_writer( state_db_ctx.as_deref(), &mut state_builder, default_provider.as_str(), + generate_memories, ) .await?; } @@ -829,6 +835,7 @@ async fn rollout_writer( Ok(()) } +#[allow(clippy::too_many_arguments)] async fn write_session_meta( mut writer: Option<&mut JsonlWriter>, session_meta: SessionMeta, @@ -837,6 +844,7 @@ async fn write_session_meta( state_db_ctx: Option<&StateRuntime>, state_builder: &mut Option, default_provider: &str, + generate_memories: bool, ) -> std::io::Result<()> { let git_info = collect_git_info(cwd).await; let session_meta_line = SessionMetaLine { @@ -858,6 +866,7 @@ async fn write_session_meta( state_builder.as_ref(), std::slice::from_ref(&rollout_item), None, + (!generate_memories).then_some("disabled"), ) .await; Ok(()) @@ -886,6 +895,7 @@ async fn write_and_reconcile_items( state_builder.as_ref(), items, "rollout_writer", + None, ) .await; Ok(()) @@ -961,35 +971,79 @@ impl From for ThreadsPage { } } -fn select_resume_path(page: &ThreadsPage, filter_cwd: Option<&Path>) -> Option { +async fn select_resume_path( + page: &ThreadsPage, + filter_cwd: Option<&Path>, + default_provider: &str, +) -> Option { match filter_cwd { - Some(cwd) => page.items.iter().find_map(|item| { - if item - .cwd - .as_ref() - .is_some_and(|session_cwd| cwd_matches(session_cwd, cwd)) - { - Some(item.path.clone()) - } else { - None + Some(cwd) => { + for item in &page.items { + if resume_candidate_matches_cwd( + item.path.as_path(), + item.cwd.as_deref(), + cwd, + default_provider, + ) + .await + { + return Some(item.path.clone()); + } } - }), + None + } None => page.items.first().map(|item| item.path.clone()), } } -fn select_resume_path_from_db_page( +async fn resume_candidate_matches_cwd( + rollout_path: &Path, + cached_cwd: Option<&Path>, + cwd: &Path, + default_provider: &str, +) -> bool { + if cached_cwd.is_some_and(|session_cwd| cwd_matches(session_cwd, cwd)) { + return true; + } + + if let Ok((items, _, _)) = RolloutRecorder::load_rollout_items(rollout_path).await + && let Some(latest_turn_context_cwd) = items.iter().rev().find_map(|item| match item { + RolloutItem::TurnContext(turn_context) => Some(turn_context.cwd.as_path()), + RolloutItem::SessionMeta(_) + | RolloutItem::ResponseItem(_) + | RolloutItem::Compacted(_) + | RolloutItem::EventMsg(_) => None, + }) + { + return cwd_matches(latest_turn_context_cwd, cwd); + } + + metadata::extract_metadata_from_rollout(rollout_path, default_provider, None) + .await + .is_ok_and(|outcome| cwd_matches(outcome.metadata.cwd.as_path(), cwd)) +} + +async fn select_resume_path_from_db_page( page: &codex_state::ThreadsPage, filter_cwd: Option<&Path>, + default_provider: &str, ) -> Option { match filter_cwd { - Some(cwd) => page.items.iter().find_map(|item| { - if cwd_matches(item.cwd.as_path(), cwd) { - Some(item.rollout_path.clone()) - } else { - None + Some(cwd) => { + for item in &page.items { + if resume_candidate_matches_cwd( + item.rollout_path.as_path(), + Some(item.cwd.as_path()), + cwd, + default_provider, + ) + .await + { + return Some(item.rollout_path.clone()); + } } - }), + None + } None => page.items.first().map(|item| item.rollout_path.clone()), } } @@ -1010,8 +1064,12 @@ mod tests { use crate::config::ConfigBuilder; use crate::features::Feature; use chrono::TimeZone; + use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use codex_protocol::protocol::AgentMessageEvent; + use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; + use codex_protocol::protocol::SandboxPolicy; + use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::UserMessageEvent; use pretty_assertions::assert_eq; use std::fs::File; @@ -1320,4 +1378,49 @@ mod tests { assert_eq!(repaired_path, Some(real_path)); Ok(()) } + + #[tokio::test] + async fn resume_candidate_matches_cwd_reads_latest_turn_context() -> std::io::Result<()> { + let home = TempDir::new().expect("temp dir"); + let stale_cwd = home.path().join("stale"); + let latest_cwd = home.path().join("latest"); + fs::create_dir_all(&stale_cwd)?; + fs::create_dir_all(&latest_cwd)?; + + let path = write_session_file(home.path(), "2025-01-03T13-00-00", Uuid::from_u128(9012))?; + let mut file = std::fs::OpenOptions::new().append(true).open(&path)?; + let turn_context = RolloutLine { + timestamp: "2025-01-03T13:00:01Z".to_string(), + item: RolloutItem::TurnContext(TurnContextItem { + turn_id: Some("turn-1".to_string()), + cwd: latest_cwd.clone(), + current_date: None, + timezone: None, + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + network: None, + model: "test-model".to_string(), + personality: None, + collaboration_mode: None, + effort: None, + summary: ReasoningSummaryConfig::Auto, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: None, + }), + }; + writeln!(file, "{}", serde_json::to_string(&turn_context)?)?; + + assert!( + resume_candidate_matches_cwd( + path.as_path(), + Some(stale_cwd.as_path()), + latest_cwd.as_path(), + "test-provider", + ) + .await + ); + Ok(()) + } } diff --git a/codex-rs/core/src/rollout/truncation.rs b/codex-rs/core/src/rollout/truncation.rs index c50eacc48b..7b316c0bb5 100644 --- a/codex-rs/core/src/rollout/truncation.rs +++ b/codex-rs/core/src/rollout/truncation.rs @@ -193,7 +193,7 @@ mod tests { #[tokio::test] async fn ignores_session_prefix_messages_when_truncating_rollout_from_start() { let (session, turn_context) = make_session_and_context().await; - let mut items = session.build_initial_context(&turn_context).await; + let mut items = session.build_initial_context(&turn_context, None).await; items.push(user_msg("feature request")); items.push(assistant_msg("ack")); items.push(user_msg("second question")); diff --git a/codex-rs/core/src/sandboxing/mod.rs b/codex-rs/core/src/sandboxing/mod.rs index 56c7bff6a6..da03f00a62 100644 --- a/codex-rs/core/src/sandboxing/mod.rs +++ b/codex-rs/core/src/sandboxing/mod.rs @@ -17,7 +17,7 @@ use crate::protocol::SandboxPolicy; #[cfg(target_os = "macos")] use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; #[cfg(target_os = "macos")] -use crate::seatbelt::create_seatbelt_command_args; +use crate::seatbelt::create_seatbelt_command_args_with_extensions; #[cfg(target_os = "macos")] use crate::spawn::CODEX_SANDBOX_ENV_VAR; use crate::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; @@ -25,6 +25,8 @@ use crate::tools::sandboxing::SandboxablePreference; use codex_network_proxy::NetworkProxy; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::FileSystemPermissions; +#[cfg(target_os = "macos")] +use codex_protocol::models::MacOsSeatbeltProfileExtensions; use codex_protocol::models::PermissionProfile; pub use codex_protocol::models::SandboxPermissions; use codex_protocol::protocol::ReadOnlyAccess; @@ -73,6 +75,8 @@ pub(crate) struct SandboxTransformRequest<'a> { // to make shared ownership explicit across runtime/sandbox plumbing. pub network: Option<&'a NetworkProxy>, pub sandbox_policy_cwd: &'a Path, + #[cfg(target_os = "macos")] + pub macos_seatbelt_profile_extensions: Option<&'a MacOsSeatbeltProfileExtensions>, pub codex_linux_sandbox_exe: Option<&'a PathBuf>, pub use_linux_sandbox_bwrap: bool, pub windows_sandbox_level: WindowsSandboxLevel, @@ -342,6 +346,8 @@ impl SandboxManager { enforce_managed_network, network, sandbox_policy_cwd, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions, codex_linux_sandbox_exe, use_linux_sandbox_bwrap, windows_sandbox_level, @@ -370,12 +376,13 @@ impl SandboxManager { SandboxType::MacosSeatbelt => { let mut seatbelt_env = HashMap::new(); seatbelt_env.insert(CODEX_SANDBOX_ENV_VAR.to_string(), "seatbelt".to_string()); - let mut args = create_seatbelt_command_args( + let mut args = create_seatbelt_command_args_with_extensions( command.clone(), &effective_policy, sandbox_policy_cwd, enforce_managed_network, network, + macos_seatbelt_profile_extensions, ); let mut full_command = Vec::with_capacity(1 + args.len()); full_command.push(MACOS_PATH_TO_SEATBELT_EXECUTABLE.to_string()); diff --git a/codex-rs/core/src/seatbelt_permissions.rs b/codex-rs/core/src/seatbelt_permissions.rs index aacc3a7784..6b9fa68188 100644 --- a/codex-rs/core/src/seatbelt_permissions.rs +++ b/codex-rs/core/src/seatbelt_permissions.rs @@ -3,34 +3,9 @@ use std::collections::BTreeSet; use std::path::PathBuf; -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub enum MacOsPreferencesPermission { - // IMPORTANT: ReadOnly needs to be the default because it's the security-sensitive default. - // it's important for allowing cf prefs to work. - #[default] - ReadOnly, - ReadWrite, - None, -} - -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub enum MacOsAutomationPermission { - #[default] - None, - All, - BundleIds(Vec), -} - -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct MacOsSeatbeltProfileExtensions { - pub macos_preferences: MacOsPreferencesPermission, - pub macos_automation: MacOsAutomationPermission, - pub macos_accessibility: bool, - pub macos_calendar: bool, -} +pub use codex_protocol::models::MacOsAutomationPermission; +pub use codex_protocol::models::MacOsPreferencesPermission; +pub use codex_protocol::models::MacOsSeatbeltProfileExtensions; #[derive(Debug, Clone, PartialEq, Eq, Default)] pub(crate) struct SeatbeltExtensionPolicy { @@ -38,25 +13,26 @@ pub(crate) struct SeatbeltExtensionPolicy { pub(crate) dir_params: Vec<(String, PathBuf)>, } -impl MacOsSeatbeltProfileExtensions { - pub fn normalized(&self) -> Self { - let mut normalized = self.clone(); - if let MacOsAutomationPermission::BundleIds(bundle_ids) = &self.macos_automation { - let bundle_ids = normalize_bundle_ids(bundle_ids); - normalized.macos_automation = if bundle_ids.is_empty() { - MacOsAutomationPermission::None - } else { - MacOsAutomationPermission::BundleIds(bundle_ids) - }; - } - normalized +fn normalized_extensions( + extensions: &MacOsSeatbeltProfileExtensions, +) -> MacOsSeatbeltProfileExtensions { + let mut normalized = extensions.clone(); + if let MacOsAutomationPermission::BundleIds(bundle_ids) = &extensions.macos_automation { + let bundle_ids = normalize_bundle_ids(bundle_ids); + normalized.macos_automation = if bundle_ids.is_empty() { + MacOsAutomationPermission::None + } else { + MacOsAutomationPermission::BundleIds(bundle_ids) + }; } + + normalized } pub(crate) fn build_seatbelt_extensions( extensions: &MacOsSeatbeltProfileExtensions, ) -> SeatbeltExtensionPolicy { - let extensions = extensions.normalized(); + let extensions = normalized_extensions(extensions); let mut clauses = Vec::new(); match extensions.macos_preferences { diff --git a/codex-rs/core/src/session_prefix.rs b/codex-rs/core/src/session_prefix.rs index 5f8516ba93..db3ac00a6d 100644 --- a/codex-rs/core/src/session_prefix.rs +++ b/codex-rs/core/src/session_prefix.rs @@ -1,28 +1,8 @@ use codex_protocol::protocol::AgentStatus; -/// Helpers for identifying model-visible "session prefix" messages. -/// -/// A session prefix is a user-role message that carries configuration or state needed by -/// follow-up turns (e.g. ``, ``). These items are persisted in -/// history so the model can see them, but they are not user intent and must not create user-turn -/// boundaries. -pub(crate) const ENVIRONMENT_CONTEXT_OPEN_TAG: &str = ""; -pub(crate) const TURN_ABORTED_OPEN_TAG: &str = ""; -pub(crate) const SUBAGENT_NOTIFICATION_OPEN_TAG: &str = ""; -pub(crate) const SUBAGENT_NOTIFICATION_CLOSE_TAG: &str = ""; - -fn starts_with_ascii_case_insensitive(text: &str, prefix: &str) -> bool { - text.get(..prefix.len()) - .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix)) -} - -/// Returns true if `text` starts with a session prefix marker (case-insensitive). -pub(crate) fn is_session_prefix(text: &str) -> bool { - let trimmed = text.trim_start(); - starts_with_ascii_case_insensitive(trimmed, ENVIRONMENT_CONTEXT_OPEN_TAG) - || starts_with_ascii_case_insensitive(trimmed, TURN_ABORTED_OPEN_TAG) - || starts_with_ascii_case_insensitive(trimmed, SUBAGENT_NOTIFICATION_OPEN_TAG) -} +/// Helpers for model-visible session state markers that are stored in user-role +/// messages but are not user intent. +use crate::contextual_user_message::SUBAGENT_NOTIFICATION_FRAGMENT; pub(crate) fn format_subagent_notification_message(agent_id: &str, status: &AgentStatus) -> String { let payload_json = serde_json::json!({ @@ -30,19 +10,12 @@ pub(crate) fn format_subagent_notification_message(agent_id: &str, status: &Agen "status": status, }) .to_string(); - format!("{SUBAGENT_NOTIFICATION_OPEN_TAG}\n{payload_json}\n{SUBAGENT_NOTIFICATION_CLOSE_TAG}") + SUBAGENT_NOTIFICATION_FRAGMENT.wrap(payload_json) } -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - - #[test] - fn is_session_prefix_is_case_insensitive() { - assert_eq!( - is_session_prefix("{}"), - true - ); +pub(crate) fn format_subagent_context_line(agent_id: &str, agent_nickname: Option<&str>) -> String { + match agent_nickname.filter(|nickname| !nickname.is_empty()) { + Some(agent_nickname) => format!("- {agent_id}: {agent_nickname}"), + None => format!("- {agent_id}"), } } diff --git a/codex-rs/core/src/skills/manager.rs b/codex-rs/core/src/skills/manager.rs index fa00fac41f..7d2c6b89fd 100644 --- a/codex-rs/core/src/skills/manager.rs +++ b/codex-rs/core/src/skills/manager.rs @@ -130,6 +130,12 @@ impl SkillsManager { }), ); let mut outcome = load_skills_from_roots(roots); + if !extra_user_roots.is_empty() { + // When extra user roots are provided, skip system skills before caching the result. + outcome + .skills + .retain(|skill| skill.scope != SkillScope::System); + } outcome.disabled_paths = disabled_paths_from_stack(&config_layer_stack); let (by_scripts_dir, by_doc_path) = build_implicit_skill_path_indexes(outcome.allowed_skills_for_implicit_invocation()); diff --git a/codex-rs/core/src/skills/permissions.rs b/codex-rs/core/src/skills/permissions.rs index 4019a31079..89c49709e3 100644 --- a/codex-rs/core/src/skills/permissions.rs +++ b/codex-rs/core/src/skills/permissions.rs @@ -8,6 +8,7 @@ use codex_protocol::models::MacOsAutomationValue; use codex_protocol::models::MacOsPermissions; #[cfg(target_os = "macos")] use codex_protocol::models::MacOsPreferencesValue; +use codex_protocol::models::MacOsSeatbeltProfileExtensions; use codex_protocol::models::PermissionProfile; use codex_utils_absolute_path::AbsolutePathBuf; use dirs::home_dir; @@ -20,10 +21,6 @@ use crate::config::types::ShellEnvironmentPolicy; use crate::protocol::AskForApproval; use crate::protocol::ReadOnlyAccess; use crate::protocol::SandboxPolicy; -#[cfg(target_os = "macos")] -use crate::seatbelt_permissions::MacOsSeatbeltProfileExtensions; -#[cfg(not(target_os = "macos"))] -type MacOsSeatbeltProfileExtensions = (); pub(crate) fn compile_permission_profile( skill_dir: &Path, diff --git a/codex-rs/core/src/state/turn.rs b/codex-rs/core/src/state/turn.rs index 3301740866..ccc50d066b 100644 --- a/codex-rs/core/src/state/turn.rs +++ b/codex-rs/core/src/state/turn.rs @@ -11,7 +11,6 @@ use tokio_util::task::AbortOnDropHandle; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::models::ResponseInputItem; use codex_protocol::request_user_input::RequestUserInputResponse; -use codex_protocol::skill_approval::SkillApprovalResponse; use tokio::sync::oneshot; use crate::codex::TurnContext; @@ -72,7 +71,6 @@ impl ActiveTurn { pub(crate) struct TurnState { pending_approvals: HashMap>, pending_user_input: HashMap>, - pending_skill_approvals: HashMap>, pending_dynamic_tools: HashMap>, pending_input: Vec, } @@ -96,7 +94,6 @@ impl TurnState { pub(crate) fn clear_pending(&mut self) { self.pending_approvals.clear(); self.pending_user_input.clear(); - self.pending_skill_approvals.clear(); self.pending_dynamic_tools.clear(); self.pending_input.clear(); } @@ -124,21 +121,6 @@ impl TurnState { self.pending_dynamic_tools.insert(key, tx) } - pub(crate) fn insert_pending_skill_approval( - &mut self, - key: String, - tx: oneshot::Sender, - ) -> Option> { - self.pending_skill_approvals.insert(key, tx) - } - - pub(crate) fn remove_pending_skill_approval( - &mut self, - key: &str, - ) -> Option> { - self.pending_skill_approvals.remove(key) - } - pub(crate) fn remove_pending_dynamic_tool( &mut self, key: &str, diff --git a/codex-rs/core/src/state_db.rs b/codex-rs/core/src/state_db.rs index f14b932fa6..93c7e75f97 100644 --- a/codex-rs/core/src/state_db.rs +++ b/codex-rs/core/src/state_db.rs @@ -345,6 +345,7 @@ pub async fn reconcile_rollout( builder: Option<&ThreadMetadataBuilder>, items: &[RolloutItem], archived_only: Option, + new_thread_memory_mode: Option<&str>, ) { let Some(ctx) = context else { return; @@ -357,6 +358,7 @@ pub async fn reconcile_rollout( builder, items, "reconcile_rollout", + new_thread_memory_mode, ) .await; return; @@ -467,6 +469,7 @@ pub async fn read_repair_rollout_path( None, &[], archived_only, + None, ) .await; } @@ -479,6 +482,7 @@ pub async fn apply_rollout_items( builder: Option<&ThreadMetadataBuilder>, items: &[RolloutItem], stage: &str, + new_thread_memory_mode: Option<&str>, ) { let Some(ctx) = context else { return; @@ -499,7 +503,10 @@ pub async fn apply_rollout_items( }; builder.rollout_path = rollout_path.to_path_buf(); builder.cwd = normalize_cwd_for_state_db(&builder.cwd); - if let Err(err) = ctx.apply_rollout_items(&builder, items, None).await { + if let Err(err) = ctx + .apply_rollout_items(&builder, items, None, new_thread_memory_mode) + .await + { warn!( "state db apply_rollout_items failed during {stage} for {}: {err}", rollout_path.display() diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 861b18a52c..97719f1049 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -21,12 +21,12 @@ use tracing::warn; use crate::AuthManager; use crate::codex::Session; use crate::codex::TurnContext; +use crate::contextual_user_message::TURN_ABORTED_OPEN_TAG; use crate::models_manager::manager::ModelsManager; use crate::protocol::EventMsg; use crate::protocol::TurnAbortReason; use crate::protocol::TurnAbortedEvent; use crate::protocol::TurnCompleteEvent; -use crate::session_prefix::TURN_ABORTED_OPEN_TAG; use crate::state::ActiveTurn; use crate::state::RunningTask; use crate::state::TaskKind; diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 5fd4cfc919..b3a95402a2 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -252,7 +252,7 @@ impl ThreadManager { } pub async fn list_thread_ids(&self) -> Vec { - self.state.threads.read().await.keys().copied().collect() + self.state.list_thread_ids().await } pub async fn refresh_mcp_servers(&self, refresh_config: McpServerRefreshConfig) { @@ -428,6 +428,10 @@ impl ThreadManager { } impl ThreadManagerState { + pub(crate) async fn list_thread_ids(&self) -> Vec { + self.threads.read().await.keys().copied().collect() + } + /// Fetch a thread by ID or return ThreadNotFound. pub(crate) async fn get_thread(&self, thread_id: ThreadId) -> CodexResult> { let threads = self.threads.read().await; @@ -511,6 +515,27 @@ impl ThreadManagerState { .await } + pub(crate) async fn fork_thread_with_source( + &self, + config: Config, + initial_history: InitialHistory, + agent_control: AgentControl, + session_source: SessionSource, + persist_extended_history: bool, + ) -> CodexResult { + self.spawn_thread_with_source( + config, + initial_history, + Arc::clone(&self.auth_manager), + agent_control, + session_source, + Vec::new(), + persist_extended_history, + None, + ) + .await + } + /// Spawn a new thread with optional history and register it with the manager. #[allow(clippy::too_many_arguments)] pub(crate) async fn spawn_thread( @@ -706,7 +731,7 @@ mod tests { #[tokio::test] async fn ignores_session_prefix_messages_when_truncating() { let (session, turn_context) = make_session_and_context().await; - let mut items = session.build_initial_context(&turn_context).await; + let mut items = session.build_initial_context(&turn_context, None).await; items.push(user_msg("feature request")); items.push(assistant_msg("ack")); items.push(user_msg("second question")); diff --git a/codex-rs/core/src/tools/context.rs b/codex-rs/core/src/tools/context.rs index 58925622a0..0700b4d013 100644 --- a/codex-rs/core/src/tools/context.rs +++ b/codex-rs/core/src/tools/context.rs @@ -95,15 +95,12 @@ impl ToolOutput { match self { ToolOutput::Function { body, success } => { // `custom_tool_call` is the Responses API item type for freeform - // tools (`ToolSpec::Freeform`, e.g. freeform `apply_patch`). - // Those payloads must round-trip as `custom_tool_call_output` - // with plain string output. + // tools (`ToolSpec::Freeform`, e.g. freeform `apply_patch` or + // `js_repl`). if matches!(payload, ToolPayload::Custom { .. }) { - // Freeform/custom tools (`custom_tool_call`) use the custom - // output wire shape and remain string-only. return ResponseInputItem::CustomToolCallOutput { call_id: call_id.to_string(), - output: body.to_text().unwrap_or_default(), + output: FunctionCallOutputPayload { body, success }, }; } @@ -183,7 +180,9 @@ mod tests { match response { ResponseInputItem::CustomToolCallOutput { call_id, output } => { assert_eq!(call_id, "call-42"); - assert_eq!(output, "patched"); + assert_eq!(output.text_content(), Some("patched")); + assert!(output.content_items().is_none()); + assert_eq!(output.success, Some(true)); } other => panic!("expected CustomToolCallOutput, got {other:?}"), } @@ -234,8 +233,21 @@ mod tests { match response { ResponseInputItem::CustomToolCallOutput { call_id, output } => { + let expected = vec![ + FunctionCallOutputContentItem::InputText { + text: "line 1".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,AAA".to_string(), + }, + FunctionCallOutputContentItem::InputText { + text: "line 2".to_string(), + }, + ]; assert_eq!(call_id, "call-99"); - assert_eq!(output, "line 1\nline 2"); + assert_eq!(output.content_items(), Some(expected.as_slice())); + assert_eq!(output.body.to_text().as_deref(), Some("line 1\nline 2")); + assert_eq!(output.success, Some(true)); } other => panic!("expected CustomToolCallOutput, got {other:?}"), } diff --git a/codex-rs/core/src/tools/handlers/js_repl.rs b/codex-rs/core/src/tools/handlers/js_repl.rs index 4488b4ea51..362d25b81e 100644 --- a/codex-rs/core/src/tools/handlers/js_repl.rs +++ b/codex-rs/core/src/tools/handlers/js_repl.rs @@ -155,9 +155,13 @@ impl ToolHandler for JsReplHandler { }; let content = result.output; - let items = vec![FunctionCallOutputContentItem::InputText { - text: content.clone(), - }]; + let mut items = Vec::with_capacity(result.content_items.len() + 1); + if !content.is_empty() { + items.push(FunctionCallOutputContentItem::InputText { + text: content.clone(), + }); + } + items.extend(result.content_items); emit_js_repl_exec_end( session.as_ref(), @@ -170,7 +174,11 @@ impl ToolHandler for JsReplHandler { .await; Ok(ToolOutput::Function { - body: FunctionCallOutputBody::ContentItems(items), + body: if items.is_empty() { + FunctionCallOutputBody::Text(content) + } else { + FunctionCallOutputBody::ContentItems(items) + }, success: Some(true), }) } diff --git a/codex-rs/core/src/tools/handlers/multi_agents.rs b/codex-rs/core/src/tools/handlers/multi_agents.rs index 6e62fd04fd..076a496a49 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents.rs @@ -91,6 +91,7 @@ impl ToolHandler for MultiAgentHandler { mod spawn { use super::*; + use crate::agent::control::SpawnAgentOptions; use crate::agent::role::DEFAULT_ROLE_NAME; use crate::agent::role::apply_role_to_config; @@ -103,6 +104,8 @@ mod spawn { message: Option, items: Option>, agent_type: Option, + #[serde(default)] + fork_context: bool, } #[derive(Debug, Serialize)] @@ -155,7 +158,7 @@ mod spawn { let result = session .services .agent_control - .spawn_agent( + .spawn_agent_with_options( config, input_items, Some(thread_spawn_source( @@ -163,6 +166,9 @@ mod spawn { child_depth, role_name, )), + SpawnAgentOptions { + fork_parent_spawn_call_id: args.fork_context.then(|| call_id.clone()), + }, ) .await .map_err(collab_spawn_error); @@ -914,7 +920,7 @@ fn build_agent_shared_config(turn: &TurnContext) -> Result, } struct KernelState { @@ -119,10 +126,37 @@ struct ExecContext { #[derive(Default)] struct ExecToolCalls { in_flight: usize, + content_items: Vec, notify: Arc, cancel: CancellationToken, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(clippy::enum_variant_names)] +enum JsReplToolCallPayloadKind { + MessageContent, + FunctionText, + FunctionContentItems, + CustomText, + CustomContentItems, + McpResult, + McpErrorResult, + Error, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct JsReplToolCallResponseSummary { + response_type: Option, + payload_kind: Option, + payload_text_preview: Option, + payload_text_length: Option, + payload_item_count: Option, + text_item_count: Option, + image_item_count: Option, + structured_content_present: Option, + result_is_error: Option, +} + enum KernelStreamEnd { Shutdown, StdoutEof, @@ -338,6 +372,21 @@ impl JsReplManager { Some(state.cancel.clone()) } + async fn record_exec_tool_call_content_items( + exec_tool_calls: &Arc>>, + exec_id: &str, + content_items: Vec, + ) { + if content_items.is_empty() { + return; + } + + let mut calls = exec_tool_calls.lock().await; + if let Some(state) = calls.get_mut(exec_id) { + state.content_items.extend(content_items); + } + } + async fn finish_exec_tool_call( exec_tool_calls: &Arc>>, exec_id: &str, @@ -404,6 +453,205 @@ impl JsReplManager { } } + fn log_tool_call_response( + req: &RunToolRequest, + ok: bool, + summary: &JsReplToolCallResponseSummary, + response: Option<&JsonValue>, + error: Option<&str>, + ) { + info!( + exec_id = %req.exec_id, + tool_call_id = %req.id, + tool_name = %req.tool_name, + ok, + summary = ?summary, + "js_repl nested tool call completed" + ); + if let Some(response) = response { + trace!( + exec_id = %req.exec_id, + tool_call_id = %req.id, + tool_name = %req.tool_name, + response_json = %response, + "js_repl nested tool call raw response" + ); + } + if let Some(error) = error { + trace!( + exec_id = %req.exec_id, + tool_call_id = %req.id, + tool_name = %req.tool_name, + error = %error, + "js_repl nested tool call raw error" + ); + } + } + + fn summarize_text_payload( + response_type: Option<&str>, + payload_kind: JsReplToolCallPayloadKind, + text: &str, + ) -> JsReplToolCallResponseSummary { + JsReplToolCallResponseSummary { + response_type: response_type.map(str::to_owned), + payload_kind: Some(payload_kind), + payload_text_preview: (!text.is_empty()).then(|| { + truncate_text( + text, + TruncationPolicy::Bytes(JS_REPL_TOOL_RESPONSE_TEXT_PREVIEW_MAX_BYTES), + ) + }), + payload_text_length: Some(text.len()), + ..Default::default() + } + } + + fn summarize_function_output_payload( + response_type: &str, + payload_kind: JsReplToolCallPayloadKind, + output: &FunctionCallOutputPayload, + ) -> JsReplToolCallResponseSummary { + let (payload_item_count, text_item_count, image_item_count) = + if let Some(items) = output.content_items() { + let text_item_count = items + .iter() + .filter(|item| matches!(item, FunctionCallOutputContentItem::InputText { .. })) + .count(); + let image_item_count = items.len().saturating_sub(text_item_count); + ( + Some(items.len()), + Some(text_item_count), + Some(image_item_count), + ) + } else { + (None, None, None) + }; + let payload_text = output.body.to_text(); + JsReplToolCallResponseSummary { + response_type: Some(response_type.to_string()), + payload_kind: Some(payload_kind), + payload_text_preview: payload_text.as_deref().and_then(|text| { + (!text.is_empty()).then(|| { + truncate_text( + text, + TruncationPolicy::Bytes(JS_REPL_TOOL_RESPONSE_TEXT_PREVIEW_MAX_BYTES), + ) + }) + }), + payload_text_length: payload_text.as_ref().map(String::len), + payload_item_count, + text_item_count, + image_item_count, + ..Default::default() + } + } + + fn summarize_message_payload(content: &[ContentItem]) -> JsReplToolCallResponseSummary { + let text_item_count = content + .iter() + .filter(|item| { + matches!( + item, + ContentItem::InputText { .. } | ContentItem::OutputText { .. } + ) + }) + .count(); + let image_item_count = content.len().saturating_sub(text_item_count); + let payload_text = content + .iter() + .filter_map(|item| match item { + ContentItem::InputText { text } | ContentItem::OutputText { text } + if !text.trim().is_empty() => + { + Some(text.as_str()) + } + ContentItem::InputText { .. } + | ContentItem::InputImage { .. } + | ContentItem::OutputText { .. } => None, + }) + .collect::>(); + let payload_text = if payload_text.is_empty() { + None + } else { + Some(payload_text.join("\n")) + }; + JsReplToolCallResponseSummary { + response_type: Some("message".to_string()), + payload_kind: Some(JsReplToolCallPayloadKind::MessageContent), + payload_text_preview: payload_text.as_deref().and_then(|text| { + (!text.is_empty()).then(|| { + truncate_text( + text, + TruncationPolicy::Bytes(JS_REPL_TOOL_RESPONSE_TEXT_PREVIEW_MAX_BYTES), + ) + }) + }), + payload_text_length: payload_text.as_ref().map(String::len), + payload_item_count: Some(content.len()), + text_item_count: Some(text_item_count), + image_item_count: Some(image_item_count), + ..Default::default() + } + } + + fn summarize_tool_call_response(response: &ResponseInputItem) -> JsReplToolCallResponseSummary { + match response { + ResponseInputItem::Message { content, .. } => Self::summarize_message_payload(content), + ResponseInputItem::FunctionCallOutput { output, .. } => { + let payload_kind = if output.content_items().is_some() { + JsReplToolCallPayloadKind::FunctionContentItems + } else { + JsReplToolCallPayloadKind::FunctionText + }; + Self::summarize_function_output_payload( + "function_call_output", + payload_kind, + output, + ) + } + ResponseInputItem::CustomToolCallOutput { output, .. } => { + let payload_kind = if output.content_items().is_some() { + JsReplToolCallPayloadKind::CustomContentItems + } else { + JsReplToolCallPayloadKind::CustomText + }; + Self::summarize_function_output_payload( + "custom_tool_call_output", + payload_kind, + output, + ) + } + ResponseInputItem::McpToolCallOutput { result, .. } => match result { + Ok(result) => { + let output = FunctionCallOutputPayload::from(result); + let mut summary = Self::summarize_function_output_payload( + "mcp_tool_call_output", + JsReplToolCallPayloadKind::McpResult, + &output, + ); + summary.payload_item_count = Some(result.content.len()); + summary.structured_content_present = Some(result.structured_content.is_some()); + summary.result_is_error = Some(result.is_error.unwrap_or(false)); + summary + } + Err(error) => { + let mut summary = Self::summarize_text_payload( + Some("mcp_tool_call_output"), + JsReplToolCallPayloadKind::McpErrorResult, + error, + ); + summary.result_is_error = Some(true); + summary + } + }, + } + } + + fn summarize_tool_call_error(error: &str) -> JsReplToolCallResponseSummary { + Self::summarize_text_payload(None, JsReplToolCallPayloadKind::Error, error) + } + pub async fn reset(&self) -> Result<(), FunctionCallError> { let _permit = self.exec_lock.clone().acquire_owned().await.map_err(|_| { FunctionCallError::RespondToModel("js_repl execution unavailable".to_string()) @@ -546,7 +794,13 @@ impl JsReplManager { }; match response { - ExecResultMessage::Ok { output } => Ok(JsExecResult { output }), + ExecResultMessage::Ok { content_items } => { + let (output, content_items) = split_exec_result_content_items(content_items); + Ok(JsExecResult { + output, + content_items, + }) + } ExecResultMessage::Err { message } => Err(FunctionCallError::RespondToModel(message)), } } @@ -556,10 +810,7 @@ impl JsReplManager { turn: Arc, thread_id: Option, ) -> Result { - let node_path = resolve_node(self.node_path.as_deref()).ok_or_else(|| { - "Node runtime not found; install Node or set CODEX_JS_REPL_NODE_PATH".to_string() - })?; - ensure_node_version(&node_path).await?; + let node_path = resolve_compatible_node(self.node_path.as_deref()).await?; let kernel_path = self .write_kernel_script() @@ -616,6 +867,8 @@ impl JsReplManager { enforce_managed_network: has_managed_network_requirements, network: None, sandbox_policy_cwd: &turn.cwd, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: turn.codex_linux_sandbox_exe.as_ref(), use_linux_sandbox_bwrap: turn .features @@ -851,10 +1104,22 @@ impl JsReplManager { error, } => { JsReplManager::wait_for_exec_tool_calls_map(&exec_tool_calls, &id).await; + let content_items = { + let calls = exec_tool_calls.lock().await; + calls + .get(&id) + .map(|state| state.content_items.clone()) + .unwrap_or_default() + }; let mut pending = pending_execs.lock().await; if let Some(tx) = pending.remove(&id) { let payload = if ok { - ExecResultMessage::Ok { output } + ExecResultMessage::Ok { + content_items: build_exec_result_content_items( + output, + content_items, + ), + } } else { ExecResultMessage::Err { message: error @@ -911,7 +1176,11 @@ impl JsReplManager { response: None, error: Some("js_repl execution reset".to_string()), }, - result = JsReplManager::run_tool_request(ctx, req) => result, + result = JsReplManager::run_tool_request( + ctx, + req, + Arc::clone(&exec_tool_calls_for_task), + ) => result, } } None => RunToolResult { @@ -1005,13 +1274,20 @@ impl JsReplManager { } } - async fn run_tool_request(exec: ExecContext, req: RunToolRequest) -> RunToolResult { + async fn run_tool_request( + exec: ExecContext, + req: RunToolRequest, + exec_tool_calls: Arc>>, + ) -> RunToolResult { if is_js_repl_internal_tool(&req.tool_name) { + let error = "js_repl cannot invoke itself".to_string(); + let summary = Self::summarize_tool_call_error(&error); + Self::log_tool_call_response(&req, false, &summary, None, Some(&error)); return RunToolResult { id: req.id, ok: false, response: None, - error: Some("js_repl cannot invoke itself".to_string()), + error: Some(error), }; } @@ -1075,62 +1351,50 @@ impl JsReplManager { .await { Ok(response) => { - if let ResponseInputItem::FunctionCallOutput { output, .. } = &response - && let Some(items) = output.content_items() - { - let mut has_image = false; - let mut content = Vec::with_capacity(items.len()); - for item in items { - match item { - FunctionCallOutputContentItem::InputText { text } => { - content.push(ContentItem::InputText { text: text.clone() }); - } - FunctionCallOutputContentItem::InputImage { image_url } => { - has_image = true; - content.push(ContentItem::InputImage { - image_url: image_url.clone(), - }); - } + if let Some(items) = response_content_items(&response) { + Self::record_exec_tool_call_content_items( + &exec_tool_calls, + &req.exec_id, + items, + ) + .await; + } + + let summary = Self::summarize_tool_call_response(&response); + match serde_json::to_value(response) { + Ok(value) => { + Self::log_tool_call_response(&req, true, &summary, Some(&value), None); + RunToolResult { + id: req.id, + ok: true, + response: Some(value), + error: None, } } - - if has_image - && session - .inject_response_items(vec![ResponseInputItem::Message { - role: "user".to_string(), - content, - }]) - .await - .is_err() - { - warn!( - tool_name = %tool_name, - "js_repl tool call returned image content but there was no active turn to attach it to" - ); + Err(err) => { + let error = format!("failed to serialize tool output: {err}"); + let summary = Self::summarize_tool_call_error(&error); + Self::log_tool_call_response(&req, false, &summary, None, Some(&error)); + RunToolResult { + id: req.id, + ok: false, + response: None, + error: Some(error), + } } } - - match serde_json::to_value(response) { - Ok(value) => RunToolResult { - id: req.id, - ok: true, - response: Some(value), - error: None, - }, - Err(err) => RunToolResult { - id: req.id, - ok: false, - response: None, - error: Some(format!("failed to serialize tool output: {err}")), - }, + } + Err(err) => { + let error = err.to_string(); + let summary = Self::summarize_tool_call_error(&error); + Self::log_tool_call_response(&req, false, &summary, None, Some(&error)); + RunToolResult { + id: req.id, + ok: false, + response: None, + error: Some(error), } } - Err(err) => RunToolResult { - id: req.id, - ok: false, - response: None, - error: Some(err.to_string()), - }, } } @@ -1168,6 +1432,50 @@ impl JsReplManager { } } +fn response_content_items( + response: &ResponseInputItem, +) -> Option> { + match response { + ResponseInputItem::FunctionCallOutput { output, .. } + | ResponseInputItem::CustomToolCallOutput { output, .. } => output + .content_items() + .map(<[FunctionCallOutputContentItem]>::to_vec), + ResponseInputItem::McpToolCallOutput { result, .. } => match result { + Ok(result) => FunctionCallOutputPayload::from(result) + .content_items() + .map(<[FunctionCallOutputContentItem]>::to_vec), + Err(_) => None, + }, + ResponseInputItem::Message { .. } => None, + } +} + +fn build_exec_result_content_items( + output: String, + content_items: Vec, +) -> Vec { + let mut all_content_items = Vec::with_capacity(content_items.len() + 1); + all_content_items.push(FunctionCallOutputContentItem::InputText { text: output }); + all_content_items.extend(content_items); + all_content_items +} + +fn split_exec_result_content_items( + mut content_items: Vec, +) -> (String, Vec) { + match content_items.first() { + Some(FunctionCallOutputContentItem::InputText { .. }) => { + let FunctionCallOutputContentItem::InputText { text } = content_items.remove(0) else { + unreachable!("first content item should be input_text"); + }; + (text, content_items) + } + Some(FunctionCallOutputContentItem::InputImage { .. }) | None => { + (String::new(), content_items) + } + } +} + fn is_freeform_tool(specs: &[ToolSpec], name: &str) -> bool { specs .iter() @@ -1223,8 +1531,12 @@ struct RunToolResult { #[derive(Debug)] enum ExecResultMessage { - Ok { output: String }, - Err { message: String }, + Ok { + content_items: Vec, + }, + Err { + message: String, + }, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -1321,6 +1633,14 @@ async fn ensure_node_version(node_path: &Path) -> Result<(), String> { Ok(()) } +pub(crate) async fn resolve_compatible_node(config_path: Option<&Path>) -> Result { + let node_path = resolve_node(config_path).ok_or_else(|| { + "Node runtime not found; install Node or set CODEX_JS_REPL_NODE_PATH".to_string() + })?; + ensure_node_version(&node_path).await?; + Ok(node_path) +} + pub(crate) fn resolve_node(config_path: Option<&Path>) -> Option { if let Some(path) = std::env::var_os("CODEX_JS_REPL_NODE_PATH") { let p = PathBuf::from(path); @@ -1354,7 +1674,8 @@ mod tests { use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; - use codex_protocol::models::ContentItem; + use codex_protocol::models::FunctionCallOutputContentItem; + use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseInputItem; use codex_protocol::openai_models::InputModality; use pretty_assertions::assert_eq; @@ -1572,6 +1893,84 @@ mod tests { ); } + #[test] + fn summarize_tool_call_response_for_multimodal_function_output() { + let response = ResponseInputItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,abcd".to_string(), + }, + ]), + }; + + let actual = JsReplManager::summarize_tool_call_response(&response); + + assert_eq!( + actual, + JsReplToolCallResponseSummary { + response_type: Some("function_call_output".to_string()), + payload_kind: Some(JsReplToolCallPayloadKind::FunctionContentItems), + payload_text_preview: None, + payload_text_length: None, + payload_item_count: Some(1), + text_item_count: Some(0), + image_item_count: Some(1), + structured_content_present: None, + result_is_error: None, + } + ); + } + + #[test] + fn summarize_tool_call_response_for_multimodal_custom_output() { + let response = ResponseInputItem::CustomToolCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,abcd".to_string(), + }, + ]), + }; + + let actual = JsReplManager::summarize_tool_call_response(&response); + + assert_eq!( + actual, + JsReplToolCallResponseSummary { + response_type: Some("custom_tool_call_output".to_string()), + payload_kind: Some(JsReplToolCallPayloadKind::CustomContentItems), + payload_text_preview: None, + payload_text_length: None, + payload_item_count: Some(1), + text_item_count: Some(0), + image_item_count: Some(1), + structured_content_present: None, + result_is_error: None, + } + ); + } + + #[test] + fn summarize_tool_call_error_marks_error_payload() { + let actual = JsReplManager::summarize_tool_call_error("tool failed"); + + assert_eq!( + actual, + JsReplToolCallResponseSummary { + response_type: None, + payload_kind: Some(JsReplToolCallPayloadKind::Error), + payload_text_preview: Some("tool failed".to_string()), + payload_text_length: Some("tool failed".len()), + payload_item_count: None, + text_item_count: None, + image_item_count: None, + structured_content_present: None, + result_is_error: None, + } + ); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn reset_clears_inflight_exec_tool_calls_without_waiting() { let manager = JsReplManager::new(None, Vec::new()) @@ -2012,20 +2411,22 @@ console.log(out.output?.body?.text ?? ""); ) .await?; assert!(result.output.contains("function_call_output")); - - let pending_input = session.get_pending_input().await; - let [ResponseInputItem::Message { role, content }] = pending_input.as_slice() else { - panic!( - "view_image should inject exactly one pending input message, got {pending_input:?}" - ); - }; - assert_eq!(role, "user"); - let [ContentItem::InputImage { image_url }] = content.as_slice() else { - panic!( - "view_image should inject exactly one input_image content item, got {content:?}" - ); + assert_eq!( + result.content_items.as_slice(), + [FunctionCallOutputContentItem::InputImage { + image_url: + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==" + .to_string(), + }] + .as_slice() + ); + let [FunctionCallOutputContentItem::InputImage { image_url }] = + result.content_items.as_slice() + else { + panic!("view_image should return exactly one input_image content item"); }; assert!(image_url.starts_with("data:image/png;base64,")); + assert!(session.get_pending_input().await.is_empty()); Ok(()) } @@ -2106,22 +2507,18 @@ console.log(out.type); response_watcher_result?; let result = result?; assert!(result.output.contains("function_call_output")); - - let pending_input = session.get_pending_input().await; assert_eq!( - pending_input, - vec![ResponseInputItem::Message { - role: "user".to_string(), - content: vec![ - ContentItem::InputText { - text: "inline image note".to_string(), - }, - ContentItem::InputImage { - image_url: image_url.to_string(), - }, - ], - }] + result.content_items, + vec![ + FunctionCallOutputContentItem::InputText { + text: "inline image note".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: image_url.to_string(), + }, + ] ); + assert!(session.get_pending_input().await.is_empty()); Ok(()) } diff --git a/codex-rs/core/src/tools/network_approval.rs b/codex-rs/core/src/tools/network_approval.rs index ca31ec4d4c..aeef59833d 100644 --- a/codex-rs/core/src/tools/network_approval.rs +++ b/codex-rs/core/src/tools/network_approval.rs @@ -331,6 +331,7 @@ impl NetworkApprovalService { protocol, }; + let available_decisions = None; let approval_decision = session .request_command_approval( turn_context.as_ref(), @@ -342,6 +343,7 @@ impl NetworkApprovalService { Some(network_approval_context.clone()), None, None, + available_decisions, ) .await; diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 04d505c584..5f2fc89e5f 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -117,7 +117,10 @@ impl ToolCallRuntime { match &call.payload { ToolPayload::Custom { .. } => ResponseInputItem::CustomToolCallOutput { call_id: call.call_id.clone(), - output: Self::abort_message(call, secs), + output: FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text(Self::abort_message(call, secs)), + ..Default::default() + }, }, ToolPayload::Mcp { .. } => ResponseInputItem::McpToolCallOutput { call_id: call.call_id.clone(), diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index 7a0be68df3..3c021cc582 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -379,6 +379,7 @@ async fn dispatch_after_tool_use_hook( .dispatch(HookPayload { session_id: session.conversation_id, cwd: turn.cwd.clone(), + client: turn.app_server_client_name.clone(), triggered_at: chrono::Utc::now(), hook_event: HookEvent::AfterToolUse { event: HookEventAfterToolUse { diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 4897b4ea6f..a55fb5fd5a 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -197,7 +197,10 @@ impl ToolRouter { if payload_outputs_custom { ResponseInputItem::CustomToolCallOutput { call_id, - output: message, + output: codex_protocol::models::FunctionCallOutputPayload { + body: FunctionCallOutputBody::Text(message), + success: Some(false), + }, } } else { ResponseInputItem::FunctionCallOutput { diff --git a/codex-rs/core/src/tools/runtimes/shell.rs b/codex-rs/core/src/tools/runtimes/shell.rs index 80d3b22ebb..101d9309fd 100644 --- a/codex-rs/core/src/tools/runtimes/shell.rs +++ b/codex-rs/core/src/tools/runtimes/shell.rs @@ -150,6 +150,7 @@ impl Approvable for ShellRuntime { let call_id = ctx.call_id.to_string(); Box::pin(async move { with_cached_approval(&session.services, "shell", keys, move || async move { + let available_decisions = None; session .request_command_approval( turn, @@ -163,6 +164,7 @@ impl Approvable for ShellRuntime { .proposed_execpolicy_amendment() .cloned(), req.additional_permissions.clone(), + available_decisions, ) .await }) diff --git a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs index 3b1a0c9067..ae1df34ece 100644 --- a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs +++ b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs @@ -12,12 +12,14 @@ use crate::skills::SkillMetadata; use crate::tools::runtimes::ExecveSessionApproval; use crate::tools::runtimes::build_command_spec; use crate::tools::sandboxing::SandboxAttempt; +use crate::tools::sandboxing::SandboxablePreference; use crate::tools::sandboxing::ToolCtx; use crate::tools::sandboxing::ToolError; use codex_execpolicy::Decision; use codex_execpolicy::Policy; use codex_execpolicy::RuleMatch; use codex_protocol::config_types::WindowsSandboxLevel; +use codex_protocol::models::MacOsSeatbeltProfileExtensions; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::NetworkPolicyRuleAction; @@ -26,11 +28,15 @@ use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::SandboxPolicy; use codex_shell_command::bash::parse_shell_lc_plain_commands; use codex_shell_command::bash::parse_shell_lc_single_command_prefix; -use codex_shell_escalation::EscalateAction; use codex_shell_escalation::EscalateServer; +use codex_shell_escalation::EscalationDecision; +use codex_shell_escalation::EscalationExecution; +use codex_shell_escalation::EscalationPermissions; use codex_shell_escalation::EscalationPolicy; use codex_shell_escalation::ExecParams; use codex_shell_escalation::ExecResult; +use codex_shell_escalation::Permissions as EscalatedPermissions; +use codex_shell_escalation::PreparedExec; use codex_shell_escalation::ShellCommandExecutor; use codex_shell_escalation::Stopwatch; use codex_utils_absolute_path::AbsolutePathBuf; @@ -105,6 +111,15 @@ pub(super) async fn try_run_zsh_fork( sandbox_permissions, justification, arg0, + sandbox_policy_cwd: ctx.turn.cwd.clone(), + macos_seatbelt_profile_extensions: ctx + .turn + .config + .permissions + .macos_seatbelt_profile_extensions + .clone(), + codex_linux_sandbox_exe: ctx.turn.codex_linux_sandbox_exe.clone(), + use_linux_sandbox_bwrap: ctx.turn.features.enabled(Feature::UseLinuxSandboxBwrap), }; let main_execve_wrapper_exe = ctx .session @@ -136,6 +151,7 @@ pub(super) async fn try_run_zsh_fork( approval_policy: ctx.turn.approval_policy.value(), sandbox_policy: attempt.policy.clone(), sandbox_permissions: req.sandbox_permissions, + prompt_permissions: req.additional_permissions.clone(), stopwatch: stopwatch.clone(), }; @@ -146,7 +162,7 @@ pub(super) async fn try_run_zsh_fork( ); let exec_result = escalate_server - .exec(exec_params, cancel_token, &command_executor) + .exec(exec_params, cancel_token, Arc::new(command_executor)) .await .map_err(|err| ToolError::Rejected(err.to_string()))?; @@ -161,6 +177,7 @@ struct CoreShellActionProvider { approval_policy: AskForApproval, sandbox_policy: SandboxPolicy, sandbox_permissions: SandboxPermissions, + prompt_permissions: Option, stopwatch: Stopwatch, } @@ -182,6 +199,55 @@ impl CoreShellActionProvider { }) } + fn shell_request_escalation_execution( + sandbox_permissions: SandboxPermissions, + sandbox_policy: &SandboxPolicy, + additional_permissions: Option<&PermissionProfile>, + macos_seatbelt_profile_extensions: Option<&MacOsSeatbeltProfileExtensions>, + ) -> EscalationExecution { + match sandbox_permissions { + SandboxPermissions::UseDefault => EscalationExecution::TurnDefault, + SandboxPermissions::RequireEscalated => EscalationExecution::Unsandboxed, + SandboxPermissions::WithAdditionalPermissions => additional_permissions + .map(|_| { + // Shell request additional permissions were already normalized and + // merged into the first-attempt sandbox policy. + EscalationExecution::Permissions(EscalationPermissions::Permissions( + EscalatedPermissions { + sandbox_policy: sandbox_policy.clone(), + macos_seatbelt_profile_extensions: macos_seatbelt_profile_extensions + .cloned(), + }, + )) + }) + .unwrap_or(EscalationExecution::TurnDefault), + } + } + + fn skill_escalation_execution(skill: &SkillMetadata) -> EscalationExecution { + skill + .permissions + .as_ref() + .map(|permissions| { + EscalationExecution::Permissions(EscalationPermissions::Permissions( + EscalatedPermissions { + sandbox_policy: permissions.sandbox_policy.get().clone(), + macos_seatbelt_profile_extensions: permissions + .macos_seatbelt_profile_extensions + .clone(), + }, + )) + }) + .or_else(|| { + skill + .permission_profile + .clone() + .map(EscalationPermissions::PermissionProfile) + .map(EscalationExecution::Permissions) + }) + .unwrap_or(EscalationExecution::TurnDefault) + } + async fn prompt( &self, program: &AbsolutePathBuf, @@ -189,6 +255,7 @@ impl CoreShellActionProvider { workdir: &AbsolutePathBuf, stopwatch: &Stopwatch, additional_permissions: Option, + decision_source: &DecisionSource, ) -> anyhow::Result { let command = join_program_and_argv(program, argv); let workdir = workdir.to_path_buf(); @@ -198,6 +265,20 @@ impl CoreShellActionProvider { let approval_id = Some(Uuid::new_v4().to_string()); Ok(stopwatch .pause_for(async move { + let available_decisions = vec![ + Some(ReviewDecision::Approved), + // Currently, ApprovedForSession is only honored for skills, + // so only offer it for skill script approvals. + if matches!(decision_source, DecisionSource::SkillScript { .. }) { + Some(ReviewDecision::ApprovedForSession) + } else { + None + }, + Some(ReviewDecision::Abort), + ] + .into_iter() + .flatten() + .collect(); session .request_command_approval( &turn, @@ -209,6 +290,7 @@ impl CoreShellActionProvider { None, None, additional_permissions, + Some(available_decisions), ) .await }) @@ -249,22 +331,21 @@ impl CoreShellActionProvider { program: &AbsolutePathBuf, argv: &[String], workdir: &AbsolutePathBuf, - additional_permissions: Option, + prompt_permissions: Option, + escalation_execution: EscalationExecution, decision_source: DecisionSource, - ) -> anyhow::Result { + ) -> anyhow::Result { let action = match decision { - Decision::Forbidden => EscalateAction::Deny { - reason: Some("Execution forbidden by policy".to_string()), - }, + Decision::Forbidden => { + EscalationDecision::deny(Some("Execution forbidden by policy".to_string())) + } Decision::Prompt => { if matches!( self.approval_policy, AskForApproval::Never | AskForApproval::Reject(RejectConfig { rules: true, .. }) ) { - EscalateAction::Deny { - reason: Some("Execution forbidden by policy".to_string()), - } + EscalationDecision::deny(Some("Execution forbidden by policy".to_string())) } else { match self .prompt( @@ -272,16 +353,17 @@ impl CoreShellActionProvider { argv, workdir, &self.stopwatch, - additional_permissions, + prompt_permissions, + &decision_source, ) .await? { ReviewDecision::Approved | ReviewDecision::ApprovedExecpolicyAmendment { .. } => { if needs_escalation { - EscalateAction::Escalate + EscalationDecision::escalate(escalation_execution.clone()) } else { - EscalateAction::Run + EscalationDecision::run() } } ReviewDecision::ApprovedForSession => { @@ -306,9 +388,9 @@ impl CoreShellActionProvider { } if needs_escalation { - EscalateAction::Escalate + EscalationDecision::escalate(escalation_execution.clone()) } else { - EscalateAction::Run + EscalationDecision::run() } } ReviewDecision::NetworkPolicyAmendment { @@ -316,29 +398,29 @@ impl CoreShellActionProvider { } => match network_policy_amendment.action { NetworkPolicyRuleAction::Allow => { if needs_escalation { - EscalateAction::Escalate + EscalationDecision::escalate(escalation_execution.clone()) } else { - EscalateAction::Run + EscalationDecision::run() } } - NetworkPolicyRuleAction::Deny => EscalateAction::Deny { - reason: Some("User denied execution".to_string()), - }, - }, - ReviewDecision::Denied => EscalateAction::Deny { - reason: Some("User denied execution".to_string()), - }, - ReviewDecision::Abort => EscalateAction::Deny { - reason: Some("User cancelled execution".to_string()), + NetworkPolicyRuleAction::Deny => { + EscalationDecision::deny(Some("User denied execution".to_string())) + } }, + ReviewDecision::Denied => { + EscalationDecision::deny(Some("User denied execution".to_string())) + } + ReviewDecision::Abort => { + EscalationDecision::deny(Some("User cancelled execution".to_string())) + } } } } Decision::Allow => { if needs_escalation { - EscalateAction::Escalate + EscalationDecision::escalate(escalation_execution) } else { - EscalateAction::Run + EscalationDecision::run() } } }; @@ -356,7 +438,7 @@ impl EscalationPolicy for CoreShellActionProvider { program: &AbsolutePathBuf, argv: &[String], workdir: &AbsolutePathBuf, - ) -> anyhow::Result { + ) -> anyhow::Result { tracing::debug!( "Determining escalation action for command {program:?} with args {argv:?} in {workdir:?}" ); @@ -377,15 +459,13 @@ impl EscalationPolicy for CoreShellActionProvider { tracing::debug!( "Found session approval for {program:?}, allowing execution without further checks" ); - // TODO(mbolin): We need to include the permissions with the - // escalation decision so it can be run with the appropriate - // permissions. - let _permissions = approval + let execution = approval .skill .as_ref() - .and_then(|s| s.permission_profile.clone()); + .map(Self::skill_escalation_execution) + .unwrap_or(EscalationExecution::TurnDefault); - return Ok(EscalateAction::Escalate); + return Ok(EscalationDecision::escalate(execution)); } // In the usual case, the execve wrapper reports the command being @@ -407,6 +487,7 @@ impl EscalationPolicy for CoreShellActionProvider { argv, workdir, skill.permission_profile.clone(), + Self::skill_escalation_execution(&skill), decision_source, ) .await; @@ -447,13 +528,24 @@ impl EscalationPolicy for CoreShellActionProvider { } else { DecisionSource::UnmatchedCommandFallback }; + let escalation_execution = Self::shell_request_escalation_execution( + self.sandbox_permissions, + &self.sandbox_policy, + self.prompt_permissions.as_ref(), + self.turn + .config + .permissions + .macos_seatbelt_profile_extensions + .as_ref(), + ); self.process_decision( evaluation.decision, needs_escalation, program, argv, workdir, - None, + self.prompt_permissions.clone(), + escalation_execution, decision_source, ) .await @@ -471,6 +563,10 @@ struct CoreShellCommandExecutor { sandbox_permissions: SandboxPermissions, justification: Option, arg0: Option, + sandbox_policy_cwd: PathBuf, + macos_seatbelt_profile_extensions: Option, + codex_linux_sandbox_exe: Option, + use_linux_sandbox_bwrap: bool, } #[async_trait::async_trait] @@ -516,6 +612,126 @@ impl ShellCommandExecutor for CoreShellCommandExecutor { timed_out: result.timed_out, }) } + + async fn prepare_escalated_exec( + &self, + program: &AbsolutePathBuf, + argv: &[String], + workdir: &AbsolutePathBuf, + env: HashMap, + execution: EscalationExecution, + ) -> anyhow::Result { + let command = join_program_and_argv(program, argv); + let Some(first_arg) = argv.first() else { + return Err(anyhow::anyhow!( + "intercepted exec request must contain argv[0]" + )); + }; + + let prepared = match execution { + EscalationExecution::Unsandboxed => PreparedExec { + command, + cwd: workdir.to_path_buf(), + env, + arg0: Some(first_arg.clone()), + }, + EscalationExecution::TurnDefault => self.prepare_sandboxed_exec( + command, + workdir, + env, + &self.sandbox_policy, + None, + self.macos_seatbelt_profile_extensions.as_ref(), + )?, + EscalationExecution::Permissions(EscalationPermissions::PermissionProfile( + permission_profile, + )) => self.prepare_sandboxed_exec( + command, + workdir, + env, + &self.sandbox_policy, + Some(permission_profile), + None, + )?, + EscalationExecution::Permissions(EscalationPermissions::Permissions(permissions)) => { + self.prepare_sandboxed_exec( + command, + workdir, + env, + &permissions.sandbox_policy, + None, + permissions.macos_seatbelt_profile_extensions.as_ref(), + )? + } + }; + + Ok(prepared) + } +} + +impl CoreShellCommandExecutor { + fn prepare_sandboxed_exec( + &self, + command: Vec, + workdir: &AbsolutePathBuf, + env: HashMap, + sandbox_policy: &SandboxPolicy, + additional_permissions: Option, + #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions: Option< + &MacOsSeatbeltProfileExtensions, + >, + #[cfg(not(target_os = "macos"))] _macos_seatbelt_profile_extensions: Option< + &MacOsSeatbeltProfileExtensions, + >, + ) -> anyhow::Result { + let (program, args) = command + .split_first() + .ok_or_else(|| anyhow::anyhow!("prepared command must not be empty"))?; + let sandbox_manager = crate::sandboxing::SandboxManager::new(); + let sandbox = sandbox_manager.select_initial( + sandbox_policy, + SandboxablePreference::Auto, + self.windows_sandbox_level, + self.network.is_some(), + ); + let mut exec_request = + sandbox_manager.transform(crate::sandboxing::SandboxTransformRequest { + spec: crate::sandboxing::CommandSpec { + program: program.clone(), + args: args.to_vec(), + cwd: workdir.to_path_buf(), + env, + expiration: ExecExpiration::DefaultTimeout, + sandbox_permissions: if additional_permissions.is_some() { + SandboxPermissions::WithAdditionalPermissions + } else { + SandboxPermissions::UseDefault + }, + additional_permissions, + justification: self.justification.clone(), + }, + policy: sandbox_policy, + sandbox, + enforce_managed_network: self.network.is_some(), + network: self.network.as_ref(), + sandbox_policy_cwd: &self.sandbox_policy_cwd, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions, + codex_linux_sandbox_exe: self.codex_linux_sandbox_exe.as_ref(), + use_linux_sandbox_bwrap: self.use_linux_sandbox_bwrap, + windows_sandbox_level: self.windows_sandbox_level, + })?; + if let Some(network) = exec_request.network.as_ref() { + network.apply_to_env(&mut exec_request.env); + } + + Ok(PreparedExec { + command: exec_request.command, + cwd: exec_request.cwd, + env: exec_request.env, + arg0: exec_request.arg0, + }) + } } #[derive(Debug, Eq, PartialEq)] @@ -583,122 +799,5 @@ fn join_program_and_argv(program: &AbsolutePathBuf, argv: &[String]) -> Vec reason, - _ => "".to_string(), - }, - "unexpected shell command format for zsh-fork execution" - ); - } - - #[test] - fn join_program_and_argv_replaces_original_argv_zero() { - assert_eq!( - join_program_and_argv( - &AbsolutePathBuf::from_absolute_path("/tmp/tool").unwrap(), - &["./tool".into(), "--flag".into(), "value".into()], - ), - vec!["/tmp/tool", "--flag", "value"] - ); - assert_eq!( - join_program_and_argv( - &AbsolutePathBuf::from_absolute_path("/tmp/tool").unwrap(), - &["./tool".into()] - ), - vec!["/tmp/tool"] - ); - } - - #[test] - fn map_exec_result_preserves_stdout_and_stderr() { - let out = map_exec_result( - SandboxType::None, - ExecResult { - exit_code: 0, - stdout: "out".to_string(), - stderr: "err".to_string(), - output: "outerr".to_string(), - duration: Duration::from_millis(1), - timed_out: false, - }, - ) - .unwrap(); - - assert_eq!(out.stdout.text, "out"); - assert_eq!(out.stderr.text, "err"); - assert_eq!(out.aggregated_output.text, "outerr"); - } -} +#[path = "unix_escalation_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/tools/runtimes/shell/unix_escalation_tests.rs b/codex-rs/core/src/tools/runtimes/shell/unix_escalation_tests.rs new file mode 100644 index 0000000000..d2fd612670 --- /dev/null +++ b/codex-rs/core/src/tools/runtimes/shell/unix_escalation_tests.rs @@ -0,0 +1,320 @@ +use super::CoreShellActionProvider; +#[cfg(target_os = "macos")] +use super::CoreShellCommandExecutor; +use super::ParsedShellCommand; +use super::extract_shell_script; +use super::join_program_and_argv; +use super::map_exec_result; +#[cfg(target_os = "macos")] +use crate::config::Constrained; +#[cfg(target_os = "macos")] +use crate::config::Permissions; +#[cfg(target_os = "macos")] +use crate::config::types::ShellEnvironmentPolicy; +use crate::exec::SandboxType; +#[cfg(target_os = "macos")] +use crate::protocol::AskForApproval; +use crate::protocol::ReadOnlyAccess; +use crate::protocol::SandboxPolicy; +#[cfg(target_os = "macos")] +use crate::sandboxing::SandboxPermissions; +#[cfg(target_os = "macos")] +use crate::seatbelt::MACOS_PATH_TO_SEATBELT_EXECUTABLE; +#[cfg(target_os = "macos")] +use codex_protocol::config_types::WindowsSandboxLevel; +use codex_protocol::models::FileSystemPermissions; +use codex_protocol::models::MacOsPreferencesPermission; +use codex_protocol::models::MacOsSeatbeltProfileExtensions; +use codex_protocol::models::PermissionProfile; +use codex_shell_escalation::EscalationExecution; +use codex_shell_escalation::EscalationPermissions; +use codex_shell_escalation::ExecResult; +use codex_shell_escalation::Permissions as EscalatedPermissions; +#[cfg(target_os = "macos")] +use codex_shell_escalation::ShellCommandExecutor; +use codex_utils_absolute_path::AbsolutePathBuf; +use pretty_assertions::assert_eq; +#[cfg(target_os = "macos")] +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::Duration; + +#[test] +fn extract_shell_script_preserves_login_flag() { + assert_eq!( + extract_shell_script(&["/bin/zsh".into(), "-lc".into(), "echo hi".into()]).unwrap(), + ParsedShellCommand { + script: "echo hi".to_string(), + login: true, + } + ); + assert_eq!( + extract_shell_script(&["/bin/zsh".into(), "-c".into(), "echo hi".into()]).unwrap(), + ParsedShellCommand { + script: "echo hi".to_string(), + login: false, + } + ); +} + +#[test] +fn extract_shell_script_supports_wrapped_command_prefixes() { + assert_eq!( + extract_shell_script(&[ + "/usr/bin/env".into(), + "CODEX_EXECVE_WRAPPER=1".into(), + "/bin/zsh".into(), + "-lc".into(), + "echo hello".into() + ]) + .unwrap(), + ParsedShellCommand { + script: "echo hello".to_string(), + login: true, + } + ); + + assert_eq!( + extract_shell_script(&[ + "sandbox-exec".into(), + "-p".into(), + "sandbox_policy".into(), + "/bin/zsh".into(), + "-c".into(), + "pwd".into(), + ]) + .unwrap(), + ParsedShellCommand { + script: "pwd".to_string(), + login: false, + } + ); +} + +#[test] +fn extract_shell_script_rejects_unsupported_shell_invocation() { + let err = extract_shell_script(&[ + "sandbox-exec".into(), + "-fc".into(), + "echo not supported".into(), + ]) + .unwrap_err(); + assert!(matches!(err, super::ToolError::Rejected(_))); + assert_eq!( + match err { + super::ToolError::Rejected(reason) => reason, + _ => "".to_string(), + }, + "unexpected shell command format for zsh-fork execution" + ); +} + +#[test] +fn join_program_and_argv_replaces_original_argv_zero() { + assert_eq!( + join_program_and_argv( + &AbsolutePathBuf::from_absolute_path("/tmp/tool").unwrap(), + &["./tool".into(), "--flag".into(), "value".into()], + ), + vec!["/tmp/tool", "--flag", "value"] + ); + assert_eq!( + join_program_and_argv( + &AbsolutePathBuf::from_absolute_path("/tmp/tool").unwrap(), + &["./tool".into()] + ), + vec!["/tmp/tool"] + ); +} + +#[test] +fn map_exec_result_preserves_stdout_and_stderr() { + let out = map_exec_result( + SandboxType::None, + ExecResult { + exit_code: 0, + stdout: "out".to_string(), + stderr: "err".to_string(), + output: "outerr".to_string(), + duration: Duration::from_millis(1), + timed_out: false, + }, + ) + .unwrap(); + + assert_eq!(out.stdout.text, "out"); + assert_eq!(out.stderr.text, "err"); + assert_eq!(out.aggregated_output.text, "outerr"); +} + +#[test] +fn shell_request_escalation_execution_is_explicit() { + let requested_permissions = PermissionProfile { + file_system: Some(FileSystemPermissions { + read: None, + write: Some(vec![PathBuf::from("./output")]), + }), + ..Default::default() + }; + let sandbox_policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![AbsolutePathBuf::from_absolute_path("/tmp/original/output").unwrap()], + read_only_access: ReadOnlyAccess::FullAccess, + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + let macos_seatbelt_profile_extensions = MacOsSeatbeltProfileExtensions { + macos_preferences: MacOsPreferencesPermission::ReadWrite, + ..Default::default() + }; + + assert_eq!( + CoreShellActionProvider::shell_request_escalation_execution( + crate::sandboxing::SandboxPermissions::UseDefault, + &sandbox_policy, + None, + Some(&macos_seatbelt_profile_extensions), + ), + EscalationExecution::TurnDefault, + ); + assert_eq!( + CoreShellActionProvider::shell_request_escalation_execution( + crate::sandboxing::SandboxPermissions::RequireEscalated, + &sandbox_policy, + None, + Some(&macos_seatbelt_profile_extensions), + ), + EscalationExecution::Unsandboxed, + ); + assert_eq!( + CoreShellActionProvider::shell_request_escalation_execution( + crate::sandboxing::SandboxPermissions::WithAdditionalPermissions, + &sandbox_policy, + Some(&requested_permissions), + Some(&macos_seatbelt_profile_extensions), + ), + EscalationExecution::Permissions(EscalationPermissions::Permissions( + EscalatedPermissions { + sandbox_policy, + macos_seatbelt_profile_extensions: Some(macos_seatbelt_profile_extensions), + }, + )), + ); +} + +#[cfg(target_os = "macos")] +#[tokio::test] +async fn prepare_escalated_exec_turn_default_preserves_macos_seatbelt_extensions() { + let cwd = AbsolutePathBuf::from_absolute_path(std::env::temp_dir()).unwrap(); + let executor = CoreShellCommandExecutor { + command: vec!["echo".to_string(), "ok".to_string()], + cwd: cwd.to_path_buf(), + env: HashMap::new(), + network: None, + sandbox: SandboxType::None, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + windows_sandbox_level: WindowsSandboxLevel::Disabled, + sandbox_permissions: SandboxPermissions::UseDefault, + justification: None, + arg0: None, + sandbox_policy_cwd: cwd.to_path_buf(), + macos_seatbelt_profile_extensions: Some(MacOsSeatbeltProfileExtensions { + macos_preferences: MacOsPreferencesPermission::ReadWrite, + ..Default::default() + }), + codex_linux_sandbox_exe: None, + use_linux_sandbox_bwrap: false, + }; + + let prepared = executor + .prepare_escalated_exec( + &AbsolutePathBuf::from_absolute_path("/bin/echo").unwrap(), + &["echo".to_string(), "ok".to_string()], + &cwd, + HashMap::new(), + EscalationExecution::TurnDefault, + ) + .await + .unwrap(); + + assert_eq!( + prepared.command.first().map(String::as_str), + Some(MACOS_PATH_TO_SEATBELT_EXECUTABLE) + ); + assert_eq!(prepared.command.get(1).map(String::as_str), Some("-p")); + assert!( + prepared + .command + .get(2) + .is_some_and(|policy| policy.contains("(allow user-preference-write)")), + "expected seatbelt policy to include macOS extension profile: {:?}", + prepared.command + ); +} + +#[cfg(target_os = "macos")] +#[tokio::test] +async fn prepare_escalated_exec_permissions_preserve_macos_seatbelt_extensions() { + let cwd = AbsolutePathBuf::from_absolute_path(std::env::temp_dir()).unwrap(); + let executor = CoreShellCommandExecutor { + command: vec!["echo".to_string(), "ok".to_string()], + cwd: cwd.to_path_buf(), + env: HashMap::new(), + network: None, + sandbox: SandboxType::None, + sandbox_policy: SandboxPolicy::DangerFullAccess, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + sandbox_permissions: SandboxPermissions::UseDefault, + justification: None, + arg0: None, + sandbox_policy_cwd: cwd.to_path_buf(), + macos_seatbelt_profile_extensions: None, + codex_linux_sandbox_exe: None, + use_linux_sandbox_bwrap: false, + }; + + let permissions = Permissions { + approval_policy: Constrained::allow_any(AskForApproval::Never), + sandbox_policy: Constrained::allow_any(SandboxPolicy::new_read_only_policy()), + network: None, + allow_login_shell: true, + shell_environment_policy: ShellEnvironmentPolicy::default(), + windows_sandbox_mode: None, + macos_seatbelt_profile_extensions: Some(MacOsSeatbeltProfileExtensions { + macos_preferences: MacOsPreferencesPermission::ReadWrite, + ..Default::default() + }), + }; + + let prepared = executor + .prepare_escalated_exec( + &AbsolutePathBuf::from_absolute_path("/bin/echo").unwrap(), + &["echo".to_string(), "ok".to_string()], + &cwd, + HashMap::new(), + EscalationExecution::Permissions(EscalationPermissions::Permissions( + EscalatedPermissions { + sandbox_policy: permissions.sandbox_policy.get().clone(), + macos_seatbelt_profile_extensions: permissions + .macos_seatbelt_profile_extensions + .clone(), + }, + )), + ) + .await + .unwrap(); + + assert_eq!( + prepared.command.first().map(String::as_str), + Some(MACOS_PATH_TO_SEATBELT_EXECUTABLE) + ); + assert_eq!(prepared.command.get(1).map(String::as_str), Some("-p")); + assert!( + prepared + .command + .get(2) + .is_some_and(|policy| policy.contains("(allow user-preference-write)")), + "expected seatbelt policy to include macOS extension profile: {:?}", + prepared.command + ); +} diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index 4b0c53d123..f4d823bf0e 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -111,6 +111,7 @@ impl Approvable for UnifiedExecRuntime<'_> { .or_else(|| req.justification.clone()); Box::pin(async move { with_cached_approval(&session.services, "unified_exec", keys, || async move { + let available_decisions = None; session .request_command_approval( turn, @@ -124,6 +125,7 @@ impl Approvable for UnifiedExecRuntime<'_> { .proposed_execpolicy_amendment() .cloned(), req.additional_permissions.clone(), + available_decisions, ) .await }) diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index 25ea101592..28d87b5bf3 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -340,6 +340,8 @@ impl<'a> SandboxAttempt<'a> { enforce_managed_network: self.enforce_managed_network, network, sandbox_policy_cwd: self.sandbox_cwd, + #[cfg(target_os = "macos")] + macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: self.codex_linux_sandbox_exe, use_linux_sandbox_bwrap: self.use_linux_sandbox_bwrap, windows_sandbox_level: self.windows_sandbox_level, diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 47682cf0e7..76730d0890 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -633,6 +633,15 @@ fn create_spawn_agent_tool(config: &ToolsConfig) -> ToolSpec { )), }, ), + ( + "fork_context".to_string(), + JsonSchema::Boolean { + description: Some( + "When true, fork the current thread history into the new agent before sending the initial prompt. This must be used when you want the new agent to have exactly the same context as you." + .to_string(), + ), + }, + ), ]); ToolSpec::Function(ResponsesApiTool { diff --git a/codex-rs/core/src/user_shell_command.rs b/codex-rs/core/src/user_shell_command.rs index 80128df006..e7921c69f3 100644 --- a/codex-rs/core/src/user_shell_command.rs +++ b/codex-rs/core/src/user_shell_command.rs @@ -1,21 +1,12 @@ use std::time::Duration; -use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use crate::codex::TurnContext; +use crate::contextual_user_message::USER_SHELL_COMMAND_FRAGMENT; use crate::exec::ExecToolCallOutput; use crate::tools::format_exec_output_str; -pub const USER_SHELL_COMMAND_OPEN: &str = ""; -pub const USER_SHELL_COMMAND_CLOSE: &str = ""; - -pub fn is_user_shell_command_text(text: &str) -> bool { - let trimmed = text.trim_start(); - let lowered = trimmed.to_ascii_lowercase(); - lowered.starts_with(USER_SHELL_COMMAND_OPEN) -} - fn format_duration_line(duration: Duration) -> String { let duration_seconds = duration.as_secs_f64(); format!("Duration: {duration_seconds:.4} seconds") @@ -48,7 +39,7 @@ pub fn format_user_shell_command_record( turn_context: &TurnContext, ) -> String { let body = format_user_shell_command_body(command, exec_output, turn_context); - format!("{USER_SHELL_COMMAND_OPEN}\n{body}\n{USER_SHELL_COMMAND_CLOSE}") + USER_SHELL_COMMAND_FRAGMENT.wrap(body) } pub fn user_shell_command_record_item( @@ -56,15 +47,11 @@ pub fn user_shell_command_record_item( exec_output: &ExecToolCallOutput, turn_context: &TurnContext, ) -> ResponseItem { - ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { - text: format_user_shell_command_record(command, exec_output, turn_context), - }], - end_turn: None, - phase: None, - } + USER_SHELL_COMMAND_FRAGMENT.into_message(format_user_shell_command_record( + command, + exec_output, + turn_context, + )) } #[cfg(test)] @@ -72,14 +59,16 @@ mod tests { use super::*; use crate::codex::make_session_and_context; use crate::exec::StreamOutput; + use codex_protocol::models::ContentItem; use pretty_assertions::assert_eq; #[test] fn detects_user_shell_command_text_variants() { - assert!(is_user_shell_command_text( - "\necho hi\n" - )); - assert!(!is_user_shell_command_text("echo hi")); + assert!( + USER_SHELL_COMMAND_FRAGMENT + .matches_text("\necho hi\n") + ); + assert!(!USER_SHELL_COMMAND_FRAGMENT.matches_text("echo hi")); } #[tokio::test] diff --git a/codex-rs/core/templates/memories/consolidation.md b/codex-rs/core/templates/memories/consolidation.md index b880bdfd65..085895a69f 100644 --- a/codex-rs/core/templates/memories/consolidation.md +++ b/codex-rs/core/templates/memories/consolidation.md @@ -106,7 +106,7 @@ Under `{{ memory_root }}/`: context. - source of rollout-level metadata needed for MEMORY.md `### rollout_summary_files` annotations; - you should be able to find `cwd` and `updated_at` there. + you should be able to find `cwd`, `rollout_path`, and `updated_at` there. - `MEMORY.md` - merged memories; produce a lightly clustered version if applicable - `rollout_summaries/*.md` @@ -121,6 +121,27 @@ Mode selection: - INCREMENTAL UPDATE: existing artifacts already exist and `raw_memories.md` mostly contains new additions. +Incremental thread diff snapshot (computed before the current artifact sync rewrites local files): + +**Diff since last consolidation:** +{{ phase2_input_selection }} + +Incremental update and forgetting mechanism: +- Use the diff provided +- Do not open raw sessions / original rollout transcripts. +- For each added thread id, search it in `raw_memories.md`, read that raw-memory section, and + read the corresponding `rollout_summaries/*.md` file only when needed for stronger evidence, + task placement, or conflict resolution. +- For each removed thread id, search it in `MEMORY.md` and delete only the memory supported by + that thread. Use `thread_id=` in `### rollout_summary_files` when available; if not, + fall back to rollout summary filenames plus the corresponding `rollout_summaries/*.md` files. +- If a `MEMORY.md` block contains both removed and undeleted threads, do not delete the whole + block. Remove only the removed thread's references and thread-local learnings, preserve shared + or still-supported content, and split or rewrite the block only if needed to keep the undeleted + threads intact. +- After `MEMORY.md` cleanup is done, revisit `memory_summary.md` and remove or rewrite stale + summary/index content that was only supported by removed thread ids. + Outputs: Under `{{ memory_root }}/`: A) `MEMORY.md` @@ -171,7 +192,7 @@ Required task-oriented body shape (strict): ## Task 1: ### rollout_summary_files -- (cwd=, updated_at=, thread_id=, ) +- (cwd=, rollout_path=, updated_at=, thread_id=, ) ### keywords @@ -236,7 +257,8 @@ Schema rules (strict): - Every `## Task ` section must include `### rollout_summary_files`, `### keywords`, and `### learnings`. - `### rollout_summary_files` must be task-local (not a block-wide catch-all list). - - Each rollout annotation must include `cwd=` and `updated_at=`. + - Each rollout annotation must include `cwd=`, `rollout_path=`, and + `updated_at=`. If missing from a rollout summary, recover them from `raw_memories.md`. - Major learnings should be traceable to rollout summaries listed in the same task section. - Order rollout references by freshness and practical usefulness. @@ -497,27 +519,42 @@ WORKFLOW conflicting task families until MEMORY blocks are richer and more useful than raw memories 3) INCREMENTAL UPDATE behavior: - - Treat `raw_memories.md` as the primary source of NEW signal. - - Read existing memory files first for continuity. + - Read existing `MEMORY.md` and `memory_summary.md` first for continuity and to locate + existing references that may need surgical cleanup. + - Use the injected thread-diff snapshot as the first routing pass: + - added thread ids = ingestion queue + - removed thread ids = forgetting / stale-cleanup queue - Build an index of rollout references already present in existing `MEMORY.md` before scanning raw memories so you can route net-new evidence into the right blocks. - - Compute net-new candidates from the raw-memory inventory (threads / rollout summaries / - updated evidence not already represented in `MEMORY.md`). + - Work in this order: + 1. For newly added thread ids, search them in `raw_memories.md`, read those sections, and + open the corresponding `rollout_summaries/*.md` files when necessary. + 2. Route the new signal into existing `MEMORY.md` blocks or create new ones when needed. + 3. For removed thread ids, search `MEMORY.md` and surgically delete or rewrite only the + unsupported thread-local memory. + 4. If a block mixes removed and undeleted threads, preserve the undeleted-thread content; + split or rewrite the block if that is the cleanest way to delete only the removed part. + 5. After `MEMORY.md` is correct, revisit `memory_summary.md` and remove or rewrite stale + summary/index content that no longer has undeleted support. - Integrate new signal into existing artifacts by: - - scanning new raw memories in recency order and identifying which existing blocks they should update + - scanning the newly added raw-memory entries in recency order and identifying which existing blocks they should update - updating existing knowledge with better/newer evidence - updating stale or contradicting guidance + - pruning or downgrading memory whose only provenance comes from removed thread ids - expanding terse old blocks when new summaries/raw memories make the task family clearer - doing light clustering and merging if needed - refreshing `MEMORY.md` top-of-file ordering so recent high-utility task families stay easy to find - rebuilding the `memory_summary.md` recent active window (last 3 memory days) from current `updated_at` coverage - updating existing skills or adding new skills only when there is clear new reusable procedure - - update `memory_summary.md` last to reflect the final state of the memory folder + - updating `memory_summary.md` last to reflect the final state of the memory folder - Minimize churn in incremental mode: if an existing `MEMORY.md` block or `## What's in Memory` topic still reflects the current evidence and points to the same task family / retrieval target, keep its wording, label, and relative order mostly stable. Rewrite/reorder/rename/ split/merge only when fixing a real problem (staleness, ambiguity, schema drift, wrong boundaries) or when meaningful new evidence materially improves retrieval clarity/searchability. + - Spend most of your deep-dive budget on newly added thread ids and on mixed blocks touched by + removed thread ids. Do not re-read unchanged older threads unless you need them for + conflict resolution, clustering, or provenance repair. 4) Evidence deep-dive rule (both modes): - `raw_memories.md` is the routing layer, not always the final authority for detail. @@ -528,6 +565,9 @@ WORKFLOW - When a task family is important, ambiguous, or duplicated across multiple rollouts, open the relevant `rollout_summaries/*.md` files and extract richer procedural detail, validation signals, and user feedback before finalizing `MEMORY.md`. + - When deleting stale memory from a mixed block, use the relevant rollout summaries to decide + which details are uniquely supported by removed threads versus still supported by undeleted + threads. - Use `updated_at` and validation strength together to resolve stale/conflicting notes. 5) For both modes, update `MEMORY.md` after skill updates: @@ -541,6 +581,8 @@ WORKFLOW 7) Final pass: - remove duplication in memory_summary, skills/, and MEMORY.md - remove stale or low-signal blocks that are less likely to be useful in the future + - remove or rewrite blocks/task sections whose supporting rollout references point only to + removed thread ids or missing rollout summary files - run a global rollout-reference audit on final `MEMORY.md` and fix accidental duplicate entries / redundant repetition, while preserving intentional multi-task or multi-block reuse when it adds distinct task-local value @@ -559,16 +601,3 @@ WORKFLOW You should dive deep and make sure you didn't miss any important information that might be useful for future agents; do not be superficial. - -============================================================ -SEARCH / REVIEW COMMANDS (RG-FIRST) -============================================================ - -Use `rg` for fast retrieval while consolidating: - -- Search durable notes: - `rg -n -i "" "{{ memory_root }}/MEMORY.md"` -- Search across memory tree: - `rg -n -i "" "{{ memory_root }}" | head -n 100` -- Locate rollout summary files: - `rg --files "{{ memory_root }}/rollout_summaries" | head -n 400` diff --git a/codex-rs/core/templates/memories/read_path.md b/codex-rs/core/templates/memories/read_path.md index 59159b2ecc..ba0e21044e 100644 --- a/codex-rs/core/templates/memories/read_path.md +++ b/codex-rs/core/templates/memories/read_path.md @@ -25,7 +25,10 @@ Memory layout (general -> specific): - scripts/ (optional helper scripts) - examples/ (optional example outputs) - templates/ (optional templates) -- {{ base_path }}/rollout_summaries/ (per-rollout recaps + evidence snippets) + - {{ base_path }}/rollout_summaries/ (per-rollout recaps + evidence snippets) + - The paths of these entries can be found in {{ base_path }}/MEMORY.md or {{ base_path }}/rollout_summaries/ as `rollout_path` + - These files are append-only `jsonl`: `session_meta.payload.id` identifies the session, `turn_context` marks turn boundaries, `event_msg` is the lightweight status stream, and `response_item` contains actual messages, tool calls, and tool outputs. + - For efficient lookup, prefer matching the filename suffix or `session_meta.payload.id`; avoid broad full-content scans unless needed. Quick memory pass (when applicable): @@ -34,7 +37,8 @@ Quick memory pass (when applicable): 3. Only if MEMORY.md directly points to rollout summaries/skills, open the 1-2 most relevant files under {{ base_path }}/rollout_summaries/ or {{ base_path }}/skills/. -4. If there are no relevant hits, stop memory lookup and continue normally. +4. If above are not clear and you need exact commands, error text, or precise evidence, search over `rollout_path` for more evidence. +5. If there are no relevant hits, stop memory lookup and continue normally. Quick-pass budget: diff --git a/codex-rs/core/tests/common/context_snapshot.rs b/codex-rs/core/tests/common/context_snapshot.rs index f9362ee30c..addb1bc31e 100644 --- a/codex-rs/core/tests/common/context_snapshot.rs +++ b/codex-rs/core/tests/common/context_snapshot.rs @@ -62,7 +62,7 @@ pub fn format_response_items_snapshot(items: &[Value], options: &ContextSnapshot match item_type { "message" => { let role = item.get("role").and_then(Value::as_str).unwrap_or("unknown"); - let text = item + let rendered_parts = item .get("content") .and_then(Value::as_array) .map(|content| { @@ -93,11 +93,27 @@ pub fn format_response_items_snapshot(items: &[Value], options: &ContextSnapshot } }) .collect::>() - .join(" | ") }) - .filter(|text| !text.is_empty()) - .unwrap_or_else(|| "".to_string()); - format!("{idx:02}:message/{role}:{text}") + .unwrap_or_default(); + let role = if rendered_parts.len() > 1 { + format!("{role}[{}]", rendered_parts.len()) + } else { + role.to_string() + }; + if rendered_parts.is_empty() { + return format!("{idx:02}:message/{role}:"); + } + if rendered_parts.len() == 1 { + return format!("{idx:02}:message/{role}:{}", rendered_parts[0]); + } + + let parts = rendered_parts + .iter() + .enumerate() + .map(|(part_idx, part)| format!(" [{:02}] {part}", part_idx + 1)) + .collect::>() + .join("\n"); + format!("{idx:02}:message/{role}:\n{parts}") } "function_call" => { let name = item.get("name").and_then(Value::as_str).unwrap_or("unknown"); @@ -222,15 +238,34 @@ fn canonicalize_snapshot_text(text: &str) -> String { return "".to_string(); } if text.starts_with("") { + let subagent_count = text + .split_once("") + .and_then(|(_, rest)| rest.split_once("")) + .map(|(subagents, _)| { + subagents + .lines() + .filter(|line| line.trim_start().starts_with("- ")) + .count() + }) + .unwrap_or(0); + let subagents_suffix = if subagent_count > 0 { + format!(":subagents={subagent_count}") + } else { + String::new() + }; if let (Some(cwd_start), Some(cwd_end)) = (text.find(""), text.find("")) { let cwd = &text[cwd_start + "".len()..cwd_end]; return if cwd.ends_with("PRETURN_CONTEXT_DIFF_CWD") { - "".to_string() + format!("") } else { - ">".to_string() + format!("{subagents_suffix}>") }; } - return "".to_string(); + return if subagent_count > 0 { + format!("") + } else { + "".to_string() + }; } if text.starts_with("You are performing a CONTEXT CHECKPOINT COMPACTION.") { return "".to_string(); @@ -258,7 +293,7 @@ mod tests { "role": "user", "content": [{ "type": "input_text", - "text": "# AGENTS.md instructions for /tmp/example" + "text": "# AGENTS.md instructions for /tmp/example\n\n\nbody\n" }] })]; @@ -269,7 +304,7 @@ mod tests { assert_eq!( rendered, - "00:message/user:# AGENTS.md instructions for /tmp/example" + r"00:message/user:# AGENTS.md instructions for /tmp/example\n\n\nbody\n" ); } @@ -280,7 +315,7 @@ mod tests { "role": "user", "content": [{ "type": "input_text", - "text": "# AGENTS.md instructions for /tmp/example" + "text": "# AGENTS.md instructions for /tmp/example\n\n\nbody\n" }] })]; @@ -292,6 +327,28 @@ mod tests { assert_eq!(rendered, "00:message/user:"); } + #[test] + fn redacted_text_mode_normalizes_environment_context_with_subagents() { + let items = vec![json!({ + "type": "message", + "role": "user", + "content": [{ + "type": "input_text", + "text": "\n /tmp/example\n bash\n \n - agent-1: atlas\n - agent-2\n \n" + }] + })]; + + let rendered = format_response_items_snapshot( + &items, + &ContextSnapshotOptions::default().render_mode(ContextSnapshotRenderMode::RedactedText), + ); + + assert_eq!( + rendered, + "00:message/user::subagents=2>" + ); + } + #[test] fn image_only_message_is_rendered_as_non_text_span() { let items = vec![json!({ @@ -333,7 +390,7 @@ mod tests { assert_eq!( rendered, - "00:message/user: | | " + "00:message/user[3]:\n [01] \n [02] \n [03] " ); } } diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index 4c7f26ddba..bb6de200a9 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -12,6 +12,7 @@ use futures::SinkExt; use futures::StreamExt; use serde_json::Value; use tokio::net::TcpListener; +use tokio::sync::Notify; use tokio::sync::oneshot; use tokio_tungstenite::accept_hdr_async_with_config; use tokio_tungstenite::tungstenite::Message; @@ -141,6 +142,32 @@ impl ResponsesRequest { .collect() } + /// Returns `input_text` spans grouped by `message` input for the provided role. + pub fn message_input_text_groups(&self, role: &str) -> Vec> { + self.inputs_of_type("message") + .into_iter() + .filter(|item| item.get("role").and_then(Value::as_str) == Some(role)) + .filter_map(|item| item.get("content").and_then(Value::as_array).cloned()) + .map(|content| { + content + .into_iter() + .filter(|span| span.get("type").and_then(Value::as_str) == Some("input_text")) + .filter_map(|span| span.get("text").and_then(Value::as_str).map(str::to_owned)) + .collect() + }) + .collect() + } + + pub fn has_message_with_input_texts( + &self, + role: &str, + predicate: impl Fn(&[String]) -> bool, + ) -> bool { + self.message_input_text_groups(role) + .iter() + .any(|texts| predicate(texts)) + } + /// Returns all `input_image` `image_url` spans from `message` inputs for the provided role. pub fn message_input_image_urls(&self, role: &str) -> Vec { self.inputs_of_type("message") @@ -237,7 +264,7 @@ impl ResponsesRequest { .cloned() .unwrap_or(Value::Null); match output { - Value::String(text) => Some((Some(text), None)), + Value::String(_) | Value::Array(_) => Some((output_value_to_text(&output), None)), Value::Object(obj) => Some(( obj.get("content") .and_then(Value::as_str) @@ -269,6 +296,87 @@ impl ResponsesRequest { } } +pub(crate) fn output_value_to_text(value: &Value) -> Option { + match value { + Value::String(text) => Some(text.clone()), + Value::Array(items) => match items.as_slice() { + [item] if item.get("type").and_then(Value::as_str) == Some("input_text") => { + item.get("text").and_then(Value::as_str).map(str::to_string) + } + [_] | [] | [_, _, ..] => None, + }, + Value::Object(_) | Value::Number(_) | Value::Bool(_) | Value::Null => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use wiremock::http::HeaderMap; + use wiremock::http::Method; + + fn request_with_input(input: Value) -> ResponsesRequest { + ResponsesRequest(wiremock::Request { + url: "http://localhost/v1/responses" + .parse() + .expect("valid request url"), + method: Method::POST, + headers: HeaderMap::new(), + body: serde_json::to_vec(&serde_json::json!({ "input": input })) + .expect("serialize request body"), + }) + } + + #[test] + fn call_output_content_and_success_returns_only_single_text_content_item() { + let single_text = request_with_input(serde_json::json!([ + { + "type": "function_call_output", + "call_id": "call-1", + "output": [{ "type": "input_text", "text": "hello" }] + }, + { + "type": "custom_tool_call_output", + "call_id": "call-2", + "output": [{ "type": "input_text", "text": "world" }] + } + ])); + assert_eq!( + single_text.function_call_output_content_and_success("call-1"), + Some((Some("hello".to_string()), None)) + ); + assert_eq!( + single_text.custom_tool_call_output_content_and_success("call-2"), + Some((Some("world".to_string()), None)) + ); + + let mixed_content = request_with_input(serde_json::json!([ + { + "type": "function_call_output", + "call_id": "call-3", + "output": [ + { "type": "input_text", "text": "hello" }, + { "type": "input_image", "image_url": "data:image/png;base64,abc" } + ] + }, + { + "type": "custom_tool_call_output", + "call_id": "call-4", + "output": [{ "type": "input_image", "image_url": "data:image/png;base64,abc" }] + } + ])); + assert_eq!( + mixed_content.function_call_output_content_and_success("call-3"), + Some((None, None)) + ); + assert_eq!( + mixed_content.custom_tool_call_output_content_and_success("call-4"), + Some((None, None)) + ); + } +} + #[derive(Debug, Clone)] pub struct WebSocketRequest { body: Value, @@ -309,6 +417,7 @@ pub struct WebSocketTestServer { uri: String, connections: Arc>>>, handshakes: Arc>>, + request_log_updated: Arc, shutdown: oneshot::Sender<()>, task: tokio::task::JoinHandle<()>, } @@ -330,6 +439,26 @@ impl WebSocketTestServer { connections.first().cloned().unwrap_or_default() } + pub async fn wait_for_request( + &self, + connection_index: usize, + request_index: usize, + ) -> WebSocketRequest { + loop { + if let Some(request) = self + .connections + .lock() + .unwrap() + .get(connection_index) + .and_then(|connection| connection.get(request_index)) + .cloned() + { + return request; + } + self.request_log_updated.notified().await; + } + } + pub fn handshakes(&self) -> Vec { self.handshakes.lock().unwrap().clone() } @@ -1043,6 +1172,7 @@ pub async fn start_websocket_server(connections: Vec>>) -> WebSoc pub async fn start_websocket_server_with_headers( connections: Vec, ) -> WebSocketTestServer { + let start = std::time::Instant::now(); let listener = TcpListener::bind("127.0.0.1:0") .await .expect("bind websocket server"); @@ -1050,8 +1180,10 @@ pub async fn start_websocket_server_with_headers( let uri = format!("ws://{addr}"); let connections_log = Arc::new(Mutex::new(Vec::new())); let handshakes_log = Arc::new(Mutex::new(Vec::new())); + let request_log_updated = Arc::new(Notify::new()); let requests = Arc::clone(&connections_log); let handshakes = Arc::clone(&handshakes_log); + let request_log = Arc::clone(&request_log_updated); let connections = Arc::new(Mutex::new(VecDeque::from(connections))); let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); @@ -1133,9 +1265,51 @@ pub async fn start_websocket_server_with_headers( let mut log = requests.lock().unwrap(); if let Some(connection_log) = log.get_mut(connection_index) { connection_log.push(WebSocketRequest { body }); + let request_index = connection_log.len() - 1; + let request = &connection_log[request_index]; + let request_body = request.body_json(); + eprintln!( + "[ws test server +{}ms] connection={} received request={} type={:?} role={:?} text={:?} data={:?}", + start.elapsed().as_millis(), + connection_index, + request_index, + request_body.get("type").and_then(Value::as_str), + request_body + .get("item") + .and_then(|item| item.get("role")) + .and_then(Value::as_str), + request_body + .get("item") + .and_then(|item| item.get("content")) + .and_then(Value::as_array) + .and_then(|content| content.first()) + .and_then(|content| content.get("text")) + .and_then(Value::as_str), + request_body + .get("item") + .and_then(|item| item.get("content")) + .and_then(Value::as_array) + .and_then(|content| content.first()) + .and_then(|content| content.get("data")) + .and_then(Value::as_str), + ); } + request_log.notify_waiters(); } + eprintln!( + "[ws test server +{}ms] connection={} sending batch_size={} event_types={:?} audio_data={:?}", + start.elapsed().as_millis(), + connection_index, + request_events.len(), + request_events + .iter() + .map(|event| event.get("type").and_then(Value::as_str)) + .collect::>(), + request_events + .iter() + .find_map(|event| event.get("delta").and_then(Value::as_str)), + ); for event in &request_events { let Ok(payload) = serde_json::to_string(event) else { continue; @@ -1158,6 +1332,7 @@ pub async fn start_websocket_server_with_headers( uri, connections: connections_log, handshakes: handshakes_log, + request_log_updated, shutdown: shutdown_tx, task, } diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index 173db5ca77..bd15c6d7e4 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -11,7 +11,6 @@ use codex_core::ThreadManager; use codex_core::built_in_model_providers; use codex_core::config::Config; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -24,6 +23,7 @@ use wiremock::MockServer; use crate::load_default_config_for_test; use crate::responses::WebSocketTestServer; +use crate::responses::output_value_to_text; use crate::responses::start_mock_server; use crate::streaming_sse::StreamingSseServer; use crate::wait_for_event; @@ -300,7 +300,7 @@ impl TestCodex { sandbox_policy, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -395,11 +395,7 @@ impl TestCodexHarness { pub async fn custom_tool_call_output(&self, call_id: &str) -> String { let bodies = self.request_bodies().await; - custom_tool_call_output(&bodies, call_id) - .get("output") - .and_then(Value::as_str) - .expect("output string") - .to_string() + custom_tool_call_output_text(&bodies, call_id) } pub async fn apply_patch_output( @@ -434,6 +430,14 @@ fn custom_tool_call_output<'a>(bodies: &'a [Value], call_id: &str) -> &'a Value panic!("custom_tool_call_output {call_id} not found"); } +fn custom_tool_call_output_text(bodies: &[Value], call_id: &str) -> String { + let output = custom_tool_call_output(bodies, call_id) + .get("output") + .unwrap_or_else(|| panic!("custom_tool_call_output {call_id} missing output")); + output_value_to_text(output) + .unwrap_or_else(|| panic!("custom_tool_call_output {call_id} missing text output")) +} + fn function_call_output<'a>(bodies: &'a [Value], call_id: &str) -> &'a Value { for body in bodies { if let Some(items) = body.get("input").and_then(Value::as_array) { @@ -457,3 +461,36 @@ pub fn test_codex() -> TestCodexBuilder { home: None, } } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde_json::json; + + #[test] + fn custom_tool_call_output_text_returns_output_text() { + let bodies = vec![json!({ + "input": [{ + "type": "custom_tool_call_output", + "call_id": "call-1", + "output": "hello" + }] + })]; + + assert_eq!(custom_tool_call_output_text(&bodies, "call-1"), "hello"); + } + + #[test] + #[should_panic(expected = "custom_tool_call_output call-2 missing output")] + fn custom_tool_call_output_text_panics_when_output_is_missing() { + let bodies = vec![json!({ + "input": [{ + "type": "custom_tool_call_output", + "call_id": "call-2" + }] + })]; + + let _ = custom_tool_call_output_text(&bodies, "call-2"); + } +} diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index 126dc2c288..f502ec08be 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -111,7 +111,14 @@ async fn responses_stream_includes_subagent_header_on_review() { }]; let mut stream = client_session - .stream(&prompt, &model_info, &otel_manager, effort, summary, None) + .stream( + &prompt, + &model_info, + &otel_manager, + effort, + summary.unwrap_or(model_info.default_reasoning_summary), + None, + ) .await .expect("stream failed"); while let Some(event) = stream.next().await { @@ -216,7 +223,14 @@ async fn responses_stream_includes_subagent_header_on_other() { }]; let mut stream = client_session - .stream(&prompt, &model_info, &otel_manager, effort, summary, None) + .stream( + &prompt, + &model_info, + &otel_manager, + effort, + summary.unwrap_or(model_info.default_reasoning_summary), + None, + ) .await .expect("stream failed"); while let Some(event) = stream.next().await { @@ -267,7 +281,7 @@ async fn responses_respects_model_info_overrides_from_config() { config.model_provider_id = provider.name.clone(); config.model_provider = provider.clone(); config.model_supports_reasoning_summaries = Some(true); - config.model_reasoning_summary = ReasoningSummary::Detailed; + config.model_reasoning_summary = Some(ReasoningSummary::Detailed); let effort = config.model_reasoning_effort; let summary = config.model_reasoning_summary; let model = config.model.clone().expect("model configured"); @@ -320,7 +334,14 @@ async fn responses_respects_model_info_overrides_from_config() { }]; let mut stream = client_session - .stream(&prompt, &model_info, &otel_manager, effort, summary, None) + .stream( + &prompt, + &model_info, + &otel_manager, + effort, + summary.unwrap_or(model_info.default_reasoning_summary), + None, + ) .await .expect("stream failed"); while let Some(event) = stream.next().await { diff --git a/codex-rs/core/tests/suite/apply_patch_cli.rs b/codex-rs/core/tests/suite/apply_patch_cli.rs index 45494124e5..7f20f468d9 100644 --- a/codex-rs/core/tests/suite/apply_patch_cli.rs +++ b/codex-rs/core/tests/suite/apply_patch_cli.rs @@ -12,7 +12,6 @@ use std::sync::atomic::AtomicI32; use std::sync::atomic::Ordering; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -312,7 +311,7 @@ async fn apply_patch_cli_move_without_content_change_has_no_turn_diff( sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -907,7 +906,7 @@ async fn apply_patch_shell_command_heredoc_with_cd_emits_turn_diff() -> Result<( sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -987,7 +986,7 @@ async fn apply_patch_shell_command_failure_propagates_error_and_skips_diff() -> sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1137,7 +1136,7 @@ async fn apply_patch_emits_turn_diff_event_with_unified_diff( sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1200,7 +1199,7 @@ async fn apply_patch_turn_diff_for_rename_with_content_change( sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1271,7 +1270,7 @@ async fn apply_patch_aggregates_diff_across_multiple_tool_calls() -> Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1342,7 +1341,7 @@ async fn apply_patch_aggregates_diff_preserves_success_after_failure() -> Result sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index 34934b3d4b..a0530809db 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -13,7 +13,6 @@ use codex_core::sandboxing::SandboxPermissions; use codex_protocol::approvals::NetworkApprovalProtocol; use codex_protocol::approvals::NetworkPolicyAmendment; use codex_protocol::approvals::NetworkPolicyRuleAction; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; @@ -551,7 +550,7 @@ async fn submit_turn( sandbox_policy, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index e040e61e77..1de0522cb5 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -31,9 +31,14 @@ use codex_protocol::models::ReasoningItemContent; use codex_protocol::models::ReasoningItemReasoningSummary; use codex_protocol::models::ResponseItem; use codex_protocol::models::WebSearchAction; +use codex_protocol::openai_models::ModelsResponse; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; +use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::RolloutLine; +use codex_protocol::protocol::SessionMeta; +use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; use core_test_support::apps_test_server::AppsTestServer; @@ -74,40 +79,14 @@ fn assert_message_role(request_body: &serde_json::Value, role: &str) { assert_eq!(request_body["role"].as_str().unwrap(), role); } -#[expect(clippy::expect_used)] -fn assert_message_equals(request_body: &serde_json::Value, text: &str) { - let content = request_body["content"][0]["text"] - .as_str() - .expect("invalid message content"); - - assert_eq!( - content, text, - "expected message content '{content}' to equal '{text}'" - ); -} - -#[expect(clippy::expect_used)] -fn assert_message_starts_with(request_body: &serde_json::Value, text: &str) { - let content = request_body["content"][0]["text"] - .as_str() - .expect("invalid message content"); - - assert!( - content.starts_with(text), - "expected message content '{content}' to start with '{text}'" - ); -} - -#[expect(clippy::expect_used)] -fn assert_message_ends_with(request_body: &serde_json::Value, text: &str) { - let content = request_body["content"][0]["text"] - .as_str() - .expect("invalid message content"); - - assert!( - content.ends_with(text), - "expected message content '{content}' to end with '{text}'" - ); +#[expect(clippy::unwrap_used)] +fn message_input_texts(item: &serde_json::Value) -> Vec<&str> { + item["content"] + .as_array() + .unwrap() + .iter() + .filter_map(|entry| entry.get("text").and_then(|text| text.as_str())) + .collect() } /// Writes an `auth.json` into the provided `codex_home` with the specified parameters. @@ -305,19 +284,15 @@ async fn resume_includes_initial_messages_and_sends_prior_items() { let request = resp_mock.single_request(); let request_body = request.body_json(); let input = request_body["input"].as_array().expect("input array"); - let messages: Vec<(String, String)> = input - .iter() - .filter_map(|item| { - let role = item.get("role")?.as_str()?; - let text = item - .get("content")? - .as_array()? - .first()? - .get("text")? - .as_str()?; - Some((role.to_string(), text.to_string())) - }) - .collect(); + let mut messages: Vec<(String, String)> = Vec::new(); + for item in input { + let Some(role) = item.get("role").and_then(|role| role.as_str()) else { + continue; + }; + for text in message_input_texts(item) { + messages.push((role.to_string(), text.to_string())); + } + } let pos_prior_user = messages .iter() .position(|(role, text)| role == "user" && text == "resumed user message") @@ -354,8 +329,7 @@ async fn resume_includes_initial_messages_and_sends_prior_items() { .position(|(role, text)| { role == "user" && text.contains("be nice") - && (text.starts_with("# AGENTS.md instructions for ") - || text.starts_with("")) + && (text.starts_with("# AGENTS.md instructions for ")) }) .expect("user instructions"); let pos_environment = messages @@ -374,6 +348,144 @@ async fn resume_includes_initial_messages_and_sends_prior_items() { assert!(pos_environment < pos_new_user); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resume_replays_legacy_js_repl_image_rollout_shapes() { + skip_if_no_network!(); + + // Early js_repl builds persisted image tool results as two separate rollout items: + // a string-valued custom_tool_call_output plus a standalone user input_image message. + // Current image tests cover today's shapes; this keeps resume compatibility for that + // legacy rollout representation. + let legacy_custom_tool_call = ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "legacy-js-call".to_string(), + name: "js_repl".to_string(), + input: "console.log('legacy image flow')".to_string(), + }; + let legacy_image_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="; + let rollout = vec![ + RolloutLine { + timestamp: "2024-01-01T00:00:00.000Z".to_string(), + item: RolloutItem::SessionMeta(SessionMetaLine { + meta: SessionMeta { + id: ThreadId::default(), + timestamp: "2024-01-01T00:00:00Z".to_string(), + cwd: ".".into(), + originator: "test_originator".to_string(), + cli_version: "test_version".to_string(), + model_provider: Some("test-provider".to_string()), + ..Default::default() + }, + git: None, + }), + }, + RolloutLine { + timestamp: "2024-01-01T00:00:01.000Z".to_string(), + item: RolloutItem::ResponseItem(legacy_custom_tool_call), + }, + RolloutLine { + timestamp: "2024-01-01T00:00:02.000Z".to_string(), + item: RolloutItem::ResponseItem(ResponseItem::CustomToolCallOutput { + call_id: "legacy-js-call".to_string(), + output: FunctionCallOutputPayload::from_text("legacy js_repl stdout".to_string()), + }), + }, + RolloutLine { + timestamp: "2024-01-01T00:00:03.000Z".to_string(), + item: RolloutItem::ResponseItem(ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputImage { + image_url: legacy_image_url.to_string(), + }], + end_turn: None, + phase: None, + }), + }, + ]; + + let tmpdir = TempDir::new().unwrap(); + let session_path = tmpdir + .path() + .join("resume-legacy-js-repl-image-rollout.jsonl"); + let mut f = std::fs::File::create(&session_path).unwrap(); + for line in rollout { + writeln!(f, "{}", serde_json::to_string(&line).unwrap()).unwrap(); + } + + let server = MockServer::start().await; + let resp_mock = mount_sse_once( + &server, + sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), + ) + .await; + + let codex_home = Arc::new(TempDir::new().unwrap()); + let mut builder = test_codex().with_model("gpt-5.1"); + let test = builder + .resume(&server, codex_home, session_path.clone()) + .await + .expect("resume conversation"); + test.submit_turn("after resume").await.unwrap(); + + let input = resp_mock.single_request().input(); + + let legacy_output_index = input + .iter() + .position(|item| { + item.get("type").and_then(|value| value.as_str()) == Some("custom_tool_call_output") + && item.get("call_id").and_then(|value| value.as_str()) == Some("legacy-js-call") + }) + .expect("legacy custom tool output should be replayed"); + assert_eq!( + input[legacy_output_index] + .get("output") + .and_then(|value| value.as_str()), + Some("legacy js_repl stdout") + ); + + let legacy_image_index = input + .iter() + .position(|item| { + item.get("type").and_then(|value| value.as_str()) == Some("message") + && item.get("role").and_then(|value| value.as_str()) == Some("user") + && item + .get("content") + .and_then(|value| value.as_array()) + .is_some_and(|content| { + content.iter().any(|entry| { + entry.get("type").and_then(|value| value.as_str()) + == Some("input_image") + && entry.get("image_url").and_then(|value| value.as_str()) + == Some(legacy_image_url) + }) + }) + }) + .expect("legacy injected image message should be replayed"); + + let new_user_index = input + .iter() + .position(|item| { + item.get("type").and_then(|value| value.as_str()) == Some("message") + && item.get("role").and_then(|value| value.as_str()) == Some("user") + && item + .get("content") + .and_then(|value| value.as_array()) + .is_some_and(|content| { + content.iter().any(|entry| { + entry.get("type").and_then(|value| value.as_str()) == Some("input_text") + && entry.get("text").and_then(|value| value.as_str()) + == Some("after resume") + }) + }) + }) + .expect("new user message should be present"); + + assert!(legacy_output_index < new_user_index); + assert!(legacy_image_index < new_user_index); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_conversation_id_and_model_headers_in_request() { skip_if_no_network!(); @@ -664,16 +776,27 @@ async fn includes_user_instructions_message_in_request() { ); assert_message_role(&request_body["input"][1], "user"); - assert_message_starts_with(&request_body["input"][1], "# AGENTS.md instructions for "); - assert_message_ends_with(&request_body["input"][1], ""); - let ui_text = request_body["input"][1]["content"][0]["text"] - .as_str() + let user_context_texts = message_input_texts(&request_body["input"][1]); + assert!( + user_context_texts + .iter() + .any(|text| text.starts_with("# AGENTS.md instructions for ")), + "expected AGENTS text in contextual user message, got {user_context_texts:?}" + ); + let ui_text = user_context_texts + .iter() + .copied() + .find(|text| text.contains("")) .expect("invalid message content"); assert!(ui_text.contains("")); assert!(ui_text.contains("be nice")); - assert_message_role(&request_body["input"][2], "user"); - assert_message_starts_with(&request_body["input"][2], ""); - assert_message_ends_with(&request_body["input"][2], ""); + assert!( + user_context_texts + .iter() + .any(|text| text.starts_with("") + && text.ends_with("")), + "expected environment context in contextual user message, got {user_context_texts:?}" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -727,10 +850,14 @@ async fn includes_apps_guidance_as_developer_message_when_enabled() { && item .get("content") .and_then(|value| value.as_array()) - .and_then(|value| value.first()) - .and_then(|value| value.get("text")) - .and_then(|value| value.as_str()) - .is_some_and(|text| text.contains(apps_snippet)) + .is_some_and(|content| { + content.iter().any(|entry| { + entry + .get("text") + .and_then(|value| value.as_str()) + .is_some_and(|text| text.contains(apps_snippet)) + }) + }) }); assert!( has_developer_apps_guidance, @@ -742,10 +869,14 @@ async fn includes_apps_guidance_as_developer_message_when_enabled() { && item .get("content") .and_then(|value| value.as_array()) - .and_then(|value| value.first()) - .and_then(|value| value.get("text")) - .and_then(|value| value.as_str()) - .is_some_and(|text| text.contains(apps_snippet)) + .is_some_and(|content| { + content.iter().any(|entry| { + entry + .get("text") + .and_then(|value| value.as_str()) + .is_some_and(|text| text.contains(apps_snippet)) + }) + }) }); assert!( !has_user_apps_guidance, @@ -992,7 +1123,11 @@ async fn user_turn_collaboration_mode_overrides_model_and_effort() -> anyhow::Re sandbox_policy: config.permissions.sandbox_policy.get().clone(), model: session_configured.model.clone(), effort: Some(ReasoningEffort::Low), - summary: config.model_reasoning_summary, + summary: Some( + config + .model_reasoning_summary + .unwrap_or(ReasoningSummary::Auto), + ), collaboration_mode: Some(collaboration_mode), final_output_json_schema: None, personality: None, @@ -1026,7 +1161,7 @@ async fn configured_reasoning_summary_is_sent() -> anyhow::Result<()> { .await; let TestCodex { codex, .. } = test_codex() .with_config(|config| { - config.model_reasoning_summary = ReasoningSummary::Concise; + config.model_reasoning_summary = Some(ReasoningSummary::Concise); }) .build(&server) .await?; @@ -1058,6 +1193,75 @@ async fn configured_reasoning_summary_is_sent() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn user_turn_explicit_reasoning_summary_overrides_model_catalog_default() -> anyhow::Result<()> +{ + skip_if_no_network!(Ok(())); + let server = MockServer::start().await; + + let resp_mock = mount_sse_once( + &server, + sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), + ) + .await; + + let mut model_catalog: ModelsResponse = + serde_json::from_str(include_str!("../../models.json")).expect("valid models.json"); + let model = model_catalog + .models + .iter_mut() + .find(|model| model.slug == "gpt-5.1") + .expect("gpt-5.1 exists in bundled models.json"); + model.supports_reasoning_summaries = true; + model.default_reasoning_summary = ReasoningSummary::Detailed; + + let TestCodex { + codex, + config, + session_configured, + .. + } = test_codex() + .with_model("gpt-5.1") + .with_config(move |config| { + config.model_catalog = Some(model_catalog); + }) + .build(&server) + .await?; + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "hello".into(), + text_elements: Vec::new(), + }], + cwd: config.cwd.clone(), + approval_policy: config.permissions.approval_policy.value(), + sandbox_policy: config.permissions.sandbox_policy.get().clone(), + model: session_configured.model, + effort: None, + summary: Some(ReasoningSummary::Concise), + collaboration_mode: None, + final_output_json_schema: None, + personality: None, + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let request_body = resp_mock.single_request().body_json(); + + pretty_assertions::assert_eq!( + request_body + .get("reasoning") + .and_then(|reasoning| reasoning.get("summary")) + .and_then(|value| value.as_str()), + Some("concise") + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn reasoning_summary_is_omitted_when_disabled() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -1070,7 +1274,7 @@ async fn reasoning_summary_is_omitted_when_disabled() -> anyhow::Result<()> { .await; let TestCodex { codex, .. } = test_codex() .with_config(|config| { - config.model_reasoning_summary = ReasoningSummary::None; + config.model_reasoning_summary = Some(ReasoningSummary::None); }) .build(&server) .await?; @@ -1101,6 +1305,60 @@ async fn reasoning_summary_is_omitted_when_disabled() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reasoning_summary_none_overrides_model_catalog_default() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + let server = MockServer::start().await; + + let resp_mock = mount_sse_once( + &server, + sse(vec![ev_response_created("resp1"), ev_completed("resp1")]), + ) + .await; + + let mut model_catalog: ModelsResponse = + serde_json::from_str(include_str!("../../models.json")).expect("valid models.json"); + let model = model_catalog + .models + .iter_mut() + .find(|model| model.slug == "gpt-5.1") + .expect("gpt-5.1 exists in bundled models.json"); + model.supports_reasoning_summaries = true; + model.default_reasoning_summary = ReasoningSummary::Detailed; + + let TestCodex { codex, .. } = test_codex() + .with_model("gpt-5.1") + .with_config(move |config| { + config.model_reasoning_summary = Some(ReasoningSummary::None); + config.model_catalog = Some(model_catalog); + }) + .build(&server) + .await?; + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "hello".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let request_body = resp_mock.single_request().body_json(); + pretty_assertions::assert_eq!( + request_body + .get("reasoning") + .and_then(|reasoning| reasoning.get("summary")), + None + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_default_verbosity_in_request() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -1283,19 +1541,42 @@ async fn includes_developer_instructions_message_in_request() { "expected permissions message to mention sandbox_mode, got {permissions_text:?}" ); - assert_message_role(&request_body["input"][1], "developer"); - assert_message_equals(&request_body["input"][1], "be useful"); - assert_message_role(&request_body["input"][2], "user"); - assert_message_starts_with(&request_body["input"][2], "# AGENTS.md instructions for "); - assert_message_ends_with(&request_body["input"][2], ""); - let ui_text = request_body["input"][2]["content"][0]["text"] - .as_str() + let developer_messages: Vec<&serde_json::Value> = request_body["input"] + .as_array() + .expect("input array") + .iter() + .filter(|item| item.get("role").and_then(|role| role.as_str()) == Some("developer")) + .collect(); + assert!( + developer_messages + .iter() + .any(|item| message_input_texts(item).contains(&"be useful")), + "expected developer instructions in a developer message, got {:?}", + request_body["input"] + ); + + assert_message_role(&request_body["input"][1], "user"); + let user_context_texts = message_input_texts(&request_body["input"][1]); + assert!( + user_context_texts + .iter() + .any(|text| text.starts_with("# AGENTS.md instructions for ")), + "expected AGENTS text in contextual user message, got {user_context_texts:?}" + ); + let ui_text = user_context_texts + .iter() + .copied() + .find(|text| text.contains("")) .expect("invalid message content"); assert!(ui_text.contains("")); assert!(ui_text.contains("be nice")); - assert_message_role(&request_body["input"][3], "user"); - assert_message_starts_with(&request_body["input"][3], ""); - assert_message_ends_with(&request_body["input"][3], ""); + assert!( + user_context_texts + .iter() + .any(|text| text.starts_with("") + && text.ends_with("")), + "expected environment context in contextual user message, got {user_context_texts:?}" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1426,11 +1707,18 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() { }); prompt.input.push(ResponseItem::CustomToolCallOutput { call_id: "custom-tool-call-id".into(), - output: "ok".into(), + output: FunctionCallOutputPayload::from_text("ok".into()), }); let mut stream = client_session - .stream(&prompt, &model_info, &otel_manager, effort, summary, None) + .stream( + &prompt, + &model_info, + &otel_manager, + effort, + summary.unwrap_or(ReasoningSummary::Auto), + None, + ) .await .expect("responses stream to start"); diff --git a/codex-rs/core/tests/suite/collaboration_instructions.rs b/codex-rs/core/tests/suite/collaboration_instructions.rs index 68d695325d..b1dbd2d660 100644 --- a/codex-rs/core/tests/suite/collaboration_instructions.rs +++ b/codex-rs/core/tests/suite/collaboration_instructions.rs @@ -169,7 +169,11 @@ async fn collaboration_instructions_added_on_user_turn() -> Result<()> { sandbox_policy: test.config.permissions.sandbox_policy.get().clone(), model: test.session_configured.model.clone(), effort: None, - summary: test.config.model_reasoning_summary, + summary: Some( + test.config + .model_reasoning_summary + .unwrap_or(codex_protocol::config_types::ReasoningSummary::Auto), + ), collaboration_mode: Some(collaboration_mode), final_output_json_schema: None, personality: None, @@ -275,7 +279,11 @@ async fn user_turn_overrides_collaboration_instructions_after_override() -> Resu sandbox_policy: test.config.permissions.sandbox_policy.get().clone(), model: test.session_configured.model.clone(), effort: None, - summary: test.config.model_reasoning_summary, + summary: Some( + test.config + .model_reasoning_summary + .unwrap_or(codex_protocol::config_types::ReasoningSummary::Auto), + ), collaboration_mode: Some(turn_mode), final_output_json_schema: None, personality: None, diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 9e9ce931e9..9e376ce494 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -5,7 +5,6 @@ use codex_core::built_in_model_providers; use codex_core::compact::SUMMARIZATION_PROMPT; use codex_core::compact::SUMMARY_PREFIX; use codex_core::config::Config; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::items::TurnItem; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelsResponse; @@ -756,16 +755,40 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { let body = requests_payloads[0].body_json(); let input = body.get("input").and_then(|v| v.as_array()).unwrap(); + fn strip_agents_parts_from_user_message( + value: &serde_json::Value, + ) -> Option { + let content = value + .get("content") + .and_then(|content| content.as_array())?; + let filtered_content = content + .iter() + .filter(|item| { + !item + .get("text") + .and_then(|text| text.as_str()) + .is_some_and(|text| text.starts_with("# AGENTS.md instructions for ")) + }) + .cloned() + .collect::>(); + if filtered_content.is_empty() { + return None; + } + let mut normalized = value.clone(); + normalized["content"] = serde_json::Value::Array(filtered_content); + Some(normalized) + } + fn normalize_inputs(values: &[serde_json::Value]) -> Vec { values .iter() - .filter(|value| { + .filter_map(|value| { if value .get("type") .and_then(|ty| ty.as_str()) .is_some_and(|ty| ty == "function_call_output") { - return false; + return None; } let text = value @@ -781,11 +804,13 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { if role == Some("developer") && text.is_some_and(|text| text.contains("`sandbox_mode`")) { - return false; + return None; } - !text.is_some_and(|text| text.starts_with("# AGENTS.md instructions for ")) + if role == Some("user") { + return strip_agents_parts_from_user_message(value); + } + Some(value.clone()) }) - .cloned() .collect() } @@ -1633,7 +1658,7 @@ async fn auto_compact_runs_after_resume_when_token_usage_is_over_limit() { sandbox_policy: SandboxPolicy::DangerFullAccess, model: resumed.session_configured.model.clone(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1722,7 +1747,7 @@ async fn pre_sampling_compact_runs_on_switch_to_smaller_context_model() { sandbox_policy: SandboxPolicy::DangerFullAccess, model: previous_model.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1745,7 +1770,7 @@ async fn pre_sampling_compact_runs_on_switch_to_smaller_context_model() { sandbox_policy: SandboxPolicy::DangerFullAccess, model: next_model.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1854,7 +1879,7 @@ async fn pre_sampling_compact_runs_after_resume_and_switch_to_smaller_model() { sandbox_policy: SandboxPolicy::DangerFullAccess, model: previous_model.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1901,7 +1926,7 @@ async fn pre_sampling_compact_runs_after_resume_and_switch_to_smaller_model() { sandbox_policy: SandboxPolicy::DangerFullAccess, model: next_model.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -3102,7 +3127,7 @@ async fn snapshot_request_shape_pre_turn_compaction_strips_incoming_model_switch sandbox_policy: SandboxPolicy::DangerFullAccess, model: previous_model.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -3125,7 +3150,7 @@ async fn snapshot_request_shape_pre_turn_compaction_strips_incoming_model_switch sandbox_policy: SandboxPolicy::DangerFullAccess, model: next_model.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -3184,7 +3209,7 @@ async fn snapshot_request_shape_pre_turn_compaction_context_window_exceeded() { ]); let mut responses = vec![first_turn]; responses.extend( - (0..6).map(|_| { + (0..5).map(|_| { sse_failed( "compact-failed", "context_length_exceeded", diff --git a/codex-rs/core/tests/suite/exec_policy.rs b/codex-rs/core/tests/suite/exec_policy.rs index 3fd06d83c7..52717f2880 100644 --- a/codex-rs/core/tests/suite/exec_policy.rs +++ b/codex-rs/core/tests/suite/exec_policy.rs @@ -4,7 +4,6 @@ use anyhow::Result; use codex_core::features::Feature; use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::ModeKind; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::Settings; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; @@ -55,7 +54,7 @@ async fn submit_user_turn( sandbox_policy, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode, personality: None, }) @@ -134,7 +133,7 @@ async fn execpolicy_blocks_shell_invocation() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs index 2349d9f5fa..c8f1f2eaf1 100644 --- a/codex-rs/core/tests/suite/image_rollout.rs +++ b/codex-rs/core/tests/suite/image_rollout.rs @@ -1,5 +1,4 @@ use anyhow::Context; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::AskForApproval; @@ -126,7 +125,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -208,7 +207,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/items.rs b/codex-rs/core/tests/suite/items.rs index 0d93278af1..dfb5ac88b4 100644 --- a/codex-rs/core/tests/suite/items.rs +++ b/codex-rs/core/tests/suite/items.rs @@ -377,7 +377,7 @@ async fn plan_mode_emits_plan_item_from_proposed_plan_block() -> anyhow::Result< sandbox_policy: codex_protocol::protocol::SandboxPolicy::DangerFullAccess, model: session_configured.model.clone(), effort: None, - summary: codex_protocol::config_types::ReasoningSummary::Auto, + summary: None, collaboration_mode: Some(collaboration_mode), personality: None, }) @@ -452,7 +452,7 @@ async fn plan_mode_strips_plan_from_agent_messages() -> anyhow::Result<()> { sandbox_policy: codex_protocol::protocol::SandboxPolicy::DangerFullAccess, model: session_configured.model.clone(), effort: None, - summary: codex_protocol::config_types::ReasoningSummary::Auto, + summary: None, collaboration_mode: Some(collaboration_mode), personality: None, }) @@ -559,7 +559,7 @@ async fn plan_mode_streaming_citations_are_stripped_across_added_deltas_and_done sandbox_policy: codex_protocol::protocol::SandboxPolicy::DangerFullAccess, model: session_configured.model.clone(), effort: None, - summary: codex_protocol::config_types::ReasoningSummary::Auto, + summary: None, collaboration_mode: Some(collaboration_mode), personality: None, }) @@ -744,7 +744,7 @@ async fn plan_mode_streaming_proposed_plan_tag_split_across_added_and_delta_is_p sandbox_policy: codex_protocol::protocol::SandboxPolicy::DangerFullAccess, model: session_configured.model.clone(), effort: None, - summary: codex_protocol::config_types::ReasoningSummary::Auto, + summary: None, collaboration_mode: Some(collaboration_mode), personality: None, }) @@ -856,7 +856,7 @@ async fn plan_mode_handles_missing_plan_close_tag() -> anyhow::Result<()> { sandbox_policy: codex_protocol::protocol::SandboxPolicy::DangerFullAccess, model: session_configured.model.clone(), effort: None, - summary: codex_protocol::config_types::ReasoningSummary::Auto, + summary: None, collaboration_mode: Some(collaboration_mode), personality: None, }) diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index ba2b52fd6d..31dbe6e60a 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -2,6 +2,7 @@ use anyhow::Result; use codex_core::features::Feature; +use codex_protocol::protocol::EventMsg; use core_test_support::responses; use core_test_support::responses::ResponseMock; use core_test_support::responses::ResponsesRequest; @@ -12,6 +13,12 @@ use core_test_support::responses::ev_response_created; use core_test_support::responses::sse; use core_test_support::skip_if_no_network; use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event_match; +use std::fs; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use tempfile::tempdir; use wiremock::MockServer; fn custom_tool_output_text_and_success( @@ -24,6 +31,45 @@ fn custom_tool_output_text_and_success( (output.unwrap_or_default(), success) } +fn tool_names(body: &serde_json::Value) -> Vec { + body["tools"] + .as_array() + .expect("tools array should be present") + .iter() + .map(|tool| { + tool.get("name") + .and_then(|value| value.as_str()) + .or_else(|| tool.get("type").and_then(|value| value.as_str())) + .expect("tool should have a name or type") + .to_string() + }) + .collect() +} + +fn write_too_old_node_script(dir: &Path) -> Result { + #[cfg(windows)] + { + let path = dir.join("old-node.cmd"); + fs::write(&path, "@echo off\r\necho v0.0.1\r\n")?; + Ok(path) + } + + #[cfg(unix)] + { + let path = dir.join("old-node.sh"); + fs::write(&path, "#!/bin/sh\necho v0.0.1\n")?; + let mut permissions = fs::metadata(&path)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&path, permissions)?; + Ok(path) + } + + #[cfg(not(any(unix, windows)))] + { + anyhow::bail!("unsupported platform for js_repl test fixture"); + } +} + async fn run_js_repl_turn( server: &MockServer, prompt: &str, @@ -54,6 +100,64 @@ async fn run_js_repl_turn( Ok(second_mock) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn js_repl_is_not_advertised_when_startup_node_is_incompatible() -> Result<()> { + skip_if_no_network!(Ok(())); + if std::env::var_os("CODEX_JS_REPL_NODE_PATH").is_some() { + return Ok(()); + } + + let server = responses::start_mock_server().await; + let temp = tempdir()?; + let old_node = write_too_old_node_script(temp.path())?; + + let mut builder = test_codex().with_config(move |config| { + config.features.enable(Feature::JsRepl); + config.js_repl_node_path = Some(old_node); + }); + let test = builder.build(&server).await?; + let warning = wait_for_event_match(&test.codex, |event| match event { + EventMsg::Warning(ev) if ev.message.contains("Disabled `js_repl` for this session") => { + Some(ev.message.clone()) + } + _ => None, + }) + .await; + assert!( + warning.contains("Node runtime"), + "warning should explain the Node compatibility issue: {warning}" + ); + + let request_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-1"), + ]), + ) + .await; + + test.submit_turn("hello").await?; + + let body = request_mock.single_request().body_json(); + let tools = tool_names(&body); + assert!( + !tools.iter().any(|tool| tool == "js_repl"), + "js_repl should be omitted when startup validation fails: {tools:?}" + ); + assert!( + !tools.iter().any(|tool| tool == "js_repl_reset"), + "js_repl_reset should be omitted when startup validation fails: {tools:?}" + ); + let instructions = body["instructions"].as_str().unwrap_or_default(); + assert!( + !instructions.contains("## JavaScript REPL (Node)"), + "startup instructions should not mention js_repl when it is disabled: {instructions}" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn js_repl_persists_top_level_bindings_and_supports_tla() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/core/tests/suite/json_result.rs b/codex-rs/core/tests/suite/json_result.rs index e32865e0c7..12c9845a75 100644 --- a/codex-rs/core/tests/suite/json_result.rs +++ b/codex-rs/core/tests/suite/json_result.rs @@ -1,6 +1,5 @@ #![cfg(not(target_os = "windows"))] -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -84,7 +83,7 @@ async fn codex_returns_json_result(model: String) -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/live_reload.rs b/codex-rs/core/tests/suite/live_reload.rs index 3ea0a3cbe9..8ee4c56dda 100644 --- a/codex-rs/core/tests/suite/live_reload.rs +++ b/codex-rs/core/tests/suite/live_reload.rs @@ -7,7 +7,6 @@ use std::time::Duration; use anyhow::Result; use codex_core::config::ProjectConfig; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::TrustLevel; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; @@ -65,7 +64,7 @@ async fn submit_skill_turn(test: &TestCodex, skill_path: PathBuf, prompt: &str) sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/memories.rs b/codex-rs/core/tests/suite/memories.rs new file mode 100644 index 0000000000..8f96178e86 --- /dev/null +++ b/codex-rs/core/tests/suite/memories.rs @@ -0,0 +1,289 @@ +use anyhow::Result; +use chrono::Duration as ChronoDuration; +use chrono::Utc; +use codex_core::features::Feature; +use codex_protocol::ThreadId; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::SessionSource; +use core_test_support::responses::ResponseMock; +use core_test_support::responses::ResponsesRequest; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_once; +use core_test_support::responses::sse; +use core_test_support::responses::start_mock_server; +use core_test_support::test_codex::TestCodex; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use pretty_assertions::assert_eq; +use std::path::Path; +use std::sync::Arc; +use tempfile::TempDir; +use tokio::time::Duration; +use tokio::time::Instant; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn memories_startup_phase2_tracks_added_and_removed_inputs_across_runs() -> Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let db = init_state_db(&home).await?; + + let now = Utc::now(); + let thread_a = seed_stage1_output( + db.as_ref(), + home.path(), + now - ChronoDuration::hours(2), + "raw memory A", + "rollout summary A", + "rollout-a", + ) + .await?; + + let first_phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2-1"), + ev_assistant_message("msg-phase2-1", "phase2 complete"), + ev_completed("resp-phase2-1"), + ]), + ) + .await; + + let first = build_test_codex(&server, home.clone()).await?; + let first_request = wait_for_single_request(&first_phase2).await; + let first_prompt = phase2_prompt_text(&first_request); + assert!( + first_prompt.contains("- selected inputs this run: 1"), + "expected selected count in first prompt: {first_prompt}" + ); + assert!( + first_prompt.contains("- newly added since the last successful Phase 2 run: 1"), + "expected added count in first prompt: {first_prompt}" + ); + assert!( + first_prompt.contains("- removed from the last successful Phase 2 run: 0"), + "expected removed count in first prompt: {first_prompt}" + ); + assert!( + first_prompt.contains(&format!("- [added] thread_id={thread_a},")), + "expected thread A to be marked added: {first_prompt}" + ); + assert!( + first_prompt.contains("Removed from the last successful Phase 2 selection:\n- none"), + "expected no removed items in first prompt: {first_prompt}" + ); + + wait_for_phase2_success(db.as_ref(), thread_a).await?; + let memory_root = home.path().join("memories"); + let raw_memories = tokio::fs::read_to_string(memory_root.join("raw_memories.md")).await?; + assert!(raw_memories.contains("raw memory A")); + assert!(!raw_memories.contains("raw memory B")); + let rollout_summaries = read_rollout_summary_bodies(&memory_root).await?; + assert_eq!(rollout_summaries.len(), 1); + assert!(rollout_summaries[0].contains("rollout summary A")); + assert!(rollout_summaries[0].contains("git_branch: branch-rollout-a")); + + shutdown_test_codex(&first).await?; + + let thread_b = seed_stage1_output( + db.as_ref(), + home.path(), + now - ChronoDuration::hours(1), + "raw memory B", + "rollout summary B", + "rollout-b", + ) + .await?; + + let second_phase2 = mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-phase2-2"), + ev_assistant_message("msg-phase2-2", "phase2 complete"), + ev_completed("resp-phase2-2"), + ]), + ) + .await; + + let second = build_test_codex(&server, home.clone()).await?; + let second_request = wait_for_single_request(&second_phase2).await; + let second_prompt = phase2_prompt_text(&second_request); + assert!( + second_prompt.contains("- selected inputs this run: 1"), + "expected selected count in second prompt: {second_prompt}" + ); + assert!( + second_prompt.contains("- newly added since the last successful Phase 2 run: 1"), + "expected added count in second prompt: {second_prompt}" + ); + assert!( + second_prompt.contains("- removed from the last successful Phase 2 run: 1"), + "expected removed count in second prompt: {second_prompt}" + ); + assert!( + second_prompt.contains(&format!("- [added] thread_id={thread_b},")), + "expected thread B to be marked added: {second_prompt}" + ); + assert!( + second_prompt.contains(&format!("- thread_id={thread_a},")), + "expected thread A to be marked removed: {second_prompt}" + ); + + wait_for_phase2_success(db.as_ref(), thread_b).await?; + let raw_memories = tokio::fs::read_to_string(memory_root.join("raw_memories.md")).await?; + assert!(raw_memories.contains("raw memory B")); + assert!(raw_memories.contains("raw memory A")); + let rollout_summaries = read_rollout_summary_bodies(&memory_root).await?; + assert_eq!(rollout_summaries.len(), 2); + assert!( + rollout_summaries + .iter() + .any(|summary| summary.contains("rollout summary B")) + ); + assert!( + rollout_summaries + .iter() + .any(|summary| summary.contains("git_branch: branch-rollout-b")) + ); + assert!( + rollout_summaries + .iter() + .any(|summary| summary.contains("rollout summary A")) + ); + + shutdown_test_codex(&second).await?; + Ok(()) +} + +async fn build_test_codex(server: &wiremock::MockServer, home: Arc) -> Result { + let mut builder = test_codex().with_home(home).with_config(|config| { + config.features.enable(Feature::Sqlite); + config.features.enable(Feature::MemoryTool); + config.memories.max_raw_memories_for_global = 1; + }); + builder.build(server).await +} + +async fn init_state_db(home: &Arc) -> Result> { + let db = + codex_state::StateRuntime::init(home.path().to_path_buf(), "test-provider".into(), None) + .await?; + db.mark_backfill_complete(None).await?; + Ok(db) +} + +async fn seed_stage1_output( + db: &codex_state::StateRuntime, + codex_home: &Path, + updated_at: chrono::DateTime, + raw_memory: &str, + rollout_summary: &str, + rollout_slug: &str, +) -> Result { + let thread_id = ThreadId::new(); + let mut metadata_builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + codex_home.join(format!("rollout-{thread_id}.jsonl")), + updated_at, + SessionSource::Cli, + ); + metadata_builder.cwd = codex_home.join(format!("workspace-{rollout_slug}")); + metadata_builder.model_provider = Some("test-provider".to_string()); + metadata_builder.git_branch = Some(format!("branch-{rollout_slug}")); + let metadata = metadata_builder.build("test-provider"); + db.upsert_thread(&metadata).await?; + + let claim = db + .try_claim_stage1_job( + thread_id, + ThreadId::new(), + updated_at.timestamp(), + 3_600, + 64, + ) + .await?; + let ownership_token = match claim { + codex_state::Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage-1 claim outcome: {other:?}"), + }; + + assert!( + db.mark_stage1_job_succeeded( + thread_id, + &ownership_token, + updated_at.timestamp(), + raw_memory, + rollout_summary, + Some(rollout_slug), + ) + .await?, + "stage-1 success should enqueue global consolidation" + ); + + Ok(thread_id) +} + +async fn wait_for_single_request(mock: &ResponseMock) -> ResponsesRequest { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let requests = mock.requests(); + if let Some(request) = requests.into_iter().next() { + return request; + } + assert!( + Instant::now() < deadline, + "timed out waiting for phase2 request" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +#[allow(clippy::expect_used)] +fn phase2_prompt_text(request: &ResponsesRequest) -> String { + request + .message_input_texts("user") + .into_iter() + .find(|text| text.contains("Current selected Phase 1 inputs:")) + .expect("phase2 prompt text") +} + +async fn wait_for_phase2_success( + db: &codex_state::StateRuntime, + expected_thread_id: ThreadId, +) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let selection = db.get_phase2_input_selection(1, 30).await?; + if selection.selected.len() == 1 + && selection.selected[0].thread_id == expected_thread_id + && selection.retained_thread_ids == vec![expected_thread_id] + && selection.removed.is_empty() + { + return Ok(()); + } + + assert!( + Instant::now() < deadline, + "timed out waiting for phase2 success for {expected_thread_id}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn read_rollout_summary_bodies(memory_root: &Path) -> Result> { + let mut dir = tokio::fs::read_dir(memory_root.join("rollout_summaries")).await?; + let mut summaries = Vec::new(); + while let Some(entry) = dir.next_entry().await? { + summaries.push(tokio::fs::read_to_string(entry.path()).await?); + } + summaries.sort(); + Ok(summaries) +} + +async fn shutdown_test_codex(test: &TestCodex) -> Result<()> { + test.codex.submit(Op::Shutdown {}).await?; + wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::ShutdownComplete)).await; + Ok(()) +} diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 4cfe955b55..e23fd9ddbb 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -83,6 +83,7 @@ mod json_result; mod list_dir; mod live_cli; mod live_reload; +mod memories; mod model_info_overrides; mod model_overrides; mod model_switching; diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index 6e06b0f224..62d0545462 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -58,7 +58,7 @@ async fn model_change_appends_model_instructions_developer_message() -> Result<( sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -91,7 +91,7 @@ async fn model_change_appends_model_instructions_developer_message() -> Result<( sandbox_policy: SandboxPolicy::new_read_only_policy(), model: next_model.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -146,7 +146,7 @@ async fn model_and_personality_change_only_appends_model_instructions() -> Resul sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -179,7 +179,7 @@ async fn model_and_personality_change_only_appends_model_instructions() -> Resul sandbox_policy: SandboxPolicy::new_read_only_policy(), model: next_model.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -234,8 +234,10 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -293,7 +295,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< sandbox_policy: SandboxPolicy::new_read_only_policy(), model: image_model_slug.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -312,7 +314,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< sandbox_policy: SandboxPolicy::new_read_only_policy(), model: text_model_slug.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -391,8 +393,10 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -469,7 +473,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< sandbox_policy: SandboxPolicy::new_read_only_policy(), model: large_model_slug.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -524,7 +528,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< sandbox_policy: SandboxPolicy::new_read_only_policy(), model: smaller_model_slug.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/model_visible_layout.rs b/codex-rs/core/tests/suite/model_visible_layout.rs index 3288eb5c05..14503b3f5b 100644 --- a/codex-rs/core/tests/suite/model_visible_layout.rs +++ b/codex-rs/core/tests/suite/model_visible_layout.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use anyhow::Result; use codex_core::config::types::Personality; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -26,6 +25,7 @@ use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; +use serde_json::json; const PRETURN_CONTEXT_DIFF_CWD: &str = "PRETURN_CONTEXT_DIFF_CWD"; @@ -53,6 +53,30 @@ fn agents_message_count(request: &ResponsesRequest) -> usize { .count() } +fn format_environment_context_subagents_snapshot(subagents: &[&str]) -> String { + let subagents_block = if subagents.is_empty() { + String::new() + } else { + let lines = subagents + .iter() + .map(|line| format!(" {line}")) + .collect::>() + .join("\n"); + format!("\n \n{lines}\n ") + }; + let items = vec![json!({ + "type": "message", + "role": "user", + "content": [{ + "type": "input_text", + "text": format!( + "\n /tmp/example\n bash{subagents_block}\n" + ), + }], + })]; + context_snapshot::format_response_items_snapshot(items.as_slice(), &context_snapshot_options()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn snapshot_model_visible_layout_turn_overrides() -> Result<()> { skip_if_no_network!(Ok(())); @@ -97,7 +121,7 @@ async fn snapshot_model_visible_layout_turn_overrides() -> Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -119,7 +143,7 @@ async fn snapshot_model_visible_layout_turn_overrides() -> Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: Some(Personality::Friendly), }) @@ -196,7 +220,7 @@ async fn snapshot_model_visible_layout_cwd_change_does_not_refresh_agents() -> R sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -218,7 +242,7 @@ async fn snapshot_model_visible_layout_cwd_change_does_not_refresh_agents() -> R sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -323,7 +347,7 @@ async fn snapshot_model_visible_layout_resume_with_personality_change() -> Resul sandbox_policy: SandboxPolicy::new_read_only_policy(), model: resumed.session_configured.model.clone(), effort: resumed.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: Some(Personality::Friendly), }) @@ -445,3 +469,23 @@ async fn snapshot_model_visible_layout_resume_override_matches_rollout_model() - Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn snapshot_model_visible_layout_environment_context_includes_one_subagent() -> Result<()> { + insta::assert_snapshot!( + "model_visible_layout_environment_context_includes_one_subagent", + format_environment_context_subagents_snapshot(&["- agent-1: Atlas"]) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn snapshot_model_visible_layout_environment_context_includes_two_subagents() -> Result<()> { + insta::assert_snapshot!( + "model_visible_layout_environment_context_includes_two_subagents", + format_environment_context_subagents_snapshot(&["- agent-1: Atlas", "- agent-2: Juniper"]) + ); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/models_cache_ttl.rs b/codex-rs/core/tests/suite/models_cache_ttl.rs index 3422e397b8..9948eb4e21 100644 --- a/codex-rs/core/tests/suite/models_cache_ttl.rs +++ b/codex-rs/core/tests/suite/models_cache_ttl.rs @@ -99,7 +99,7 @@ async fn renews_cache_ttl_on_matching_models_etag() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: test.session_configured.model.clone(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -336,8 +336,10 @@ fn test_remote_model(slug: &str, priority: i32) -> ModelInfo { base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/models_etag_responses.rs b/codex-rs/core/tests/suite/models_etag_responses.rs index 7c40bec987..1cdc549012 100644 --- a/codex-rs/core/tests/suite/models_etag_responses.rs +++ b/codex-rs/core/tests/suite/models_etag_responses.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use anyhow::Result; use codex_core::CodexAuth; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::openai_models::ModelsResponse; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; @@ -106,7 +105,7 @@ async fn refresh_models_on_models_etag_mismatch_and_avoid_duplicate_models_fetch sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/personality.rs b/codex-rs/core/tests/suite/personality.rs index 0b4d485cfb..97ce2360ba 100644 --- a/codex-rs/core/tests/suite/personality.rs +++ b/codex-rs/core/tests/suite/personality.rs @@ -97,7 +97,7 @@ async fn user_turn_personality_none_does_not_add_update_message() -> anyhow::Res sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -143,7 +143,7 @@ async fn config_personality_some_sets_instructions_template() -> anyhow::Result< sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -196,7 +196,7 @@ async fn config_personality_none_sends_no_personality() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -255,7 +255,7 @@ async fn default_personality_is_pragmatic_without_config_toml() -> anyhow::Resul sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -302,7 +302,7 @@ async fn user_turn_personality_some_adds_update_message() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -336,7 +336,7 @@ async fn user_turn_personality_some_adds_update_message() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -398,7 +398,7 @@ async fn user_turn_personality_same_value_does_not_add_update_message() -> anyho sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -432,7 +432,7 @@ async fn user_turn_personality_same_value_does_not_add_update_message() -> anyho sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -504,7 +504,7 @@ async fn user_turn_personality_skips_if_feature_disabled() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -538,7 +538,7 @@ async fn user_turn_personality_skips_if_feature_disabled() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::new_read_only_policy(), model: test.session_configured.model.clone(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -599,8 +599,10 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow }), }), supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -646,7 +648,7 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow sandbox_policy: SandboxPolicy::new_read_only_policy(), model: remote_slug.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: Some(Personality::Friendly), }) @@ -706,8 +708,10 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - }), }), supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -756,7 +760,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - sandbox_policy: SandboxPolicy::new_read_only_policy(), model: remote_slug.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -790,7 +794,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - sandbox_policy: SandboxPolicy::new_read_only_policy(), model: remote_slug.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/prompt_caching.rs b/codex-rs/core/tests/suite/prompt_caching.rs index 38a8145d50..54ee11fba4 100644 --- a/codex-rs/core/tests/suite/prompt_caching.rs +++ b/codex-rs/core/tests/suite/prompt_caching.rs @@ -30,21 +30,46 @@ use pretty_assertions::assert_eq; use tempfile::TempDir; fn text_user_input(text: String) -> serde_json::Value { + text_user_input_parts(vec![text]) +} + +fn text_user_input_parts(texts: Vec) -> serde_json::Value { serde_json::json!({ "type": "message", "role": "user", - "content": [ { "type": "input_text", "text": text } ] + "content": texts + .into_iter() + .map(|text| serde_json::json!({ "type": "input_text", "text": text })) + .collect::>() }) } -fn default_env_context_str(cwd: &str, shell: &Shell) -> String { +fn assert_default_env_context(text: &str, cwd: &str, shell: &Shell) { let shell_name = shell.name(); - format!( - r#" - {cwd} - {shell_name} -"# - ) + assert!( + text.starts_with(ENVIRONMENT_CONTEXT_OPEN_TAG), + "expected environment context fragment: {text}" + ); + assert!( + text.contains(&format!("{cwd}")), + "expected cwd in environment context: {text}" + ); + assert!( + text.contains(&format!("{shell_name}")), + "expected shell in environment context: {text}" + ); + assert!( + text.contains("") && text.contains(""), + "expected current_date in environment context: {text}" + ); + assert!( + text.contains("") && text.contains(""), + "expected timezone in environment context: {text}" + ); + assert!( + text.ends_with(""), + "expected closing environment_context tag: {text}" + ); } fn assert_tool_names(body: &serde_json::Value, expected_names: &[&str]) { @@ -297,8 +322,8 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests let input1 = body1["input"].as_array().expect("input array"); assert_eq!( input1.len(), - 4, - "expected permissions + cached prefix + env + user msg" + 3, + "expected permissions + cached contextual user prefix + user msg" ); let ui_text = input1[1]["content"][0]["text"] @@ -311,13 +336,16 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests let shell = default_user_shell(); let cwd_str = config.cwd.to_string_lossy(); - let expected_env_text = default_env_context_str(&cwd_str, &shell); + let env_text = input1[1]["content"][1]["text"] + .as_str() + .expect("environment context text"); + assert_default_env_context(env_text, &cwd_str, &shell); assert_eq!( - input1[2], - text_user_input(expected_env_text), - "expected environment context after UI message" + input1[1]["content"][1]["type"].as_str(), + Some("input_text"), + "expected environment context bundled after UI message in cached contextual message" ); - assert_eq!(input1[3], text_user_input("hello 1".to_string())); + assert_eq!(input1[2], text_user_input("hello 1".to_string())); let body2 = req2.single_request().body_json(); let input2 = body2["input"].as_array().expect("input array"); @@ -402,8 +430,10 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; - let body1 = req1.single_request().body_json(); - let body2 = req2.single_request().body_json(); + let request1 = req1.single_request(); + let request2 = req2.single_request(); + let body1 = request1.body_json(); + let body2 = request2.body_json(); // prompt_cache_key should remain constant across overrides assert_eq!( body1["prompt_cache_key"], body2["prompt_cache_key"], @@ -500,17 +530,32 @@ async fn override_before_first_turn_emits_environment_context() -> anyhow::Resul let env_texts: Vec<&str> = input .iter() .filter_map(|msg| { - msg["content"] - .as_array() - .and_then(|content| content.first()) - .and_then(|item| item["text"].as_str()) + msg["content"].as_array().map(|content| { + content + .iter() + .filter_map(|item| item["text"].as_str()) + .collect::>() + }) }) + .flatten() .filter(|text| text.starts_with(ENVIRONMENT_CONTEXT_OPEN_TAG)) .collect(); assert!( !env_texts.is_empty(), "expected environment context to be emitted: {env_texts:?}" ); + assert!( + env_texts + .iter() + .any(|text| text.contains("") && text.contains("")), + "expected current_date in environment context: {env_texts:?}" + ); + assert!( + env_texts + .iter() + .any(|text| text.contains("") && text.contains("")), + "expected timezone in environment context: {env_texts:?}" + ); let env_count = input .iter() @@ -559,11 +604,14 @@ async fn override_before_first_turn_emits_environment_context() -> anyhow::Resul let user_texts: Vec<&str> = input .iter() .filter_map(|msg| { - msg["content"] - .as_array() - .and_then(|content| content.first()) - .and_then(|item| item["text"].as_str()) + msg["content"].as_array().map(|content| { + content + .iter() + .filter_map(|item| item["text"].as_str()) + .collect::>() + }) }) + .flatten() .collect(); assert!( user_texts.contains(&"first message"), @@ -631,7 +679,7 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res sandbox_policy: new_policy.clone(), model: "o3".to_string(), effort: Some(ReasoningEffort::High), - summary: ReasoningSummary::Detailed, + summary: Some(ReasoningSummary::Detailed), collaboration_mode: None, final_output_json_schema: None, personality: None, @@ -639,8 +687,10 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; - let body1 = req1.single_request().body_json(); - let body2 = req2.single_request().body_json(); + let request1 = req1.single_request(); + let request2 = req2.single_request(); + let body1 = request1.body_json(); + let body2 = request2.body_json(); // prompt_cache_key should remain constant across per-turn overrides assert_eq!( @@ -655,43 +705,34 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res "role": "user", "content": [ { "type": "input_text", "text": "hello 2" } ] }); - let shell = default_user_shell(); - - let expected_env_text_2 = format!( - r#" - {} - {} -"#, - new_cwd.path().display(), - shell.name() - ); - let expected_env_msg_2 = serde_json::json!({ - "type": "message", - "role": "user", - "content": [ { "type": "input_text", "text": expected_env_text_2 } ] - }); let expected_permissions_msg = body1["input"][0].clone(); let body1_input = body1["input"].as_array().expect("input array"); - let expected_model_switch_msg = body2["input"][body1_input.len()].clone(); + let expected_settings_update_msg = body2["input"][body1_input.len()].clone(); + assert_ne!( + expected_settings_update_msg, expected_permissions_msg, + "expected updated permissions message after per-turn override" + ); assert_eq!( - expected_model_switch_msg["role"].as_str(), + expected_settings_update_msg["role"].as_str(), Some("developer") ); assert!( - expected_model_switch_msg["content"][0]["text"] - .as_str() - .is_some_and(|text| text.contains("")), - "expected model switch message after model override: {expected_model_switch_msg:?}" - ); - let expected_permissions_msg_2 = body2["input"][body1_input.len() + 2].clone(); - assert_ne!( - expected_permissions_msg_2, expected_permissions_msg, - "expected updated permissions message after per-turn override" + request2.has_message_with_input_texts("developer", |texts| { + texts.iter().any(|text| text.contains("")) + }), + "expected model switch section after model override: {expected_settings_update_msg:?}" ); + let expected_env_msg_2 = body2["input"][body1_input.len() + 1].clone(); + assert_eq!(expected_env_msg_2["role"].as_str(), Some("user")); + let env_text = expected_env_msg_2["content"][0]["text"] + .as_str() + .expect("environment context text"); + let shell = default_user_shell(); + let expected_cwd = new_cwd.path().display().to_string(); + assert_default_env_context(env_text, &expected_cwd, &shell); let mut expected_body2 = body1_input.to_vec(); - expected_body2.push(expected_model_switch_msg); + expected_body2.push(expected_settings_update_msg); expected_body2.push(expected_env_msg_2); - expected_body2.push(expected_permissions_msg_2); expected_body2.push(expected_user_message_2); assert_eq!(body2["input"], serde_json::Value::Array(expected_body2)); @@ -746,7 +787,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a sandbox_policy: default_sandbox_policy.clone(), model: default_model.clone(), effort: default_effort, - summary: default_summary, + summary: Some(default_summary.unwrap_or(ReasoningSummary::Auto)), collaboration_mode: None, final_output_json_schema: None, personality: None, @@ -765,7 +806,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a sandbox_policy: default_sandbox_policy.clone(), model: default_model.clone(), effort: default_effort, - summary: default_summary, + summary: Some(default_summary.unwrap_or(ReasoningSummary::Auto)), collaboration_mode: None, final_output_json_schema: None, personality: None, @@ -773,22 +814,34 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; - let body1 = req1.single_request().body_json(); - let body2 = req2.single_request().body_json(); + let request1 = req1.single_request(); + let request2 = req2.single_request(); + let body1 = request1.body_json(); + let body2 = request2.body_json(); let expected_permissions_msg = body1["input"][0].clone(); let expected_ui_msg = body1["input"][1].clone(); let shell = default_user_shell(); let default_cwd_lossy = default_cwd.to_string_lossy(); + let expected_env_text_1 = expected_ui_msg["content"][1]["text"] + .as_str() + .expect("cached environment context text") + .to_string(); + assert_default_env_context(&expected_env_text_1, &default_cwd_lossy, &shell); - let expected_env_msg_1 = text_user_input(default_env_context_str(&default_cwd_lossy, &shell)); + let expected_contextual_user_msg_1 = text_user_input_parts(vec![ + expected_ui_msg["content"][0]["text"] + .as_str() + .expect("cached user instructions text") + .to_string(), + expected_env_text_1, + ]); let expected_user_message_1 = text_user_input("hello 1".to_string()); let expected_input_1 = serde_json::Value::Array(vec![ expected_permissions_msg.clone(), - expected_ui_msg.clone(), - expected_env_msg_1.clone(), + expected_contextual_user_msg_1.clone(), expected_user_message_1.clone(), ]); assert_eq!(body1["input"], expected_input_1); @@ -796,8 +849,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a let expected_user_message_2 = text_user_input("hello 2".to_string()); let expected_input_2 = serde_json::Value::Array(vec![ expected_permissions_msg, - expected_ui_msg, - expected_env_msg_1, + expected_contextual_user_msg_1, expected_user_message_1, expected_user_message_2, ]); @@ -854,7 +906,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu sandbox_policy: default_sandbox_policy.clone(), model: default_model, effort: default_effort, - summary: default_summary, + summary: Some(default_summary.unwrap_or(ReasoningSummary::Auto)), collaboration_mode: None, final_output_json_schema: None, personality: None, @@ -873,7 +925,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu sandbox_policy: SandboxPolicy::DangerFullAccess, model: "o3".to_string(), effort: Some(ReasoningEffort::High), - summary: ReasoningSummary::Detailed, + summary: Some(ReasoningSummary::Detailed), collaboration_mode: None, final_output_json_schema: None, personality: None, @@ -881,49 +933,57 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; - let body1 = req1.single_request().body_json(); - let body2 = req2.single_request().body_json(); + let request1 = req1.single_request(); + let request2 = req2.single_request(); + let body1 = request1.body_json(); + let body2 = request2.body_json(); let expected_permissions_msg = body1["input"][0].clone(); let expected_ui_msg = body1["input"][1].clone(); let shell = default_user_shell(); - let expected_env_text_1 = default_env_context_str(&default_cwd.to_string_lossy(), &shell); - let expected_env_msg_1 = text_user_input(expected_env_text_1); + let expected_env_text_1 = expected_ui_msg["content"][1]["text"] + .as_str() + .expect("cached environment context text") + .to_string(); + assert_default_env_context(&expected_env_text_1, &default_cwd.to_string_lossy(), &shell); + let expected_contextual_user_msg_1 = text_user_input_parts(vec![ + expected_ui_msg["content"][0]["text"] + .as_str() + .expect("cached user instructions text") + .to_string(), + expected_env_text_1, + ]); let expected_user_message_1 = text_user_input("hello 1".to_string()); let expected_input_1 = serde_json::Value::Array(vec![ expected_permissions_msg.clone(), - expected_ui_msg.clone(), - expected_env_msg_1.clone(), + expected_contextual_user_msg_1.clone(), expected_user_message_1.clone(), ]); assert_eq!(body1["input"], expected_input_1); let body1_input = body1["input"].as_array().expect("input array"); - let expected_model_switch_msg = body2["input"][body1_input.len()].clone(); + let expected_settings_update_msg = body2["input"][body1_input.len()].clone(); + assert_ne!( + expected_settings_update_msg, expected_permissions_msg, + "expected updated permissions message after policy change" + ); assert_eq!( - expected_model_switch_msg["role"].as_str(), + expected_settings_update_msg["role"].as_str(), Some("developer") ); assert!( - expected_model_switch_msg["content"][0]["text"] - .as_str() - .is_some_and(|text| text.contains("")), - "expected model switch message after model override: {expected_model_switch_msg:?}" - ); - let expected_permissions_msg_2 = body2["input"][body1_input.len() + 1].clone(); - assert_ne!( - expected_permissions_msg_2, expected_permissions_msg, - "expected updated permissions message after policy change" + request2.has_message_with_input_texts("developer", |texts| { + texts.iter().any(|text| text.contains("")) + }), + "expected model switch section after model override: {expected_settings_update_msg:?}" ); let expected_user_message_2 = text_user_input("hello 2".to_string()); let expected_input_2 = serde_json::Value::Array(vec![ expected_permissions_msg, - expected_ui_msg, - expected_env_msg_1, + expected_contextual_user_msg_1, expected_user_message_1, - expected_model_switch_msg, - expected_permissions_msg_2, + expected_settings_update_msg, expected_user_message_2, ]); assert_eq!(body2["input"], expected_input_2); diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 98848e911c..1f7504d41b 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -715,6 +715,7 @@ async fn inbound_realtime_text_ignores_user_role_and_still_forwards_audio() -> R async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_audio() -> Result<()> { skip_if_no_network!(Ok(())); + let start = std::time::Instant::now(); let (gate_completed_tx, gate_completed_rx) = oneshot::channel(); let first_chunks = vec![ @@ -806,18 +807,45 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au _ => None, }) .await; + eprintln!( + "[realtime test +{}ms] saw trigger text={:?}", + start.elapsed().as_millis(), + "delegate now" + ); - let audio_out = tokio::time::timeout( - Duration::from_millis(500), - wait_for_event_match(&test.codex, |msg| match msg { - EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent { - payload: RealtimeEvent::AudioOut(frame), - }) => Some(frame.clone()), - _ => None, - }), - ) - .await - .expect("timed out waiting for realtime audio after echoed user-role message"); + let mirrored_request = realtime_server.wait_for_request(0, 1).await; + let mirrored_request_body = mirrored_request.body_json(); + eprintln!( + "[realtime test +{}ms] saw mirrored request type={:?} role={:?} text={:?} data={:?}", + start.elapsed().as_millis(), + mirrored_request_body["type"].as_str(), + mirrored_request_body["item"]["role"].as_str(), + mirrored_request_body["item"]["content"][0]["text"].as_str(), + mirrored_request_body["item"]["content"][0]["data"].as_str(), + ); + assert_eq!( + mirrored_request_body["type"].as_str(), + Some("conversation.item.create") + ); + assert_eq!( + mirrored_request_body["item"]["content"][0]["text"].as_str(), + Some("assistant says hi") + ); + + let audio_out = wait_for_event_match(&test.codex, |msg| match msg { + EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent { + payload: RealtimeEvent::AudioOut(frame), + }) => Some(frame.clone()), + _ => None, + }) + .await; + eprintln!( + "[realtime test +{}ms] saw audio out data={} sample_rate={} num_channels={}", + start.elapsed().as_millis(), + audio_out.data, + audio_out.sample_rate, + audio_out.num_channels + ); assert_eq!(audio_out.data, "AQID"); let completion = completions @@ -828,6 +856,10 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au completion .await .expect("delegated turn request did not complete"); + eprintln!( + "[realtime test +{}ms] delegated completion resolved", + start.elapsed().as_millis() + ); wait_for_event(&test.codex, |event| { matches!(event, EventMsg::TurnComplete(_)) }) diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index 166f0ceac6..c304cc25fa 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -139,6 +139,7 @@ async fn remote_models_long_model_slug_is_sent_with_high_reasoning() -> Result<( }, ]; remote_model.supports_reasoning_summaries = true; + remote_model.default_reasoning_summary = ReasoningSummary::Detailed; mount_models_once( &server, ModelsResponse { @@ -175,7 +176,7 @@ async fn remote_models_long_model_slug_is_sent_with_high_reasoning() -> Result<( sandbox_policy: config.permissions.sandbox_policy.get().clone(), model: requested_model.to_string(), effort: None, - summary: config.model_reasoning_summary, + summary: None, collaboration_mode: None, personality: None, }) @@ -189,8 +190,13 @@ async fn remote_models_long_model_slug_is_sent_with_high_reasoning() -> Result<( .get("reasoning") .and_then(|reasoning| reasoning.get("effort")) .and_then(|value| value.as_str()); + let reasoning_summary = body + .get("reasoning") + .and_then(|reasoning| reasoning.get("summary")) + .and_then(|value| value.as_str()); assert_eq!(body["model"].as_str(), Some(requested_model)); assert_eq!(reasoning_effort, Some("high")); + assert_eq!(reasoning_summary, Some("detailed")); Ok(()) } @@ -227,7 +233,11 @@ async fn namespaced_model_slug_uses_catalog_metadata_without_fallback_warning() sandbox_policy: config.permissions.sandbox_policy.get().clone(), model: requested_model.to_string(), effort: None, - summary: config.model_reasoning_summary, + summary: Some( + config + .model_reasoning_summary + .unwrap_or(ReasoningSummary::Auto), + ), collaboration_mode: None, personality: None, }) @@ -284,8 +294,10 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -379,7 +391,7 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: REMOTE_MODEL_SLUG.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: Some(ReasoningSummary::Auto), collaboration_mode: None, personality: None, }) @@ -520,8 +532,10 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { base_instructions: remote_base.to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -590,7 +604,7 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: model.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: Some(ReasoningSummary::Auto), collaboration_mode: None, personality: None, }) @@ -980,8 +994,10 @@ fn test_remote_model_with_policy( base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/request_permissions.rs b/codex-rs/core/tests/suite/request_permissions.rs index cda482079a..43ed8941d1 100644 --- a/codex-rs/core/tests/suite/request_permissions.rs +++ b/codex-rs/core/tests/suite/request_permissions.rs @@ -4,7 +4,6 @@ use anyhow::Result; use codex_core::config::Constrained; use codex_core::features::Feature; use codex_core::sandboxing::SandboxPermissions; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; @@ -111,7 +110,7 @@ async fn submit_turn( sandbox_policy, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/request_user_input.rs b/codex-rs/core/tests/suite/request_user_input.rs index 64f2e0b6db..1f20f5dfdf 100644 --- a/codex-rs/core/tests/suite/request_user_input.rs +++ b/codex-rs/core/tests/suite/request_user_input.rs @@ -5,7 +5,6 @@ use std::collections::HashMap; use codex_core::features::Feature; use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::ModeKind; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::Settings; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; @@ -138,7 +137,7 @@ async fn request_user_input_round_trip_for_mode(mode: ModeKind) -> anyhow::Resul sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: Some(CollaborationMode { mode, settings: Settings { @@ -254,7 +253,7 @@ where sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: Some(collaboration_mode), personality: None, }) diff --git a/codex-rs/core/tests/suite/resume_warning.rs b/codex-rs/core/tests/suite/resume_warning.rs index b0503d5929..4742b417ae 100644 --- a/codex-rs/core/tests/suite/resume_warning.rs +++ b/codex-rs/core/tests/suite/resume_warning.rs @@ -4,6 +4,7 @@ use codex_core::CodexAuth; use codex_core::NewThread; use codex_protocol::ThreadId; use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::ResumedHistory; @@ -27,6 +28,8 @@ fn resume_history( let turn_ctx = TurnContextItem { turn_id: Some(turn_id.clone()), cwd: config.cwd.clone(), + current_date: None, + timezone: None, approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, @@ -34,7 +37,9 @@ fn resume_history( personality: None, collaboration_mode: None, effort: config.model_reasoning_effort, - summary: config.model_reasoning_summary, + summary: config + .model_reasoning_summary + .unwrap_or(ReasoningSummary::Auto), user_instructions: None, developer_instructions: None, final_output_json_schema: None, diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index a123981444..d4b8ba80a4 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -605,7 +605,9 @@ async fn review_input_isolated_from_parent_history() { let env_text = input .iter() - .filter_map(|msg| msg["content"][0]["text"].as_str()) + .filter_map(|msg| msg.get("content").and_then(|content| content.as_array())) + .flat_map(|content| content.iter()) + .filter_map(|entry| entry.get("text").and_then(|text| text.as_str())) .find(|text| text.starts_with(ENVIRONMENT_CONTEXT_OPEN_TAG)) .expect("env text"); assert!( @@ -615,7 +617,9 @@ async fn review_input_isolated_from_parent_history() { let review_text = input .iter() - .filter_map(|msg| msg["content"][0]["text"].as_str()) + .filter_map(|msg| msg.get("content").and_then(|content| content.as_array())) + .flat_map(|content| content.iter()) + .filter_map(|entry| entry.get("text").and_then(|text| text.as_str())) .find(|text| *text == review_prompt) .expect("review prompt text"); assert_eq!( diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index df389c5ef7..dfbac85a09 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -104,6 +104,7 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -128,7 +129,7 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -246,6 +247,7 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -295,7 +297,7 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -398,8 +400,10 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -462,6 +466,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -492,7 +497,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re sandbox_policy: SandboxPolicy::new_read_only_policy(), model: text_only_model_slug.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -580,6 +585,7 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -604,7 +610,7 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -739,6 +745,7 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -763,7 +770,7 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -958,6 +965,7 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -982,7 +990,7 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/safety_check_downgrade.rs b/codex-rs/core/tests/suite/safety_check_downgrade.rs index 247cfdaedc..263e4b96dc 100644 --- a/codex-rs/core/tests/suite/safety_check_downgrade.rs +++ b/codex-rs/core/tests/suite/safety_check_downgrade.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::AskForApproval; @@ -49,7 +48,7 @@ async fn openai_model_header_mismatch_emits_warning_event_and_warning_item() -> sandbox_policy: SandboxPolicy::DangerFullAccess, model: REQUESTED_MODEL.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -146,7 +145,7 @@ async fn response_model_field_mismatch_emits_warning_when_header_matches_request sandbox_policy: SandboxPolicy::DangerFullAccess, model: REQUESTED_MODEL.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -230,7 +229,7 @@ async fn openai_model_header_mismatch_only_emits_one_warning_per_turn() -> Resul sandbox_policy: SandboxPolicy::DangerFullAccess, model: REQUESTED_MODEL.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -278,7 +277,7 @@ async fn openai_model_header_casing_only_mismatch_does_not_warn() -> Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: REQUESTED_MODEL.to_string(), effort: test.config.model_reasoning_effort, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index 1caab45b12..d852c9cfb3 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -133,6 +133,7 @@ fn rmcp_server_config(command: String) -> McpServerConfig { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, } } diff --git a/codex-rs/core/tests/suite/shell_snapshot.rs b/codex-rs/core/tests/suite/shell_snapshot.rs index 3be9d2b579..a6f8ada10f 100644 --- a/codex-rs/core/tests/suite/shell_snapshot.rs +++ b/codex-rs/core/tests/suite/shell_snapshot.rs @@ -1,6 +1,5 @@ use anyhow::Result; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ExecCommandBeginEvent; @@ -162,7 +161,7 @@ async fn run_snapshot_command_with_options( sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -248,7 +247,7 @@ async fn run_shell_command_snapshot_with_options( sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -317,7 +316,7 @@ async fn run_tool_turn_on_harness( sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -535,7 +534,7 @@ async fn shell_command_snapshot_still_intercepts_apply_patch() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/skill_approval.rs b/codex-rs/core/tests/suite/skill_approval.rs index 8abd8f5924..a51c69bcfa 100644 --- a/codex-rs/core/tests/suite/skill_approval.rs +++ b/codex-rs/core/tests/suite/skill_approval.rs @@ -2,13 +2,15 @@ #![cfg(unix)] use anyhow::Result; +use codex_core::config::Config; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::FileSystemPermissions; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ExecApprovalRequestEvent; use codex_protocol::protocol::Op; +use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; use core_test_support::responses::mount_function_call_agent_response; @@ -56,7 +58,7 @@ async fn submit_turn_with_policies( sandbox_policy, model: test.session_configured.model.clone(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -64,18 +66,25 @@ async fn submit_turn_with_policies( Ok(()) } -async fn wait_for_turn_complete_without_skill_approval(test: &TestCodex) { - wait_for_event_match(test.codex.as_ref(), |event| match event { - EventMsg::SkillRequestApproval(request) => { - panic!("unexpected skill approval request: {request:?}"); - } - EventMsg::TurnComplete(_) => Some(()), - _ => None, - }) - .await; +fn write_skill_with_shell_script(home: &Path, name: &str, script_name: &str) -> Result { + write_skill_with_shell_script_contents( + home, + name, + script_name, + r#"#!/bin/sh +echo 'zsh-fork-stdout' +echo 'zsh-fork-stderr' >&2 +"#, + ) } -fn write_skill_with_shell_script(home: &Path, name: &str, script_name: &str) -> Result { +#[cfg(unix)] +fn write_skill_with_shell_script_contents( + home: &Path, + name: &str, + script_name: &str, + script_contents: &str, +) -> Result { use std::os::unix::fs::PermissionsExt; let skill_dir = home.join("skills").join(name); @@ -93,13 +102,7 @@ description: {name} skill )?; let script_path = scripts_dir.join(script_name); - fs::write( - &script_path, - r#"#!/bin/sh -echo 'zsh-fork-stdout' -echo 'zsh-fork-stderr' >&2 -"#, - )?; + fs::write(&script_path, script_contents)?; let mut permissions = fs::metadata(&script_path)?.permissions(); permissions.set_mode(0o755); fs::set_permissions(&script_path, permissions)?; @@ -140,34 +143,134 @@ fn supports_exec_wrapper_intercept(zsh_path: &Path) -> bool { } } -#[cfg(unix)] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn shell_zsh_fork_prompts_for_skill_script_execution() -> Result<()> { - use codex_config::Constrained; - use codex_protocol::protocol::ReviewDecision; +#[derive(Clone)] +struct ZshForkRuntime { + zsh_path: PathBuf, + main_execve_wrapper_exe: PathBuf, +} - skip_if_no_network!(Ok(())); +impl ZshForkRuntime { + fn apply_to_config( + &self, + config: &mut Config, + approval_policy: AskForApproval, + sandbox_policy: SandboxPolicy, + ) { + use codex_config::Constrained; + config.features.enable(Feature::ShellTool); + config.features.enable(Feature::ShellZshFork); + config.zsh_path = Some(self.zsh_path.clone()); + config.main_execve_wrapper_exe = Some(self.main_execve_wrapper_exe.clone()); + config.permissions.allow_login_shell = false; + config.permissions.approval_policy = Constrained::allow_any(approval_policy); + config.permissions.sandbox_policy = Constrained::allow_any(sandbox_policy); + } +} + +fn restrictive_workspace_write_policy() -> SandboxPolicy { + SandboxPolicy::WorkspaceWrite { + writable_roots: Vec::new(), + read_only_access: Default::default(), + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + } +} + +fn zsh_fork_runtime(test_name: &str) -> Result> { let Some(zsh_path) = find_test_zsh_path()? else { - return Ok(()); + return Ok(None); }; if !supports_exec_wrapper_intercept(&zsh_path) { eprintln!( - "skipping zsh-fork skill test: zsh does not support EXEC_WRAPPER intercepts ({})", + "skipping {test_name}: zsh does not support EXEC_WRAPPER intercepts ({})", zsh_path.display() ); - return Ok(()); + return Ok(None); } let Ok(main_execve_wrapper_exe) = codex_utils_cargo_bin::cargo_bin("codex-execve-wrapper") else { - eprintln!("skipping zsh-fork skill test: unable to resolve `codex-execve-wrapper` binary"); + eprintln!("skipping {test_name}: unable to resolve `codex-execve-wrapper` binary"); + return Ok(None); + }; + + Ok(Some(ZshForkRuntime { + zsh_path, + main_execve_wrapper_exe, + })) +} + +async fn build_zsh_fork_test( + server: &wiremock::MockServer, + runtime: ZshForkRuntime, + approval_policy: AskForApproval, + sandbox_policy: SandboxPolicy, + pre_build_hook: F, +) -> Result +where + F: FnOnce(&Path) + Send + 'static, +{ + let mut builder = test_codex() + .with_pre_build_hook(pre_build_hook) + .with_config(move |config| { + runtime.apply_to_config(config, approval_policy, sandbox_policy); + }); + builder.build(server).await +} + +fn skill_script_command(test: &TestCodex, script_name: &str) -> Result<(String, String)> { + let script_path = fs::canonicalize( + test.codex_home_path() + .join("skills/mbolin-test-skill/scripts") + .join(script_name), + )?; + let script_path_str = script_path.to_string_lossy().into_owned(); + let command = shlex::try_join([script_path_str.as_str()])?; + Ok((script_path_str, command)) +} + +async fn wait_for_exec_approval_request(test: &TestCodex) -> Option { + wait_for_event_match(test.codex.as_ref(), |event| match event { + EventMsg::ExecApprovalRequest(request) => Some(Some(request.clone())), + EventMsg::TurnComplete(_) => Some(None), + _ => None, + }) + .await +} + +async fn wait_for_turn_complete(test: &TestCodex) { + wait_for_event(test.codex.as_ref(), |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; +} + +fn output_shows_sandbox_denial(output: &str) -> bool { + output.contains("Permission denied") + || output.contains("Operation not permitted") + || output.contains("Read-only file system") +} + +/// Focus on the approval payload: the skill should prompt before execution and +/// only advertise the permissions declared in its metadata. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shell_zsh_fork_prompts_for_skill_script_execution() -> Result<()> { + skip_if_no_network!(Ok(())); + + let Some(runtime) = zsh_fork_runtime("zsh-fork skill prompt test")? else { return Ok(()); }; let server = start_mock_server().await; let tool_call_id = "zsh-fork-skill-call"; - let mut builder = test_codex() - .with_pre_build_hook(|home| { + let test = build_zsh_fork_test( + &server, + runtime, + AskForApproval::OnRequest, + SandboxPolicy::new_workspace_write_policy(), + |home| { write_skill_with_shell_script(home, "mbolin-test-skill", "hello-mbolin.sh").unwrap(); write_skill_metadata( home, @@ -182,25 +285,11 @@ permissions: "#, ) .unwrap(); - }) - .with_config(move |config| { - config.features.enable(Feature::ShellTool); - config.features.enable(Feature::ShellZshFork); - config.zsh_path = Some(zsh_path.clone()); - config.main_execve_wrapper_exe = Some(main_execve_wrapper_exe); - config.permissions.allow_login_shell = false; - config.permissions.approval_policy = Constrained::allow_any(AskForApproval::OnRequest); - config.permissions.sandbox_policy = - Constrained::allow_any(SandboxPolicy::new_workspace_write_policy()); - }); - let test = builder.build(&server).await?; + }, + ) + .await?; - let script_path = fs::canonicalize( - test.codex_home_path() - .join("skills/mbolin-test-skill/scripts/hello-mbolin.sh"), - )?; - let script_path_str = script_path.to_string_lossy().into_owned(); - let command = shlex::try_join([script_path_str.as_str()])?; + let (script_path_str, command) = skill_script_command(&test, "hello-mbolin.sh")?; let arguments = shell_command_arguments(&command)?; let mocks = mount_function_call_agent_response(&server, tool_call_id, &arguments, "shell_command") @@ -214,12 +303,7 @@ permissions: ) .await?; - let maybe_approval = wait_for_event_match(test.codex.as_ref(), |event| match event { - EventMsg::ExecApprovalRequest(request) => Some(Some(request.clone())), - EventMsg::TurnComplete(_) => Some(None), - _ => None, - }) - .await; + let maybe_approval = wait_for_exec_approval_request(&test).await; let approval = match maybe_approval { Some(approval) => approval, None => { @@ -234,6 +318,14 @@ permissions: }; assert_eq!(approval.call_id, tool_call_id); assert_eq!(approval.command, vec![script_path_str.clone()]); + assert_eq!( + approval.available_decisions, + Some(vec![ + ReviewDecision::Approved, + ReviewDecision::ApprovedForSession, + ReviewDecision::Abort, + ]) + ); assert_eq!( approval.additional_permissions, Some(PermissionProfile { @@ -249,14 +341,11 @@ permissions: .submit(Op::ExecApproval { id: approval.effective_approval_id(), turn_id: None, - decision: ReviewDecision::Approved, + decision: ReviewDecision::Denied, }) .await?; - wait_for_event(test.codex.as_ref(), |event| { - matches!(event, EventMsg::TurnComplete(_)) - }) - .await; + wait_for_turn_complete(&test).await; let call_output = mocks .completion @@ -264,69 +353,357 @@ permissions: .function_call_output(tool_call_id); let output = call_output["output"].as_str().unwrap_or_default(); assert!( - output.contains("zsh-fork-stdout"), - "expected stdout marker in function_call_output: {output:?}" - ); - assert!( - output.contains("zsh-fork-stderr"), - "expected stderr marker in function_call_output: {output:?}" + output.contains("Execution denied: User denied execution"), + "expected rejection marker in function_call_output: {output:?}" ); Ok(()) } +/// Look for `additional_permissions == None`, then verify that both the first +/// run and the cached session-approval rerun stay inside the turn sandbox. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shell_zsh_fork_skill_without_permissions_inherits_turn_sandbox() -> Result<()> { + skip_if_no_network!(Ok(())); + + let Some(runtime) = zsh_fork_runtime("zsh-fork inherited skill sandbox test")? else { + return Ok(()); + }; + + let outside_dir = tempfile::tempdir_in(std::env::current_dir()?)?; + let outside_path = outside_dir + .path() + .join("zsh-fork-skill-inherited-sandbox.txt"); + let outside_path_quoted = shlex::try_join([outside_path.to_string_lossy().as_ref()])?; + let script_contents = format!( + "#!/bin/sh\nprintf '%s' forbidden > {outside_path_quoted}\ncat {outside_path_quoted}\n" + ); + let outside_path_for_hook = outside_path.clone(); + let script_contents_for_hook = script_contents.clone(); + let workspace_write_policy = restrictive_workspace_write_policy(); + + let server = start_mock_server().await; + let test = build_zsh_fork_test( + &server, + runtime, + AskForApproval::OnRequest, + workspace_write_policy.clone(), + move |home| { + let _ = fs::remove_file(&outside_path_for_hook); + write_skill_with_shell_script_contents( + home, + "mbolin-test-skill", + "sandboxed.sh", + &script_contents_for_hook, + ) + .unwrap(); + }, + ) + .await?; + + let (script_path_str, command) = skill_script_command(&test, "sandboxed.sh")?; + + let first_call_id = "zsh-fork-skill-permissions-1"; + let first_arguments = shell_command_arguments(&command)?; + let first_mocks = mount_function_call_agent_response( + &server, + first_call_id, + &first_arguments, + "shell_command", + ) + .await; + + submit_turn_with_policies( + &test, + "use $mbolin-test-skill", + AskForApproval::OnRequest, + workspace_write_policy.clone(), + ) + .await?; + + let maybe_approval = wait_for_exec_approval_request(&test).await; + let approval = match maybe_approval { + Some(approval) => approval, + None => panic!("expected exec approval request before completion"), + }; + assert_eq!(approval.call_id, first_call_id); + assert_eq!(approval.command, vec![script_path_str.clone()]); + assert_eq!(approval.additional_permissions, None); + + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::ApprovedForSession, + }) + .await?; + + wait_for_turn_complete(&test).await; + + let first_output = first_mocks + .completion + .single_request() + .function_call_output(first_call_id)["output"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!( + output_shows_sandbox_denial(&first_output) || !first_output.contains("forbidden"), + "expected inherited turn sandbox denial on first run, got output: {first_output:?}" + ); + assert!( + !outside_path.exists(), + "first run should not write outside the turn sandbox" + ); + + let second_call_id = "zsh-fork-skill-permissions-2"; + let second_arguments = shell_command_arguments(&command)?; + let second_mocks = mount_function_call_agent_response( + &server, + second_call_id, + &second_arguments, + "shell_command", + ) + .await; + + submit_turn_with_policies( + &test, + "use $mbolin-test-skill", + AskForApproval::OnRequest, + workspace_write_policy, + ) + .await?; + + let cached_approval = wait_for_exec_approval_request(&test).await; + assert!( + cached_approval.is_none(), + "expected second run to reuse the cached session approval" + ); + + let second_output = second_mocks + .completion + .single_request() + .function_call_output(second_call_id)["output"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!( + output_shows_sandbox_denial(&second_output) || !second_output.contains("forbidden"), + "expected cached skill approval to retain inherited turn sandboxing, got output: {second_output:?}" + ); + assert!( + !outside_path.exists(), + "cached session approval should not widen a permissionless skill to full access" + ); + + Ok(()) +} + +/// The validation to focus on is: writes to the skill-approved folder succeed, +/// and writes to an unrelated folder fail, both before and after cached approval. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shell_zsh_fork_skill_session_approval_enforces_skill_permissions() -> Result<()> { + skip_if_no_network!(Ok(())); + + let Some(runtime) = zsh_fork_runtime("zsh-fork explicit skill sandbox test")? else { + return Ok(()); + }; + + let outside_dir = tempfile::tempdir_in(std::env::current_dir()?)?; + let allowed_dir = outside_dir.path().join("allowed-output"); + let blocked_dir = outside_dir.path().join("blocked-output"); + fs::create_dir_all(&allowed_dir)?; + fs::create_dir_all(&blocked_dir)?; + + let allowed_path = allowed_dir.join("allowed.txt"); + let blocked_path = blocked_dir.join("blocked.txt"); + let allowed_path_quoted = shlex::try_join([allowed_path.to_string_lossy().as_ref()])?; + let blocked_path_quoted = shlex::try_join([blocked_path.to_string_lossy().as_ref()])?; + let script_contents = format!( + "#!/bin/sh\nprintf '%s' allowed > {allowed_path_quoted}\ncat {allowed_path_quoted}\nprintf '%s' forbidden > {blocked_path_quoted}\nif [ -f {blocked_path_quoted} ]; then echo blocked-created; fi\n" + ); + let allowed_dir_for_hook = allowed_dir.clone(); + let allowed_path_for_hook = allowed_path.clone(); + let blocked_path_for_hook = blocked_path.clone(); + let script_contents_for_hook = script_contents.clone(); + + let permissions_yaml = format!( + "permissions:\n file_system:\n write:\n - \"{}\"\n", + allowed_dir.display() + ); + + let workspace_write_policy = restrictive_workspace_write_policy(); + let server = start_mock_server().await; + let test = build_zsh_fork_test( + &server, + runtime, + AskForApproval::OnRequest, + workspace_write_policy.clone(), + move |home| { + let _ = fs::remove_file(&allowed_path_for_hook); + let _ = fs::remove_file(&blocked_path_for_hook); + fs::create_dir_all(&allowed_dir_for_hook).unwrap(); + fs::create_dir_all(blocked_path_for_hook.parent().unwrap()).unwrap(); + write_skill_with_shell_script_contents( + home, + "mbolin-test-skill", + "sandboxed.sh", + &script_contents_for_hook, + ) + .unwrap(); + write_skill_metadata(home, "mbolin-test-skill", &permissions_yaml).unwrap(); + }, + ) + .await?; + + let (script_path_str, command) = skill_script_command(&test, "sandboxed.sh")?; + + let first_call_id = "zsh-fork-skill-permissions-1"; + let first_arguments = shell_command_arguments(&command)?; + let first_mocks = mount_function_call_agent_response( + &server, + first_call_id, + &first_arguments, + "shell_command", + ) + .await; + + submit_turn_with_policies( + &test, + "use $mbolin-test-skill", + AskForApproval::OnRequest, + workspace_write_policy.clone(), + ) + .await?; + + let maybe_approval = wait_for_exec_approval_request(&test).await; + let approval = match maybe_approval { + Some(approval) => approval, + None => panic!("expected exec approval request before completion"), + }; + assert_eq!(approval.call_id, first_call_id); + assert_eq!(approval.command, vec![script_path_str.clone()]); + assert_eq!( + approval.additional_permissions, + Some(PermissionProfile { + file_system: Some(FileSystemPermissions { + read: None, + write: Some(vec![allowed_dir.clone()]), + }), + ..Default::default() + }) + ); + + test.codex + .submit(Op::ExecApproval { + id: approval.effective_approval_id(), + turn_id: None, + decision: ReviewDecision::ApprovedForSession, + }) + .await?; + + wait_for_turn_complete(&test).await; + + let first_output = first_mocks + .completion + .single_request() + .function_call_output(first_call_id)["output"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!( + first_output.contains("allowed"), + "expected skill sandbox to permit writes to the approved folder, got output: {first_output:?}" + ); + assert_eq!(fs::read_to_string(&allowed_path)?, "allowed"); + assert!( + !blocked_path.exists(), + "first run should not write outside the explicit skill sandbox" + ); + assert!( + !first_output.contains("blocked-created"), + "blocked path should not have been created: {first_output:?}" + ); + + let second_call_id = "zsh-fork-skill-permissions-2"; + let second_arguments = shell_command_arguments(&command)?; + let second_mocks = mount_function_call_agent_response( + &server, + second_call_id, + &second_arguments, + "shell_command", + ) + .await; + + let _ = fs::remove_file(&allowed_path); + let _ = fs::remove_file(&blocked_path); + + submit_turn_with_policies( + &test, + "use $mbolin-test-skill", + AskForApproval::OnRequest, + workspace_write_policy, + ) + .await?; + + let cached_approval = wait_for_exec_approval_request(&test).await; + assert!( + cached_approval.is_none(), + "expected second run to reuse the cached session approval" + ); + + let second_output = second_mocks + .completion + .single_request() + .function_call_output(second_call_id)["output"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!( + second_output.contains("allowed"), + "expected cached skill approval to retain the explicit skill sandbox, got output: {second_output:?}" + ); + assert_eq!(fs::read_to_string(&allowed_path)?, "allowed"); + assert!( + !blocked_path.exists(), + "cached session approval should not widen skill execution beyond the explicit skill sandbox" + ); + assert!( + !second_output.contains("blocked-created"), + "blocked path should not have been created after cached approval: {second_output:?}" + ); + + Ok(()) +} + +/// This stays narrow on purpose: the important check is that `WorkspaceWrite` +/// continues to deny writes outside the workspace even under `zsh-fork`. #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shell_zsh_fork_still_enforces_workspace_write_sandbox() -> Result<()> { - use codex_config::Constrained; - use codex_protocol::protocol::AskForApproval; - skip_if_no_network!(Ok(())); - let Some(zsh_path) = find_test_zsh_path()? else { - return Ok(()); - }; - if !supports_exec_wrapper_intercept(&zsh_path) { - eprintln!( - "skipping zsh-fork sandbox test: zsh does not support EXEC_WRAPPER intercepts ({})", - zsh_path.display() - ); - return Ok(()); - } - let Ok(main_execve_wrapper_exe) = codex_utils_cargo_bin::cargo_bin("codex-execve-wrapper") - else { - eprintln!( - "skipping zsh-fork sandbox test: unable to resolve `codex-execve-wrapper` binary" - ); + let Some(runtime) = zsh_fork_runtime("zsh-fork workspace sandbox test")? else { return Ok(()); }; let server = start_mock_server().await; let tool_call_id = "zsh-fork-workspace-write-deny"; let outside_path = "/tmp/codex-zsh-fork-workspace-write-deny.txt"; - let workspace_write_policy = SandboxPolicy::WorkspaceWrite { - writable_roots: Vec::new(), - read_only_access: Default::default(), - network_access: false, - exclude_tmpdir_env_var: true, - exclude_slash_tmp: true, - }; - let policy_for_config = workspace_write_policy.clone(); + let workspace_write_policy = restrictive_workspace_write_policy(); let _ = fs::remove_file(outside_path); - let mut builder = test_codex() - .with_pre_build_hook(move |_| { + let test = build_zsh_fork_test( + &server, + runtime, + AskForApproval::Never, + workspace_write_policy.clone(), + move |_| { let _ = fs::remove_file(outside_path); - }) - .with_config(move |config| { - config.features.enable(Feature::ShellTool); - config.features.enable(Feature::ShellZshFork); - config.zsh_path = Some(zsh_path.clone()); - config.main_execve_wrapper_exe = Some(main_execve_wrapper_exe); - config.permissions.allow_login_shell = false; - config.permissions.approval_policy = Constrained::allow_any(AskForApproval::Never); - config.permissions.sandbox_policy = Constrained::allow_any(policy_for_config); - }); - let test = builder.build(&server).await?; + }, + ) + .await?; let command = format!("touch {outside_path}"); let arguments = shell_command_arguments(&command)?; @@ -342,7 +719,7 @@ async fn shell_zsh_fork_still_enforces_workspace_write_sandbox() -> Result<()> { ) .await?; - wait_for_turn_complete_without_skill_approval(&test).await; + wait_for_turn_complete(&test).await; let call_output = mocks .completion @@ -350,9 +727,7 @@ async fn shell_zsh_fork_still_enforces_workspace_write_sandbox() -> Result<()> { .function_call_output(tool_call_id); let output = call_output["output"].as_str().unwrap_or_default(); assert!( - output.contains("Permission denied") - || output.contains("Operation not permitted") - || output.contains("Read-only file system"), + output_shows_sandbox_denial(output), "expected sandbox denial, got output: {output:?}" ); assert!( diff --git a/codex-rs/core/tests/suite/skills.rs b/codex-rs/core/tests/suite/skills.rs index f681ca1b64..2f786590b1 100644 --- a/codex-rs/core/tests/suite/skills.rs +++ b/codex-rs/core/tests/suite/skills.rs @@ -77,7 +77,7 @@ async fn user_turn_includes_skill_instructions() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: codex_protocol::config_types::ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_with_history_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_with_history_shapes.snap index c88ff33122..e15d55aab2 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_with_history_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_with_history_shapes.snap @@ -6,16 +6,18 @@ Scenario: Manual /compact with prior user history compacts existing history and ## Local Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:first manual turn -04:message/assistant:FIRST_REPLY -05:message/user: +01:message/user[2]: + [01] + [02] > +02:message/user:first manual turn +03:message/assistant:FIRST_REPLY +04:message/user: ## Local Post-Compaction History Layout 00:message/user:first manual turn 01:message/user:\nFIRST_MANUAL_SUMMARY 02:message/developer: -03:message/user: -04:message/user:> -05:message/user:second manual turn +03:message/user[2]: + [01] + [02] > +04:message/user:second manual turn diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_without_prev_user_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_without_prev_user_shapes.snap index 0c8f87d04c..ca07006eae 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_without_prev_user_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__manual_compact_without_prev_user_shapes.snap @@ -6,13 +6,15 @@ Scenario: Manual /compact with no prior user turn currently still issues a compa ## Local Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user: +01:message/user[2]: + [01] + [02] > +02:message/user: ## Local Post-Compaction History Layout 00:message/user:\nMANUAL_EMPTY_SUMMARY 01:message/developer: -02:message/user: -03:message/user:> -04:message/user:AFTER_MANUAL_EMPTY_COMPACT +02:message/user[2]: + [01] + [02] > +03:message/user:AFTER_MANUAL_EMPTY_COMPACT diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__mid_turn_compaction_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__mid_turn_compaction_shapes.snap index c13f78aff9..f59fdf4b96 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact__mid_turn_compaction_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__mid_turn_compaction_shapes.snap @@ -1,22 +1,23 @@ --- source: core/tests/suite/compact.rs -assertion_line: 2646 expression: "format_labeled_requests_snapshot(\"True mid-turn continuation compaction after tool output: compact request includes tool artifacts, and the continuation request includes the summary in the same turn.\",\n&[(\"Local Compaction Request\", &auto_compact_mock.single_request()),\n(\"Local Post-Compaction History Layout\",\n&post_auto_compact_mock.single_request()),])" --- Scenario: True mid-turn continuation compaction after tool output: compact request includes tool artifacts, and the continuation request includes the summary in the same turn. ## Local Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:function call limit push -04:function_call/test_tool -05:function_call_output:unsupported call: test_tool -06:message/user: +01:message/user[2]: + [01] + [02] > +02:message/user:function call limit push +03:function_call/test_tool +04:function_call_output:unsupported call: test_tool +05:message/user: ## Local Post-Compaction History Layout 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:function call limit push -04:message/user:\nAUTO_SUMMARY +01:message/user[2]: + [01] + [02] > +02:message/user:function call limit push +03:message/user:\nAUTO_SUMMARY diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_sampling_model_switch_compaction_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_sampling_model_switch_compaction_shapes.snap index 552c117c98..7f61d7ed5e 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_sampling_model_switch_compaction_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_sampling_model_switch_compaction_shapes.snap @@ -1,30 +1,35 @@ --- source: core/tests/suite/compact.rs +assertion_line: 1791 expression: "format_labeled_requests_snapshot(\"Pre-sampling compaction on model switch to a smaller context window: current behavior compacts using prior-turn history only (incoming user message excluded), and the follow-up request carries compacted history plus the new user message.\",\n&[(\"Initial Request (Previous Model)\", &requests[0]),\n(\"Pre-sampling Compaction Request\", &requests[1]),\n(\"Post-Compaction Follow-up Request (Next Model)\", &requests[2]),])" --- Scenario: Pre-sampling compaction on model switch to a smaller context window: current behavior compacts using prior-turn history only (incoming user message excluded), and the follow-up request carries compacted history plus the new user message. ## Initial Request (Previous Model) 00:message/developer: -01:message/user: -02:message/user:> -03:message/developer: -04:message/user:before switch +01:message/user[2]: + [01] + [02] > +02:message/developer: +03:message/user:before switch ## Pre-sampling Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/developer: -04:message/user:before switch -05:message/assistant:before switch -06:message/user: +01:message/user[2]: + [01] + [02] > +02:message/developer: +03:message/user:before switch +04:message/assistant:before switch +05:message/user: ## Post-Compaction Follow-up Request (Next Model) 00:message/user:before switch 01:message/user:\nPRE_SAMPLING_SUMMARY -02:message/developer:\nThe user was previously using a different model.... -03:message/developer: -04:message/user: -05:message/user:> -06:message/user:after switch +02:message/developer[2]: + [01] \nThe user was previously using a different model.... + [02] +03:message/user[2]: + [01] + [02] > +04:message/user:after switch diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_context_window_exceeded_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_context_window_exceeded_shapes.snap index da30b01bd7..0de8baeebe 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_context_window_exceeded_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_context_window_exceeded_shapes.snap @@ -6,8 +6,9 @@ Scenario: Pre-turn auto-compaction context-window failure: compaction request ex ## Local Compaction Request (Incoming User Excluded) 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE -04:message/assistant:FIRST_REPLY -05:message/user: +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE +03:message/assistant:FIRST_REPLY +04:message/user: diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_including_incoming_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_including_incoming_shapes.snap index ea4fea2f21..8712df5833 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_including_incoming_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_including_incoming_shapes.snap @@ -6,19 +6,25 @@ Scenario: Pre-turn auto-compaction with a context override emits the context dif ## Local Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE -04:message/assistant:FIRST_REPLY -05:message/user:USER_TWO -06:message/assistant:SECOND_REPLY -07:message/user: +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE +03:message/assistant:FIRST_REPLY +04:message/user:USER_TWO +05:message/assistant:SECOND_REPLY +06:message/user: ## Local Post-Compaction History Layout 00:message/user:USER_ONE 01:message/user:USER_TWO 02:message/user:\nPRE_TURN_SUMMARY 03:message/developer: -04:message/user: -05:message/user: -06:message/user: | | | USER_THREE +04:message/user[2]: + [01] + [02] +05:message/user[4]: + [01] + [02] + [03] + [04] USER_THREE diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_strips_incoming_model_switch_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_strips_incoming_model_switch_shapes.snap index 59d3785a43..46d76bb100 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_strips_incoming_model_switch_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact__pre_turn_compaction_strips_incoming_model_switch_shapes.snap @@ -1,31 +1,36 @@ --- source: core/tests/suite/compact.rs +assertion_line: 3188 expression: "format_labeled_requests_snapshot(\"Pre-turn compaction during model switch (without pre-sampling model-switch compaction): current behavior strips incoming from the compact request and restores it in the post-compaction follow-up request.\",\n&[(\"Initial Request (Previous Model)\", &requests[0]),\n(\"Local Compaction Request\", &requests[1]),\n(\"Local Post-Compaction History Layout\", &requests[2]),])" --- Scenario: Pre-turn compaction during model switch (without pre-sampling model-switch compaction): current behavior strips incoming from the compact request and restores it in the post-compaction follow-up request. ## Initial Request (Previous Model) 00:message/developer: -01:message/user: -02:message/user:> -03:message/developer: -04:message/user:BEFORE_SWITCH_USER +01:message/user[2]: + [01] + [02] > +02:message/developer: +03:message/user:BEFORE_SWITCH_USER ## Local Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/developer: -04:message/user:BEFORE_SWITCH_USER -05:message/assistant:BEFORE_SWITCH_REPLY -06:message/user: +01:message/user[2]: + [01] + [02] > +02:message/developer: +03:message/user:BEFORE_SWITCH_USER +04:message/assistant:BEFORE_SWITCH_REPLY +05:message/user: ## Local Post-Compaction History Layout 00:message/user:BEFORE_SWITCH_USER 01:message/user:\nPRETURN_SWITCH_SUMMARY -02:message/developer:\nThe user was previously using a different model.... -03:message/developer: -04:message/developer: The user has requested a new communication st... -05:message/user: -06:message/user:> -07:message/user:AFTER_SWITCH_USER +02:message/developer[3]: + [01] \nThe user was previously using a different model.... + [02] + [03] The user has requested a new communication st... +03:message/user[2]: + [01] + [02] > +04:message/user:AFTER_SWITCH_USER diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_with_history_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_with_history_shapes.snap index 701e676761..83bec30fb6 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_with_history_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_with_history_shapes.snap @@ -6,14 +6,16 @@ Scenario: Remote manual /compact where remote compact output is compaction-only: ## Remote Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:hello remote compact -04:message/assistant:FIRST_REMOTE_REPLY +01:message/user[2]: + [01] + [02] > +02:message/user:hello remote compact +03:message/assistant:FIRST_REMOTE_REPLY ## Remote Post-Compaction History Layout 00:compaction:encrypted=true 01:message/developer: -02:message/user: -03:message/user:> -04:message/user:after compact +02:message/user[2]: + [01] + [02] > +03:message/user:after compact diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_without_prev_user_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_without_prev_user_shapes.snap index ce62806d99..6ec8149c07 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_without_prev_user_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_manual_compact_without_prev_user_shapes.snap @@ -6,11 +6,13 @@ Scenario: Remote manual /compact with no prior user turn still issues a compact ## Remote Compaction Request 00:message/developer: -01:message/user: -02:message/user:> +01:message/user[2]: + [01] + [02] > ## Remote Post-Compaction History Layout 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_multi_summary_reinjects_above_last_summary_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_multi_summary_reinjects_above_last_summary_shapes.snap index 7d5bcde902..ccc5a55811 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_multi_summary_reinjects_above_last_summary_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_multi_summary_reinjects_above_last_summary_shapes.snap @@ -1,6 +1,5 @@ --- source: core/tests/suite/compact_remote.rs -assertion_line: 1876 expression: "format_labeled_requests_snapshot(\"After a prior manual /compact produced an older remote compaction item, the next turn hits remote auto-compaction before the next sampling request. The compact request carries forward that earlier compaction item, and the next sampling request shows the latest compaction item with context reinjected before USER_TWO.\",\n&[(\"Remote Compaction Request\", &compact_request),\n(\"Second Turn Request (After Compaction)\", &second_turn_request),])" --- Scenario: After a prior manual /compact produced an older remote compaction item, the next turn hits remote auto-compaction before the next sampling request. The compact request carries forward that earlier compaction item, and the next sampling request shows the latest compaction item with context reinjected before USER_TWO. @@ -13,6 +12,7 @@ Scenario: After a prior manual /compact produced an older remote compaction item 00:message/user:USER_ONE 01:compaction:encrypted=true 02:message/developer: -03:message/user: -04:message/user:> -05:message/user:USER_TWO +03:message/user[2]: + [01] + [02] > +04:message/user:USER_TWO diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_shapes.snap index f5533b94a9..8e3e4235fa 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_shapes.snap @@ -6,15 +6,17 @@ Scenario: Remote mid-turn continuation compaction after tool output: compact req ## Remote Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE -04:function_call/test_tool -05:function_call_output:unsupported call: test_tool +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE +03:function_call/test_tool +04:function_call_output:unsupported call: test_tool ## Remote Post-Compaction History Layout 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE -04:compaction:encrypted=true +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE +03:compaction:encrypted=true diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_summary_only_reinjects_context_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_summary_only_reinjects_context_shapes.snap index 3f8c77e975..0f5886be13 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_summary_only_reinjects_context_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_mid_turn_compaction_summary_only_reinjects_context_shapes.snap @@ -1,19 +1,21 @@ --- source: core/tests/suite/compact_remote.rs -expression: "format_labeled_requests_snapshot(\"Remote mid-turn compaction where compact output has only a compaction item: continuation layout reinjects context before that compaction item.\",\n&[(\"Remote Compaction Request\", &compact_request),\n(\"Remote Post-Compaction History Layout\", &requests[1]),])" +expression: "format_labeled_requests_snapshot(\"Remote mid-turn compaction where compact output has only a compaction item: continuation layout reinjects context before that compaction item.\",\n&[(\"Remote Compaction Request\", &compact_request),\n(\"Remote Post-Compaction History Layout\", &post_compact_turn_request),])" --- Scenario: Remote mid-turn compaction where compact output has only a compaction item: continuation layout reinjects context before that compaction item. ## Remote Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE -04:function_call/test_tool -05:function_call_output:unsupported call: test_tool +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE +03:function_call/test_tool +04:function_call_output:unsupported call: test_tool ## Remote Post-Compaction History Layout 00:message/developer: -01:message/user: -02:message/user:> -03:compaction:encrypted=true +01:message/user[2]: + [01] + [02] > +02:compaction:encrypted=true diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_context_window_exceeded_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_context_window_exceeded_shapes.snap index e718bfda88..88e5c0bd23 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_context_window_exceeded_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_context_window_exceeded_shapes.snap @@ -6,7 +6,8 @@ Scenario: Remote pre-turn auto-compaction context-window failure: compaction req ## Remote Compaction Request (Incoming User Excluded) 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE -04:message/assistant:REMOTE_FIRST_REPLY +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE +03:message/assistant:REMOTE_FIRST_REPLY diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_failure_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_failure_shapes.snap index f8aa74e9a1..224f6dbba7 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_failure_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_failure_shapes.snap @@ -6,7 +6,8 @@ Scenario: Remote pre-turn auto-compaction parse failure: compaction request excl ## Remote Compaction Request (Incoming User Excluded) 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:turn that exceeds token threshold -04:message/assistant:initial turn complete +01:message/user[2]: + [01] + [02] > +02:message/user:turn that exceeds token threshold +03:message/assistant:initial turn complete diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_including_incoming_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_including_incoming_shapes.snap index 2236984a67..5a6f270d3d 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_including_incoming_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_including_incoming_shapes.snap @@ -6,18 +6,20 @@ Scenario: Remote pre-turn auto-compaction with a context override emits the cont ## Remote Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:USER_ONE -04:message/assistant:REMOTE_FIRST_REPLY -05:message/user:USER_TWO -06:message/assistant:REMOTE_SECOND_REPLY +01:message/user[2]: + [01] + [02] > +02:message/user:USER_ONE +03:message/assistant:REMOTE_FIRST_REPLY +04:message/user:USER_TWO +05:message/assistant:REMOTE_SECOND_REPLY ## Remote Post-Compaction History Layout 00:message/user:USER_ONE 01:message/user:USER_TWO 02:compaction:encrypted=true 03:message/developer: -04:message/user: -05:message/user: -06:message/user:USER_THREE +04:message/user[2]: + [01] + [02] +05:message/user:USER_THREE diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_strips_incoming_model_switch_shapes.snap b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_strips_incoming_model_switch_shapes.snap index 15beec5f27..ebab84f4e0 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_strips_incoming_model_switch_shapes.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__compact_remote__remote_pre_turn_compaction_strips_incoming_model_switch_shapes.snap @@ -1,28 +1,33 @@ --- source: core/tests/suite/compact_remote.rs +assertion_line: 1514 expression: "format_labeled_requests_snapshot(\"Remote pre-turn compaction during model switch currently excludes incoming user input, strips incoming from the compact request payload, and restores it in the post-compaction follow-up request.\",\n&[(\"Initial Request (Previous Model)\", &initial_turn_request),\n(\"Remote Compaction Request\", &compact_request),\n(\"Remote Post-Compaction History Layout\", &post_compact_turn_request),])" --- Scenario: Remote pre-turn compaction during model switch currently excludes incoming user input, strips incoming from the compact request payload, and restores it in the post-compaction follow-up request. ## Initial Request (Previous Model) 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:BEFORE_SWITCH_USER +01:message/user[2]: + [01] + [02] > +02:message/user:BEFORE_SWITCH_USER ## Remote Compaction Request 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:BEFORE_SWITCH_USER -04:message/assistant:BEFORE_SWITCH_REPLY +01:message/user[2]: + [01] + [02] > +02:message/user:BEFORE_SWITCH_USER +03:message/assistant:BEFORE_SWITCH_REPLY ## Remote Post-Compaction History Layout 00:message/user:BEFORE_SWITCH_USER 01:compaction:encrypted=true -02:message/developer:\nThe user was previously using a different model.... -03:message/developer: -04:message/developer: The user has requested a new communication st... -05:message/user: -06:message/user:> -07:message/user:AFTER_SWITCH_USER +02:message/developer[3]: + [01] \nThe user was previously using a different model.... + [02] + [03] The user has requested a new communication st... +03:message/user[2]: + [01] + [02] > +04:message/user:AFTER_SWITCH_USER diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_cwd_change_does_not_refresh_agents.snap b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_cwd_change_does_not_refresh_agents.snap index 9b6291809a..65dffc556c 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_cwd_change_does_not_refresh_agents.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_cwd_change_does_not_refresh_agents.snap @@ -6,19 +6,21 @@ Scenario: Second turn changes cwd to a directory with different AGENTS.md; curre ## First Request (agents_one) 00:message/developer: -01:message/user: -02:message/user:> +01:message/user[2]: + [01] + [02] > +02:message/developer: 03:message/user:> -04:message/developer: -05:message/user:first turn in agents_one +04:message/user:first turn in agents_one ## Second Request (agents_two cwd) 00:message/developer: -01:message/user: -02:message/user:> +01:message/user[2]: + [01] + [02] > +02:message/developer: 03:message/user:> -04:message/developer: -05:message/user:first turn in agents_one -06:message/assistant:turn one complete -07:message/user:> -08:message/user:second turn in agents_two +04:message/user:first turn in agents_one +05:message/assistant:turn one complete +06:message/user:> +07:message/user:second turn in agents_two diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_environment_context_includes_one_subagent.snap b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_environment_context_includes_one_subagent.snap new file mode 100644 index 0000000000..3436943cd2 --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_environment_context_includes_one_subagent.snap @@ -0,0 +1,6 @@ +--- +source: core/tests/suite/model_visible_layout.rs +assertion_line: 476 +expression: "format_environment_context_subagents_snapshot(&[\"- agent-1: Atlas\"])" +--- +00:message/user::subagents=1> diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_environment_context_includes_two_subagents.snap b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_environment_context_includes_two_subagents.snap new file mode 100644 index 0000000000..105c28515b --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_environment_context_includes_two_subagents.snap @@ -0,0 +1,6 @@ +--- +source: core/tests/suite/model_visible_layout.rs +assertion_line: 486 +expression: "format_environment_context_subagents_snapshot(&[\"- agent-1: Atlas\",\n\"- agent-2: Juniper\",])" +--- +00:message/user::subagents=2> diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_override_matches_rollout_model.snap b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_override_matches_rollout_model.snap index 06269eff76..045e97706b 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_override_matches_rollout_model.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_override_matches_rollout_model.snap @@ -1,21 +1,22 @@ --- source: core/tests/suite/model_visible_layout.rs -assertion_line: 435 expression: "format_labeled_requests_snapshot(\"First post-resume turn where pre-turn override sets model to rollout model; no model-switch update should appear.\",\n&[(\"Last Request Before Resume\", &initial_request),\n(\"First Request After Resume + Override\", &resumed_request),])" --- Scenario: First post-resume turn where pre-turn override sets model to rollout model; no model-switch update should appear. ## Last Request Before Resume 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:seed resume history +01:message/user[2]: + [01] + [02] > +02:message/user:seed resume history ## First Request After Resume + Override 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:seed resume history -04:message/assistant:recorded before resume -05:message/user: -06:message/user:first resumed turn after model override +01:message/user[2]: + [01] + [02] > +02:message/user:seed resume history +03:message/assistant:recorded before resume +04:message/user: +05:message/user:first resumed turn after model override diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_with_personality_change.snap b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_with_personality_change.snap index 54417c6de2..3918fafa65 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_with_personality_change.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_resume_with_personality_change.snap @@ -1,23 +1,25 @@ --- source: core/tests/suite/model_visible_layout.rs -assertion_line: 337 expression: "format_labeled_requests_snapshot(\"First post-resume turn where resumed config model differs from rollout and personality changes.\",\n&[(\"Last Request Before Resume\", &initial_request),\n(\"First Request After Resume\", &resumed_request),])" --- Scenario: First post-resume turn where resumed config model differs from rollout and personality changes. ## Last Request Before Resume 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:seed resume history +01:message/user[2]: + [01] + [02] > +02:message/user:seed resume history ## First Request After Resume 00:message/developer: -01:message/user: -02:message/user:> -03:message/user:seed resume history -04:message/assistant:recorded before resume -05:message/developer:\nThe user was previously using a different model. Please continue the conversatio... -06:message/user: -07:message/developer: -08:message/user:resume and change personality +01:message/user[2]: + [01] + [02] > +02:message/user:seed resume history +03:message/assistant:recorded before resume +04:message/developer[2]: + [01] \nThe user was previously using a different model. Please continue the conversatio... + [02] +05:message/user: +06:message/user:resume and change personality diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_turn_overrides.snap b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_turn_overrides.snap index 9ec0b2e7d0..2172d7399f 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_turn_overrides.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__model_visible_layout__model_visible_layout_turn_overrides.snap @@ -6,19 +6,22 @@ Scenario: Second turn changes cwd, approval policy, and personality while keepin ## First Request (Baseline) 00:message/developer: -01:message/user: -02:message/user:> -03:message/developer: -04:message/user:first turn +01:message/user[2]: + [01] + [02] > +02:message/developer: +03:message/user:first turn ## Second Request (Turn Overrides) 00:message/developer: -01:message/user: -02:message/user:> -03:message/developer: -04:message/user:first turn -05:message/assistant:turn one complete +01:message/user[2]: + [01] + [02] > +02:message/developer: +03:message/user:first turn +04:message/assistant:turn one complete +05:message/developer[2]: + [01] + [02] The user has requested a new communication style. Future messages should adhe... 06:message/user: -07:message/developer: -08:message/developer: The user has requested a new communication style. Future messages should adhe... -09:message/user:second turn with context updates +07:message/user:second turn with context updates diff --git a/codex-rs/core/tests/suite/subagent_notifications.rs b/codex-rs/core/tests/suite/subagent_notifications.rs index 422510f328..3dc463c4ab 100644 --- a/codex-rs/core/tests/suite/subagent_notifications.rs +++ b/codex-rs/core/tests/suite/subagent_notifications.rs @@ -20,6 +20,8 @@ use tokio::time::sleep; use wiremock::MockServer; const SPAWN_CALL_ID: &str = "spawn-call-1"; +const FORKED_SPAWN_AGENT_OUTPUT_MESSAGE: &str = "You are the newly spawned agent. The prior conversation history was forked from your parent agent. Treat the next user message as your new task, and use the forked history only as background context."; +const TURN_0_FORK_PROMPT: &str = "seed fork context"; const TURN_1_PROMPT: &str = "spawn a child and continue"; const TURN_2_NO_WAIT_PROMPT: &str = "follow up without wait"; const CHILD_PROMPT: &str = "child: do work"; @@ -194,3 +196,116 @@ async fn subagent_notification_is_included_without_wait() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn spawned_child_receives_forked_parent_context() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + + let seed_turn = mount_sse_once_match( + &server, + |req: &wiremock::Request| body_contains(req, TURN_0_FORK_PROMPT), + sse(vec![ + ev_response_created("resp-seed-1"), + ev_assistant_message("msg-seed-1", "seeded"), + ev_completed("resp-seed-1"), + ]), + ) + .await; + + let spawn_args = serde_json::to_string(&json!({ + "message": CHILD_PROMPT, + "fork_context": true, + }))?; + let spawn_turn = mount_sse_once_match( + &server, + |req: &wiremock::Request| body_contains(req, TURN_1_PROMPT), + sse(vec![ + ev_response_created("resp-turn1-1"), + ev_function_call(SPAWN_CALL_ID, "spawn_agent", &spawn_args), + ev_completed("resp-turn1-1"), + ]), + ) + .await; + + let _child_request_log = mount_sse_once_match( + &server, + |req: &wiremock::Request| body_contains(req, CHILD_PROMPT), + sse(vec![ + ev_response_created("resp-child-1"), + ev_assistant_message("msg-child-1", "child done"), + ev_completed("resp-child-1"), + ]), + ) + .await; + + let _turn1_followup = mount_sse_once_match( + &server, + |req: &wiremock::Request| body_contains(req, SPAWN_CALL_ID), + sse(vec![ + ev_response_created("resp-turn1-2"), + ev_assistant_message("msg-turn1-2", "parent done"), + ev_completed("resp-turn1-2"), + ]), + ) + .await; + + let mut builder = test_codex().with_config(|config| { + config.features.enable(Feature::Collab); + }); + let test = builder.build(&server).await?; + + test.submit_turn(TURN_0_FORK_PROMPT).await?; + let _ = seed_turn.single_request(); + + test.submit_turn(TURN_1_PROMPT).await?; + let _ = spawn_turn.single_request(); + + let deadline = Instant::now() + Duration::from_secs(2); + let child_request = loop { + if let Some(request) = server + .received_requests() + .await + .unwrap_or_default() + .into_iter() + .find(|request| { + body_contains(request, CHILD_PROMPT) + && body_contains(request, FORKED_SPAWN_AGENT_OUTPUT_MESSAGE) + }) + { + break request; + } + if Instant::now() >= deadline { + anyhow::bail!("timed out waiting for forked child request"); + } + sleep(Duration::from_millis(10)).await; + }; + assert!(body_contains(&child_request, TURN_0_FORK_PROMPT)); + assert!(body_contains(&child_request, "seeded")); + + let child_body = child_request + .body_json::() + .expect("forked child request body should be json"); + let function_call_output = child_body["input"] + .as_array() + .and_then(|items| { + items.iter().find(|item| { + item["type"].as_str() == Some("function_call_output") + && item["call_id"].as_str() == Some(SPAWN_CALL_ID) + }) + }) + .unwrap_or_else(|| panic!("expected forked child request to include spawn_agent output")); + let (content, success) = match &function_call_output["output"] { + serde_json::Value::String(text) => (Some(text.as_str()), None), + serde_json::Value::Object(output) => ( + output.get("content").and_then(serde_json::Value::as_str), + output.get("success").and_then(serde_json::Value::as_bool), + ), + _ => (None, None), + }; + assert_eq!(content, Some(FORKED_SPAWN_AGENT_OUTPUT_MESSAGE)); + assert_ne!(success, Some(false)); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/tool_harness.rs b/codex-rs/core/tests/suite/tool_harness.rs index 507c8eb068..13191892ce 100644 --- a/codex-rs/core/tests/suite/tool_harness.rs +++ b/codex-rs/core/tests/suite/tool_harness.rs @@ -4,7 +4,6 @@ use std::fs; use assert_matches::assert_matches; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::plan_tool::StepStatus; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; @@ -89,7 +88,7 @@ async fn shell_tool_executes_command_and_streams_output() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -158,7 +157,7 @@ async fn update_plan_tool_emits_plan_update_event() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -237,7 +236,7 @@ async fn update_plan_tool_rejects_malformed_payload() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -328,7 +327,7 @@ async fn apply_patch_tool_executes_and_emits_patch_events() -> anyhow::Result<() sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -427,7 +426,7 @@ async fn apply_patch_reports_parse_diagnostics() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/tool_parallelism.rs b/codex-rs/core/tests/suite/tool_parallelism.rs index 357dcff0fd..1678e58349 100644 --- a/codex-rs/core/tests/suite/tool_parallelism.rs +++ b/codex-rs/core/tests/suite/tool_parallelism.rs @@ -5,7 +5,6 @@ use std::fs; use std::time::Duration; use std::time::Instant; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -46,7 +45,7 @@ async fn run_turn(test: &TestCodex, prompt: &str) -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -361,7 +360,7 @@ async fn shell_tools_start_before_response_completed_when_stream_delayed() -> an sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 8ede6c621f..6ba6e53b1b 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -5,7 +5,6 @@ use anyhow::Context; use anyhow::Result; use codex_core::config::types::McpServerConfig; use codex_core::config::types::McpServerTransportConfig; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -371,6 +370,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -465,6 +465,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config @@ -488,7 +489,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -730,6 +731,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }, ); config diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 705e415620..55bc19e680 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -6,7 +6,6 @@ use std::sync::OnceLock; use anyhow::Context; use anyhow::Result; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ExecCommandSource; @@ -209,7 +208,7 @@ async fn unified_exec_intercepts_apply_patch_exec_command() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -338,7 +337,7 @@ async fn unified_exec_emits_exec_command_begin_event() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -416,7 +415,7 @@ async fn unified_exec_resolves_relative_workdir() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -497,7 +496,7 @@ async fn unified_exec_respects_workdir_override() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -590,7 +589,7 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -665,7 +664,7 @@ async fn unified_exec_emits_output_delta_for_exec_command() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -741,7 +740,7 @@ async fn unified_exec_full_lifecycle_with_background_end_event() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -871,7 +870,7 @@ async fn unified_exec_emits_terminal_interaction_for_write_stdin() -> Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1008,7 +1007,7 @@ async fn unified_exec_terminal_interaction_captures_delayed_output() -> Result<( sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1168,7 +1167,7 @@ async fn unified_exec_emits_one_begin_and_one_end_event() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1266,7 +1265,7 @@ async fn exec_command_reports_chunk_and_exit_metadata() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1384,7 +1383,7 @@ async fn unified_exec_defaults_to_pipe() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1474,7 +1473,7 @@ async fn unified_exec_can_enable_tty() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1555,7 +1554,7 @@ async fn unified_exec_respects_early_exit_notifications() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1686,7 +1685,7 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1854,7 +1853,7 @@ async fn unified_exec_emits_end_event_when_session_dies_via_stdin() -> Result<() sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -1931,7 +1930,7 @@ async fn unified_exec_keeps_long_running_session_after_turn_end() -> Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2019,7 +2018,7 @@ async fn unified_exec_interrupt_terminates_long_running_session() -> Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2116,7 +2115,7 @@ async fn unified_exec_reuses_session_via_stdin() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2251,7 +2250,7 @@ PY sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2365,7 +2364,7 @@ async fn unified_exec_timeout_and_followup_poll() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2461,7 +2460,7 @@ PY sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2543,7 +2542,7 @@ async fn unified_exec_runs_under_sandbox() -> Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2647,7 +2646,7 @@ async fn unified_exec_python_prompt_under_seatbelt() -> Result<()> { sandbox_policy: SandboxPolicy::new_read_only_policy(), model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2742,7 +2741,7 @@ async fn unified_exec_runs_on_all_platforms() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -2877,7 +2876,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/user_notification.rs b/codex-rs/core/tests/suite/user_notification.rs index 5ef939776b..a2303c0ecf 100644 --- a/codex-rs/core/tests/suite/user_notification.rs +++ b/codex-rs/core/tests/suite/user_notification.rs @@ -23,10 +23,6 @@ use responses::start_mock_server; use std::time::Duration; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "flaky on ubuntu-24.04-arm - aarch64-unknown-linux-gnu"] -// The notify script gets far enough to create (and therefore surface) the file, -// but hasn’t flushed the JSON yet. Reading an empty file produces EOF while parsing -// a value at line 1 column 0. May be caused by a slow runner. async fn summarize_context_three_requests_and_instructions() -> anyhow::Result<()> { skip_if_no_network!(Ok(())); @@ -43,7 +39,10 @@ async fn summarize_context_three_requests_and_instructions() -> anyhow::Result<( ¬ify_script, r#"#!/bin/bash set -e -echo -n "${@: -1}" > $(dirname "${0}")/notify.txt"#, +payload_path="$(dirname "${0}")/notify.txt" +tmp_path="${payload_path}.tmp" +echo -n "${@: -1}" > "${tmp_path}" +mv "${tmp_path}" "${payload_path}""#, )?; std::fs::set_permissions(¬ify_script, std::fs::Permissions::from_mode(0o755))?; diff --git a/codex-rs/core/tests/suite/user_shell_cmd.rs b/codex-rs/core/tests/suite/user_shell_cmd.rs index c38b86e443..766d79abb8 100644 --- a/codex-rs/core/tests/suite/user_shell_cmd.rs +++ b/codex-rs/core/tests/suite/user_shell_cmd.rs @@ -1,6 +1,5 @@ use anyhow::Context; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ExecCommandEndEvent; @@ -178,7 +177,7 @@ async fn user_shell_command_does_not_replace_active_turn() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: fixture.session_configured.model.clone(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index 36ccf1a25e..4521ed45f8 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -112,7 +112,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -214,7 +214,7 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -344,7 +344,7 @@ console.log(out.output?.body?.text ?? ""); sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -358,40 +358,26 @@ console.log(out.output?.body?.text ?? ""); .await; let req = mock.single_request(); - let (js_repl_output, js_repl_success) = req - .custom_tool_call_output_content_and_success(call_id) - .expect("custom tool output present"); - let js_repl_output = js_repl_output.expect("custom tool output text present"); - assert_ne!( - js_repl_success, - Some(false), - "js_repl call failed unexpectedly: {js_repl_output}" + let body = req.body_json(); + assert_eq!( + image_messages(&body).len(), + 0, + "js_repl view_image should not inject a pending input image message" ); - let body = req.body_json(); - let image_messages = image_messages(&body); - assert_eq!( - image_messages.len(), - 1, - "js_repl view_image should inject exactly one pending input image message" - ); - let image_message = image_messages - .into_iter() - .next() - .expect("pending input image message not included in request"); - let image_url = image_message - .get("content") + let custom_output = req.custom_tool_call_output(call_id); + let output_items = custom_output + .get("output") .and_then(Value::as_array) - .and_then(|content| { - content.iter().find_map(|span| { - if span.get("type").and_then(Value::as_str) == Some("input_image") { - span.get("image_url").and_then(Value::as_str) - } else { - None - } - }) + .expect("custom_tool_call_output should be a content item array"); + let image_url = output_items + .iter() + .find_map(|item| { + (item.get("type").and_then(Value::as_str) == Some("input_image")) + .then(|| item.get("image_url").and_then(Value::as_str)) + .flatten() }) - .expect("image_url present"); + .expect("image_url present in js_repl custom tool output"); assert!( image_url.starts_with("data:image/png;base64,"), "expected png data URL, got {image_url}" @@ -447,7 +433,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -522,7 +508,7 @@ async fn view_image_tool_placeholder_for_non_image_files() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -614,7 +600,7 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -675,8 +661,10 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an base_instructions: "base instructions".to_string(), model_messages: None, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, + availability_nux: None, apply_patch_tool_type: None, truncation_policy: TruncationPolicyConfig::bytes(10_000), supports_parallel_tool_calls: false, @@ -736,7 +724,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an sandbox_policy: SandboxPolicy::DangerFullAccess, model: model_slug.to_string(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) @@ -812,7 +800,7 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_model, effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/core/tests/suite/websocket_fallback.rs b/codex-rs/core/tests/suite/websocket_fallback.rs index 20c7b3b2f2..9ff5cbe001 100644 --- a/codex-rs/core/tests/suite/websocket_fallback.rs +++ b/codex-rs/core/tests/suite/websocket_fallback.rs @@ -1,6 +1,5 @@ use anyhow::Result; use codex_core::features::Feature; -use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; @@ -161,7 +160,7 @@ async fn websocket_fallback_hides_first_websocket_retry_stream_error() -> Result sandbox_policy: SandboxPolicy::DangerFullAccess, model: session_configured.model.clone(), effort: None, - summary: ReasoningSummary::Auto, + summary: None, collaboration_mode: None, personality: None, }) diff --git a/codex-rs/docs/codex_mcp_interface.md b/codex-rs/docs/codex_mcp_interface.md index 3a3abe9a9e..763f2602ad 100644 --- a/codex-rs/docs/codex_mcp_interface.md +++ b/codex-rs/docs/codex_mcp_interface.md @@ -90,20 +90,26 @@ directory (it returns the restored thread summary). Fetch the catalog of models available in the current Codex build with `model/list`. The request accepts optional pagination inputs: -- `pageSize` – number of models to return (defaults to a server-selected value) +- `limit` – number of models to return (defaults to a server-selected value) - `cursor` – opaque string from the previous response’s `nextCursor` Each response yields: -- `items` – ordered list of models. A model includes: +- `data` – ordered list of models. A model includes: - `id`, `model`, `displayName`, `description` - `supportedReasoningEfforts` – array of objects with: - - `reasoningEffort` – one of `minimal|low|medium|high` + - `reasoningEffort` – one of `none|minimal|low|medium|high|xhigh` - `description` – human-friendly label for the effort - `defaultReasoningEffort` – suggested effort for the UI + - `inputModalities` – accepted input types for the model - `supportsPersonality` – whether the model supports personality-specific instructions - `isDefault` – whether the model is recommended for most users - `upgrade` – optional recommended upgrade model id + - `upgradeInfo` – optional upgrade metadata object with: + - `model` – recommended upgrade model id + - `upgradeCopy` – optional display copy for the upgrade recommendation + - `modelLink` – optional link for the upgrade recommendation + - `migrationMarkdown` – optional markdown shown when presenting the upgrade - `nextCursor` – pass into the next request to continue paging (optional) ## Collaboration modes (experimental) diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index eac514a8f1..a3a29852b3 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -854,8 +854,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { | EventMsg::RealtimeConversationRealtime(_) | EventMsg::RealtimeConversationClosed(_) | EventMsg::DynamicToolCallRequest(_) - | EventMsg::DynamicToolCallResponse(_) - | EventMsg::SkillRequestApproval(_) => {} + | EventMsg::DynamicToolCallResponse(_) => {} } CodexStatus::Running } diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 1dca110aff..e8ab6f8af7 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -377,7 +377,6 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result let default_approval_policy = config.permissions.approval_policy.value(); let default_sandbox_policy = config.permissions.sandbox_policy.get(); let default_effort = config.model_reasoning_effort; - let default_summary = config.model_reasoning_summary; // When --yolo (dangerously_bypass_approvals_and_sandbox) is set, also skip the git repo check // since the user is explicitly running in an externally sandboxed environment. @@ -560,7 +559,7 @@ pub async fn run_main(cli: Cli, arg0_paths: Arg0DispatchPaths) -> anyhow::Result sandbox_policy: default_sandbox_policy.clone(), model: default_model, effort: default_effort, - summary: default_summary, + summary: None, final_output_json_schema: output_schema, collaboration_mode: None, personality: None, diff --git a/codex-rs/exec/tests/suite/resume.rs b/codex-rs/exec/tests/suite/resume.rs index 8cd8fffb40..a85183620e 100644 --- a/codex-rs/exec/tests/suite/resume.rs +++ b/codex-rs/exec/tests/suite/resume.rs @@ -321,6 +321,9 @@ fn exec_resume_last_respects_cwd_filter_and_all_flag() -> anyhow::Result<()> { let resumed_path_cwd = find_session_file_containing_marker(&sessions_dir, &marker_a2) .expect("no resumed session file containing marker_a2"); + // The `--all` resume above appends a new turn to `path_b` while running from `dir_a`, so the + // session's latest cwd now matches `dir_a`. A subsequent `resume --last` should therefore pick + // the newest matching session (`path_b`). assert_eq!( resumed_path_cwd, path_b, "resume --last should prefer sessions whose latest turn context matches the current cwd" diff --git a/codex-rs/hooks/src/registry.rs b/codex-rs/hooks/src/registry.rs index 6568f5374f..1648fa0462 100644 --- a/codex-rs/hooks/src/registry.rs +++ b/codex-rs/hooks/src/registry.rs @@ -104,6 +104,7 @@ mod tests { HookPayload { session_id: ThreadId::new(), cwd: PathBuf::from(CWD), + client: None, triggered_at: Utc .with_ymd_and_hms(2025, 1, 1, 0, 0, 0) .single() @@ -172,6 +173,7 @@ mod tests { HookPayload { session_id: ThreadId::new(), cwd: PathBuf::from(CWD), + client: None, triggered_at: Utc .with_ymd_and_hms(2025, 1, 1, 0, 0, 0) .single() diff --git a/codex-rs/hooks/src/types.rs b/codex-rs/hooks/src/types.rs index 5131580128..b7ef3c2141 100644 --- a/codex-rs/hooks/src/types.rs +++ b/codex-rs/hooks/src/types.rs @@ -65,6 +65,8 @@ impl Hook { pub struct HookPayload { pub session_id: ThreadId, pub cwd: PathBuf, + #[serde(skip_serializing_if = "Option::is_none")] + pub client: Option, #[serde(serialize_with = "serialize_triggered_at")] pub triggered_at: DateTime, pub hook_event: HookEvent, @@ -181,6 +183,7 @@ mod tests { let payload = HookPayload { session_id, cwd: PathBuf::from("tmp"), + client: None, triggered_at: Utc .with_ymd_and_hms(2025, 1, 1, 0, 0, 0) .single() @@ -218,6 +221,7 @@ mod tests { let payload = HookPayload { session_id, cwd: PathBuf::from("tmp"), + client: None, triggered_at: Utc .with_ymd_and_hms(2025, 1, 1, 0, 0, 0) .single() diff --git a/codex-rs/hooks/src/user_notification.rs b/codex-rs/hooks/src/user_notification.rs index 50472225e4..caeca59b50 100644 --- a/codex-rs/hooks/src/user_notification.rs +++ b/codex-rs/hooks/src/user_notification.rs @@ -1,4 +1,3 @@ -use std::path::Path; use std::process::Stdio; use std::sync::Arc; @@ -19,6 +18,8 @@ enum UserNotification { thread_id: String, turn_id: String, cwd: String, + #[serde(skip_serializing_if = "Option::is_none")] + client: Option, /// Messages that the user sent to the agent to initiate the turn. input_messages: Vec, @@ -28,13 +29,14 @@ enum UserNotification { }, } -pub fn legacy_notify_json(hook_event: &HookEvent, cwd: &Path) -> Result { - match hook_event { +pub fn legacy_notify_json(payload: &HookPayload) -> Result { + match &payload.hook_event { HookEvent::AfterAgent { event } => { serde_json::to_string(&UserNotification::AgentTurnComplete { thread_id: event.thread_id.to_string(), turn_id: event.turn_id.clone(), - cwd: cwd.display().to_string(), + cwd: payload.cwd.display().to_string(), + client: payload.client.clone(), input_messages: event.input_messages.clone(), last_assistant_message: event.last_assistant_message.clone(), }) @@ -56,7 +58,7 @@ pub fn notify_hook(argv: Vec) -> Hook { Some(command) => command, None => return HookResult::Success, }; - if let Ok(notify_payload) = legacy_notify_json(&payload.hook_event, &payload.cwd) { + if let Ok(notify_payload) = legacy_notify_json(payload) { command.arg(notify_payload); } @@ -91,6 +93,7 @@ mod tests { "thread-id": "b5f6c1c2-1111-2222-3333-444455556666", "turn-id": "12345", "cwd": "/Users/example/project", + "client": "codex-tui", "input-messages": ["Rename `foo` to `bar` and update the callsites."], "last-assistant-message": "Rename complete and verified `cargo build` succeeds.", }) @@ -102,6 +105,7 @@ mod tests { thread_id: "b5f6c1c2-1111-2222-3333-444455556666".to_string(), turn_id: "12345".to_string(), cwd: "/Users/example/project".to_string(), + client: Some("codex-tui".to_string()), input_messages: vec!["Rename `foo` to `bar` and update the callsites.".to_string()], last_assistant_message: Some( "Rename complete and verified `cargo build` succeeds.".to_string(), @@ -115,19 +119,27 @@ mod tests { #[test] fn legacy_notify_json_matches_historical_wire_shape() -> Result<()> { - let hook_event = HookEvent::AfterAgent { - event: crate::HookEventAfterAgent { - thread_id: ThreadId::from_string("b5f6c1c2-1111-2222-3333-444455556666") - .expect("valid thread id"), - turn_id: "12345".to_string(), - input_messages: vec!["Rename `foo` to `bar` and update the callsites.".to_string()], - last_assistant_message: Some( - "Rename complete and verified `cargo build` succeeds.".to_string(), - ), + let payload = HookPayload { + session_id: ThreadId::new(), + cwd: std::path::Path::new("/Users/example/project").to_path_buf(), + client: Some("codex-tui".to_string()), + triggered_at: chrono::Utc::now(), + hook_event: HookEvent::AfterAgent { + event: crate::HookEventAfterAgent { + thread_id: ThreadId::from_string("b5f6c1c2-1111-2222-3333-444455556666") + .expect("valid thread id"), + turn_id: "12345".to_string(), + input_messages: vec![ + "Rename `foo` to `bar` and update the callsites.".to_string(), + ], + last_assistant_message: Some( + "Rename complete and verified `cargo build` succeeds.".to_string(), + ), + }, }, }; - let serialized = legacy_notify_json(&hook_event, Path::new("/Users/example/project"))?; + let serialized = legacy_notify_json(&payload)?; let actual: Value = serde_json::from_str(&serialized)?; assert_eq!(actual, expected_notification_json()); diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 805ac0e6ea..e624fe2b1b 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -227,6 +227,7 @@ async fn run_codex_tool_session_inner( parsed_cmd, network_approval_context: _, additional_permissions: _, + available_decisions: _, } = ev; handle_exec_approval_request( command, @@ -364,7 +365,6 @@ async fn run_codex_tool_session_inner( | EventMsg::RequestUserInput(_) | EventMsg::DynamicToolCallRequest(_) | EventMsg::DynamicToolCallResponse(_) - | EventMsg::SkillRequestApproval(_) | EventMsg::ContextCompacted(_) | EventMsg::ModelReroute(_) | EventMsg::ThreadRolledBack(_) diff --git a/codex-rs/node-version.txt b/codex-rs/node-version.txt index 32f8c50de0..85e502778f 100644 --- a/codex-rs/node-version.txt +++ b/codex-rs/node-version.txt @@ -1 +1 @@ -24.13.1 +22.22.0 diff --git a/codex-rs/protocol/src/approvals.rs b/codex-rs/protocol/src/approvals.rs index f742dc6328..12bd65ec59 100644 --- a/codex-rs/protocol/src/approvals.rs +++ b/codex-rs/protocol/src/approvals.rs @@ -2,14 +2,30 @@ use std::collections::HashMap; use std::path::PathBuf; use crate::mcp::RequestId; +use crate::models::MacOsSeatbeltProfileExtensions; use crate::models::PermissionProfile; use crate::parse_command::ParsedCommand; use crate::protocol::FileChange; +use crate::protocol::ReviewDecision; +use crate::protocol::SandboxPolicy; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use ts_rs::TS; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Permissions { + pub sandbox_policy: SandboxPolicy, + pub macos_seatbelt_profile_extensions: Option, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EscalationPermissions { + PermissionProfile(PermissionProfile), + Permissions(Permissions), +} + /// Proposed execpolicy change to allow commands starting with this prefix. /// /// The `command` tokens form the prefix that would be added as an execpolicy @@ -107,6 +123,13 @@ pub struct ExecApprovalRequestEvent { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub additional_permissions: Option, + /// Ordered list of decisions the client may present for this prompt. + /// + /// When absent, clients should derive the legacy default set from the + /// other fields on this request. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub available_decisions: Option>, pub parsed_cmd: Vec, } @@ -116,6 +139,55 @@ impl ExecApprovalRequestEvent { .clone() .unwrap_or_else(|| self.call_id.clone()) } + + pub fn effective_available_decisions(&self) -> Vec { + // available_decisions is a new field that may not be populated by older + // senders, so we fall back to the legacy logic if it's not present. + match &self.available_decisions { + Some(decisions) => decisions.clone(), + None => Self::default_available_decisions( + self.network_approval_context.as_ref(), + self.proposed_execpolicy_amendment.as_ref(), + self.proposed_network_policy_amendments.as_deref(), + self.additional_permissions.as_ref(), + ), + } + } + + pub fn default_available_decisions( + network_approval_context: Option<&NetworkApprovalContext>, + proposed_execpolicy_amendment: Option<&ExecPolicyAmendment>, + proposed_network_policy_amendments: Option<&[NetworkPolicyAmendment]>, + additional_permissions: Option<&PermissionProfile>, + ) -> Vec { + if network_approval_context.is_some() { + let mut decisions = vec![ReviewDecision::Approved, ReviewDecision::ApprovedForSession]; + if let Some(amendment) = proposed_network_policy_amendments.and_then(|amendments| { + amendments + .iter() + .find(|amendment| amendment.action == NetworkPolicyRuleAction::Allow) + }) { + decisions.push(ReviewDecision::NetworkPolicyAmendment { + network_policy_amendment: amendment.clone(), + }); + } + decisions.push(ReviewDecision::Abort); + return decisions; + } + + if additional_permissions.is_some() { + return vec![ReviewDecision::Approved, ReviewDecision::Abort]; + } + + let mut decisions = vec![ReviewDecision::Approved]; + if let Some(prefix) = proposed_execpolicy_amendment { + decisions.push(ReviewDecision::ApprovedExecpolicyAmendment { + proposed_execpolicy_amendment: prefix.clone(), + }); + } + decisions.push(ReviewDecision::Abort); + decisions + } } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index c942a717d7..5841b1187e 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -15,5 +15,4 @@ pub mod parse_command; pub mod plan_tool; pub mod protocol; pub mod request_user_input; -pub mod skill_approval; pub mod user_input; diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index ff5030c75e..14cb968281 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -95,6 +95,32 @@ pub enum MacOsAutomationValue { BundleIds(Vec), } +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum MacOsPreferencesPermission { + // IMPORTANT: ReadOnly needs to be the default because it's the + // security-sensitive default and keeps cf prefs working. + #[default] + ReadOnly, + ReadWrite, + None, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum MacOsAutomationPermission { + #[default] + None, + All, + BundleIds(Vec), +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct MacOsSeatbeltProfileExtensions { + pub macos_preferences: MacOsPreferencesPermission, + pub macos_automation: MacOsAutomationPermission, + pub macos_accessibility: bool, + pub macos_calendar: bool, +} + #[derive(Debug, Clone, Default, Eq, Hash, PartialEq, Serialize, Deserialize, JsonSchema, TS)] pub struct PermissionProfile { pub network: Option, @@ -135,7 +161,7 @@ pub enum ResponseInputItem { }, CustomToolCallOutput { call_id: String, - output: String, + output: FunctionCallOutputPayload, }, } @@ -235,9 +261,12 @@ pub enum ResponseItem { name: String, input: String, }, + // `custom_tool_call_output.output` uses the same wire encoding as + // `function_call_output.output` so freeform tools can return either plain + // text or structured content items. CustomToolCallOutput { call_id: String, - output: String, + output: FunctionCallOutputPayload, }, // Emitted by the Responses API when the agent triggers a web search. // Example payload (from SSE `response.output_item.done`): @@ -1512,6 +1541,26 @@ mod tests { Ok(()) } + #[test] + fn serializes_custom_tool_image_outputs_as_array() -> Result<()> { + let item = ResponseInputItem::CustomToolCallOutput { + call_id: "call1".into(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputImage { + image_url: "data:image/png;base64,BASE64".into(), + }, + ]), + }; + + let json = serde_json::to_string(&item)?; + let v: serde_json::Value = serde_json::from_str(&json)?; + + let output = v.get("output").expect("output field"); + assert!(output.is_array(), "expected array output"); + + Ok(()) + } + #[test] fn preserves_existing_image_data_urls() -> Result<()> { let call_tool_result = CallToolResult { diff --git a/codex-rs/protocol/src/openai_models.rs b/codex-rs/protocol/src/openai_models.rs index 8d01e10d0b..b63f137442 100644 --- a/codex-rs/protocol/src/openai_models.rs +++ b/codex-rs/protocol/src/openai_models.rs @@ -15,6 +15,7 @@ use tracing::warn; use ts_rs::TS; use crate::config_types::Personality; +use crate::config_types::ReasoningSummary; use crate::config_types::Verbosity; const PERSONALITY_PLACEHOLDER: &str = "{{ personality }}"; @@ -98,6 +99,11 @@ pub struct ModelUpgrade { pub migration_markdown: Option, } +#[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq, Eq)] +pub struct ModelAvailabilityNux { + pub message: String, +} + /// Metadata describing a Codex-supported model. #[derive(Debug, Clone, Deserialize, Serialize, TS, JsonSchema, PartialEq)] pub struct ModelPreset { @@ -122,6 +128,8 @@ pub struct ModelPreset { pub upgrade: Option, /// Whether this preset should appear in the picker UI. pub show_in_picker: bool, + /// Availability NUX shown when this preset becomes accessible to the user. + pub availability_nux: Option, /// whether this model is supported in the api pub supported_in_api: bool, /// Input modalities accepted when composing user turns for this preset. @@ -224,11 +232,14 @@ pub struct ModelInfo { pub visibility: ModelVisibility, pub supported_in_api: bool, pub priority: i32, + pub availability_nux: Option, pub upgrade: Option, pub base_instructions: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub model_messages: Option, pub supports_reasoning_summaries: bool, + #[serde(default)] + pub default_reasoning_summary: ReasoningSummary, pub support_verbosity: bool, pub default_verbosity: Option, pub apply_patch_tool_type: Option, @@ -407,6 +418,7 @@ impl From for ModelPreset { migration_markdown: Some(upgrade.migration_markdown.clone()), }), show_in_picker: info.visibility == ModelVisibility::List, + availability_nux: info.availability_nux, supported_in_api: info.supported_in_api, input_modalities: info.input_modalities, } @@ -492,10 +504,12 @@ mod tests { visibility: ModelVisibility::List, supported_in_api: true, priority: 1, + availability_nux: None, upgrade: None, base_instructions: "base".to_string(), model_messages: spec, supports_reasoning_summaries: false, + default_reasoning_summary: ReasoningSummary::Auto, support_verbosity: false, default_verbosity: None, apply_patch_tool_type: None, @@ -664,4 +678,57 @@ mod tests { ); assert_eq!(personality_variables.get_personality_message(None), None); } + + #[test] + fn model_info_defaults_availability_nux_to_none_when_omitted() { + let model: ModelInfo = serde_json::from_value(serde_json::json!({ + "slug": "test-model", + "display_name": "Test Model", + "description": null, + "supported_reasoning_levels": [], + "shell_type": "shell_command", + "visibility": "list", + "supported_in_api": true, + "priority": 1, + "upgrade": null, + "base_instructions": "base", + "model_messages": null, + "supports_reasoning_summaries": false, + "default_reasoning_summary": "auto", + "support_verbosity": false, + "default_verbosity": null, + "apply_patch_tool_type": null, + "truncation_policy": { + "mode": "bytes", + "limit": 10000 + }, + "supports_parallel_tool_calls": false, + "context_window": null, + "auto_compact_token_limit": null, + "effective_context_window_percent": 95, + "experimental_supported_tools": [], + "input_modalities": ["text", "image"], + "prefer_websockets": false + })) + .expect("deserialize model info"); + + assert_eq!(model.availability_nux, None); + } + + #[test] + fn model_preset_preserves_availability_nux() { + let preset = ModelPreset::from(ModelInfo { + availability_nux: Some(ModelAvailabilityNux { + message: "Try Spark.".to_string(), + }), + ..test_model(None) + }); + + assert_eq!( + preset.availability_nux, + Some(ModelAvailabilityNux { + message: "Try Spark.".to_string(), + }) + ); + } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 0e2e86f8ca..b5aaf47ea6 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -41,7 +41,6 @@ use crate::openai_models::ReasoningEffort as ReasoningEffortConfig; use crate::parse_command::ParsedCommand; use crate::plan_tool::UpdatePlanArgs; use crate::request_user_input::RequestUserInputResponse; -use crate::skill_approval::SkillApprovalResponse; use crate::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; use schemars::JsonSchema; @@ -62,7 +61,6 @@ pub use crate::approvals::NetworkApprovalProtocol; pub use crate::approvals::NetworkPolicyAmendment; pub use crate::approvals::NetworkPolicyRuleAction; pub use crate::request_user_input::RequestUserInputEvent; -pub use crate::skill_approval::SkillRequestApprovalEvent; /// Open/close tags for special user-input blocks. Used across crates to avoid /// duplicated hardcoded strings. @@ -187,7 +185,11 @@ pub enum Op { effort: Option, /// Will only be honored if the model is configured to use reasoning. - summary: ReasoningSummaryConfig, + /// + /// When omitted, the session keeps the current setting (which allows core to + /// fall back to the selected model's default on new sessions). + #[serde(default, skip_serializing_if = "Option::is_none")] + summary: Option, // The JSON schema to use for the final assistant message final_output_json_schema: Option, @@ -296,14 +298,6 @@ pub enum Op { response: DynamicToolResponse, }, - /// Resolve a skill approval request. - SkillApproval { - /// Item id for the in-flight request. - id: String, - /// User decision. - response: SkillApprovalResponse, - }, - /// Append an entry to the persistent cross-session message history. /// /// Note the entry is not guaranteed to be logged if the user has @@ -1056,8 +1050,6 @@ pub enum EventMsg { DynamicToolCallResponse(DynamicToolCallResponseEvent), - SkillRequestApproval(SkillRequestApprovalEvent), - ElicitationRequest(ElicitationRequestEvent), ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent), @@ -2000,7 +1992,7 @@ impl SessionSource { agent_nickname.clone() } SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) => { - Some("morpheus".to_string()) + Some("Morpheus".to_string()) } _ => None, } @@ -2141,6 +2133,10 @@ pub struct TurnContextItem { #[serde(default, skip_serializing_if = "Option::is_none")] pub turn_id: Option, pub cwd: PathBuf, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_date: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timezone: Option, pub approval_policy: AskForApproval, pub sandbox_policy: SandboxPolicy, #[serde(skip_serializing_if = "Option::is_none")] @@ -2789,8 +2785,8 @@ pub enum ReviewDecision { proposed_execpolicy_amendment: ExecPolicyAmendment, }, - /// User has approved this command and wants to automatically approve any - /// future identical instances (`command` and `cwd` match exactly) for the + /// User has approved this request and wants future prompts in the same + /// session-scoped approval cache to be automatically approved for the /// remainder of the session. ApprovedForSession, @@ -3369,6 +3365,8 @@ mod tests { let item = TurnContextItem { turn_id: None, cwd: PathBuf::from("/tmp"), + current_date: None, + timezone: None, approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, network: Some(TurnContextNetworkItem { diff --git a/codex-rs/protocol/src/skill_approval.rs b/codex-rs/protocol/src/skill_approval.rs deleted file mode 100644 index 152e248186..0000000000 --- a/codex-rs/protocol/src/skill_approval.rs +++ /dev/null @@ -1,15 +0,0 @@ -use schemars::JsonSchema; -use serde::Deserialize; -use serde::Serialize; -use ts_rs::TS; - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] -pub struct SkillRequestApprovalEvent { - pub item_id: String, - pub skill_name: String, -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] -pub struct SkillApprovalResponse { - pub approved: bool, -} diff --git a/codex-rs/protocol/src/user_input.rs b/codex-rs/protocol/src/user_input.rs index d40511f342..c1c22dcaf9 100644 --- a/codex-rs/protocol/src/user_input.rs +++ b/codex-rs/protocol/src/user_input.rs @@ -3,6 +3,9 @@ use serde::Deserialize; use serde::Serialize; use ts_rs::TS; +/// Conservative cap so one user message cannot monopolize a large context window. +pub const MAX_USER_INPUT_TEXT_CHARS: usize = 1 << 20; + /// User input #[non_exhaustive] #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, TS, JsonSchema)] diff --git a/codex-rs/rmcp-client/src/perform_oauth_login.rs b/codex-rs/rmcp-client/src/perform_oauth_login.rs index 62a9c3b019..c71799c629 100644 --- a/codex-rs/rmcp-client/src/perform_oauth_login.rs +++ b/codex-rs/rmcp-client/src/perform_oauth_login.rs @@ -47,6 +47,7 @@ pub async fn perform_oauth_login( http_headers: Option>, env_http_headers: Option>, scopes: &[String], + oauth_resource: Option<&str>, callback_port: Option, callback_url: Option<&str>, ) -> Result<()> { @@ -60,6 +61,7 @@ pub async fn perform_oauth_login( store_mode, headers, scopes, + oauth_resource, true, callback_port, callback_url, @@ -78,6 +80,7 @@ pub async fn perform_oauth_login_return_url( http_headers: Option>, env_http_headers: Option>, scopes: &[String], + oauth_resource: Option<&str>, timeout_secs: Option, callback_port: Option, callback_url: Option<&str>, @@ -92,6 +95,7 @@ pub async fn perform_oauth_login_return_url( store_mode, headers, scopes, + oauth_resource, false, callback_port, callback_url, @@ -303,6 +307,7 @@ impl OauthLoginFlow { store_mode: OAuthCredentialsStoreMode, headers: OauthHeaders, scopes: &[String], + oauth_resource: Option<&str>, launch_browser: bool, callback_port: Option, callback_url: Option<&str>, @@ -340,7 +345,11 @@ impl OauthLoginFlow { oauth_state .start_authorization(&scope_refs, &redirect_uri, Some("Codex")) .await?; - let auth_url = oauth_state.get_authorization_url().await?; + let auth_url = append_query_param( + &oauth_state.get_authorization_url().await?, + "resource", + oauth_resource, + ); let timeout_secs = timeout_secs.unwrap_or(DEFAULT_OAUTH_TIMEOUT_SECS).max(1); let timeout = Duration::from_secs(timeout_secs as u64); @@ -431,9 +440,29 @@ impl OauthLoginFlow { } } +fn append_query_param(url: &str, key: &str, value: Option<&str>) -> String { + let Some(value) = value else { + return url.to_string(); + }; + let value = value.trim(); + if value.is_empty() { + return url.to_string(); + } + if let Ok(mut parsed) = Url::parse(url) { + parsed.query_pairs_mut().append_pair(key, value); + return parsed.to_string(); + } + let encoded = urlencoding::encode(value); + let separator = if url.contains('?') { "&" } else { "?" }; + format!("{url}{separator}{key}={encoded}") +} + #[cfg(test)] mod tests { + use pretty_assertions::assert_eq; + use super::CallbackOutcome; + use super::append_query_param; use super::callback_path_from_redirect_uri; use super::parse_oauth_callback; @@ -461,4 +490,36 @@ mod tests { .expect("redirect URI should parse"); assert_eq!(path, "/oauth/callback"); } + + #[test] + fn append_query_param_adds_resource_to_absolute_url() { + let url = append_query_param( + "https://example.com/authorize?scope=read", + "resource", + Some("https://api.example.com"), + ); + + assert_eq!( + url, + "https://example.com/authorize?scope=read&resource=https%3A%2F%2Fapi.example.com" + ); + } + + #[test] + fn append_query_param_ignores_empty_values() { + let url = append_query_param( + "https://example.com/authorize?scope=read", + "resource", + Some(" "), + ); + + assert_eq!(url, "https://example.com/authorize?scope=read"); + } + + #[test] + fn append_query_param_handles_unparseable_url() { + let url = append_query_param("not a url", "resource", Some("api/resource")); + + assert_eq!(url, "not a url?resource=api%2Fresource"); + } } diff --git a/codex-rs/shell-escalation/Cargo.toml b/codex-rs/shell-escalation/Cargo.toml index 85075cfb51..fbc5bcd8c7 100644 --- a/codex-rs/shell-escalation/Cargo.toml +++ b/codex-rs/shell-escalation/Cargo.toml @@ -12,6 +12,7 @@ path = "src/bin/main_execve_wrapper.rs" anyhow = { workspace = true } async-trait = { workspace = true } clap = { workspace = true, features = ["derive"] } +codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } libc = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/codex-rs/shell-escalation/src/lib.rs b/codex-rs/shell-escalation/src/lib.rs index 48bc116581..1cc42a46db 100644 --- a/codex-rs/shell-escalation/src/lib.rs +++ b/codex-rs/shell-escalation/src/lib.rs @@ -6,12 +6,22 @@ pub use unix::EscalateAction; #[cfg(unix)] pub use unix::EscalateServer; #[cfg(unix)] +pub use unix::EscalationDecision; +#[cfg(unix)] +pub use unix::EscalationExecution; +#[cfg(unix)] +pub use unix::EscalationPermissions; +#[cfg(unix)] pub use unix::EscalationPolicy; #[cfg(unix)] pub use unix::ExecParams; #[cfg(unix)] pub use unix::ExecResult; #[cfg(unix)] +pub use unix::Permissions; +#[cfg(unix)] +pub use unix::PreparedExec; +#[cfg(unix)] pub use unix::ShellCommandExecutor; #[cfg(unix)] pub use unix::Stopwatch; diff --git a/codex-rs/shell-escalation/src/unix/escalate_protocol.rs b/codex-rs/shell-escalation/src/unix/escalate_protocol.rs index 6002fdc57b..ea38c7b2a6 100644 --- a/codex-rs/shell-escalation/src/unix/escalate_protocol.rs +++ b/codex-rs/shell-escalation/src/unix/escalate_protocol.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::os::fd::RawFd; use std::path::PathBuf; +use codex_protocol::approvals::EscalationPermissions; use codex_utils_absolute_path::AbsolutePathBuf; use serde::Deserialize; use serde::Serialize; @@ -35,6 +36,38 @@ pub struct EscalateResponse { pub action: EscalateAction, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EscalationDecision { + Run, + Escalate(EscalationExecution), + Deny { reason: Option }, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EscalationExecution { + /// Rerun the intercepted command outside any sandbox wrapper. + Unsandboxed, + /// Rerun using the turn's current sandbox configuration. + TurnDefault, + /// Rerun using an explicit sandbox configuration attached to the request. + Permissions(EscalationPermissions), +} + +impl EscalationDecision { + pub fn run() -> Self { + Self::Run + } + + pub fn escalate(execution: EscalationExecution) -> Self { + Self::Escalate(execution) + } + + pub fn deny(reason: Option) -> Self { + Self::Deny { reason } + } +} + #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] pub enum EscalateAction { /// The command should be run directly by the client. diff --git a/codex-rs/shell-escalation/src/unix/escalate_server.rs b/codex-rs/shell-escalation/src/unix/escalate_server.rs index 2951f584a6..bb45c3ed71 100644 --- a/codex-rs/shell-escalation/src/unix/escalate_server.rs +++ b/codex-rs/shell-escalation/src/unix/escalate_server.rs @@ -15,6 +15,8 @@ use crate::unix::escalate_protocol::EXEC_WRAPPER_ENV_VAR; use crate::unix::escalate_protocol::EscalateAction; use crate::unix::escalate_protocol::EscalateRequest; use crate::unix::escalate_protocol::EscalateResponse; +use crate::unix::escalate_protocol::EscalationDecision; +use crate::unix::escalate_protocol::EscalationExecution; use crate::unix::escalate_protocol::LEGACY_BASH_EXEC_WRAPPER_ENV_VAR; use crate::unix::escalate_protocol::SuperExecMessage; use crate::unix::escalate_protocol::SuperExecResult; @@ -37,6 +39,16 @@ pub trait ShellCommandExecutor: Send + Sync { env: HashMap, cancel_rx: CancellationToken, ) -> anyhow::Result; + + /// Prepares an escalated subcommand for execution on the server side. + async fn prepare_escalated_exec( + &self, + program: &AbsolutePathBuf, + argv: &[String], + workdir: &AbsolutePathBuf, + env: HashMap, + execution: EscalationExecution, + ) -> anyhow::Result; } #[derive(Debug, serde::Deserialize, serde::Serialize)] @@ -62,6 +74,14 @@ pub struct ExecResult { pub timed_out: bool, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PreparedExec { + pub command: Vec, + pub cwd: PathBuf, + pub env: HashMap, + pub arg0: Option, +} + pub struct EscalateServer { bash_path: PathBuf, execve_wrapper: PathBuf, @@ -69,9 +89,9 @@ pub struct EscalateServer { } impl EscalateServer { - pub fn new

(bash_path: PathBuf, execve_wrapper: PathBuf, policy: P) -> Self + pub fn new(bash_path: PathBuf, execve_wrapper: PathBuf, policy: Policy) -> Self where - P: EscalationPolicy + Send + Sync + 'static, + Policy: EscalationPolicy + Send + Sync + 'static, { Self { bash_path, @@ -84,13 +104,17 @@ impl EscalateServer { &self, params: ExecParams, cancel_rx: CancellationToken, - command_executor: &dyn ShellCommandExecutor, + command_executor: Arc, ) -> anyhow::Result { let (escalate_server, escalate_client) = AsyncDatagramSocket::pair()?; let client_socket = escalate_client.into_inner(); // Only the client endpoint should cross exec into the wrapper process. client_socket.set_cloexec(false)?; - let escalate_task = tokio::spawn(escalate_task(escalate_server, self.policy.clone())); + let escalate_task = tokio::spawn(escalate_task( + escalate_server, + Arc::clone(&self.policy), + Arc::clone(&command_executor), + )); let mut env = std::env::vars().collect::>(); env.insert( ESCALATE_SOCKET_ENV_VAR.to_string(), @@ -126,6 +150,7 @@ impl EscalateServer { async fn escalate_task( socket: AsyncDatagramSocket, policy: Arc, + command_executor: Arc, ) -> anyhow::Result<()> { loop { let (_, mut fds) = socket.receive_with_fds().await?; @@ -134,9 +159,12 @@ async fn escalate_task( continue; } let stream_socket = AsyncSocket::from_fd(fds.remove(0))?; - let policy = policy.clone(); + let policy = Arc::clone(&policy); + let command_executor = Arc::clone(&command_executor); tokio::spawn(async move { - if let Err(err) = handle_escalate_session_with_policy(stream_socket, policy).await { + if let Err(err) = + handle_escalate_session_with_policy(stream_socket, policy, command_executor).await + { tracing::error!("escalate session failed: {err:?}"); } }); @@ -146,6 +174,7 @@ async fn escalate_task( async fn handle_escalate_session_with_policy( socket: AsyncSocket, policy: Arc, + command_executor: Arc, ) -> anyhow::Result<()> { let EscalateRequest { file, @@ -154,22 +183,22 @@ async fn handle_escalate_session_with_policy( env, } = socket.receive::().await?; let program = AbsolutePathBuf::resolve_path_against_base(file, workdir.as_path())?; - let action = policy + let decision = policy .determine_action(&program, &argv, &workdir) .await .context("failed to determine escalation action")?; - tracing::debug!("decided {action:?} for {program:?} {argv:?} {workdir:?}"); + tracing::debug!("decided {decision:?} for {program:?} {argv:?} {workdir:?}"); - match action { - EscalateAction::Run => { + match decision { + EscalationDecision::Run => { socket .send(EscalateResponse { action: EscalateAction::Run, }) .await?; } - EscalateAction::Escalate => { + EscalationDecision::Escalate(execution) => { socket .send(EscalateResponse { action: EscalateAction::Escalate, @@ -197,12 +226,23 @@ async fn handle_escalate_session_with_policy( )); } - let mut command = Command::new(program.as_path()); + let PreparedExec { + command, + cwd, + env, + arg0, + } = command_executor + .prepare_escalated_exec(&program, &argv, &workdir, env, execution) + .await?; + let (program, args) = command + .split_first() + .ok_or_else(|| anyhow::anyhow!("prepared escalated command must not be empty"))?; + let mut command = Command::new(program); command - .args(&argv[1..]) - .arg0(argv[0].clone()) + .args(args) + .arg0(arg0.unwrap_or_else(|| program.clone())) .envs(&env) - .current_dir(&workdir) + .current_dir(&cwd) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()); @@ -222,7 +262,7 @@ async fn handle_escalate_session_with_policy( }) .await?; } - EscalateAction::Deny { reason } => { + EscalationDecision::Deny { reason } => { socket .send(EscalateResponse { action: EscalateAction::Deny { reason }, @@ -236,13 +276,15 @@ async fn handle_escalate_session_with_policy( #[cfg(test)] mod tests { use super::*; + use codex_protocol::approvals::EscalationPermissions; + use codex_protocol::models::PermissionProfile; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use std::collections::HashMap; use std::path::PathBuf; struct DeterministicEscalationPolicy { - action: EscalateAction, + decision: EscalationDecision, } #[async_trait::async_trait] @@ -252,8 +294,8 @@ mod tests { _file: &AbsolutePathBuf, _argv: &[String], _workdir: &AbsolutePathBuf, - ) -> anyhow::Result { - Ok(self.action.clone()) + ) -> anyhow::Result { + Ok(self.decision.clone()) } } @@ -269,10 +311,82 @@ mod tests { file: &AbsolutePathBuf, _argv: &[String], workdir: &AbsolutePathBuf, - ) -> anyhow::Result { + ) -> anyhow::Result { assert_eq!(file, &self.expected_file); assert_eq!(workdir, &self.expected_workdir); - Ok(EscalateAction::Run) + Ok(EscalationDecision::run()) + } + } + + struct ForwardingShellCommandExecutor; + + #[async_trait::async_trait] + impl ShellCommandExecutor for ForwardingShellCommandExecutor { + async fn run( + &self, + _command: Vec, + _cwd: PathBuf, + _env: HashMap, + _cancel_rx: CancellationToken, + ) -> anyhow::Result { + unreachable!("run() is not used by handle_escalate_session_with_policy() tests") + } + + async fn prepare_escalated_exec( + &self, + program: &AbsolutePathBuf, + argv: &[String], + workdir: &AbsolutePathBuf, + env: HashMap, + _execution: EscalationExecution, + ) -> anyhow::Result { + Ok(PreparedExec { + command: std::iter::once(program.to_string_lossy().to_string()) + .chain(argv.iter().skip(1).cloned()) + .collect(), + cwd: workdir.to_path_buf(), + env, + arg0: argv.first().cloned(), + }) + } + } + + struct PermissionAssertingShellCommandExecutor { + expected_permissions: EscalationPermissions, + } + + #[async_trait::async_trait] + impl ShellCommandExecutor for PermissionAssertingShellCommandExecutor { + async fn run( + &self, + _command: Vec, + _cwd: PathBuf, + _env: HashMap, + _cancel_rx: CancellationToken, + ) -> anyhow::Result { + unreachable!("run() is not used by handle_escalate_session_with_policy() tests") + } + + async fn prepare_escalated_exec( + &self, + program: &AbsolutePathBuf, + argv: &[String], + workdir: &AbsolutePathBuf, + env: HashMap, + execution: EscalationExecution, + ) -> anyhow::Result { + assert_eq!( + execution, + EscalationExecution::Permissions(self.expected_permissions.clone()) + ); + Ok(PreparedExec { + command: std::iter::once(program.to_string_lossy().to_string()) + .chain(argv.iter().skip(1).cloned()) + .collect(), + cwd: workdir.to_path_buf(), + env, + arg0: argv.first().cloned(), + }) } } @@ -282,8 +396,9 @@ mod tests { let server_task = tokio::spawn(handle_escalate_session_with_policy( server, Arc::new(DeterministicEscalationPolicy { - action: EscalateAction::Run, + decision: EscalationDecision::run(), }), + Arc::new(ForwardingShellCommandExecutor), )); let mut env = HashMap::new(); @@ -326,6 +441,7 @@ mod tests { expected_file, expected_workdir: workdir.clone(), }), + Arc::new(ForwardingShellCommandExecutor), )); client @@ -353,8 +469,9 @@ mod tests { let server_task = tokio::spawn(handle_escalate_session_with_policy( server, Arc::new(DeterministicEscalationPolicy { - action: EscalateAction::Escalate, + decision: EscalationDecision::escalate(EscalationExecution::Unsandboxed), }), + Arc::new(ForwardingShellCommandExecutor), )); client @@ -387,4 +504,52 @@ mod tests { server_task.await? } + + #[tokio::test] + async fn handle_escalate_session_passes_permissions_to_executor() -> anyhow::Result<()> { + let (server, client) = AsyncSocket::pair()?; + let server_task = tokio::spawn(handle_escalate_session_with_policy( + server, + Arc::new(DeterministicEscalationPolicy { + decision: EscalationDecision::escalate(EscalationExecution::Permissions( + EscalationPermissions::PermissionProfile(PermissionProfile { + network: Some(true), + ..Default::default() + }), + )), + }), + Arc::new(PermissionAssertingShellCommandExecutor { + expected_permissions: EscalationPermissions::PermissionProfile(PermissionProfile { + network: Some(true), + ..Default::default() + }), + }), + )); + + client + .send(EscalateRequest { + file: PathBuf::from("/bin/sh"), + argv: vec!["sh".to_string(), "-c".to_string(), "exit 0".to_string()], + workdir: AbsolutePathBuf::current_dir()?, + env: HashMap::new(), + }) + .await?; + + let response = client.receive::().await?; + assert_eq!( + EscalateResponse { + action: EscalateAction::Escalate, + }, + response + ); + + client + .send_with_fds(SuperExecMessage { fds: Vec::new() }, &[]) + .await?; + + let result = client.receive::().await?; + assert_eq!(0, result.exit_code); + + server_task.await? + } } diff --git a/codex-rs/shell-escalation/src/unix/escalation_policy.rs b/codex-rs/shell-escalation/src/unix/escalation_policy.rs index 28e04e6f59..c3da252dd9 100644 --- a/codex-rs/shell-escalation/src/unix/escalation_policy.rs +++ b/codex-rs/shell-escalation/src/unix/escalation_policy.rs @@ -1,6 +1,6 @@ use codex_utils_absolute_path::AbsolutePathBuf; -use crate::unix::escalate_protocol::EscalateAction; +use crate::unix::escalate_protocol::EscalationDecision; /// Decides what action to take in response to an execve request from a client. #[async_trait::async_trait] @@ -10,5 +10,5 @@ pub trait EscalationPolicy: Send + Sync { file: &AbsolutePathBuf, argv: &[String], workdir: &AbsolutePathBuf, - ) -> anyhow::Result; + ) -> anyhow::Result; } diff --git a/codex-rs/shell-escalation/src/unix/mod.rs b/codex-rs/shell-escalation/src/unix/mod.rs index 37e29e8775..6de12297a4 100644 --- a/codex-rs/shell-escalation/src/unix/mod.rs +++ b/codex-rs/shell-escalation/src/unix/mod.rs @@ -63,10 +63,15 @@ pub mod stopwatch; pub use self::escalate_client::run_shell_escalation_execve_wrapper; pub use self::escalate_protocol::EscalateAction; +pub use self::escalate_protocol::EscalationDecision; +pub use self::escalate_protocol::EscalationExecution; pub use self::escalate_server::EscalateServer; pub use self::escalate_server::ExecParams; pub use self::escalate_server::ExecResult; +pub use self::escalate_server::PreparedExec; pub use self::escalate_server::ShellCommandExecutor; pub use self::escalation_policy::EscalationPolicy; pub use self::execve_wrapper::main_execve_wrapper; pub use self::stopwatch::Stopwatch; +pub use codex_protocol::approvals::EscalationPermissions; +pub use codex_protocol::approvals::Permissions; diff --git a/codex-rs/state/migrations/0018_phase2_selection_snapshot.sql b/codex-rs/state/migrations/0018_phase2_selection_snapshot.sql new file mode 100644 index 0000000000..f175980fc7 --- /dev/null +++ b/codex-rs/state/migrations/0018_phase2_selection_snapshot.sql @@ -0,0 +1,3 @@ +ALTER TABLE stage1_outputs +ADD COLUMN selected_for_phase2_source_updated_at INTEGER; +ALTER TABLE threads ADD COLUMN memory_mode TEXT NOT NULL DEFAULT 'enabled'; diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index 0ba3df6790..f978238979 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -56,7 +56,9 @@ fn apply_session_meta_from_item(metadata: &mut ThreadMetadata, meta_line: &Sessi } fn apply_turn_context(metadata: &mut ThreadMetadata, turn_ctx: &TurnContextItem) { - metadata.cwd = turn_ctx.cwd.clone(); + if metadata.cwd.as_os_str().is_empty() { + metadata.cwd = turn_ctx.cwd.clone(); + } metadata.sandbox_policy = enum_to_string(&turn_ctx.sandbox_policy); metadata.approval_mode = enum_to_string(&turn_ctx.approval_policy); } @@ -125,10 +127,17 @@ mod tests { use chrono::DateTime; use chrono::Utc; use codex_protocol::ThreadId; + use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; + use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::RolloutItem; + use codex_protocol::protocol::SandboxPolicy; + use codex_protocol::protocol::SessionMeta; + use codex_protocol::protocol::SessionMetaLine; + use codex_protocol::protocol::SessionSource; + use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::USER_MESSAGE_BEGIN; use codex_protocol::protocol::UserMessageEvent; @@ -209,6 +218,97 @@ mod tests { assert_eq!(metadata.title, ""); } + #[test] + fn turn_context_does_not_override_session_cwd() { + let mut metadata = metadata_for_test(); + metadata.cwd = PathBuf::new(); + let thread_id = metadata.id; + + apply_rollout_item( + &mut metadata, + &RolloutItem::SessionMeta(SessionMetaLine { + meta: SessionMeta { + id: thread_id, + forked_from_id: Some( + ThreadId::from_string(&Uuid::now_v7().to_string()).expect("thread id"), + ), + timestamp: "2026-02-26T00:00:00.000Z".to_string(), + cwd: PathBuf::from("/child/worktree"), + originator: "codex_cli_rs".to_string(), + cli_version: "0.0.0".to_string(), + source: SessionSource::Cli, + agent_nickname: None, + agent_role: None, + model_provider: Some("openai".to_string()), + base_instructions: None, + dynamic_tools: None, + }, + git: None, + }), + "test-provider", + ); + apply_rollout_item( + &mut metadata, + &RolloutItem::TurnContext(TurnContextItem { + turn_id: Some("turn-1".to_string()), + cwd: PathBuf::from("/parent/workspace"), + current_date: None, + timezone: None, + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + network: None, + model: "gpt-5".to_string(), + personality: None, + collaboration_mode: None, + effort: None, + summary: ReasoningSummary::Auto, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: None, + }), + "test-provider", + ); + + assert_eq!(metadata.cwd, PathBuf::from("/child/worktree")); + assert_eq!( + metadata.sandbox_policy, + super::enum_to_string(&SandboxPolicy::DangerFullAccess) + ); + assert_eq!(metadata.approval_mode, "never"); + } + + #[test] + fn turn_context_sets_cwd_when_session_cwd_missing() { + let mut metadata = metadata_for_test(); + metadata.cwd = PathBuf::new(); + + apply_rollout_item( + &mut metadata, + &RolloutItem::TurnContext(TurnContextItem { + turn_id: Some("turn-1".to_string()), + cwd: PathBuf::from("/fallback/workspace"), + current_date: None, + timezone: None, + approval_policy: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + network: None, + model: "gpt-5".to_string(), + personality: None, + collaboration_mode: None, + effort: None, + summary: ReasoningSummary::Auto, + user_instructions: None, + developer_instructions: None, + final_output_json_schema: None, + truncation_policy: None, + }), + "test-provider", + ); + + assert_eq!(metadata.cwd, PathBuf::from("/fallback/workspace")); + } + fn metadata_for_test() -> ThreadMetadata { let id = ThreadId::from_string(&Uuid::from_u128(42).to_string()).expect("thread id"); let created_at = DateTime::::from_timestamp(1_735_689_600, 0).expect("timestamp"); diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index 59607f3d9b..a6d1deeb84 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -14,6 +14,7 @@ mod runtime; pub use model::LogEntry; pub use model::LogQuery; pub use model::LogRow; +pub use model::Phase2InputSelection; pub use model::Phase2JobClaimOutcome; /// Preferred entrypoint: owns configuration and metrics. pub use runtime::StateRuntime; @@ -38,6 +39,7 @@ pub use model::SortKey; pub use model::Stage1JobClaim; pub use model::Stage1JobClaimOutcome; pub use model::Stage1Output; +pub use model::Stage1OutputRef; pub use model::Stage1StartupClaimParams; pub use model::ThreadMetadata; pub use model::ThreadMetadataBuilder; diff --git a/codex-rs/state/src/model/memories.rs b/codex-rs/state/src/model/memories.rs index 813c999395..0e663bf904 100644 --- a/codex-rs/state/src/model/memories.rs +++ b/codex-rs/state/src/model/memories.rs @@ -12,22 +12,41 @@ use super::ThreadMetadata; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Stage1Output { pub thread_id: ThreadId, + pub rollout_path: PathBuf, pub source_updated_at: DateTime, pub raw_memory: String, pub rollout_summary: String, pub rollout_slug: Option, pub cwd: PathBuf, + pub git_branch: Option, pub generated_at: DateTime, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Stage1OutputRef { + pub thread_id: ThreadId, + pub source_updated_at: DateTime, + pub rollout_slug: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Phase2InputSelection { + pub selected: Vec, + pub previous_selected: Vec, + pub retained_thread_ids: Vec, + pub removed: Vec, +} + #[derive(Debug)] pub(crate) struct Stage1OutputRow { thread_id: String, + rollout_path: String, source_updated_at: i64, raw_memory: String, rollout_summary: String, rollout_slug: Option, cwd: String, + git_branch: Option, generated_at: i64, } @@ -35,11 +54,13 @@ impl Stage1OutputRow { pub(crate) fn try_from_row(row: &SqliteRow) -> Result { Ok(Self { thread_id: row.try_get("thread_id")?, + rollout_path: row.try_get("rollout_path")?, source_updated_at: row.try_get("source_updated_at")?, raw_memory: row.try_get("raw_memory")?, rollout_summary: row.try_get("rollout_summary")?, rollout_slug: row.try_get("rollout_slug")?, cwd: row.try_get("cwd")?, + git_branch: row.try_get("git_branch")?, generated_at: row.try_get("generated_at")?, }) } @@ -51,11 +72,13 @@ impl TryFrom for Stage1Output { fn try_from(row: Stage1OutputRow) -> std::result::Result { Ok(Self { thread_id: ThreadId::try_from(row.thread_id)?, + rollout_path: PathBuf::from(row.rollout_path), source_updated_at: epoch_seconds_to_datetime(row.source_updated_at)?, raw_memory: row.raw_memory, rollout_summary: row.rollout_summary, rollout_slug: row.rollout_slug, cwd: PathBuf::from(row.cwd), + git_branch: row.git_branch, generated_at: epoch_seconds_to_datetime(row.generated_at)?, }) } @@ -66,6 +89,18 @@ fn epoch_seconds_to_datetime(secs: i64) -> Result> { .ok_or_else(|| anyhow::anyhow!("invalid unix timestamp: {secs}")) } +pub(crate) fn stage1_output_ref_from_parts( + thread_id: String, + source_updated_at: i64, + rollout_slug: Option, +) -> Result { + Ok(Stage1OutputRef { + thread_id: ThreadId::try_from(thread_id)?, + source_updated_at: epoch_seconds_to_datetime(source_updated_at)?, + rollout_slug, + }) +} + /// Result of trying to claim a stage-1 memory extraction job. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Stage1JobClaimOutcome { diff --git a/codex-rs/state/src/model/mod.rs b/codex-rs/state/src/model/mod.rs index 816c036f8b..efaf3f787e 100644 --- a/codex-rs/state/src/model/mod.rs +++ b/codex-rs/state/src/model/mod.rs @@ -16,10 +16,12 @@ pub use backfill_state::BackfillStatus; pub use log::LogEntry; pub use log::LogQuery; pub use log::LogRow; +pub use memories::Phase2InputSelection; pub use memories::Phase2JobClaimOutcome; pub use memories::Stage1JobClaim; pub use memories::Stage1JobClaimOutcome; pub use memories::Stage1Output; +pub use memories::Stage1OutputRef; pub use memories::Stage1StartupClaimParams; pub use thread_metadata::Anchor; pub use thread_metadata::BackfillStats; @@ -32,6 +34,7 @@ pub use thread_metadata::ThreadsPage; pub(crate) use agent_job::AgentJobItemRow; pub(crate) use agent_job::AgentJobRow; pub(crate) use memories::Stage1OutputRow; +pub(crate) use memories::stage1_output_ref_from_parts; pub(crate) use thread_metadata::ThreadRow; pub(crate) use thread_metadata::anchor_from_item; pub(crate) use thread_metadata::datetime_to_epoch_seconds; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index f0c6230508..42d235fc76 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -18,7 +18,6 @@ use crate::ThreadMetadataBuilder; use crate::ThreadsPage; use crate::apply_rollout_item; use crate::migrations::MIGRATOR; -use crate::model::AgentJobItemRow; use crate::model::AgentJobRow; use crate::model::ThreadRow; use crate::model::anchor_from_item; @@ -48,10 +47,14 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use tracing::warn; -use uuid::Uuid; +mod agent_jobs; +mod backfill; +mod logs; mod memories; -// Memory-specific CRUD and phase job lifecycle methods live in `runtime/memories.rs`. +#[cfg(test)] +mod test_support; +mod threads; // "Partition" is the retention bucket we cap at 10 MiB: // - one bucket per non-null thread_id @@ -107,1461 +110,6 @@ impl StateRuntime { pub fn codex_home(&self) -> &Path { self.codex_home.as_path() } - - /// Get persisted rollout metadata backfill state. - pub async fn get_backfill_state(&self) -> anyhow::Result { - self.ensure_backfill_state_row().await?; - let row = sqlx::query( - r#" -SELECT status, last_watermark, last_success_at -FROM backfill_state -WHERE id = 1 - "#, - ) - .fetch_one(self.pool.as_ref()) - .await?; - crate::BackfillState::try_from_row(&row) - } - - /// Attempt to claim ownership of rollout metadata backfill. - /// - /// Returns `true` when this runtime claimed the backfill worker slot. - /// Returns `false` if backfill is already complete or currently owned by a - /// non-expired worker. - pub async fn try_claim_backfill(&self, lease_seconds: i64) -> anyhow::Result { - self.ensure_backfill_state_row().await?; - let now = Utc::now().timestamp(); - let lease_cutoff = now.saturating_sub(lease_seconds.max(0)); - let result = sqlx::query( - r#" -UPDATE backfill_state -SET status = ?, updated_at = ? -WHERE id = 1 - AND status != ? - AND (status != ? OR updated_at <= ?) - "#, - ) - .bind(crate::BackfillStatus::Running.as_str()) - .bind(now) - .bind(crate::BackfillStatus::Complete.as_str()) - .bind(crate::BackfillStatus::Running.as_str()) - .bind(lease_cutoff) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() == 1) - } - - /// Mark rollout metadata backfill as running. - pub async fn mark_backfill_running(&self) -> anyhow::Result<()> { - self.ensure_backfill_state_row().await?; - sqlx::query( - r#" -UPDATE backfill_state -SET status = ?, updated_at = ? -WHERE id = 1 - "#, - ) - .bind(crate::BackfillStatus::Running.as_str()) - .bind(Utc::now().timestamp()) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } - - /// Persist rollout metadata backfill progress. - pub async fn checkpoint_backfill(&self, watermark: &str) -> anyhow::Result<()> { - self.ensure_backfill_state_row().await?; - sqlx::query( - r#" -UPDATE backfill_state -SET status = ?, last_watermark = ?, updated_at = ? -WHERE id = 1 - "#, - ) - .bind(crate::BackfillStatus::Running.as_str()) - .bind(watermark) - .bind(Utc::now().timestamp()) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } - - /// Mark rollout metadata backfill as complete. - pub async fn mark_backfill_complete(&self, last_watermark: Option<&str>) -> anyhow::Result<()> { - self.ensure_backfill_state_row().await?; - let now = Utc::now().timestamp(); - sqlx::query( - r#" -UPDATE backfill_state -SET - status = ?, - last_watermark = COALESCE(?, last_watermark), - last_success_at = ?, - updated_at = ? -WHERE id = 1 - "#, - ) - .bind(crate::BackfillStatus::Complete.as_str()) - .bind(last_watermark) - .bind(now) - .bind(now) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } - - /// Load thread metadata by id using the underlying database. - pub async fn get_thread(&self, id: ThreadId) -> anyhow::Result> { - let row = sqlx::query( - r#" -SELECT - id, - rollout_path, - created_at, - updated_at, - source, - agent_nickname, - agent_role, - model_provider, - cwd, - cli_version, - title, - sandbox_policy, - approval_mode, - tokens_used, - first_user_message, - archived_at, - git_sha, - git_branch, - git_origin_url -FROM threads -WHERE id = ? - "#, - ) - .bind(id.to_string()) - .fetch_optional(self.pool.as_ref()) - .await?; - row.map(|row| ThreadRow::try_from_row(&row).and_then(ThreadMetadata::try_from)) - .transpose() - } - - /// Get dynamic tools for a thread, if present. - pub async fn get_dynamic_tools( - &self, - thread_id: ThreadId, - ) -> anyhow::Result>> { - let rows = sqlx::query( - r#" -SELECT name, description, input_schema -FROM thread_dynamic_tools -WHERE thread_id = ? -ORDER BY position ASC - "#, - ) - .bind(thread_id.to_string()) - .fetch_all(self.pool.as_ref()) - .await?; - if rows.is_empty() { - return Ok(None); - } - let mut tools = Vec::with_capacity(rows.len()); - for row in rows { - let input_schema: String = row.try_get("input_schema")?; - let input_schema = serde_json::from_str::(input_schema.as_str())?; - tools.push(DynamicToolSpec { - name: row.try_get("name")?, - description: row.try_get("description")?, - input_schema, - }); - } - Ok(Some(tools)) - } - - /// Find a rollout path by thread id using the underlying database. - pub async fn find_rollout_path_by_id( - &self, - id: ThreadId, - archived_only: Option, - ) -> anyhow::Result> { - let mut builder = - QueryBuilder::::new("SELECT rollout_path FROM threads WHERE id = "); - builder.push_bind(id.to_string()); - match archived_only { - Some(true) => { - builder.push(" AND archived = 1"); - } - Some(false) => { - builder.push(" AND archived = 0"); - } - None => {} - } - let row = builder.build().fetch_optional(self.pool.as_ref()).await?; - Ok(row - .and_then(|r| r.try_get::("rollout_path").ok()) - .map(PathBuf::from)) - } - - /// List threads using the underlying database. - #[allow(clippy::too_many_arguments)] - pub async fn list_threads( - &self, - page_size: usize, - anchor: Option<&crate::Anchor>, - sort_key: crate::SortKey, - allowed_sources: &[String], - model_providers: Option<&[String]>, - archived_only: bool, - search_term: Option<&str>, - ) -> anyhow::Result { - let limit = page_size.saturating_add(1); - - let mut builder = QueryBuilder::::new( - r#" -SELECT - id, - rollout_path, - created_at, - updated_at, - source, - agent_nickname, - agent_role, - model_provider, - cwd, - cli_version, - title, - sandbox_policy, - approval_mode, - tokens_used, - first_user_message, - archived_at, - git_sha, - git_branch, - git_origin_url -FROM threads - "#, - ); - push_thread_filters( - &mut builder, - archived_only, - allowed_sources, - model_providers, - anchor, - sort_key, - search_term, - ); - push_thread_order_and_limit(&mut builder, sort_key, limit); - - let rows = builder.build().fetch_all(self.pool.as_ref()).await?; - let mut items = rows - .into_iter() - .map(|row| ThreadRow::try_from_row(&row).and_then(ThreadMetadata::try_from)) - .collect::, _>>()?; - let num_scanned_rows = items.len(); - let next_anchor = if items.len() > page_size { - items.pop(); - items - .last() - .and_then(|item| anchor_from_item(item, sort_key)) - } else { - None - }; - Ok(ThreadsPage { - items, - next_anchor, - num_scanned_rows, - }) - } - - /// Insert one log entry into the logs table. - pub async fn insert_log(&self, entry: &LogEntry) -> anyhow::Result<()> { - self.insert_logs(std::slice::from_ref(entry)).await - } - - /// Insert a batch of log entries into the logs table. - pub async fn insert_logs(&self, entries: &[LogEntry]) -> anyhow::Result<()> { - if entries.is_empty() { - return Ok(()); - } - - let mut tx = self.pool.begin().await?; - let mut builder = QueryBuilder::::new( - "INSERT INTO logs (ts, ts_nanos, level, target, message, thread_id, process_uuid, module_path, file, line, estimated_bytes) ", - ); - builder.push_values(entries, |mut row, entry| { - let estimated_bytes = entry.message.as_ref().map_or(0, String::len) as i64 - + entry.level.len() as i64 - + entry.target.len() as i64 - + entry.module_path.as_ref().map_or(0, String::len) as i64 - + entry.file.as_ref().map_or(0, String::len) as i64; - row.push_bind(entry.ts) - .push_bind(entry.ts_nanos) - .push_bind(&entry.level) - .push_bind(&entry.target) - .push_bind(&entry.message) - .push_bind(&entry.thread_id) - .push_bind(&entry.process_uuid) - .push_bind(&entry.module_path) - .push_bind(&entry.file) - .push_bind(entry.line) - .push_bind(estimated_bytes); - }); - builder.build().execute(&mut *tx).await?; - self.prune_logs_after_insert(entries, &mut tx).await?; - tx.commit().await?; - Ok(()) - } - - /// Enforce per-partition log size caps after a successful batch insert. - /// - /// We maintain two independent budgets: - /// - Thread logs: rows with `thread_id IS NOT NULL`, capped per `thread_id`. - /// - Threadless process logs: rows with `thread_id IS NULL` ("threadless"), - /// capped per `process_uuid` (including `process_uuid IS NULL` as its own - /// threadless partition). - /// - /// "Threadless" means the log row is not associated with any conversation - /// thread, so retention is keyed by process identity instead. - /// - /// This runs inside the same transaction as the insert so callers never - /// observe "inserted but not yet pruned" rows. - async fn prune_logs_after_insert( - &self, - entries: &[LogEntry], - tx: &mut SqliteConnection, - ) -> anyhow::Result<()> { - let thread_ids: BTreeSet<&str> = entries - .iter() - .filter_map(|entry| entry.thread_id.as_deref()) - .collect(); - if !thread_ids.is_empty() { - // Cheap precheck: only run the heavier window-function prune for - // threads that are currently above the cap. - let mut over_limit_threads_query = - QueryBuilder::::new("SELECT thread_id FROM logs WHERE thread_id IN ("); - { - let mut separated = over_limit_threads_query.separated(", "); - for thread_id in &thread_ids { - separated.push_bind(*thread_id); - } - } - over_limit_threads_query.push(") GROUP BY thread_id HAVING SUM("); - over_limit_threads_query.push("estimated_bytes"); - over_limit_threads_query.push(") > "); - over_limit_threads_query.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); - let over_limit_thread_ids: Vec = over_limit_threads_query - .build() - .fetch_all(&mut *tx) - .await? - .into_iter() - .map(|row| row.try_get("thread_id")) - .collect::>()?; - if !over_limit_thread_ids.is_empty() { - // Enforce a strict per-thread cap by deleting every row whose - // newest-first cumulative bytes exceed the partition budget. - let mut prune_threads = QueryBuilder::::new( - r#" -DELETE FROM logs -WHERE id IN ( - SELECT id - FROM ( - SELECT - id, - SUM( -"#, - ); - prune_threads.push("estimated_bytes"); - prune_threads.push( - r#" - ) OVER ( - PARTITION BY thread_id - ORDER BY ts DESC, ts_nanos DESC, id DESC - ) AS cumulative_bytes - FROM logs - WHERE thread_id IN ( -"#, - ); - { - let mut separated = prune_threads.separated(", "); - for thread_id in &over_limit_thread_ids { - separated.push_bind(thread_id); - } - } - prune_threads.push( - r#" - ) - ) - WHERE cumulative_bytes > -"#, - ); - prune_threads.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); - prune_threads.push("\n)"); - prune_threads.build().execute(&mut *tx).await?; - } - } - - let threadless_process_uuids: BTreeSet<&str> = entries - .iter() - .filter(|entry| entry.thread_id.is_none()) - .filter_map(|entry| entry.process_uuid.as_deref()) - .collect(); - let has_threadless_null_process_uuid = entries - .iter() - .any(|entry| entry.thread_id.is_none() && entry.process_uuid.is_none()); - if !threadless_process_uuids.is_empty() { - // Threadless logs are budgeted separately per process UUID. - let mut over_limit_processes_query = QueryBuilder::::new( - "SELECT process_uuid FROM logs WHERE thread_id IS NULL AND process_uuid IN (", - ); - { - let mut separated = over_limit_processes_query.separated(", "); - for process_uuid in &threadless_process_uuids { - separated.push_bind(*process_uuid); - } - } - over_limit_processes_query.push(") GROUP BY process_uuid HAVING SUM("); - over_limit_processes_query.push("estimated_bytes"); - over_limit_processes_query.push(") > "); - over_limit_processes_query.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); - let over_limit_process_uuids: Vec = over_limit_processes_query - .build() - .fetch_all(&mut *tx) - .await? - .into_iter() - .map(|row| row.try_get("process_uuid")) - .collect::>()?; - if !over_limit_process_uuids.is_empty() { - // Same strict cap policy as thread pruning, but only for - // threadless rows in the affected process UUIDs. - let mut prune_threadless_process_logs = QueryBuilder::::new( - r#" -DELETE FROM logs -WHERE id IN ( - SELECT id - FROM ( - SELECT - id, - SUM( -"#, - ); - prune_threadless_process_logs.push("estimated_bytes"); - prune_threadless_process_logs.push( - r#" - ) OVER ( - PARTITION BY process_uuid - ORDER BY ts DESC, ts_nanos DESC, id DESC - ) AS cumulative_bytes - FROM logs - WHERE thread_id IS NULL - AND process_uuid IN ( -"#, - ); - { - let mut separated = prune_threadless_process_logs.separated(", "); - for process_uuid in &over_limit_process_uuids { - separated.push_bind(process_uuid); - } - } - prune_threadless_process_logs.push( - r#" - ) - ) - WHERE cumulative_bytes > -"#, - ); - prune_threadless_process_logs.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); - prune_threadless_process_logs.push("\n)"); - prune_threadless_process_logs - .build() - .execute(&mut *tx) - .await?; - } - } - if has_threadless_null_process_uuid { - // Rows without a process UUID still need a cap; treat NULL as its - // own threadless partition. - let mut null_process_usage_query = QueryBuilder::::new("SELECT SUM("); - null_process_usage_query.push("estimated_bytes"); - null_process_usage_query.push( - ") AS total_bytes FROM logs WHERE thread_id IS NULL AND process_uuid IS NULL", - ); - let total_null_process_bytes: Option = null_process_usage_query - .build() - .fetch_one(&mut *tx) - .await? - .try_get("total_bytes")?; - - if total_null_process_bytes.unwrap_or(0) > LOG_PARTITION_SIZE_LIMIT_BYTES { - let mut prune_threadless_null_process_logs = QueryBuilder::::new( - r#" -DELETE FROM logs -WHERE id IN ( - SELECT id - FROM ( - SELECT - id, - SUM( -"#, - ); - prune_threadless_null_process_logs.push("estimated_bytes"); - prune_threadless_null_process_logs.push( - r#" - ) OVER ( - PARTITION BY process_uuid - ORDER BY ts DESC, ts_nanos DESC, id DESC - ) AS cumulative_bytes - FROM logs - WHERE thread_id IS NULL - AND process_uuid IS NULL - ) - WHERE cumulative_bytes > -"#, - ); - prune_threadless_null_process_logs.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); - prune_threadless_null_process_logs.push("\n)"); - prune_threadless_null_process_logs - .build() - .execute(&mut *tx) - .await?; - } - } - Ok(()) - } - - pub(crate) async fn delete_logs_before(&self, cutoff_ts: i64) -> anyhow::Result { - let result = sqlx::query("DELETE FROM logs WHERE ts < ?") - .bind(cutoff_ts) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected()) - } - - /// Query logs with optional filters. - pub async fn query_logs(&self, query: &LogQuery) -> anyhow::Result> { - let mut builder = QueryBuilder::::new( - "SELECT id, ts, ts_nanos, level, target, message, thread_id, process_uuid, file, line FROM logs WHERE 1 = 1", - ); - push_log_filters(&mut builder, query); - if query.descending { - builder.push(" ORDER BY id DESC"); - } else { - builder.push(" ORDER BY id ASC"); - } - if let Some(limit) = query.limit { - builder.push(" LIMIT ").push_bind(limit as i64); - } - - let rows = builder - .build_query_as::() - .fetch_all(self.pool.as_ref()) - .await?; - Ok(rows) - } - - /// Return the max log id matching optional filters. - pub async fn max_log_id(&self, query: &LogQuery) -> anyhow::Result { - let mut builder = - QueryBuilder::::new("SELECT MAX(id) AS max_id FROM logs WHERE 1 = 1"); - push_log_filters(&mut builder, query); - let row = builder.build().fetch_one(self.pool.as_ref()).await?; - let max_id: Option = row.try_get("max_id")?; - Ok(max_id.unwrap_or(0)) - } - - /// List thread ids using the underlying database (no rollout scanning). - pub async fn list_thread_ids( - &self, - limit: usize, - anchor: Option<&crate::Anchor>, - sort_key: crate::SortKey, - allowed_sources: &[String], - model_providers: Option<&[String]>, - archived_only: bool, - ) -> anyhow::Result> { - let mut builder = QueryBuilder::::new("SELECT id FROM threads"); - push_thread_filters( - &mut builder, - archived_only, - allowed_sources, - model_providers, - anchor, - sort_key, - None, - ); - push_thread_order_and_limit(&mut builder, sort_key, limit); - - let rows = builder.build().fetch_all(self.pool.as_ref()).await?; - rows.into_iter() - .map(|row| { - let id: String = row.try_get("id")?; - Ok(ThreadId::try_from(id)?) - }) - .collect() - } - - /// Insert or replace thread metadata directly. - pub async fn upsert_thread(&self, metadata: &crate::ThreadMetadata) -> anyhow::Result<()> { - sqlx::query( - r#" -INSERT INTO threads ( - id, - rollout_path, - created_at, - updated_at, - source, - agent_nickname, - agent_role, - model_provider, - cwd, - cli_version, - title, - sandbox_policy, - approval_mode, - tokens_used, - first_user_message, - archived, - archived_at, - git_sha, - git_branch, - git_origin_url -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -ON CONFLICT(id) DO UPDATE SET - rollout_path = excluded.rollout_path, - created_at = excluded.created_at, - updated_at = excluded.updated_at, - source = excluded.source, - agent_nickname = excluded.agent_nickname, - agent_role = excluded.agent_role, - model_provider = excluded.model_provider, - cwd = excluded.cwd, - cli_version = excluded.cli_version, - title = excluded.title, - sandbox_policy = excluded.sandbox_policy, - approval_mode = excluded.approval_mode, - tokens_used = excluded.tokens_used, - first_user_message = excluded.first_user_message, - archived = excluded.archived, - archived_at = excluded.archived_at, - git_sha = excluded.git_sha, - git_branch = excluded.git_branch, - git_origin_url = excluded.git_origin_url - "#, - ) - .bind(metadata.id.to_string()) - .bind(metadata.rollout_path.display().to_string()) - .bind(datetime_to_epoch_seconds(metadata.created_at)) - .bind(datetime_to_epoch_seconds(metadata.updated_at)) - .bind(metadata.source.as_str()) - .bind(metadata.agent_nickname.as_deref()) - .bind(metadata.agent_role.as_deref()) - .bind(metadata.model_provider.as_str()) - .bind(metadata.cwd.display().to_string()) - .bind(metadata.cli_version.as_str()) - .bind(metadata.title.as_str()) - .bind(metadata.sandbox_policy.as_str()) - .bind(metadata.approval_mode.as_str()) - .bind(metadata.tokens_used) - .bind(metadata.first_user_message.as_deref().unwrap_or_default()) - .bind(metadata.archived_at.is_some()) - .bind(metadata.archived_at.map(datetime_to_epoch_seconds)) - .bind(metadata.git_sha.as_deref()) - .bind(metadata.git_branch.as_deref()) - .bind(metadata.git_origin_url.as_deref()) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } - - /// Persist dynamic tools for a thread if none have been stored yet. - /// - /// Dynamic tools are defined at thread start and should not change afterward. - /// This only writes the first time we see tools for a given thread. - pub async fn persist_dynamic_tools( - &self, - thread_id: ThreadId, - tools: Option<&[DynamicToolSpec]>, - ) -> anyhow::Result<()> { - let Some(tools) = tools else { - return Ok(()); - }; - if tools.is_empty() { - return Ok(()); - } - let thread_id = thread_id.to_string(); - let mut tx = self.pool.begin().await?; - for (idx, tool) in tools.iter().enumerate() { - let position = i64::try_from(idx).unwrap_or(i64::MAX); - let input_schema = serde_json::to_string(&tool.input_schema)?; - sqlx::query( - r#" -INSERT INTO thread_dynamic_tools ( - thread_id, - position, - name, - description, - input_schema -) VALUES (?, ?, ?, ?, ?) -ON CONFLICT(thread_id, position) DO NOTHING - "#, - ) - .bind(thread_id.as_str()) - .bind(position) - .bind(tool.name.as_str()) - .bind(tool.description.as_str()) - .bind(input_schema) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; - Ok(()) - } - - /// Apply rollout items incrementally using the underlying database. - pub async fn apply_rollout_items( - &self, - builder: &ThreadMetadataBuilder, - items: &[RolloutItem], - otel: Option<&OtelManager>, - ) -> anyhow::Result<()> { - if items.is_empty() { - return Ok(()); - } - let mut metadata = self - .get_thread(builder.id) - .await? - .unwrap_or_else(|| builder.build(&self.default_provider)); - metadata.rollout_path = builder.rollout_path.clone(); - for item in items { - apply_rollout_item(&mut metadata, item, &self.default_provider); - } - if let Some(updated_at) = file_modified_time_utc(builder.rollout_path.as_path()).await { - metadata.updated_at = updated_at; - } - // Keep the thread upsert before dynamic tools to satisfy the foreign key constraint: - // thread_dynamic_tools.thread_id -> threads.id. - if let Err(err) = self.upsert_thread(&metadata).await { - if let Some(otel) = otel { - otel.counter(DB_ERROR_METRIC, 1, &[("stage", "apply_rollout_items")]); - } - return Err(err); - } - let dynamic_tools = extract_dynamic_tools(items); - if let Some(dynamic_tools) = dynamic_tools - && let Err(err) = self - .persist_dynamic_tools(builder.id, dynamic_tools.as_deref()) - .await - { - if let Some(otel) = otel { - otel.counter(DB_ERROR_METRIC, 1, &[("stage", "persist_dynamic_tools")]); - } - return Err(err); - } - Ok(()) - } - - /// Mark a thread as archived using the underlying database. - pub async fn mark_archived( - &self, - thread_id: ThreadId, - rollout_path: &Path, - archived_at: DateTime, - ) -> anyhow::Result<()> { - let Some(mut metadata) = self.get_thread(thread_id).await? else { - return Ok(()); - }; - metadata.archived_at = Some(archived_at); - metadata.rollout_path = rollout_path.to_path_buf(); - if let Some(updated_at) = file_modified_time_utc(rollout_path).await { - metadata.updated_at = updated_at; - } - if metadata.id != thread_id { - warn!( - "thread id mismatch during archive: expected {thread_id}, got {}", - metadata.id - ); - } - self.upsert_thread(&metadata).await - } - - /// Mark a thread as unarchived using the underlying database. - pub async fn mark_unarchived( - &self, - thread_id: ThreadId, - rollout_path: &Path, - ) -> anyhow::Result<()> { - let Some(mut metadata) = self.get_thread(thread_id).await? else { - return Ok(()); - }; - metadata.archived_at = None; - metadata.rollout_path = rollout_path.to_path_buf(); - if let Some(updated_at) = file_modified_time_utc(rollout_path).await { - metadata.updated_at = updated_at; - } - if metadata.id != thread_id { - warn!( - "thread id mismatch during unarchive: expected {thread_id}, got {}", - metadata.id - ); - } - self.upsert_thread(&metadata).await - } - - /// Delete a thread metadata row by id. - pub async fn delete_thread(&self, thread_id: ThreadId) -> anyhow::Result { - let result = sqlx::query("DELETE FROM threads WHERE id = ?") - .bind(thread_id.to_string()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected()) - } - - pub async fn create_agent_job( - &self, - params: &AgentJobCreateParams, - items: &[AgentJobItemCreateParams], - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let input_headers_json = serde_json::to_string(¶ms.input_headers)?; - let output_schema_json = params - .output_schema_json - .as_ref() - .map(serde_json::to_string) - .transpose()?; - let max_runtime_seconds = params - .max_runtime_seconds - .map(i64::try_from) - .transpose() - .map_err(|_| anyhow::anyhow!("invalid max_runtime_seconds value"))?; - let mut tx = self.pool.begin().await?; - sqlx::query( - r#" -INSERT INTO agent_jobs ( - id, - name, - status, - instruction, - auto_export, - max_runtime_seconds, - output_schema_json, - input_headers_json, - input_csv_path, - output_csv_path, - created_at, - updated_at, - started_at, - completed_at, - last_error -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL) - "#, - ) - .bind(params.id.as_str()) - .bind(params.name.as_str()) - .bind(AgentJobStatus::Pending.as_str()) - .bind(params.instruction.as_str()) - .bind(i64::from(params.auto_export)) - .bind(max_runtime_seconds) - .bind(output_schema_json) - .bind(input_headers_json) - .bind(params.input_csv_path.as_str()) - .bind(params.output_csv_path.as_str()) - .bind(now) - .bind(now) - .execute(&mut *tx) - .await?; - - for item in items { - let row_json = serde_json::to_string(&item.row_json)?; - sqlx::query( - r#" -INSERT INTO agent_job_items ( - job_id, - item_id, - row_index, - source_id, - row_json, - status, - assigned_thread_id, - attempt_count, - result_json, - last_error, - created_at, - updated_at, - completed_at, - reported_at -) VALUES (?, ?, ?, ?, ?, ?, NULL, 0, NULL, NULL, ?, ?, NULL, NULL) - "#, - ) - .bind(params.id.as_str()) - .bind(item.item_id.as_str()) - .bind(item.row_index) - .bind(item.source_id.as_deref()) - .bind(row_json) - .bind(AgentJobItemStatus::Pending.as_str()) - .bind(now) - .bind(now) - .execute(&mut *tx) - .await?; - } - - tx.commit().await?; - - let job_id = params.id.as_str(); - self.get_agent_job(job_id) - .await? - .ok_or_else(|| anyhow::anyhow!("failed to load created agent job {job_id}")) - } - - pub async fn get_agent_job(&self, job_id: &str) -> anyhow::Result> { - let row = sqlx::query_as::<_, AgentJobRow>( - r#" -SELECT - id, - name, - status, - instruction, - auto_export, - max_runtime_seconds, - output_schema_json, - input_headers_json, - input_csv_path, - output_csv_path, - created_at, - updated_at, - started_at, - completed_at, - last_error -FROM agent_jobs -WHERE id = ? - "#, - ) - .bind(job_id) - .fetch_optional(self.pool.as_ref()) - .await?; - row.map(AgentJob::try_from).transpose() - } - - pub async fn list_agent_job_items( - &self, - job_id: &str, - status: Option, - limit: Option, - ) -> anyhow::Result> { - let mut builder = QueryBuilder::::new( - r#" -SELECT - job_id, - item_id, - row_index, - source_id, - row_json, - status, - assigned_thread_id, - attempt_count, - result_json, - last_error, - created_at, - updated_at, - completed_at, - reported_at -FROM agent_job_items -WHERE job_id = - "#, - ); - builder.push_bind(job_id); - if let Some(status) = status { - builder.push(" AND status = "); - builder.push_bind(status.as_str()); - } - builder.push(" ORDER BY row_index ASC"); - if let Some(limit) = limit { - builder.push(" LIMIT "); - builder.push_bind(limit as i64); - } - let rows = builder - .build_query_as::() - .fetch_all(self.pool.as_ref()) - .await?; - rows.into_iter().map(AgentJobItem::try_from).collect() - } - - pub async fn get_agent_job_item( - &self, - job_id: &str, - item_id: &str, - ) -> anyhow::Result> { - let row = sqlx::query_as::<_, AgentJobItemRow>( - r#" -SELECT - job_id, - item_id, - row_index, - source_id, - row_json, - status, - assigned_thread_id, - attempt_count, - result_json, - last_error, - created_at, - updated_at, - completed_at, - reported_at -FROM agent_job_items -WHERE job_id = ? AND item_id = ? - "#, - ) - .bind(job_id) - .bind(item_id) - .fetch_optional(self.pool.as_ref()) - .await?; - row.map(AgentJobItem::try_from).transpose() - } - - pub async fn mark_agent_job_running(&self, job_id: &str) -> anyhow::Result<()> { - let now = Utc::now().timestamp(); - sqlx::query( - r#" -UPDATE agent_jobs -SET - status = ?, - updated_at = ?, - started_at = COALESCE(started_at, ?), - completed_at = NULL, - last_error = NULL -WHERE id = ? - "#, - ) - .bind(AgentJobStatus::Running.as_str()) - .bind(now) - .bind(now) - .bind(job_id) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } - - pub async fn mark_agent_job_completed(&self, job_id: &str) -> anyhow::Result<()> { - let now = Utc::now().timestamp(); - sqlx::query( - r#" -UPDATE agent_jobs -SET status = ?, updated_at = ?, completed_at = ?, last_error = NULL -WHERE id = ? - "#, - ) - .bind(AgentJobStatus::Completed.as_str()) - .bind(now) - .bind(now) - .bind(job_id) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } - - pub async fn mark_agent_job_failed( - &self, - job_id: &str, - error_message: &str, - ) -> anyhow::Result<()> { - let now = Utc::now().timestamp(); - sqlx::query( - r#" -UPDATE agent_jobs -SET status = ?, updated_at = ?, completed_at = ?, last_error = ? -WHERE id = ? - "#, - ) - .bind(AgentJobStatus::Failed.as_str()) - .bind(now) - .bind(now) - .bind(error_message) - .bind(job_id) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } - - pub async fn mark_agent_job_cancelled( - &self, - job_id: &str, - reason: &str, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let result = sqlx::query( - r#" -UPDATE agent_jobs -SET status = ?, updated_at = ?, completed_at = ?, last_error = ? -WHERE id = ? AND status IN (?, ?) - "#, - ) - .bind(AgentJobStatus::Cancelled.as_str()) - .bind(now) - .bind(now) - .bind(reason) - .bind(job_id) - .bind(AgentJobStatus::Pending.as_str()) - .bind(AgentJobStatus::Running.as_str()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn is_agent_job_cancelled(&self, job_id: &str) -> anyhow::Result { - let row = sqlx::query( - r#" -SELECT status -FROM agent_jobs -WHERE id = ? - "#, - ) - .bind(job_id) - .fetch_optional(self.pool.as_ref()) - .await?; - let Some(row) = row else { - return Ok(false); - }; - let status: String = row.try_get("status")?; - Ok(AgentJobStatus::parse(status.as_str())? == AgentJobStatus::Cancelled) - } - - pub async fn mark_agent_job_item_running( - &self, - job_id: &str, - item_id: &str, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let result = sqlx::query( - r#" -UPDATE agent_job_items -SET - status = ?, - assigned_thread_id = NULL, - attempt_count = attempt_count + 1, - updated_at = ?, - last_error = NULL -WHERE job_id = ? AND item_id = ? AND status = ? - "#, - ) - .bind(AgentJobItemStatus::Running.as_str()) - .bind(now) - .bind(job_id) - .bind(item_id) - .bind(AgentJobItemStatus::Pending.as_str()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn mark_agent_job_item_running_with_thread( - &self, - job_id: &str, - item_id: &str, - thread_id: &str, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let result = sqlx::query( - r#" -UPDATE agent_job_items -SET - status = ?, - assigned_thread_id = ?, - attempt_count = attempt_count + 1, - updated_at = ?, - last_error = NULL -WHERE job_id = ? AND item_id = ? AND status = ? - "#, - ) - .bind(AgentJobItemStatus::Running.as_str()) - .bind(thread_id) - .bind(now) - .bind(job_id) - .bind(item_id) - .bind(AgentJobItemStatus::Pending.as_str()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn mark_agent_job_item_pending( - &self, - job_id: &str, - item_id: &str, - error_message: Option<&str>, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let result = sqlx::query( - r#" -UPDATE agent_job_items -SET - status = ?, - assigned_thread_id = NULL, - updated_at = ?, - last_error = ? -WHERE job_id = ? AND item_id = ? AND status = ? - "#, - ) - .bind(AgentJobItemStatus::Pending.as_str()) - .bind(now) - .bind(error_message) - .bind(job_id) - .bind(item_id) - .bind(AgentJobItemStatus::Running.as_str()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn set_agent_job_item_thread( - &self, - job_id: &str, - item_id: &str, - thread_id: &str, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let result = sqlx::query( - r#" -UPDATE agent_job_items -SET assigned_thread_id = ?, updated_at = ? -WHERE job_id = ? AND item_id = ? AND status = ? - "#, - ) - .bind(thread_id) - .bind(now) - .bind(job_id) - .bind(item_id) - .bind(AgentJobItemStatus::Running.as_str()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn report_agent_job_item_result( - &self, - job_id: &str, - item_id: &str, - reporting_thread_id: &str, - result_json: &Value, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let serialized = serde_json::to_string(result_json)?; - let result = sqlx::query( - r#" -UPDATE agent_job_items -SET - result_json = ?, - reported_at = ?, - updated_at = ?, - last_error = NULL -WHERE - job_id = ? - AND item_id = ? - AND status = ? - AND assigned_thread_id = ? - "#, - ) - .bind(serialized) - .bind(now) - .bind(now) - .bind(job_id) - .bind(item_id) - .bind(AgentJobItemStatus::Running.as_str()) - .bind(reporting_thread_id) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn mark_agent_job_item_completed( - &self, - job_id: &str, - item_id: &str, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let result = sqlx::query( - r#" -UPDATE agent_job_items -SET - status = ?, - completed_at = ?, - updated_at = ?, - assigned_thread_id = NULL -WHERE - job_id = ? - AND item_id = ? - AND status = ? - AND result_json IS NOT NULL - "#, - ) - .bind(AgentJobItemStatus::Completed.as_str()) - .bind(now) - .bind(now) - .bind(job_id) - .bind(item_id) - .bind(AgentJobItemStatus::Running.as_str()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn mark_agent_job_item_failed( - &self, - job_id: &str, - item_id: &str, - error_message: &str, - ) -> anyhow::Result { - let now = Utc::now().timestamp(); - let result = sqlx::query( - r#" -UPDATE agent_job_items -SET - status = ?, - completed_at = ?, - updated_at = ?, - last_error = ?, - assigned_thread_id = NULL -WHERE - job_id = ? - AND item_id = ? - AND status = ? - "#, - ) - .bind(AgentJobItemStatus::Failed.as_str()) - .bind(now) - .bind(now) - .bind(error_message) - .bind(job_id) - .bind(item_id) - .bind(AgentJobItemStatus::Running.as_str()) - .execute(self.pool.as_ref()) - .await?; - Ok(result.rows_affected() > 0) - } - - pub async fn get_agent_job_progress(&self, job_id: &str) -> anyhow::Result { - let row = sqlx::query( - r#" -SELECT - COUNT(*) AS total_items, - SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS pending_items, - SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS running_items, - SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS completed_items, - SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS failed_items -FROM agent_job_items -WHERE job_id = ? - "#, - ) - .bind(AgentJobItemStatus::Pending.as_str()) - .bind(AgentJobItemStatus::Running.as_str()) - .bind(AgentJobItemStatus::Completed.as_str()) - .bind(AgentJobItemStatus::Failed.as_str()) - .bind(job_id) - .fetch_one(self.pool.as_ref()) - .await?; - - let total_items: i64 = row.try_get("total_items")?; - let pending_items: Option = row.try_get("pending_items")?; - let running_items: Option = row.try_get("running_items")?; - let completed_items: Option = row.try_get("completed_items")?; - let failed_items: Option = row.try_get("failed_items")?; - Ok(AgentJobProgress { - total_items: usize::try_from(total_items).unwrap_or_default(), - pending_items: usize::try_from(pending_items.unwrap_or_default()).unwrap_or_default(), - running_items: usize::try_from(running_items.unwrap_or_default()).unwrap_or_default(), - completed_items: usize::try_from(completed_items.unwrap_or_default()) - .unwrap_or_default(), - failed_items: usize::try_from(failed_items.unwrap_or_default()).unwrap_or_default(), - }) - } - - async fn ensure_backfill_state_row(&self) -> anyhow::Result<()> { - sqlx::query( - r#" -INSERT INTO backfill_state (id, status, last_watermark, last_success_at, updated_at) -VALUES (?, ?, NULL, NULL, ?) -ON CONFLICT(id) DO NOTHING - "#, - ) - .bind(1_i64) - .bind(crate::BackfillStatus::Pending.as_str()) - .bind(Utc::now().timestamp()) - .execute(self.pool.as_ref()) - .await?; - Ok(()) - } -} - -fn push_log_filters<'a>(builder: &mut QueryBuilder<'a, Sqlite>, query: &'a LogQuery) { - if let Some(level_upper) = query.level_upper.as_ref() { - builder - .push(" AND UPPER(level) = ") - .push_bind(level_upper.as_str()); - } - if let Some(from_ts) = query.from_ts { - builder.push(" AND ts >= ").push_bind(from_ts); - } - if let Some(to_ts) = query.to_ts { - builder.push(" AND ts <= ").push_bind(to_ts); - } - push_like_filters(builder, "module_path", &query.module_like); - push_like_filters(builder, "file", &query.file_like); - let has_thread_filter = !query.thread_ids.is_empty() || query.include_threadless; - if has_thread_filter { - builder.push(" AND ("); - let mut needs_or = false; - for thread_id in &query.thread_ids { - if needs_or { - builder.push(" OR "); - } - builder.push("thread_id = ").push_bind(thread_id.as_str()); - needs_or = true; - } - if query.include_threadless { - if needs_or { - builder.push(" OR "); - } - builder.push("thread_id IS NULL"); - } - builder.push(")"); - } - if let Some(after_id) = query.after_id { - builder.push(" AND id > ").push_bind(after_id); - } - if let Some(search) = query.search.as_ref() { - builder.push(" AND INSTR(message, "); - builder.push_bind(search.as_str()); - builder.push(") > 0"); - } -} - -fn push_like_filters<'a>( - builder: &mut QueryBuilder<'a, Sqlite>, - column: &str, - filters: &'a [String], -) { - if filters.is_empty() { - return; - } - builder.push(" AND ("); - for (idx, filter) in filters.iter().enumerate() { - if idx > 0 { - builder.push(" OR "); - } - builder - .push(column) - .push(" LIKE '%' || ") - .push_bind(filter.as_str()) - .push(" || '%'"); - } - builder.push(")"); -} - -fn extract_dynamic_tools(items: &[RolloutItem]) -> Option>> { - items.iter().find_map(|item| match item { - RolloutItem::SessionMeta(meta_line) => Some(meta_line.meta.dynamic_tools.clone()), - RolloutItem::ResponseItem(_) - | RolloutItem::Compacted(_) - | RolloutItem::TurnContext(_) - | RolloutItem::EventMsg(_) => None, - }) } async fn open_sqlite(path: &Path) -> anyhow::Result { @@ -1650,2254 +198,3 @@ fn should_remove_state_file(file_name: &str, current_name: &str) -> bool { }; !version_suffix.is_empty() && version_suffix.chars().all(|ch| ch.is_ascii_digit()) } - -fn push_thread_filters<'a>( - builder: &mut QueryBuilder<'a, Sqlite>, - archived_only: bool, - allowed_sources: &'a [String], - model_providers: Option<&'a [String]>, - anchor: Option<&crate::Anchor>, - sort_key: SortKey, - search_term: Option<&'a str>, -) { - builder.push(" WHERE 1 = 1"); - if archived_only { - builder.push(" AND archived = 1"); - } else { - builder.push(" AND archived = 0"); - } - builder.push(" AND first_user_message <> ''"); - if !allowed_sources.is_empty() { - builder.push(" AND source IN ("); - let mut separated = builder.separated(", "); - for source in allowed_sources { - separated.push_bind(source); - } - separated.push_unseparated(")"); - } - if let Some(model_providers) = model_providers - && !model_providers.is_empty() - { - builder.push(" AND model_provider IN ("); - let mut separated = builder.separated(", "); - for provider in model_providers { - separated.push_bind(provider); - } - separated.push_unseparated(")"); - } - if let Some(search_term) = search_term { - builder.push(" AND instr(title, "); - builder.push_bind(search_term); - builder.push(") > 0"); - } - if let Some(anchor) = anchor { - let anchor_ts = datetime_to_epoch_seconds(anchor.ts); - let column = match sort_key { - SortKey::CreatedAt => "created_at", - SortKey::UpdatedAt => "updated_at", - }; - builder.push(" AND ("); - builder.push(column); - builder.push(" < "); - builder.push_bind(anchor_ts); - builder.push(" OR ("); - builder.push(column); - builder.push(" = "); - builder.push_bind(anchor_ts); - builder.push(" AND id < "); - builder.push_bind(anchor.id.to_string()); - builder.push("))"); - } -} - -fn push_thread_order_and_limit( - builder: &mut QueryBuilder<'_, Sqlite>, - sort_key: SortKey, - limit: usize, -) { - let order_column = match sort_key { - SortKey::CreatedAt => "created_at", - SortKey::UpdatedAt => "updated_at", - }; - builder.push(" ORDER BY "); - builder.push(order_column); - builder.push(" DESC, id DESC"); - builder.push(" LIMIT "); - builder.push_bind(limit as i64); -} - -#[cfg(test)] -mod tests { - use super::StateRuntime; - use super::ThreadMetadata; - use super::state_db_filename; - use crate::LogEntry; - use crate::LogQuery; - use crate::STATE_DB_FILENAME; - use crate::STATE_DB_VERSION; - use crate::model::Phase2JobClaimOutcome; - use crate::model::Stage1JobClaimOutcome; - use crate::model::Stage1StartupClaimParams; - use chrono::DateTime; - use chrono::Duration; - use chrono::Utc; - use codex_protocol::ThreadId; - use codex_protocol::protocol::AskForApproval; - use codex_protocol::protocol::SandboxPolicy; - use pretty_assertions::assert_eq; - use sqlx::Row; - use std::path::Path; - use std::path::PathBuf; - use std::sync::Arc; - use std::time::SystemTime; - use std::time::UNIX_EPOCH; - use uuid::Uuid; - - fn unique_temp_dir() -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |duration| duration.as_nanos()); - std::env::temp_dir().join(format!( - "codex-state-runtime-test-{nanos}-{}", - Uuid::new_v4() - )) - } - - #[tokio::test] - async fn init_removes_legacy_state_db_files() { - let codex_home = unique_temp_dir(); - tokio::fs::create_dir_all(&codex_home) - .await - .expect("create codex_home"); - - let current_name = state_db_filename(); - let previous_version = STATE_DB_VERSION.saturating_sub(1); - let unversioned_name = format!("{STATE_DB_FILENAME}.sqlite"); - for suffix in ["", "-wal", "-shm", "-journal"] { - let path = codex_home.join(format!("{unversioned_name}{suffix}")); - tokio::fs::write(path, b"legacy") - .await - .expect("write legacy"); - let old_version_path = codex_home.join(format!( - "{STATE_DB_FILENAME}_{previous_version}.sqlite{suffix}" - )); - tokio::fs::write(old_version_path, b"old_version") - .await - .expect("write old version"); - } - let unrelated_path = codex_home.join("state.sqlite_backup"); - tokio::fs::write(&unrelated_path, b"keep") - .await - .expect("write unrelated"); - let numeric_path = codex_home.join("123"); - tokio::fs::write(&numeric_path, b"keep") - .await - .expect("write numeric"); - - let _runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - for suffix in ["", "-wal", "-shm", "-journal"] { - let legacy_path = codex_home.join(format!("{unversioned_name}{suffix}")); - assert_eq!( - tokio::fs::try_exists(&legacy_path) - .await - .expect("check legacy path"), - false - ); - let old_version_path = codex_home.join(format!( - "{STATE_DB_FILENAME}_{previous_version}.sqlite{suffix}" - )); - assert_eq!( - tokio::fs::try_exists(&old_version_path) - .await - .expect("check old version path"), - false - ); - } - assert_eq!( - tokio::fs::try_exists(codex_home.join(current_name)) - .await - .expect("check new db path"), - true - ); - assert_eq!( - tokio::fs::try_exists(&unrelated_path) - .await - .expect("check unrelated path"), - true - ); - assert_eq!( - tokio::fs::try_exists(&numeric_path) - .await - .expect("check numeric path"), - true - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn backfill_state_persists_progress_and_completion() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let initial = runtime - .get_backfill_state() - .await - .expect("get initial backfill state"); - assert_eq!(initial.status, crate::BackfillStatus::Pending); - assert_eq!(initial.last_watermark, None); - assert_eq!(initial.last_success_at, None); - - runtime - .mark_backfill_running() - .await - .expect("mark backfill running"); - runtime - .checkpoint_backfill("sessions/2026/01/27/rollout-a.jsonl") - .await - .expect("checkpoint backfill"); - - let running = runtime - .get_backfill_state() - .await - .expect("get running backfill state"); - assert_eq!(running.status, crate::BackfillStatus::Running); - assert_eq!( - running.last_watermark, - Some("sessions/2026/01/27/rollout-a.jsonl".to_string()) - ); - assert_eq!(running.last_success_at, None); - - runtime - .mark_backfill_complete(Some("sessions/2026/01/28/rollout-b.jsonl")) - .await - .expect("mark backfill complete"); - let completed = runtime - .get_backfill_state() - .await - .expect("get completed backfill state"); - assert_eq!(completed.status, crate::BackfillStatus::Complete); - assert_eq!( - completed.last_watermark, - Some("sessions/2026/01/28/rollout-b.jsonl".to_string()) - ); - assert!(completed.last_success_at.is_some()); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn backfill_claim_is_singleton_until_stale_and_blocked_when_complete() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let claimed = runtime - .try_claim_backfill(3600) - .await - .expect("initial backfill claim"); - assert_eq!(claimed, true); - - let duplicate_claim = runtime - .try_claim_backfill(3600) - .await - .expect("duplicate backfill claim"); - assert_eq!(duplicate_claim, false); - - let stale_updated_at = Utc::now().timestamp().saturating_sub(10_000); - sqlx::query( - r#" -UPDATE backfill_state -SET status = ?, updated_at = ? -WHERE id = 1 - "#, - ) - .bind(crate::BackfillStatus::Running.as_str()) - .bind(stale_updated_at) - .execute(runtime.pool.as_ref()) - .await - .expect("force stale backfill lease"); - - let stale_claim = runtime - .try_claim_backfill(10) - .await - .expect("stale backfill claim"); - assert_eq!(stale_claim, true); - - runtime - .mark_backfill_complete(None) - .await - .expect("mark complete"); - let claim_after_complete = runtime - .try_claim_backfill(3600) - .await - .expect("claim after complete"); - assert_eq!(claim_after_complete, false); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn stage1_claim_skips_when_up_to_date() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let metadata = test_thread_metadata(&codex_home, thread_id, codex_home.join("a")); - runtime - .upsert_thread(&metadata) - .await - .expect("upsert thread"); - - let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - - let claim = runtime - .try_claim_stage1_job(thread_id, owner_a, 100, 3600, 64) - .await - .expect("claim stage1 job"); - let ownership_token = match claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected claim outcome: {other:?}"), - }; - - assert!( - runtime - .mark_stage1_job_succeeded( - thread_id, - ownership_token.as_str(), - 100, - "raw", - "sum", - None, - ) - .await - .expect("mark stage1 succeeded"), - "stage1 success should finalize for current token" - ); - - let up_to_date = runtime - .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) - .await - .expect("claim stage1 up-to-date"); - assert_eq!(up_to_date, Stage1JobClaimOutcome::SkippedUpToDate); - - let needs_rerun = runtime - .try_claim_stage1_job(thread_id, owner_b, 101, 3600, 64) - .await - .expect("claim stage1 newer source"); - assert!( - matches!(needs_rerun, Stage1JobClaimOutcome::Claimed { .. }), - "newer source_updated_at should be claimable" - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn stage1_running_stale_can_be_stolen_but_fresh_running_is_skipped() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let cwd = codex_home.join("workspace"); - runtime - .upsert_thread(&test_thread_metadata(&codex_home, thread_id, cwd)) - .await - .expect("upsert thread"); - - let claim_a = runtime - .try_claim_stage1_job(thread_id, owner_a, 100, 3600, 64) - .await - .expect("claim a"); - assert!(matches!(claim_a, Stage1JobClaimOutcome::Claimed { .. })); - - let claim_b_fresh = runtime - .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) - .await - .expect("claim b fresh"); - assert_eq!(claim_b_fresh, Stage1JobClaimOutcome::SkippedRunning); - - sqlx::query("UPDATE jobs SET lease_until = 0 WHERE kind = 'memory_stage1' AND job_key = ?") - .bind(thread_id.to_string()) - .execute(runtime.pool.as_ref()) - .await - .expect("force stale lease"); - - let claim_b_stale = runtime - .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) - .await - .expect("claim b stale"); - assert!(matches!( - claim_b_stale, - Stage1JobClaimOutcome::Claimed { .. } - )); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn stage1_concurrent_claim_for_same_thread_is_conflict_safe() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id, - codex_home.join("workspace"), - )) - .await - .expect("upsert thread"); - - let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let thread_id_a = thread_id; - let thread_id_b = thread_id; - let runtime_a = Arc::clone(&runtime); - let runtime_b = Arc::clone(&runtime); - let claim_with_retry = |runtime: Arc, - thread_id: ThreadId, - owner: ThreadId| async move { - for attempt in 0..5 { - match runtime - .try_claim_stage1_job(thread_id, owner, 100, 3_600, 64) - .await - { - Ok(outcome) => return outcome, - Err(err) if err.to_string().contains("database is locked") && attempt < 4 => { - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - Err(err) => panic!("claim stage1 should not fail: {err}"), - } - } - panic!("claim stage1 should have returned within retry budget") - }; - - let (claim_a, claim_b) = tokio::join!( - claim_with_retry(runtime_a, thread_id_a, owner_a), - claim_with_retry(runtime_b, thread_id_b, owner_b), - ); - - let claim_outcomes = vec![claim_a, claim_b]; - let claimed_count = claim_outcomes - .iter() - .filter(|outcome| matches!(outcome, Stage1JobClaimOutcome::Claimed { .. })) - .count(); - assert_eq!(claimed_count, 1); - assert!( - claim_outcomes.iter().all(|outcome| { - matches!( - outcome, - Stage1JobClaimOutcome::Claimed { .. } | Stage1JobClaimOutcome::SkippedRunning - ) - }), - "unexpected claim outcomes: {claim_outcomes:?}" - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn stage1_concurrent_claims_respect_running_cap() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_a, - codex_home.join("workspace-a"), - )) - .await - .expect("upsert thread a"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_b, - codex_home.join("workspace-b"), - )) - .await - .expect("upsert thread b"); - - let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let runtime_a = Arc::clone(&runtime); - let runtime_b = Arc::clone(&runtime); - - let (claim_a, claim_b) = tokio::join!( - async move { - runtime_a - .try_claim_stage1_job(thread_a, owner_a, 100, 3_600, 1) - .await - .expect("claim stage1 thread a") - }, - async move { - runtime_b - .try_claim_stage1_job(thread_b, owner_b, 101, 3_600, 1) - .await - .expect("claim stage1 thread b") - }, - ); - - let claim_outcomes = vec![claim_a, claim_b]; - let claimed_count = claim_outcomes - .iter() - .filter(|outcome| matches!(outcome, Stage1JobClaimOutcome::Claimed { .. })) - .count(); - assert_eq!(claimed_count, 1); - assert!( - claim_outcomes - .iter() - .any(|outcome| { matches!(outcome, Stage1JobClaimOutcome::SkippedRunning) }), - "one concurrent claim should be throttled by running cap: {claim_outcomes:?}" - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn claim_stage1_jobs_filters_by_age_idle_and_current_thread() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let now = Utc::now(); - let fresh_at = now - Duration::hours(1); - let just_under_idle_at = now - Duration::hours(12) + Duration::minutes(1); - let eligible_idle_at = now - Duration::hours(12) - Duration::minutes(1); - let old_at = now - Duration::days(31); - - let current_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); - let fresh_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("fresh thread id"); - let just_under_idle_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("just under idle thread id"); - let eligible_idle_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("eligible idle thread id"); - let old_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("old thread id"); - - let mut current = - test_thread_metadata(&codex_home, current_thread_id, codex_home.join("current")); - current.created_at = now; - current.updated_at = now; - runtime - .upsert_thread(¤t) - .await - .expect("upsert current"); - - let mut fresh = - test_thread_metadata(&codex_home, fresh_thread_id, codex_home.join("fresh")); - fresh.created_at = fresh_at; - fresh.updated_at = fresh_at; - runtime.upsert_thread(&fresh).await.expect("upsert fresh"); - - let mut just_under_idle = test_thread_metadata( - &codex_home, - just_under_idle_thread_id, - codex_home.join("just-under-idle"), - ); - just_under_idle.created_at = just_under_idle_at; - just_under_idle.updated_at = just_under_idle_at; - runtime - .upsert_thread(&just_under_idle) - .await - .expect("upsert just-under-idle"); - - let mut eligible_idle = test_thread_metadata( - &codex_home, - eligible_idle_thread_id, - codex_home.join("eligible-idle"), - ); - eligible_idle.created_at = eligible_idle_at; - eligible_idle.updated_at = eligible_idle_at; - runtime - .upsert_thread(&eligible_idle) - .await - .expect("upsert eligible-idle"); - - let mut old = test_thread_metadata(&codex_home, old_thread_id, codex_home.join("old")); - old.created_at = old_at; - old.updated_at = old_at; - runtime.upsert_thread(&old).await.expect("upsert old"); - - let allowed_sources = vec!["cli".to_string()]; - let claims = runtime - .claim_stage1_jobs_for_startup( - current_thread_id, - Stage1StartupClaimParams { - scan_limit: 1, - max_claimed: 5, - max_age_days: 30, - min_rollout_idle_hours: 12, - allowed_sources: allowed_sources.as_slice(), - lease_seconds: 3600, - }, - ) - .await - .expect("claim stage1 jobs"); - - assert_eq!(claims.len(), 1); - assert_eq!(claims[0].thread.id, eligible_idle_thread_id); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn claim_stage1_jobs_prefilters_threads_with_up_to_date_memory() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let now = Utc::now(); - let eligible_newer_at = now - Duration::hours(13); - let eligible_older_at = now - Duration::hours(14); - - let current_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); - let up_to_date_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("up-to-date thread id"); - let stale_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("stale thread id"); - let worker_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("worker id"); - - let mut current = - test_thread_metadata(&codex_home, current_thread_id, codex_home.join("current")); - current.created_at = now; - current.updated_at = now; - runtime - .upsert_thread(¤t) - .await - .expect("upsert current thread"); - - let mut up_to_date = test_thread_metadata( - &codex_home, - up_to_date_thread_id, - codex_home.join("up-to-date"), - ); - up_to_date.created_at = eligible_newer_at; - up_to_date.updated_at = eligible_newer_at; - runtime - .upsert_thread(&up_to_date) - .await - .expect("upsert up-to-date thread"); - - let up_to_date_claim = runtime - .try_claim_stage1_job( - up_to_date_thread_id, - worker_id, - up_to_date.updated_at.timestamp(), - 3600, - 64, - ) - .await - .expect("claim up-to-date thread for seed"); - let up_to_date_token = match up_to_date_claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected seed claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded( - up_to_date_thread_id, - up_to_date_token.as_str(), - up_to_date.updated_at.timestamp(), - "raw", - "summary", - None, - ) - .await - .expect("mark up-to-date thread succeeded"), - "seed stage1 success should complete for up-to-date thread" - ); - - let mut stale = - test_thread_metadata(&codex_home, stale_thread_id, codex_home.join("stale")); - stale.created_at = eligible_older_at; - stale.updated_at = eligible_older_at; - runtime - .upsert_thread(&stale) - .await - .expect("upsert stale thread"); - - let allowed_sources = vec!["cli".to_string()]; - let claims = runtime - .claim_stage1_jobs_for_startup( - current_thread_id, - Stage1StartupClaimParams { - scan_limit: 1, - max_claimed: 1, - max_age_days: 30, - min_rollout_idle_hours: 12, - allowed_sources: allowed_sources.as_slice(), - lease_seconds: 3600, - }, - ) - .await - .expect("claim stage1 startup jobs"); - assert_eq!(claims.len(), 1); - assert_eq!(claims[0].thread.id, stale_thread_id); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn claim_stage1_jobs_enforces_global_running_cap() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let current_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - current_thread_id, - codex_home.join("current"), - )) - .await - .expect("upsert current"); - - let now = Utc::now(); - let started_at = now.timestamp(); - let lease_until = started_at + 3600; - let eligible_at = now - Duration::hours(13); - let existing_running = 10usize; - let total_candidates = 80usize; - - for idx in 0..total_candidates { - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let mut metadata = test_thread_metadata( - &codex_home, - thread_id, - codex_home.join(format!("thread-{idx}")), - ); - metadata.created_at = eligible_at - Duration::seconds(idx as i64); - metadata.updated_at = eligible_at - Duration::seconds(idx as i64); - runtime - .upsert_thread(&metadata) - .await - .expect("upsert thread"); - - if idx < existing_running { - sqlx::query( - r#" -INSERT INTO jobs ( - kind, - job_key, - status, - worker_id, - ownership_token, - started_at, - finished_at, - lease_until, - retry_at, - retry_remaining, - last_error, - input_watermark, - last_success_watermark -) VALUES (?, ?, 'running', ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, NULL) - "#, - ) - .bind("memory_stage1") - .bind(thread_id.to_string()) - .bind(current_thread_id.to_string()) - .bind(Uuid::new_v4().to_string()) - .bind(started_at) - .bind(lease_until) - .bind(3) - .bind(metadata.updated_at.timestamp()) - .execute(runtime.pool.as_ref()) - .await - .expect("seed running stage1 job"); - } - } - - let allowed_sources = vec!["cli".to_string()]; - let claims = runtime - .claim_stage1_jobs_for_startup( - current_thread_id, - Stage1StartupClaimParams { - scan_limit: 200, - max_claimed: 64, - max_age_days: 30, - min_rollout_idle_hours: 12, - allowed_sources: allowed_sources.as_slice(), - lease_seconds: 3600, - }, - ) - .await - .expect("claim stage1 jobs"); - assert_eq!(claims.len(), 54); - - let running_count = sqlx::query( - r#" -SELECT COUNT(*) AS count -FROM jobs -WHERE kind = 'memory_stage1' - AND status = 'running' - AND lease_until IS NOT NULL - AND lease_until > ? - "#, - ) - .bind(Utc::now().timestamp()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("count running stage1 jobs") - .try_get::("count") - .expect("running count value"); - assert_eq!(running_count, 64); - - let more_claims = runtime - .claim_stage1_jobs_for_startup( - current_thread_id, - Stage1StartupClaimParams { - scan_limit: 200, - max_claimed: 64, - max_age_days: 30, - min_rollout_idle_hours: 12, - allowed_sources: allowed_sources.as_slice(), - lease_seconds: 3600, - }, - ) - .await - .expect("claim stage1 jobs with cap reached"); - assert_eq!(more_claims.len(), 0); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn claim_stage1_jobs_processes_two_full_batches_across_startup_passes() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let current_thread_id = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); - let mut current = - test_thread_metadata(&codex_home, current_thread_id, codex_home.join("current")); - current.created_at = Utc::now(); - current.updated_at = Utc::now(); - runtime - .upsert_thread(¤t) - .await - .expect("upsert current"); - - let eligible_at = Utc::now() - Duration::hours(13); - for idx in 0..200 { - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let mut metadata = test_thread_metadata( - &codex_home, - thread_id, - codex_home.join(format!("thread-{idx}")), - ); - metadata.created_at = eligible_at - Duration::seconds(idx as i64); - metadata.updated_at = eligible_at - Duration::seconds(idx as i64); - runtime - .upsert_thread(&metadata) - .await - .expect("upsert eligible thread"); - } - - let allowed_sources = vec!["cli".to_string()]; - let first_claims = runtime - .claim_stage1_jobs_for_startup( - current_thread_id, - Stage1StartupClaimParams { - scan_limit: 5_000, - max_claimed: 64, - max_age_days: 30, - min_rollout_idle_hours: 12, - allowed_sources: allowed_sources.as_slice(), - lease_seconds: 3_600, - }, - ) - .await - .expect("first stage1 startup claim"); - assert_eq!(first_claims.len(), 64); - - for claim in first_claims { - assert!( - runtime - .mark_stage1_job_succeeded( - claim.thread.id, - claim.ownership_token.as_str(), - claim.thread.updated_at.timestamp(), - "raw", - "summary", - None, - ) - .await - .expect("mark first-batch stage1 success"), - "first batch stage1 completion should succeed" - ); - } - - let second_claims = runtime - .claim_stage1_jobs_for_startup( - current_thread_id, - Stage1StartupClaimParams { - scan_limit: 5_000, - max_claimed: 64, - max_age_days: 30, - min_rollout_idle_hours: 12, - allowed_sources: allowed_sources.as_slice(), - lease_seconds: 3_600, - }, - ) - .await - .expect("second stage1 startup claim"); - assert_eq!(second_claims.len(), 64); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn stage1_output_cascades_on_thread_delete() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let cwd = codex_home.join("workspace"); - runtime - .upsert_thread(&test_thread_metadata(&codex_home, thread_id, cwd)) - .await - .expect("upsert thread"); - - let claim = runtime - .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) - .await - .expect("claim stage1"); - let ownership_token = match claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded( - thread_id, - ownership_token.as_str(), - 100, - "raw", - "sum", - None, - ) - .await - .expect("mark stage1 succeeded"), - "mark stage1 succeeded should write stage1_outputs" - ); - - let count_before = - sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") - .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("count before delete") - .try_get::("count") - .expect("count value"); - assert_eq!(count_before, 1); - - sqlx::query("DELETE FROM threads WHERE id = ?") - .bind(thread_id.to_string()) - .execute(runtime.pool.as_ref()) - .await - .expect("delete thread"); - - let count_after = - sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") - .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("count after delete") - .try_get::("count") - .expect("count value"); - assert_eq!(count_after, 0); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn mark_stage1_job_succeeded_no_output_skips_phase2_when_output_was_already_absent() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id, - codex_home.join("workspace"), - )) - .await - .expect("upsert thread"); - - let claim = runtime - .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) - .await - .expect("claim stage1"); - let ownership_token = match claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded_no_output(thread_id, ownership_token.as_str()) - .await - .expect("mark stage1 succeeded without output"), - "stage1 no-output success should complete the job" - ); - - let output_row_count = - sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") - .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("load stage1 output count") - .try_get::("count") - .expect("stage1 output count"); - assert_eq!( - output_row_count, 0, - "stage1 no-output success should not persist empty stage1 outputs" - ); - - let up_to_date = runtime - .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) - .await - .expect("claim stage1 up-to-date"); - assert_eq!(up_to_date, Stage1JobClaimOutcome::SkippedUpToDate); - - let global_job_row_count = sqlx::query("SELECT COUNT(*) AS count FROM jobs WHERE kind = ?") - .bind("memory_consolidate_global") - .fetch_one(runtime.pool.as_ref()) - .await - .expect("load phase2 job row count") - .try_get::("count") - .expect("phase2 job row count"); - assert_eq!( - global_job_row_count, 0, - "no-output without an existing stage1 output should not enqueue phase2" - ); - - let claim_phase2 = runtime - .try_claim_global_phase2_job(owner, 3600) - .await - .expect("claim phase2"); - assert_eq!( - claim_phase2, - Phase2JobClaimOutcome::SkippedNotDirty, - "phase2 should remain clean when no-output deleted nothing" - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn mark_stage1_job_succeeded_no_output_enqueues_phase2_when_deleting_output() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id, - codex_home.join("workspace"), - )) - .await - .expect("upsert thread"); - - let first_claim = runtime - .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) - .await - .expect("claim initial stage1"); - let first_token = match first_claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected initial stage1 claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded(thread_id, first_token.as_str(), 100, "raw", "sum", None) - .await - .expect("mark initial stage1 succeeded"), - "initial stage1 success should create stage1 output" - ); - - let phase2_claim = runtime - .try_claim_global_phase2_job(owner, 3600) - .await - .expect("claim phase2 after initial output"); - let (phase2_token, phase2_input_watermark) = match phase2_claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, - input_watermark, - } => (ownership_token, input_watermark), - other => panic!("unexpected phase2 claim after initial output: {other:?}"), - }; - assert_eq!(phase2_input_watermark, 100); - assert!( - runtime - .mark_global_phase2_job_succeeded(phase2_token.as_str(), phase2_input_watermark) - .await - .expect("mark initial phase2 succeeded"), - "initial phase2 success should clear global dirty state" - ); - - let no_output_claim = runtime - .try_claim_stage1_job(thread_id, owner_b, 101, 3600, 64) - .await - .expect("claim stage1 for no-output delete"); - let no_output_token = match no_output_claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected no-output stage1 claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded_no_output(thread_id, no_output_token.as_str()) - .await - .expect("mark stage1 no-output after existing output"), - "no-output should succeed when deleting an existing stage1 output" - ); - - let output_row_count = - sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") - .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("load stage1 output count after delete") - .try_get::("count") - .expect("stage1 output count"); - assert_eq!(output_row_count, 0); - - let claim_phase2 = runtime - .try_claim_global_phase2_job(owner, 3600) - .await - .expect("claim phase2 after no-output deletion"); - let (phase2_token, phase2_input_watermark) = match claim_phase2 { - Phase2JobClaimOutcome::Claimed { - ownership_token, - input_watermark, - } => (ownership_token, input_watermark), - other => panic!("unexpected phase2 claim after no-output deletion: {other:?}"), - }; - assert_eq!(phase2_input_watermark, 101); - assert!( - runtime - .mark_global_phase2_job_succeeded(phase2_token.as_str(), phase2_input_watermark) - .await - .expect("mark phase2 succeeded after no-output delete") - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn stage1_retry_exhaustion_does_not_block_newer_watermark() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id, - codex_home.join("workspace"), - )) - .await - .expect("upsert thread"); - - for attempt in 0..3 { - let claim = runtime - .try_claim_stage1_job(thread_id, owner, 100, 3_600, 64) - .await - .expect("claim stage1 for retry exhaustion"); - let ownership_token = match claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!( - "attempt {} should claim stage1 before retries are exhausted: {other:?}", - attempt + 1 - ), - }; - assert!( - runtime - .mark_stage1_job_failed(thread_id, ownership_token.as_str(), "boom", 0) - .await - .expect("mark stage1 failed"), - "attempt {} should decrement retry budget", - attempt + 1 - ); - } - - let exhausted_claim = runtime - .try_claim_stage1_job(thread_id, owner, 100, 3_600, 64) - .await - .expect("claim stage1 after retry exhaustion"); - assert_eq!( - exhausted_claim, - Stage1JobClaimOutcome::SkippedRetryExhausted - ); - - let newer_source_claim = runtime - .try_claim_stage1_job(thread_id, owner, 101, 3_600, 64) - .await - .expect("claim stage1 with newer source watermark"); - assert!( - matches!(newer_source_claim, Stage1JobClaimOutcome::Claimed { .. }), - "newer source watermark should reset retry budget and be claimable" - ); - - let job_row = sqlx::query( - "SELECT retry_remaining, input_watermark FROM jobs WHERE kind = ? AND job_key = ?", - ) - .bind("memory_stage1") - .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("load stage1 job row after newer-source claim"); - assert_eq!( - job_row - .try_get::("retry_remaining") - .expect("retry_remaining"), - 3 - ); - assert_eq!( - job_row - .try_get::("input_watermark") - .expect("input_watermark"), - 101 - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn phase2_global_consolidation_reruns_when_watermark_advances() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - - runtime - .enqueue_global_consolidation(100) - .await - .expect("enqueue global consolidation"); - - let claim = runtime - .try_claim_global_phase2_job(owner, 3600) - .await - .expect("claim phase2"); - let (ownership_token, input_watermark) = match claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, - input_watermark, - } => (ownership_token, input_watermark), - other => panic!("unexpected phase2 claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_global_phase2_job_succeeded(ownership_token.as_str(), input_watermark) - .await - .expect("mark phase2 succeeded"), - "phase2 success should finalize for current token" - ); - - let claim_up_to_date = runtime - .try_claim_global_phase2_job(owner, 3600) - .await - .expect("claim phase2 up-to-date"); - assert_eq!(claim_up_to_date, Phase2JobClaimOutcome::SkippedNotDirty); - - runtime - .enqueue_global_consolidation(101) - .await - .expect("enqueue global consolidation again"); - - let claim_rerun = runtime - .try_claim_global_phase2_job(owner, 3600) - .await - .expect("claim phase2 rerun"); - assert!( - matches!(claim_rerun, Phase2JobClaimOutcome::Claimed { .. }), - "advanced watermark should be claimable" - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn list_stage1_outputs_for_global_returns_latest_outputs() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let thread_id_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id_a, - codex_home.join("workspace-a"), - )) - .await - .expect("upsert thread a"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id_b, - codex_home.join("workspace-b"), - )) - .await - .expect("upsert thread b"); - - let claim = runtime - .try_claim_stage1_job(thread_id_a, owner, 100, 3600, 64) - .await - .expect("claim stage1 a"); - let ownership_token = match claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage1 claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded( - thread_id_a, - ownership_token.as_str(), - 100, - "raw memory a", - "summary a", - None, - ) - .await - .expect("mark stage1 succeeded a"), - "stage1 success should persist output a" - ); - - let claim = runtime - .try_claim_stage1_job(thread_id_b, owner, 101, 3600, 64) - .await - .expect("claim stage1 b"); - let ownership_token = match claim { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage1 claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded( - thread_id_b, - ownership_token.as_str(), - 101, - "raw memory b", - "summary b", - Some("rollout-b"), - ) - .await - .expect("mark stage1 succeeded b"), - "stage1 success should persist output b" - ); - - let outputs = runtime - .list_stage1_outputs_for_global(10) - .await - .expect("list stage1 outputs for global"); - assert_eq!(outputs.len(), 2); - assert_eq!(outputs[0].thread_id, thread_id_b); - assert_eq!(outputs[0].rollout_summary, "summary b"); - assert_eq!(outputs[0].rollout_slug.as_deref(), Some("rollout-b")); - assert_eq!(outputs[0].cwd, codex_home.join("workspace-b")); - assert_eq!(outputs[1].thread_id, thread_id_a); - assert_eq!(outputs[1].rollout_summary, "summary a"); - assert_eq!(outputs[1].rollout_slug, None); - assert_eq!(outputs[1].cwd, codex_home.join("workspace-a")); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn list_stage1_outputs_for_global_skips_empty_payloads() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_id_non_empty = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - let thread_id_empty = - ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id_non_empty, - codex_home.join("workspace-non-empty"), - )) - .await - .expect("upsert non-empty thread"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_id_empty, - codex_home.join("workspace-empty"), - )) - .await - .expect("upsert empty thread"); - - sqlx::query( - r#" -INSERT INTO stage1_outputs (thread_id, source_updated_at, raw_memory, rollout_summary, generated_at) -VALUES (?, ?, ?, ?, ?) - "#, - ) - .bind(thread_id_non_empty.to_string()) - .bind(100_i64) - .bind("raw memory") - .bind("summary") - .bind(100_i64) - .execute(runtime.pool.as_ref()) - .await - .expect("insert non-empty stage1 output"); - sqlx::query( - r#" -INSERT INTO stage1_outputs (thread_id, source_updated_at, raw_memory, rollout_summary, generated_at) -VALUES (?, ?, ?, ?, ?) - "#, - ) - .bind(thread_id_empty.to_string()) - .bind(101_i64) - .bind("") - .bind("") - .bind(101_i64) - .execute(runtime.pool.as_ref()) - .await - .expect("insert empty stage1 output"); - - let outputs = runtime - .list_stage1_outputs_for_global(1) - .await - .expect("list stage1 outputs for global"); - assert_eq!(outputs.len(), 1); - assert_eq!(outputs[0].thread_id, thread_id_non_empty); - assert_eq!(outputs[0].rollout_summary, "summary"); - assert_eq!(outputs[0].cwd, codex_home.join("workspace-non-empty")); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn record_stage1_output_usage_updates_usage_metadata() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id a"); - let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id b"); - let missing = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("missing id"); - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_a, - codex_home.join("workspace-a"), - )) - .await - .expect("upsert thread a"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_b, - codex_home.join("workspace-b"), - )) - .await - .expect("upsert thread b"); - - let claim_a = runtime - .try_claim_stage1_job(thread_a, owner, 100, 3600, 64) - .await - .expect("claim stage1 a"); - let token_a = match claim_a { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage1 claim outcome for a: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded(thread_a, token_a.as_str(), 100, "raw a", "sum a", None) - .await - .expect("mark stage1 succeeded a") - ); - - let claim_b = runtime - .try_claim_stage1_job(thread_b, owner, 101, 3600, 64) - .await - .expect("claim stage1 b"); - let token_b = match claim_b { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage1 claim outcome for b: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded(thread_b, token_b.as_str(), 101, "raw b", "sum b", None) - .await - .expect("mark stage1 succeeded b") - ); - - let updated_rows = runtime - .record_stage1_output_usage(&[thread_a, thread_a, thread_b, missing]) - .await - .expect("record stage1 output usage"); - assert_eq!(updated_rows, 3); - - let row_a = - sqlx::query("SELECT usage_count, last_usage FROM stage1_outputs WHERE thread_id = ?") - .bind(thread_a.to_string()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("load stage1 usage row a"); - let row_b = - sqlx::query("SELECT usage_count, last_usage FROM stage1_outputs WHERE thread_id = ?") - .bind(thread_b.to_string()) - .fetch_one(runtime.pool.as_ref()) - .await - .expect("load stage1 usage row b"); - - assert_eq!( - row_a - .try_get::("usage_count") - .expect("usage_count a"), - 2 - ); - assert_eq!( - row_b - .try_get::("usage_count") - .expect("usage_count b"), - 1 - ); - - let last_usage_a = row_a.try_get::("last_usage").expect("last_usage a"); - let last_usage_b = row_b.try_get::("last_usage").expect("last_usage b"); - assert_eq!(last_usage_a, last_usage_b); - assert!(last_usage_a > 0); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn mark_stage1_job_succeeded_enqueues_global_consolidation() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id a"); - let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id b"); - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); - - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_a, - codex_home.join("workspace-a"), - )) - .await - .expect("upsert thread a"); - runtime - .upsert_thread(&test_thread_metadata( - &codex_home, - thread_b, - codex_home.join("workspace-b"), - )) - .await - .expect("upsert thread b"); - - let claim_a = runtime - .try_claim_stage1_job(thread_a, owner, 100, 3600, 64) - .await - .expect("claim stage1 a"); - let token_a = match claim_a { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage1 claim outcome for thread a: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded( - thread_a, - token_a.as_str(), - 100, - "raw-a", - "summary-a", - None, - ) - .await - .expect("mark stage1 succeeded a"), - "stage1 success should persist output for thread a" - ); - - let claim_b = runtime - .try_claim_stage1_job(thread_b, owner, 101, 3600, 64) - .await - .expect("claim stage1 b"); - let token_b = match claim_b { - Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, - other => panic!("unexpected stage1 claim outcome for thread b: {other:?}"), - }; - assert!( - runtime - .mark_stage1_job_succeeded( - thread_b, - token_b.as_str(), - 101, - "raw-b", - "summary-b", - None, - ) - .await - .expect("mark stage1 succeeded b"), - "stage1 success should persist output for thread b" - ); - - let claim = runtime - .try_claim_global_phase2_job(owner, 3600) - .await - .expect("claim global consolidation"); - let input_watermark = match claim { - Phase2JobClaimOutcome::Claimed { - input_watermark, .. - } => input_watermark, - other => panic!("unexpected global consolidation claim outcome: {other:?}"), - }; - assert_eq!(input_watermark, 101); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn phase2_global_lock_allows_only_one_fresh_runner() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - runtime - .enqueue_global_consolidation(200) - .await - .expect("enqueue global consolidation"); - - let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner a"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner b"); - - let running_claim = runtime - .try_claim_global_phase2_job(owner_a, 3600) - .await - .expect("claim global lock"); - assert!( - matches!(running_claim, Phase2JobClaimOutcome::Claimed { .. }), - "first owner should claim global lock" - ); - - let second_claim = runtime - .try_claim_global_phase2_job(owner_b, 3600) - .await - .expect("claim global lock from second owner"); - assert_eq!(second_claim, Phase2JobClaimOutcome::SkippedRunning); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn phase2_global_lock_stale_lease_allows_takeover() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - runtime - .enqueue_global_consolidation(300) - .await - .expect("enqueue global consolidation"); - - let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner a"); - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner b"); - - let initial_claim = runtime - .try_claim_global_phase2_job(owner_a, 3600) - .await - .expect("claim initial global lock"); - let token_a = match initial_claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, .. - } => ownership_token, - other => panic!("unexpected initial claim outcome: {other:?}"), - }; - - sqlx::query("UPDATE jobs SET lease_until = ? WHERE kind = ? AND job_key = ?") - .bind(Utc::now().timestamp() - 1) - .bind("memory_consolidate_global") - .bind("global") - .execute(runtime.pool.as_ref()) - .await - .expect("expire global consolidation lease"); - - let takeover_claim = runtime - .try_claim_global_phase2_job(owner_b, 3600) - .await - .expect("claim stale global lock"); - let (token_b, input_watermark) = match takeover_claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, - input_watermark, - } => (ownership_token, input_watermark), - other => panic!("unexpected takeover claim outcome: {other:?}"), - }; - assert_ne!(token_a, token_b); - assert_eq!(input_watermark, 300); - - assert_eq!( - runtime - .mark_global_phase2_job_succeeded(token_a.as_str(), 300) - .await - .expect("mark stale owner success result"), - false, - "stale owner should lose finalization ownership after takeover" - ); - assert!( - runtime - .mark_global_phase2_job_succeeded(token_b.as_str(), 300) - .await - .expect("mark takeover owner success"), - "takeover owner should finalize consolidation" - ); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn phase2_backfilled_inputs_below_last_success_still_become_dirty() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - runtime - .enqueue_global_consolidation(500) - .await - .expect("enqueue initial consolidation"); - let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner a"); - let claim_a = runtime - .try_claim_global_phase2_job(owner_a, 3_600) - .await - .expect("claim initial consolidation"); - let token_a = match claim_a { - Phase2JobClaimOutcome::Claimed { - ownership_token, - input_watermark, - } => { - assert_eq!(input_watermark, 500); - ownership_token - } - other => panic!("unexpected initial phase2 claim outcome: {other:?}"), - }; - assert!( - runtime - .mark_global_phase2_job_succeeded(token_a.as_str(), 500) - .await - .expect("mark initial phase2 success"), - "initial phase2 success should finalize" - ); - - runtime - .enqueue_global_consolidation(400) - .await - .expect("enqueue backfilled consolidation"); - - let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner b"); - let claim_b = runtime - .try_claim_global_phase2_job(owner_b, 3_600) - .await - .expect("claim backfilled consolidation"); - match claim_b { - Phase2JobClaimOutcome::Claimed { - input_watermark, .. - } => { - assert!( - input_watermark > 500, - "backfilled enqueue should advance dirty watermark beyond last success" - ); - } - other => panic!("unexpected backfilled phase2 claim outcome: {other:?}"), - } - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn phase2_failure_fallback_updates_unowned_running_job() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - runtime - .enqueue_global_consolidation(400) - .await - .expect("enqueue global consolidation"); - - let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner"); - let claim = runtime - .try_claim_global_phase2_job(owner, 3_600) - .await - .expect("claim global consolidation"); - let ownership_token = match claim { - Phase2JobClaimOutcome::Claimed { - ownership_token, .. - } => ownership_token, - other => panic!("unexpected claim outcome: {other:?}"), - }; - - sqlx::query("UPDATE jobs SET ownership_token = NULL WHERE kind = ? AND job_key = ?") - .bind("memory_consolidate_global") - .bind("global") - .execute(runtime.pool.as_ref()) - .await - .expect("clear ownership token"); - - assert_eq!( - runtime - .mark_global_phase2_job_failed(ownership_token.as_str(), "lost", 3_600) - .await - .expect("mark phase2 failed with strict ownership"), - false, - "strict failure update should not match unowned running job" - ); - assert!( - runtime - .mark_global_phase2_job_failed_if_unowned(ownership_token.as_str(), "lost", 3_600) - .await - .expect("fallback failure update should match unowned running job"), - "fallback failure update should transition the unowned running job" - ); - - let claim = runtime - .try_claim_global_phase2_job(ThreadId::new(), 3_600) - .await - .expect("claim after fallback failure"); - assert_eq!(claim, Phase2JobClaimOutcome::SkippedNotDirty); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn query_logs_with_search_matches_substring() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - runtime - .insert_logs(&[ - LogEntry { - ts: 1_700_000_001, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some("alpha".to_string()), - thread_id: Some("thread-1".to_string()), - process_uuid: None, - file: Some("main.rs".to_string()), - line: Some(42), - module_path: None, - }, - LogEntry { - ts: 1_700_000_002, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some("alphabet".to_string()), - thread_id: Some("thread-1".to_string()), - process_uuid: None, - file: Some("main.rs".to_string()), - line: Some(43), - module_path: None, - }, - ]) - .await - .expect("insert test logs"); - - let rows = runtime - .query_logs(&LogQuery { - search: Some("alphab".to_string()), - ..Default::default() - }) - .await - .expect("query matching logs"); - - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].message.as_deref(), Some("alphabet")); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn insert_logs_prunes_old_rows_when_thread_exceeds_size_limit() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let six_mebibytes = "a".repeat(6 * 1024 * 1024); - runtime - .insert_logs(&[ - LogEntry { - ts: 1, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(six_mebibytes.clone()), - thread_id: Some("thread-1".to_string()), - process_uuid: Some("proc-1".to_string()), - file: Some("main.rs".to_string()), - line: Some(1), - module_path: Some("mod".to_string()), - }, - LogEntry { - ts: 2, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(six_mebibytes.clone()), - thread_id: Some("thread-1".to_string()), - process_uuid: Some("proc-1".to_string()), - file: Some("main.rs".to_string()), - line: Some(2), - module_path: Some("mod".to_string()), - }, - ]) - .await - .expect("insert test logs"); - - let rows = runtime - .query_logs(&LogQuery { - thread_ids: vec!["thread-1".to_string()], - ..Default::default() - }) - .await - .expect("query thread logs"); - - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].ts, 2); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn insert_logs_prunes_single_thread_row_when_it_exceeds_size_limit() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let eleven_mebibytes = "d".repeat(11 * 1024 * 1024); - runtime - .insert_logs(&[LogEntry { - ts: 1, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(eleven_mebibytes), - thread_id: Some("thread-oversized".to_string()), - process_uuid: Some("proc-1".to_string()), - file: Some("main.rs".to_string()), - line: Some(1), - module_path: Some("mod".to_string()), - }]) - .await - .expect("insert test log"); - - let rows = runtime - .query_logs(&LogQuery { - thread_ids: vec!["thread-oversized".to_string()], - ..Default::default() - }) - .await - .expect("query thread logs"); - - assert!(rows.is_empty()); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn insert_logs_prunes_threadless_rows_per_process_uuid_only() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let six_mebibytes = "b".repeat(6 * 1024 * 1024); - runtime - .insert_logs(&[ - LogEntry { - ts: 1, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(six_mebibytes.clone()), - thread_id: None, - process_uuid: Some("proc-1".to_string()), - file: Some("main.rs".to_string()), - line: Some(1), - module_path: Some("mod".to_string()), - }, - LogEntry { - ts: 2, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(six_mebibytes.clone()), - thread_id: None, - process_uuid: Some("proc-1".to_string()), - file: Some("main.rs".to_string()), - line: Some(2), - module_path: Some("mod".to_string()), - }, - LogEntry { - ts: 3, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(six_mebibytes), - thread_id: Some("thread-1".to_string()), - process_uuid: Some("proc-1".to_string()), - file: Some("main.rs".to_string()), - line: Some(3), - module_path: Some("mod".to_string()), - }, - ]) - .await - .expect("insert test logs"); - - let rows = runtime - .query_logs(&LogQuery { - thread_ids: vec!["thread-1".to_string()], - include_threadless: true, - ..Default::default() - }) - .await - .expect("query thread and threadless logs"); - - let mut timestamps: Vec = rows.into_iter().map(|row| row.ts).collect(); - timestamps.sort_unstable(); - assert_eq!(timestamps, vec![2, 3]); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn insert_logs_prunes_single_threadless_process_row_when_it_exceeds_size_limit() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let eleven_mebibytes = "e".repeat(11 * 1024 * 1024); - runtime - .insert_logs(&[LogEntry { - ts: 1, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(eleven_mebibytes), - thread_id: None, - process_uuid: Some("proc-oversized".to_string()), - file: Some("main.rs".to_string()), - line: Some(1), - module_path: Some("mod".to_string()), - }]) - .await - .expect("insert test log"); - - let rows = runtime - .query_logs(&LogQuery { - include_threadless: true, - ..Default::default() - }) - .await - .expect("query threadless logs"); - - assert!(rows.is_empty()); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn insert_logs_prunes_threadless_rows_with_null_process_uuid() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let six_mebibytes = "c".repeat(6 * 1024 * 1024); - runtime - .insert_logs(&[ - LogEntry { - ts: 1, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(six_mebibytes.clone()), - thread_id: None, - process_uuid: None, - file: Some("main.rs".to_string()), - line: Some(1), - module_path: Some("mod".to_string()), - }, - LogEntry { - ts: 2, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(six_mebibytes), - thread_id: None, - process_uuid: None, - file: Some("main.rs".to_string()), - line: Some(2), - module_path: Some("mod".to_string()), - }, - LogEntry { - ts: 3, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some("small".to_string()), - thread_id: None, - process_uuid: Some("proc-1".to_string()), - file: Some("main.rs".to_string()), - line: Some(3), - module_path: Some("mod".to_string()), - }, - ]) - .await - .expect("insert test logs"); - - let rows = runtime - .query_logs(&LogQuery { - include_threadless: true, - ..Default::default() - }) - .await - .expect("query threadless logs"); - - let mut timestamps: Vec = rows.into_iter().map(|row| row.ts).collect(); - timestamps.sort_unstable(); - assert_eq!(timestamps, vec![2, 3]); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - #[tokio::test] - async fn insert_logs_prunes_single_threadless_null_process_row_when_it_exceeds_limit() { - let codex_home = unique_temp_dir(); - let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) - .await - .expect("initialize runtime"); - - let eleven_mebibytes = "f".repeat(11 * 1024 * 1024); - runtime - .insert_logs(&[LogEntry { - ts: 1, - ts_nanos: 0, - level: "INFO".to_string(), - target: "cli".to_string(), - message: Some(eleven_mebibytes), - thread_id: None, - process_uuid: None, - file: Some("main.rs".to_string()), - line: Some(1), - module_path: Some("mod".to_string()), - }]) - .await - .expect("insert test log"); - - let rows = runtime - .query_logs(&LogQuery { - include_threadless: true, - ..Default::default() - }) - .await - .expect("query threadless logs"); - - assert!(rows.is_empty()); - - let _ = tokio::fs::remove_dir_all(codex_home).await; - } - - fn test_thread_metadata( - codex_home: &Path, - thread_id: ThreadId, - cwd: PathBuf, - ) -> ThreadMetadata { - let now = DateTime::::from_timestamp(1_700_000_000, 0).expect("timestamp"); - ThreadMetadata { - id: thread_id, - rollout_path: codex_home.join(format!("rollout-{thread_id}.jsonl")), - created_at: now, - updated_at: now, - source: "cli".to_string(), - agent_nickname: None, - agent_role: None, - model_provider: "test-provider".to_string(), - cwd, - cli_version: "0.0.0".to_string(), - title: String::new(), - sandbox_policy: crate::extract::enum_to_string(&SandboxPolicy::new_read_only_policy()), - approval_mode: crate::extract::enum_to_string(&AskForApproval::OnRequest), - tokens_used: 0, - first_user_message: Some("hello".to_string()), - archived_at: None, - git_sha: None, - git_branch: None, - git_origin_url: None, - } - } -} diff --git a/codex-rs/state/src/runtime/agent_jobs.rs b/codex-rs/state/src/runtime/agent_jobs.rs new file mode 100644 index 0000000000..c685605945 --- /dev/null +++ b/codex-rs/state/src/runtime/agent_jobs.rs @@ -0,0 +1,562 @@ +use super::*; +use crate::model::AgentJobItemRow; + +impl StateRuntime { + pub async fn create_agent_job( + &self, + params: &AgentJobCreateParams, + items: &[AgentJobItemCreateParams], + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let input_headers_json = serde_json::to_string(¶ms.input_headers)?; + let output_schema_json = params + .output_schema_json + .as_ref() + .map(serde_json::to_string) + .transpose()?; + let max_runtime_seconds = params + .max_runtime_seconds + .map(i64::try_from) + .transpose() + .map_err(|_| anyhow::anyhow!("invalid max_runtime_seconds value"))?; + let mut tx = self.pool.begin().await?; + sqlx::query( + r#" +INSERT INTO agent_jobs ( + id, + name, + status, + instruction, + auto_export, + max_runtime_seconds, + output_schema_json, + input_headers_json, + input_csv_path, + output_csv_path, + created_at, + updated_at, + started_at, + completed_at, + last_error +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL) + "#, + ) + .bind(params.id.as_str()) + .bind(params.name.as_str()) + .bind(AgentJobStatus::Pending.as_str()) + .bind(params.instruction.as_str()) + .bind(i64::from(params.auto_export)) + .bind(max_runtime_seconds) + .bind(output_schema_json) + .bind(input_headers_json) + .bind(params.input_csv_path.as_str()) + .bind(params.output_csv_path.as_str()) + .bind(now) + .bind(now) + .execute(&mut *tx) + .await?; + + for item in items { + let row_json = serde_json::to_string(&item.row_json)?; + sqlx::query( + r#" +INSERT INTO agent_job_items ( + job_id, + item_id, + row_index, + source_id, + row_json, + status, + assigned_thread_id, + attempt_count, + result_json, + last_error, + created_at, + updated_at, + completed_at, + reported_at +) VALUES (?, ?, ?, ?, ?, ?, NULL, 0, NULL, NULL, ?, ?, NULL, NULL) + "#, + ) + .bind(params.id.as_str()) + .bind(item.item_id.as_str()) + .bind(item.row_index) + .bind(item.source_id.as_deref()) + .bind(row_json) + .bind(AgentJobItemStatus::Pending.as_str()) + .bind(now) + .bind(now) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + + let job_id = params.id.as_str(); + self.get_agent_job(job_id) + .await? + .ok_or_else(|| anyhow::anyhow!("failed to load created agent job {job_id}")) + } + + pub async fn get_agent_job(&self, job_id: &str) -> anyhow::Result> { + let row = sqlx::query_as::<_, AgentJobRow>( + r#" +SELECT + id, + name, + status, + instruction, + auto_export, + max_runtime_seconds, + output_schema_json, + input_headers_json, + input_csv_path, + output_csv_path, + created_at, + updated_at, + started_at, + completed_at, + last_error +FROM agent_jobs +WHERE id = ? + "#, + ) + .bind(job_id) + .fetch_optional(self.pool.as_ref()) + .await?; + row.map(AgentJob::try_from).transpose() + } + + pub async fn list_agent_job_items( + &self, + job_id: &str, + status: Option, + limit: Option, + ) -> anyhow::Result> { + let mut builder = QueryBuilder::::new( + r#" +SELECT + job_id, + item_id, + row_index, + source_id, + row_json, + status, + assigned_thread_id, + attempt_count, + result_json, + last_error, + created_at, + updated_at, + completed_at, + reported_at +FROM agent_job_items +WHERE job_id = + "#, + ); + builder.push_bind(job_id); + if let Some(status) = status { + builder.push(" AND status = "); + builder.push_bind(status.as_str()); + } + builder.push(" ORDER BY row_index ASC"); + if let Some(limit) = limit { + builder.push(" LIMIT "); + builder.push_bind(limit as i64); + } + let rows: Vec = builder + .build_query_as::() + .fetch_all(self.pool.as_ref()) + .await?; + rows.into_iter().map(AgentJobItem::try_from).collect() + } + + pub async fn get_agent_job_item( + &self, + job_id: &str, + item_id: &str, + ) -> anyhow::Result> { + let row: Option = sqlx::query_as::<_, AgentJobItemRow>( + r#" +SELECT + job_id, + item_id, + row_index, + source_id, + row_json, + status, + assigned_thread_id, + attempt_count, + result_json, + last_error, + created_at, + updated_at, + completed_at, + reported_at +FROM agent_job_items +WHERE job_id = ? AND item_id = ? + "#, + ) + .bind(job_id) + .bind(item_id) + .fetch_optional(self.pool.as_ref()) + .await?; + row.map(AgentJobItem::try_from).transpose() + } + + pub async fn mark_agent_job_running(&self, job_id: &str) -> anyhow::Result<()> { + let now = Utc::now().timestamp(); + sqlx::query( + r#" +UPDATE agent_jobs +SET + status = ?, + updated_at = ?, + started_at = COALESCE(started_at, ?), + completed_at = NULL, + last_error = NULL +WHERE id = ? + "#, + ) + .bind(AgentJobStatus::Running.as_str()) + .bind(now) + .bind(now) + .bind(job_id) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } + + pub async fn mark_agent_job_completed(&self, job_id: &str) -> anyhow::Result<()> { + let now = Utc::now().timestamp(); + sqlx::query( + r#" +UPDATE agent_jobs +SET status = ?, updated_at = ?, completed_at = ?, last_error = NULL +WHERE id = ? + "#, + ) + .bind(AgentJobStatus::Completed.as_str()) + .bind(now) + .bind(now) + .bind(job_id) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } + + pub async fn mark_agent_job_failed( + &self, + job_id: &str, + error_message: &str, + ) -> anyhow::Result<()> { + let now = Utc::now().timestamp(); + sqlx::query( + r#" +UPDATE agent_jobs +SET status = ?, updated_at = ?, completed_at = ?, last_error = ? +WHERE id = ? + "#, + ) + .bind(AgentJobStatus::Failed.as_str()) + .bind(now) + .bind(now) + .bind(error_message) + .bind(job_id) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } + + pub async fn mark_agent_job_cancelled( + &self, + job_id: &str, + reason: &str, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let result = sqlx::query( + r#" +UPDATE agent_jobs +SET status = ?, updated_at = ?, completed_at = ?, last_error = ? +WHERE id = ? AND status IN (?, ?) + "#, + ) + .bind(AgentJobStatus::Cancelled.as_str()) + .bind(now) + .bind(now) + .bind(reason) + .bind(job_id) + .bind(AgentJobStatus::Pending.as_str()) + .bind(AgentJobStatus::Running.as_str()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn is_agent_job_cancelled(&self, job_id: &str) -> anyhow::Result { + let row = sqlx::query( + r#" +SELECT status +FROM agent_jobs +WHERE id = ? + "#, + ) + .bind(job_id) + .fetch_optional(self.pool.as_ref()) + .await?; + let Some(row) = row else { + return Ok(false); + }; + let status: String = row.try_get("status")?; + Ok(AgentJobStatus::parse(status.as_str())? == AgentJobStatus::Cancelled) + } + + pub async fn mark_agent_job_item_running( + &self, + job_id: &str, + item_id: &str, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let result = sqlx::query( + r#" +UPDATE agent_job_items +SET + status = ?, + assigned_thread_id = NULL, + attempt_count = attempt_count + 1, + updated_at = ?, + last_error = NULL +WHERE job_id = ? AND item_id = ? AND status = ? + "#, + ) + .bind(AgentJobItemStatus::Running.as_str()) + .bind(now) + .bind(job_id) + .bind(item_id) + .bind(AgentJobItemStatus::Pending.as_str()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn mark_agent_job_item_running_with_thread( + &self, + job_id: &str, + item_id: &str, + thread_id: &str, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let result = sqlx::query( + r#" +UPDATE agent_job_items +SET + status = ?, + assigned_thread_id = ?, + attempt_count = attempt_count + 1, + updated_at = ?, + last_error = NULL +WHERE job_id = ? AND item_id = ? AND status = ? + "#, + ) + .bind(AgentJobItemStatus::Running.as_str()) + .bind(thread_id) + .bind(now) + .bind(job_id) + .bind(item_id) + .bind(AgentJobItemStatus::Pending.as_str()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn mark_agent_job_item_pending( + &self, + job_id: &str, + item_id: &str, + error_message: Option<&str>, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let result = sqlx::query( + r#" +UPDATE agent_job_items +SET + status = ?, + assigned_thread_id = NULL, + updated_at = ?, + last_error = ? +WHERE job_id = ? AND item_id = ? AND status = ? + "#, + ) + .bind(AgentJobItemStatus::Pending.as_str()) + .bind(now) + .bind(error_message) + .bind(job_id) + .bind(item_id) + .bind(AgentJobItemStatus::Running.as_str()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn set_agent_job_item_thread( + &self, + job_id: &str, + item_id: &str, + thread_id: &str, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let result = sqlx::query( + r#" +UPDATE agent_job_items +SET assigned_thread_id = ?, updated_at = ? +WHERE job_id = ? AND item_id = ? AND status = ? + "#, + ) + .bind(thread_id) + .bind(now) + .bind(job_id) + .bind(item_id) + .bind(AgentJobItemStatus::Running.as_str()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn report_agent_job_item_result( + &self, + job_id: &str, + item_id: &str, + reporting_thread_id: &str, + result_json: &Value, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let serialized = serde_json::to_string(result_json)?; + let result = sqlx::query( + r#" +UPDATE agent_job_items +SET + result_json = ?, + reported_at = ?, + updated_at = ?, + last_error = NULL +WHERE + job_id = ? + AND item_id = ? + AND status = ? + AND assigned_thread_id = ? + "#, + ) + .bind(serialized) + .bind(now) + .bind(now) + .bind(job_id) + .bind(item_id) + .bind(AgentJobItemStatus::Running.as_str()) + .bind(reporting_thread_id) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn mark_agent_job_item_completed( + &self, + job_id: &str, + item_id: &str, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let result = sqlx::query( + r#" +UPDATE agent_job_items +SET + status = ?, + completed_at = ?, + updated_at = ?, + assigned_thread_id = NULL +WHERE + job_id = ? + AND item_id = ? + AND status = ? + AND result_json IS NOT NULL + "#, + ) + .bind(AgentJobItemStatus::Completed.as_str()) + .bind(now) + .bind(now) + .bind(job_id) + .bind(item_id) + .bind(AgentJobItemStatus::Running.as_str()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn mark_agent_job_item_failed( + &self, + job_id: &str, + item_id: &str, + error_message: &str, + ) -> anyhow::Result { + let now = Utc::now().timestamp(); + let result = sqlx::query( + r#" +UPDATE agent_job_items +SET + status = ?, + completed_at = ?, + updated_at = ?, + last_error = ?, + assigned_thread_id = NULL +WHERE + job_id = ? + AND item_id = ? + AND status = ? + "#, + ) + .bind(AgentJobItemStatus::Failed.as_str()) + .bind(now) + .bind(now) + .bind(error_message) + .bind(job_id) + .bind(item_id) + .bind(AgentJobItemStatus::Running.as_str()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn get_agent_job_progress(&self, job_id: &str) -> anyhow::Result { + let row = sqlx::query( + r#" +SELECT + COUNT(*) AS total_items, + SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS pending_items, + SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS running_items, + SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS completed_items, + SUM(CASE WHEN status = ? THEN 1 ELSE 0 END) AS failed_items +FROM agent_job_items +WHERE job_id = ? + "#, + ) + .bind(AgentJobItemStatus::Pending.as_str()) + .bind(AgentJobItemStatus::Running.as_str()) + .bind(AgentJobItemStatus::Completed.as_str()) + .bind(AgentJobItemStatus::Failed.as_str()) + .bind(job_id) + .fetch_one(self.pool.as_ref()) + .await?; + + let total_items: i64 = row.try_get("total_items")?; + let pending_items: Option = row.try_get("pending_items")?; + let running_items: Option = row.try_get("running_items")?; + let completed_items: Option = row.try_get("completed_items")?; + let failed_items: Option = row.try_get("failed_items")?; + Ok(AgentJobProgress { + total_items: usize::try_from(total_items).unwrap_or_default(), + pending_items: usize::try_from(pending_items.unwrap_or_default()).unwrap_or_default(), + running_items: usize::try_from(running_items.unwrap_or_default()).unwrap_or_default(), + completed_items: usize::try_from(completed_items.unwrap_or_default()) + .unwrap_or_default(), + failed_items: usize::try_from(failed_items.unwrap_or_default()).unwrap_or_default(), + }) + } +} diff --git a/codex-rs/state/src/runtime/backfill.rs b/codex-rs/state/src/runtime/backfill.rs new file mode 100644 index 0000000000..93e1569872 --- /dev/null +++ b/codex-rs/state/src/runtime/backfill.rs @@ -0,0 +1,311 @@ +use super::*; + +impl StateRuntime { + pub async fn get_backfill_state(&self) -> anyhow::Result { + self.ensure_backfill_state_row().await?; + let row = sqlx::query( + r#" +SELECT status, last_watermark, last_success_at +FROM backfill_state +WHERE id = 1 + "#, + ) + .fetch_one(self.pool.as_ref()) + .await?; + crate::BackfillState::try_from_row(&row) + } + + /// Attempt to claim ownership of rollout metadata backfill. + /// + /// Returns `true` when this runtime claimed the backfill worker slot. + /// Returns `false` if backfill is already complete or currently owned by a + /// non-expired worker. + pub async fn try_claim_backfill(&self, lease_seconds: i64) -> anyhow::Result { + self.ensure_backfill_state_row().await?; + let now = Utc::now().timestamp(); + let lease_cutoff = now.saturating_sub(lease_seconds.max(0)); + let result = sqlx::query( + r#" +UPDATE backfill_state +SET status = ?, updated_at = ? +WHERE id = 1 + AND status != ? + AND (status != ? OR updated_at <= ?) + "#, + ) + .bind(crate::BackfillStatus::Running.as_str()) + .bind(now) + .bind(crate::BackfillStatus::Complete.as_str()) + .bind(crate::BackfillStatus::Running.as_str()) + .bind(lease_cutoff) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected() == 1) + } + + /// Mark rollout metadata backfill as running. + pub async fn mark_backfill_running(&self) -> anyhow::Result<()> { + self.ensure_backfill_state_row().await?; + sqlx::query( + r#" +UPDATE backfill_state +SET status = ?, updated_at = ? +WHERE id = 1 + "#, + ) + .bind(crate::BackfillStatus::Running.as_str()) + .bind(Utc::now().timestamp()) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } + + /// Persist rollout metadata backfill progress. + pub async fn checkpoint_backfill(&self, watermark: &str) -> anyhow::Result<()> { + self.ensure_backfill_state_row().await?; + sqlx::query( + r#" +UPDATE backfill_state +SET status = ?, last_watermark = ?, updated_at = ? +WHERE id = 1 + "#, + ) + .bind(crate::BackfillStatus::Running.as_str()) + .bind(watermark) + .bind(Utc::now().timestamp()) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } + + /// Mark rollout metadata backfill as complete. + pub async fn mark_backfill_complete(&self, last_watermark: Option<&str>) -> anyhow::Result<()> { + self.ensure_backfill_state_row().await?; + let now = Utc::now().timestamp(); + sqlx::query( + r#" +UPDATE backfill_state +SET + status = ?, + last_watermark = COALESCE(?, last_watermark), + last_success_at = ?, + updated_at = ? +WHERE id = 1 + "#, + ) + .bind(crate::BackfillStatus::Complete.as_str()) + .bind(last_watermark) + .bind(now) + .bind(now) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } + + async fn ensure_backfill_state_row(&self) -> anyhow::Result<()> { + sqlx::query( + r#" +INSERT INTO backfill_state (id, status, last_watermark, last_success_at, updated_at) +VALUES (?, ?, NULL, NULL, ?) +ON CONFLICT(id) DO NOTHING + "#, + ) + .bind(1_i64) + .bind(crate::BackfillStatus::Pending.as_str()) + .bind(Utc::now().timestamp()) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::StateRuntime; + use super::state_db_filename; + use super::test_support::unique_temp_dir; + use crate::STATE_DB_FILENAME; + use crate::STATE_DB_VERSION; + use chrono::Utc; + use pretty_assertions::assert_eq; + + #[tokio::test] + async fn init_removes_legacy_state_db_files() { + let codex_home = unique_temp_dir(); + tokio::fs::create_dir_all(&codex_home) + .await + .expect("create codex_home"); + + let current_name = state_db_filename(); + let previous_version = STATE_DB_VERSION.saturating_sub(1); + let unversioned_name = format!("{STATE_DB_FILENAME}.sqlite"); + for suffix in ["", "-wal", "-shm", "-journal"] { + let path = codex_home.join(format!("{unversioned_name}{suffix}")); + tokio::fs::write(path, b"legacy") + .await + .expect("write legacy"); + let old_version_path = codex_home.join(format!( + "{STATE_DB_FILENAME}_{previous_version}.sqlite{suffix}" + )); + tokio::fs::write(old_version_path, b"old_version") + .await + .expect("write old version"); + } + let unrelated_path = codex_home.join("state.sqlite_backup"); + tokio::fs::write(&unrelated_path, b"keep") + .await + .expect("write unrelated"); + let numeric_path = codex_home.join("123"); + tokio::fs::write(&numeric_path, b"keep") + .await + .expect("write numeric"); + + let _runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + for suffix in ["", "-wal", "-shm", "-journal"] { + let legacy_path = codex_home.join(format!("{unversioned_name}{suffix}")); + assert_eq!( + tokio::fs::try_exists(&legacy_path) + .await + .expect("check legacy path"), + false + ); + let old_version_path = codex_home.join(format!( + "{STATE_DB_FILENAME}_{previous_version}.sqlite{suffix}" + )); + assert_eq!( + tokio::fs::try_exists(&old_version_path) + .await + .expect("check old version path"), + false + ); + } + assert_eq!( + tokio::fs::try_exists(codex_home.join(current_name)) + .await + .expect("check new db path"), + true + ); + assert_eq!( + tokio::fs::try_exists(&unrelated_path) + .await + .expect("check unrelated path"), + true + ); + assert_eq!( + tokio::fs::try_exists(&numeric_path) + .await + .expect("check numeric path"), + true + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn backfill_state_persists_progress_and_completion() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let initial = runtime + .get_backfill_state() + .await + .expect("get initial backfill state"); + assert_eq!(initial.status, crate::BackfillStatus::Pending); + assert_eq!(initial.last_watermark, None); + assert_eq!(initial.last_success_at, None); + + runtime + .mark_backfill_running() + .await + .expect("mark backfill running"); + runtime + .checkpoint_backfill("sessions/2026/01/27/rollout-a.jsonl") + .await + .expect("checkpoint backfill"); + + let running = runtime + .get_backfill_state() + .await + .expect("get running backfill state"); + assert_eq!(running.status, crate::BackfillStatus::Running); + assert_eq!( + running.last_watermark, + Some("sessions/2026/01/27/rollout-a.jsonl".to_string()) + ); + assert_eq!(running.last_success_at, None); + + runtime + .mark_backfill_complete(Some("sessions/2026/01/28/rollout-b.jsonl")) + .await + .expect("mark backfill complete"); + let completed = runtime + .get_backfill_state() + .await + .expect("get completed backfill state"); + assert_eq!(completed.status, crate::BackfillStatus::Complete); + assert_eq!( + completed.last_watermark, + Some("sessions/2026/01/28/rollout-b.jsonl".to_string()) + ); + assert!(completed.last_success_at.is_some()); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn backfill_claim_is_singleton_until_stale_and_blocked_when_complete() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let claimed = runtime + .try_claim_backfill(3600) + .await + .expect("initial backfill claim"); + assert_eq!(claimed, true); + + let duplicate_claim = runtime + .try_claim_backfill(3600) + .await + .expect("duplicate backfill claim"); + assert_eq!(duplicate_claim, false); + + let stale_updated_at = Utc::now().timestamp().saturating_sub(10_000); + sqlx::query( + r#" +UPDATE backfill_state +SET status = ?, updated_at = ? +WHERE id = 1 + "#, + ) + .bind(crate::BackfillStatus::Running.as_str()) + .bind(stale_updated_at) + .execute(runtime.pool.as_ref()) + .await + .expect("force stale backfill lease"); + + let stale_claim = runtime + .try_claim_backfill(10) + .await + .expect("stale backfill claim"); + assert_eq!(stale_claim, true); + + runtime + .mark_backfill_complete(None) + .await + .expect("mark complete"); + let claim_after_complete = runtime + .try_claim_backfill(3600) + .await + .expect("claim after complete"); + assert_eq!(claim_after_complete, false); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } +} diff --git a/codex-rs/state/src/runtime/logs.rs b/codex-rs/state/src/runtime/logs.rs new file mode 100644 index 0000000000..762f68f1a9 --- /dev/null +++ b/codex-rs/state/src/runtime/logs.rs @@ -0,0 +1,715 @@ +use super::*; + +impl StateRuntime { + pub async fn insert_log(&self, entry: &LogEntry) -> anyhow::Result<()> { + self.insert_logs(std::slice::from_ref(entry)).await + } + + /// Insert a batch of log entries into the logs table. + pub async fn insert_logs(&self, entries: &[LogEntry]) -> anyhow::Result<()> { + if entries.is_empty() { + return Ok(()); + } + + let mut tx = self.pool.begin().await?; + let mut builder = QueryBuilder::::new( + "INSERT INTO logs (ts, ts_nanos, level, target, message, thread_id, process_uuid, module_path, file, line, estimated_bytes) ", + ); + builder.push_values(entries, |mut row, entry| { + let estimated_bytes = entry.message.as_ref().map_or(0, String::len) as i64 + + entry.level.len() as i64 + + entry.target.len() as i64 + + entry.module_path.as_ref().map_or(0, String::len) as i64 + + entry.file.as_ref().map_or(0, String::len) as i64; + row.push_bind(entry.ts) + .push_bind(entry.ts_nanos) + .push_bind(&entry.level) + .push_bind(&entry.target) + .push_bind(&entry.message) + .push_bind(&entry.thread_id) + .push_bind(&entry.process_uuid) + .push_bind(&entry.module_path) + .push_bind(&entry.file) + .push_bind(entry.line) + .push_bind(estimated_bytes); + }); + builder.build().execute(&mut *tx).await?; + self.prune_logs_after_insert(entries, &mut tx).await?; + tx.commit().await?; + Ok(()) + } + + /// Enforce per-partition log size caps after a successful batch insert. + /// + /// We maintain two independent budgets: + /// - Thread logs: rows with `thread_id IS NOT NULL`, capped per `thread_id`. + /// - Threadless process logs: rows with `thread_id IS NULL` ("threadless"), + /// capped per `process_uuid` (including `process_uuid IS NULL` as its own + /// threadless partition). + /// + /// "Threadless" means the log row is not associated with any conversation + /// thread, so retention is keyed by process identity instead. + /// + /// This runs inside the same transaction as the insert so callers never + /// observe "inserted but not yet pruned" rows. + async fn prune_logs_after_insert( + &self, + entries: &[LogEntry], + tx: &mut SqliteConnection, + ) -> anyhow::Result<()> { + let thread_ids: BTreeSet<&str> = entries + .iter() + .filter_map(|entry| entry.thread_id.as_deref()) + .collect(); + if !thread_ids.is_empty() { + // Cheap precheck: only run the heavier window-function prune for + // threads that are currently above the cap. + let mut over_limit_threads_query = + QueryBuilder::::new("SELECT thread_id FROM logs WHERE thread_id IN ("); + { + let mut separated = over_limit_threads_query.separated(", "); + for thread_id in &thread_ids { + separated.push_bind(*thread_id); + } + } + over_limit_threads_query.push(") GROUP BY thread_id HAVING SUM("); + over_limit_threads_query.push("estimated_bytes"); + over_limit_threads_query.push(") > "); + over_limit_threads_query.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); + let over_limit_thread_ids: Vec = over_limit_threads_query + .build() + .fetch_all(&mut *tx) + .await? + .into_iter() + .map(|row| row.try_get("thread_id")) + .collect::>()?; + if !over_limit_thread_ids.is_empty() { + // Enforce a strict per-thread cap by deleting every row whose + // newest-first cumulative bytes exceed the partition budget. + let mut prune_threads = QueryBuilder::::new( + r#" +DELETE FROM logs +WHERE id IN ( + SELECT id + FROM ( + SELECT + id, + SUM( +"#, + ); + prune_threads.push("estimated_bytes"); + prune_threads.push( + r#" + ) OVER ( + PARTITION BY thread_id + ORDER BY ts DESC, ts_nanos DESC, id DESC + ) AS cumulative_bytes + FROM logs + WHERE thread_id IN ( +"#, + ); + { + let mut separated = prune_threads.separated(", "); + for thread_id in &over_limit_thread_ids { + separated.push_bind(thread_id); + } + } + prune_threads.push( + r#" + ) + ) + WHERE cumulative_bytes > +"#, + ); + prune_threads.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); + prune_threads.push("\n)"); + prune_threads.build().execute(&mut *tx).await?; + } + } + + let threadless_process_uuids: BTreeSet<&str> = entries + .iter() + .filter(|entry| entry.thread_id.is_none()) + .filter_map(|entry| entry.process_uuid.as_deref()) + .collect(); + let has_threadless_null_process_uuid = entries + .iter() + .any(|entry| entry.thread_id.is_none() && entry.process_uuid.is_none()); + if !threadless_process_uuids.is_empty() { + // Threadless logs are budgeted separately per process UUID. + let mut over_limit_processes_query = QueryBuilder::::new( + "SELECT process_uuid FROM logs WHERE thread_id IS NULL AND process_uuid IN (", + ); + { + let mut separated = over_limit_processes_query.separated(", "); + for process_uuid in &threadless_process_uuids { + separated.push_bind(*process_uuid); + } + } + over_limit_processes_query.push(") GROUP BY process_uuid HAVING SUM("); + over_limit_processes_query.push("estimated_bytes"); + over_limit_processes_query.push(") > "); + over_limit_processes_query.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); + let over_limit_process_uuids: Vec = over_limit_processes_query + .build() + .fetch_all(&mut *tx) + .await? + .into_iter() + .map(|row| row.try_get("process_uuid")) + .collect::>()?; + if !over_limit_process_uuids.is_empty() { + // Same strict cap policy as thread pruning, but only for + // threadless rows in the affected process UUIDs. + let mut prune_threadless_process_logs = QueryBuilder::::new( + r#" +DELETE FROM logs +WHERE id IN ( + SELECT id + FROM ( + SELECT + id, + SUM( +"#, + ); + prune_threadless_process_logs.push("estimated_bytes"); + prune_threadless_process_logs.push( + r#" + ) OVER ( + PARTITION BY process_uuid + ORDER BY ts DESC, ts_nanos DESC, id DESC + ) AS cumulative_bytes + FROM logs + WHERE thread_id IS NULL + AND process_uuid IN ( +"#, + ); + { + let mut separated = prune_threadless_process_logs.separated(", "); + for process_uuid in &over_limit_process_uuids { + separated.push_bind(process_uuid); + } + } + prune_threadless_process_logs.push( + r#" + ) + ) + WHERE cumulative_bytes > +"#, + ); + prune_threadless_process_logs.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); + prune_threadless_process_logs.push("\n)"); + prune_threadless_process_logs + .build() + .execute(&mut *tx) + .await?; + } + } + if has_threadless_null_process_uuid { + // Rows without a process UUID still need a cap; treat NULL as its + // own threadless partition. + let mut null_process_usage_query = QueryBuilder::::new("SELECT SUM("); + null_process_usage_query.push("estimated_bytes"); + null_process_usage_query.push( + ") AS total_bytes FROM logs WHERE thread_id IS NULL AND process_uuid IS NULL", + ); + let total_null_process_bytes: Option = null_process_usage_query + .build() + .fetch_one(&mut *tx) + .await? + .try_get("total_bytes")?; + + if total_null_process_bytes.unwrap_or(0) > LOG_PARTITION_SIZE_LIMIT_BYTES { + let mut prune_threadless_null_process_logs = QueryBuilder::::new( + r#" +DELETE FROM logs +WHERE id IN ( + SELECT id + FROM ( + SELECT + id, + SUM( +"#, + ); + prune_threadless_null_process_logs.push("estimated_bytes"); + prune_threadless_null_process_logs.push( + r#" + ) OVER ( + PARTITION BY process_uuid + ORDER BY ts DESC, ts_nanos DESC, id DESC + ) AS cumulative_bytes + FROM logs + WHERE thread_id IS NULL + AND process_uuid IS NULL + ) + WHERE cumulative_bytes > +"#, + ); + prune_threadless_null_process_logs.push_bind(LOG_PARTITION_SIZE_LIMIT_BYTES); + prune_threadless_null_process_logs.push("\n)"); + prune_threadless_null_process_logs + .build() + .execute(&mut *tx) + .await?; + } + } + Ok(()) + } + + pub(crate) async fn delete_logs_before(&self, cutoff_ts: i64) -> anyhow::Result { + let result = sqlx::query("DELETE FROM logs WHERE ts < ?") + .bind(cutoff_ts) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected()) + } + + /// Query logs with optional filters. + pub async fn query_logs(&self, query: &LogQuery) -> anyhow::Result> { + let mut builder = QueryBuilder::::new( + "SELECT id, ts, ts_nanos, level, target, message, thread_id, process_uuid, file, line FROM logs WHERE 1 = 1", + ); + push_log_filters(&mut builder, query); + if query.descending { + builder.push(" ORDER BY id DESC"); + } else { + builder.push(" ORDER BY id ASC"); + } + if let Some(limit) = query.limit { + builder.push(" LIMIT ").push_bind(limit as i64); + } + + let rows = builder + .build_query_as::() + .fetch_all(self.pool.as_ref()) + .await?; + Ok(rows) + } + + /// Return the max log id matching optional filters. + pub async fn max_log_id(&self, query: &LogQuery) -> anyhow::Result { + let mut builder = + QueryBuilder::::new("SELECT MAX(id) AS max_id FROM logs WHERE 1 = 1"); + push_log_filters(&mut builder, query); + let row = builder.build().fetch_one(self.pool.as_ref()).await?; + let max_id: Option = row.try_get("max_id")?; + Ok(max_id.unwrap_or(0)) + } +} + +fn push_log_filters<'a>(builder: &mut QueryBuilder<'a, Sqlite>, query: &'a LogQuery) { + if let Some(level_upper) = query.level_upper.as_ref() { + builder + .push(" AND UPPER(level) = ") + .push_bind(level_upper.as_str()); + } + if let Some(from_ts) = query.from_ts { + builder.push(" AND ts >= ").push_bind(from_ts); + } + if let Some(to_ts) = query.to_ts { + builder.push(" AND ts <= ").push_bind(to_ts); + } + push_like_filters(builder, "module_path", &query.module_like); + push_like_filters(builder, "file", &query.file_like); + let has_thread_filter = !query.thread_ids.is_empty() || query.include_threadless; + if has_thread_filter { + builder.push(" AND ("); + let mut needs_or = false; + for thread_id in &query.thread_ids { + if needs_or { + builder.push(" OR "); + } + builder.push("thread_id = ").push_bind(thread_id.as_str()); + needs_or = true; + } + if query.include_threadless { + if needs_or { + builder.push(" OR "); + } + builder.push("thread_id IS NULL"); + } + builder.push(")"); + } + if let Some(after_id) = query.after_id { + builder.push(" AND id > ").push_bind(after_id); + } + if let Some(search) = query.search.as_ref() { + builder.push(" AND INSTR(message, "); + builder.push_bind(search.as_str()); + builder.push(") > 0"); + } +} + +fn push_like_filters<'a>( + builder: &mut QueryBuilder<'a, Sqlite>, + column: &str, + filters: &'a [String], +) { + if filters.is_empty() { + return; + } + builder.push(" AND ("); + for (idx, filter) in filters.iter().enumerate() { + if idx > 0 { + builder.push(" OR "); + } + builder + .push(column) + .push(" LIKE '%' || ") + .push_bind(filter.as_str()) + .push(" || '%'"); + } + builder.push(")"); +} + +#[cfg(test)] +mod tests { + use super::StateRuntime; + use super::test_support::unique_temp_dir; + use crate::LogEntry; + use crate::LogQuery; + use pretty_assertions::assert_eq; + #[tokio::test] + async fn query_logs_with_search_matches_substring() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + runtime + .insert_logs(&[ + LogEntry { + ts: 1_700_000_001, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some("alpha".to_string()), + thread_id: Some("thread-1".to_string()), + process_uuid: None, + file: Some("main.rs".to_string()), + line: Some(42), + module_path: None, + }, + LogEntry { + ts: 1_700_000_002, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some("alphabet".to_string()), + thread_id: Some("thread-1".to_string()), + process_uuid: None, + file: Some("main.rs".to_string()), + line: Some(43), + module_path: None, + }, + ]) + .await + .expect("insert test logs"); + + let rows = runtime + .query_logs(&LogQuery { + search: Some("alphab".to_string()), + ..Default::default() + }) + .await + .expect("query matching logs"); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].message.as_deref(), Some("alphabet")); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn insert_logs_prunes_old_rows_when_thread_exceeds_size_limit() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let six_mebibytes = "a".repeat(6 * 1024 * 1024); + runtime + .insert_logs(&[ + LogEntry { + ts: 1, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(six_mebibytes.clone()), + thread_id: Some("thread-1".to_string()), + process_uuid: Some("proc-1".to_string()), + file: Some("main.rs".to_string()), + line: Some(1), + module_path: Some("mod".to_string()), + }, + LogEntry { + ts: 2, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(six_mebibytes.clone()), + thread_id: Some("thread-1".to_string()), + process_uuid: Some("proc-1".to_string()), + file: Some("main.rs".to_string()), + line: Some(2), + module_path: Some("mod".to_string()), + }, + ]) + .await + .expect("insert test logs"); + + let rows = runtime + .query_logs(&LogQuery { + thread_ids: vec!["thread-1".to_string()], + ..Default::default() + }) + .await + .expect("query thread logs"); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].ts, 2); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn insert_logs_prunes_single_thread_row_when_it_exceeds_size_limit() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let eleven_mebibytes = "d".repeat(11 * 1024 * 1024); + runtime + .insert_logs(&[LogEntry { + ts: 1, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(eleven_mebibytes), + thread_id: Some("thread-oversized".to_string()), + process_uuid: Some("proc-1".to_string()), + file: Some("main.rs".to_string()), + line: Some(1), + module_path: Some("mod".to_string()), + }]) + .await + .expect("insert test log"); + + let rows = runtime + .query_logs(&LogQuery { + thread_ids: vec!["thread-oversized".to_string()], + ..Default::default() + }) + .await + .expect("query thread logs"); + + assert!(rows.is_empty()); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn insert_logs_prunes_threadless_rows_per_process_uuid_only() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let six_mebibytes = "b".repeat(6 * 1024 * 1024); + runtime + .insert_logs(&[ + LogEntry { + ts: 1, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(six_mebibytes.clone()), + thread_id: None, + process_uuid: Some("proc-1".to_string()), + file: Some("main.rs".to_string()), + line: Some(1), + module_path: Some("mod".to_string()), + }, + LogEntry { + ts: 2, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(six_mebibytes.clone()), + thread_id: None, + process_uuid: Some("proc-1".to_string()), + file: Some("main.rs".to_string()), + line: Some(2), + module_path: Some("mod".to_string()), + }, + LogEntry { + ts: 3, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(six_mebibytes), + thread_id: Some("thread-1".to_string()), + process_uuid: Some("proc-1".to_string()), + file: Some("main.rs".to_string()), + line: Some(3), + module_path: Some("mod".to_string()), + }, + ]) + .await + .expect("insert test logs"); + + let rows = runtime + .query_logs(&LogQuery { + thread_ids: vec!["thread-1".to_string()], + include_threadless: true, + ..Default::default() + }) + .await + .expect("query thread and threadless logs"); + + let mut timestamps: Vec = rows.into_iter().map(|row| row.ts).collect(); + timestamps.sort_unstable(); + assert_eq!(timestamps, vec![2, 3]); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn insert_logs_prunes_single_threadless_process_row_when_it_exceeds_size_limit() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let eleven_mebibytes = "e".repeat(11 * 1024 * 1024); + runtime + .insert_logs(&[LogEntry { + ts: 1, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(eleven_mebibytes), + thread_id: None, + process_uuid: Some("proc-oversized".to_string()), + file: Some("main.rs".to_string()), + line: Some(1), + module_path: Some("mod".to_string()), + }]) + .await + .expect("insert test log"); + + let rows = runtime + .query_logs(&LogQuery { + include_threadless: true, + ..Default::default() + }) + .await + .expect("query threadless logs"); + + assert!(rows.is_empty()); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn insert_logs_prunes_threadless_rows_with_null_process_uuid() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let six_mebibytes = "c".repeat(6 * 1024 * 1024); + runtime + .insert_logs(&[ + LogEntry { + ts: 1, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(six_mebibytes.clone()), + thread_id: None, + process_uuid: None, + file: Some("main.rs".to_string()), + line: Some(1), + module_path: Some("mod".to_string()), + }, + LogEntry { + ts: 2, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(six_mebibytes), + thread_id: None, + process_uuid: None, + file: Some("main.rs".to_string()), + line: Some(2), + module_path: Some("mod".to_string()), + }, + LogEntry { + ts: 3, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some("small".to_string()), + thread_id: None, + process_uuid: Some("proc-1".to_string()), + file: Some("main.rs".to_string()), + line: Some(3), + module_path: Some("mod".to_string()), + }, + ]) + .await + .expect("insert test logs"); + + let rows = runtime + .query_logs(&LogQuery { + include_threadless: true, + ..Default::default() + }) + .await + .expect("query threadless logs"); + + let mut timestamps: Vec = rows.into_iter().map(|row| row.ts).collect(); + timestamps.sort_unstable(); + assert_eq!(timestamps, vec![2, 3]); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn insert_logs_prunes_single_threadless_null_process_row_when_it_exceeds_limit() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let eleven_mebibytes = "f".repeat(11 * 1024 * 1024); + runtime + .insert_logs(&[LogEntry { + ts: 1, + ts_nanos: 0, + level: "INFO".to_string(), + target: "cli".to_string(), + message: Some(eleven_mebibytes), + thread_id: None, + process_uuid: None, + file: Some("main.rs".to_string()), + line: Some(1), + module_path: Some("mod".to_string()), + }]) + .await + .expect("insert test log"); + + let rows = runtime + .query_logs(&LogQuery { + include_threadless: true, + ..Default::default() + }) + .await + .expect("query threadless logs"); + + assert!(rows.is_empty()); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } +} diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 6d5ffc888a..908919d0aa 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -1,4 +1,7 @@ +use super::threads::push_thread_filters; +use super::threads::push_thread_order_and_limit; use super::*; +use crate::model::Phase2InputSelection; use crate::model::Phase2JobClaimOutcome; use crate::model::Stage1JobClaim; use crate::model::Stage1JobClaimOutcome; @@ -6,10 +9,13 @@ use crate::model::Stage1Output; use crate::model::Stage1OutputRow; use crate::model::Stage1StartupClaimParams; use crate::model::ThreadRow; +use crate::model::stage1_output_ref_from_parts; use chrono::Duration; use sqlx::Executor; use sqlx::QueryBuilder; use sqlx::Sqlite; +use std::collections::HashSet; +use uuid::Uuid; const JOB_KIND_MEMORY_STAGE1: &str = "memory_stage1"; const JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL: &str = "memory_consolidate_global"; @@ -91,6 +97,7 @@ WHERE thread_id = ? /// Query behavior: /// - starts from `threads` filtered to active threads and allowed sources /// (`push_thread_filters`) + /// - excludes threads with `memory_mode != 'enabled'` /// - excludes the current thread id /// - keeps only threads in the age window: /// `updated_at >= now - max_age_days` and `updated_at <= now - min_rollout_idle_hours` @@ -168,6 +175,7 @@ LEFT JOIN jobs SortKey::UpdatedAt, None, ); + builder.push(" AND threads.memory_mode = 'enabled'"); builder .push(" AND id != ") .push_bind(current_thread_id.as_str()); @@ -218,7 +226,7 @@ LEFT JOIN jobs /// /// Query behavior: /// - filters out rows where both `raw_memory` and `rollout_summary` are blank - /// - joins `threads` to include thread `cwd` + /// - joins `threads` to include thread `cwd`, `rollout_path`, and `git_branch` /// - orders by `source_updated_at DESC, thread_id DESC` /// - applies `LIMIT n` pub async fn list_stage1_outputs_for_global( @@ -233,12 +241,14 @@ LEFT JOIN jobs r#" SELECT so.thread_id, + COALESCE(t.rollout_path, '') AS rollout_path, so.source_updated_at, so.raw_memory, so.rollout_summary, so.rollout_slug, - so.generated_at - , COALESCE(t.cwd, '') AS cwd + so.generated_at, + COALESCE(t.cwd, '') AS cwd, + t.git_branch AS git_branch FROM stage1_outputs AS so LEFT JOIN threads AS t ON t.id = so.thread_id @@ -256,6 +266,136 @@ LIMIT ? .collect::, _>>() } + /// Returns the current phase-2 input set along with its diff against the + /// last successful phase-2 selection. + /// + /// Query behavior: + /// - current selection keeps only non-empty stage-1 outputs whose + /// `last_usage` is within `max_unused_days`, or whose + /// `source_updated_at` is within that window when the memory has never + /// been used + /// - eligible rows are ordered by `usage_count DESC`, + /// `COALESCE(last_usage, source_updated_at) DESC`, `source_updated_at DESC`, + /// `thread_id DESC` + /// - previously selected rows are identified by `selected_for_phase2 = 1` + /// - `previous_selected` contains the current persisted rows that belonged + /// to the last successful phase-2 baseline + /// - `retained_thread_ids` records which current rows still match the exact + /// snapshot selected in the last successful phase-2 run + /// - removed rows are previously selected rows that are still present in + /// `stage1_outputs` but fall outside the current top-`n` selection + pub async fn get_phase2_input_selection( + &self, + n: usize, + max_unused_days: i64, + ) -> anyhow::Result { + if n == 0 { + return Ok(Phase2InputSelection::default()); + } + let cutoff = (Utc::now() - Duration::days(max_unused_days.max(0))).timestamp(); + + let current_rows = sqlx::query( + r#" +SELECT + so.thread_id, + COALESCE(t.rollout_path, '') AS rollout_path, + so.source_updated_at, + so.raw_memory, + so.rollout_summary, + so.rollout_slug, + so.generated_at, + COALESCE(t.cwd, '') AS cwd, + t.git_branch AS git_branch, + so.selected_for_phase2, + so.selected_for_phase2_source_updated_at +FROM stage1_outputs AS so +LEFT JOIN threads AS t + ON t.id = so.thread_id +WHERE (length(trim(so.raw_memory)) > 0 OR length(trim(so.rollout_summary)) > 0) + AND ( + (so.last_usage IS NOT NULL AND so.last_usage >= ?) + OR (so.last_usage IS NULL AND so.source_updated_at >= ?) + ) +ORDER BY + COALESCE(so.usage_count, 0) DESC, + COALESCE(so.last_usage, so.source_updated_at) DESC, + so.source_updated_at DESC, + so.thread_id DESC +LIMIT ? + "#, + ) + .bind(cutoff) + .bind(cutoff) + .bind(n as i64) + .fetch_all(self.pool.as_ref()) + .await?; + + let mut current_thread_ids = HashSet::with_capacity(current_rows.len()); + let mut selected = Vec::with_capacity(current_rows.len()); + let mut retained_thread_ids = Vec::new(); + for row in current_rows { + let thread_id = row.try_get::("thread_id")?; + current_thread_ids.insert(thread_id.clone()); + let source_updated_at = row.try_get::("source_updated_at")?; + if row.try_get::("selected_for_phase2")? != 0 + && row.try_get::, _>("selected_for_phase2_source_updated_at")? + == Some(source_updated_at) + { + retained_thread_ids.push(ThreadId::try_from(thread_id.clone())?); + } + selected.push(Stage1Output::try_from(Stage1OutputRow::try_from_row( + &row, + )?)?); + } + + let previous_rows = sqlx::query( + r#" +SELECT + so.thread_id, + COALESCE(t.rollout_path, '') AS rollout_path, + so.source_updated_at, + so.raw_memory, + so.rollout_summary, + so.rollout_slug, + so.generated_at, + COALESCE(t.cwd, '') AS cwd, + t.git_branch AS git_branch +FROM stage1_outputs AS so +LEFT JOIN threads AS t + ON t.id = so.thread_id +WHERE so.selected_for_phase2 = 1 +ORDER BY so.source_updated_at DESC, so.thread_id DESC + "#, + ) + .fetch_all(self.pool.as_ref()) + .await?; + + let previous_selected = previous_rows + .iter() + .map(Stage1OutputRow::try_from_row) + .map(|row| row.and_then(Stage1Output::try_from)) + .collect::, _>>()?; + let mut removed = Vec::new(); + for row in previous_rows { + let thread_id = row.try_get::("thread_id")?; + if current_thread_ids.contains(thread_id.as_str()) { + continue; + } + removed.push(stage1_output_ref_from_parts( + thread_id, + row.try_get("source_updated_at")?, + row.try_get("rollout_slug")?, + )?); + } + + Ok(Phase2InputSelection { + selected, + previous_selected, + retained_thread_ids, + removed, + }) + } + /// Attempts to claim a stage-1 job for a thread at `source_updated_at`. /// /// Claim semantics: @@ -453,6 +593,9 @@ WHERE kind = ? AND job_key = ? /// - sets `status='done'` and `last_success_watermark = input_watermark` /// - upserts `stage1_outputs` for the thread, replacing existing output only /// when `source_updated_at` is newer or equal + /// - preserves any existing `selected_for_phase2` baseline until the next + /// successful phase-2 run rewrites the baseline selection, including the + /// snapshot timestamp chosen during that run /// - persists optional `rollout_slug` for rollout summary artifact naming /// - enqueues/advances the global phase-2 job watermark using /// `source_updated_at` @@ -805,12 +948,18 @@ WHERE kind = ? AND job_key = ? /// - sets `status='done'`, clears lease/errors /// - advances `last_success_watermark` to /// `max(existing_last_success_watermark, completed_watermark)` + /// - rewrites `selected_for_phase2` so only the exact selected stage-1 + /// snapshots remain marked as part of the latest successful phase-2 + /// selection, and persists each selected snapshot's + /// `source_updated_at` for future retained-vs-added diffing pub async fn mark_global_phase2_job_succeeded( &self, ownership_token: &str, completed_watermark: i64, + selected_outputs: &[Stage1Output], ) -> anyhow::Result { let now = Utc::now().timestamp(); + let mut tx = self.pool.begin().await?; let rows_affected = sqlx::query( r#" UPDATE jobs @@ -829,11 +978,46 @@ WHERE kind = ? AND job_key = ? .bind(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL) .bind(MEMORY_CONSOLIDATION_JOB_KEY) .bind(ownership_token) - .execute(self.pool.as_ref()) + .execute(&mut *tx) .await? .rows_affected(); - Ok(rows_affected > 0) + if rows_affected == 0 { + tx.commit().await?; + return Ok(false); + } + + sqlx::query( + r#" +UPDATE stage1_outputs +SET + selected_for_phase2 = 0, + selected_for_phase2_source_updated_at = NULL +WHERE selected_for_phase2 != 0 OR selected_for_phase2_source_updated_at IS NOT NULL + "#, + ) + .execute(&mut *tx) + .await?; + + for output in selected_outputs { + sqlx::query( + r#" +UPDATE stage1_outputs +SET + selected_for_phase2 = 1, + selected_for_phase2_source_updated_at = ? +WHERE thread_id = ? AND source_updated_at = ? + "#, + ) + .bind(output.source_updated_at.timestamp()) + .bind(output.thread_id.to_string()) + .bind(output.source_updated_at.timestamp()) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + Ok(true) } /// Marks the owned running global phase-2 job as failed and schedules retry. @@ -971,3 +1155,2586 @@ ON CONFLICT(kind, job_key) DO UPDATE SET Ok(()) } + +#[cfg(test)] +mod tests { + use super::StateRuntime; + use super::test_support::test_thread_metadata; + use super::test_support::unique_temp_dir; + use crate::model::Phase2JobClaimOutcome; + use crate::model::Stage1JobClaimOutcome; + use crate::model::Stage1StartupClaimParams; + use chrono::Duration; + use chrono::Utc; + use codex_protocol::ThreadId; + use pretty_assertions::assert_eq; + use sqlx::Row; + use std::sync::Arc; + use uuid::Uuid; + + #[tokio::test] + async fn stage1_claim_skips_when_up_to_date() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let metadata = test_thread_metadata(&codex_home, thread_id, codex_home.join("a")); + runtime + .upsert_thread(&metadata) + .await + .expect("upsert thread"); + + let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + + let claim = runtime + .try_claim_stage1_job(thread_id, owner_a, 100, 3600, 64) + .await + .expect("claim stage1 job"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected claim outcome: {other:?}"), + }; + + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + 100, + "raw", + "sum", + None, + ) + .await + .expect("mark stage1 succeeded"), + "stage1 success should finalize for current token" + ); + + let up_to_date = runtime + .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) + .await + .expect("claim stage1 up-to-date"); + assert_eq!(up_to_date, Stage1JobClaimOutcome::SkippedUpToDate); + + let needs_rerun = runtime + .try_claim_stage1_job(thread_id, owner_b, 101, 3600, 64) + .await + .expect("claim stage1 newer source"); + assert!( + matches!(needs_rerun, Stage1JobClaimOutcome::Claimed { .. }), + "newer source_updated_at should be claimable" + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn stage1_running_stale_can_be_stolen_but_fresh_running_is_skipped() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let cwd = codex_home.join("workspace"); + runtime + .upsert_thread(&test_thread_metadata(&codex_home, thread_id, cwd)) + .await + .expect("upsert thread"); + + let claim_a = runtime + .try_claim_stage1_job(thread_id, owner_a, 100, 3600, 64) + .await + .expect("claim a"); + assert!(matches!(claim_a, Stage1JobClaimOutcome::Claimed { .. })); + + let claim_b_fresh = runtime + .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) + .await + .expect("claim b fresh"); + assert_eq!(claim_b_fresh, Stage1JobClaimOutcome::SkippedRunning); + + sqlx::query("UPDATE jobs SET lease_until = 0 WHERE kind = 'memory_stage1' AND job_key = ?") + .bind(thread_id.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("force stale lease"); + + let claim_b_stale = runtime + .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) + .await + .expect("claim b stale"); + assert!(matches!( + claim_b_stale, + Stage1JobClaimOutcome::Claimed { .. } + )); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn stage1_concurrent_claim_for_same_thread_is_conflict_safe() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let thread_id_a = thread_id; + let thread_id_b = thread_id; + let runtime_a = Arc::clone(&runtime); + let runtime_b = Arc::clone(&runtime); + let claim_with_retry = |runtime: Arc, + thread_id: ThreadId, + owner: ThreadId| async move { + for attempt in 0..5 { + match runtime + .try_claim_stage1_job(thread_id, owner, 100, 3_600, 64) + .await + { + Ok(outcome) => return outcome, + Err(err) if err.to_string().contains("database is locked") && attempt < 4 => { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + Err(err) => panic!("claim stage1 should not fail: {err}"), + } + } + panic!("claim stage1 should have returned within retry budget") + }; + + let (claim_a, claim_b) = tokio::join!( + claim_with_retry(runtime_a, thread_id_a, owner_a), + claim_with_retry(runtime_b, thread_id_b, owner_b), + ); + + let claim_outcomes = vec![claim_a, claim_b]; + let claimed_count = claim_outcomes + .iter() + .filter(|outcome| matches!(outcome, Stage1JobClaimOutcome::Claimed { .. })) + .count(); + assert_eq!(claimed_count, 1); + assert!( + claim_outcomes.iter().all(|outcome| { + matches!( + outcome, + Stage1JobClaimOutcome::Claimed { .. } | Stage1JobClaimOutcome::SkippedRunning + ) + }), + "unexpected claim outcomes: {claim_outcomes:?}" + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn stage1_concurrent_claims_respect_running_cap() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_a, + codex_home.join("workspace-a"), + )) + .await + .expect("upsert thread a"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_b, + codex_home.join("workspace-b"), + )) + .await + .expect("upsert thread b"); + + let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let runtime_a = Arc::clone(&runtime); + let runtime_b = Arc::clone(&runtime); + + let (claim_a, claim_b) = tokio::join!( + async move { + runtime_a + .try_claim_stage1_job(thread_a, owner_a, 100, 3_600, 1) + .await + .expect("claim stage1 thread a") + }, + async move { + runtime_b + .try_claim_stage1_job(thread_b, owner_b, 101, 3_600, 1) + .await + .expect("claim stage1 thread b") + }, + ); + + let claim_outcomes = vec![claim_a, claim_b]; + let claimed_count = claim_outcomes + .iter() + .filter(|outcome| matches!(outcome, Stage1JobClaimOutcome::Claimed { .. })) + .count(); + assert_eq!(claimed_count, 1); + assert!( + claim_outcomes + .iter() + .any(|outcome| { matches!(outcome, Stage1JobClaimOutcome::SkippedRunning) }), + "one concurrent claim should be throttled by running cap: {claim_outcomes:?}" + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn claim_stage1_jobs_filters_by_age_idle_and_current_thread() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let now = Utc::now(); + let fresh_at = now - Duration::hours(1); + let just_under_idle_at = now - Duration::hours(12) + Duration::minutes(1); + let eligible_idle_at = now - Duration::hours(12) - Duration::minutes(1); + let old_at = now - Duration::days(31); + + let current_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); + let fresh_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("fresh thread id"); + let just_under_idle_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("just under idle thread id"); + let eligible_idle_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("eligible idle thread id"); + let old_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("old thread id"); + + let mut current = + test_thread_metadata(&codex_home, current_thread_id, codex_home.join("current")); + current.created_at = now; + current.updated_at = now; + runtime + .upsert_thread(¤t) + .await + .expect("upsert current"); + + let mut fresh = + test_thread_metadata(&codex_home, fresh_thread_id, codex_home.join("fresh")); + fresh.created_at = fresh_at; + fresh.updated_at = fresh_at; + runtime.upsert_thread(&fresh).await.expect("upsert fresh"); + + let mut just_under_idle = test_thread_metadata( + &codex_home, + just_under_idle_thread_id, + codex_home.join("just-under-idle"), + ); + just_under_idle.created_at = just_under_idle_at; + just_under_idle.updated_at = just_under_idle_at; + runtime + .upsert_thread(&just_under_idle) + .await + .expect("upsert just-under-idle"); + + let mut eligible_idle = test_thread_metadata( + &codex_home, + eligible_idle_thread_id, + codex_home.join("eligible-idle"), + ); + eligible_idle.created_at = eligible_idle_at; + eligible_idle.updated_at = eligible_idle_at; + runtime + .upsert_thread(&eligible_idle) + .await + .expect("upsert eligible-idle"); + + let mut old = test_thread_metadata(&codex_home, old_thread_id, codex_home.join("old")); + old.created_at = old_at; + old.updated_at = old_at; + runtime.upsert_thread(&old).await.expect("upsert old"); + + let allowed_sources = vec!["cli".to_string()]; + let claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 1, + max_claimed: 5, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3600, + }, + ) + .await + .expect("claim stage1 jobs"); + + assert_eq!(claims.len(), 1); + assert_eq!(claims[0].thread.id, eligible_idle_thread_id); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn claim_stage1_jobs_prefilters_threads_with_up_to_date_memory() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let now = Utc::now(); + let eligible_newer_at = now - Duration::hours(13); + let eligible_older_at = now - Duration::hours(14); + + let current_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); + let up_to_date_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("up-to-date thread id"); + let stale_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("stale thread id"); + let worker_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("worker id"); + + let mut current = + test_thread_metadata(&codex_home, current_thread_id, codex_home.join("current")); + current.created_at = now; + current.updated_at = now; + runtime + .upsert_thread(¤t) + .await + .expect("upsert current thread"); + + let mut up_to_date = test_thread_metadata( + &codex_home, + up_to_date_thread_id, + codex_home.join("up-to-date"), + ); + up_to_date.created_at = eligible_newer_at; + up_to_date.updated_at = eligible_newer_at; + runtime + .upsert_thread(&up_to_date) + .await + .expect("upsert up-to-date thread"); + + let up_to_date_claim = runtime + .try_claim_stage1_job( + up_to_date_thread_id, + worker_id, + up_to_date.updated_at.timestamp(), + 3600, + 64, + ) + .await + .expect("claim up-to-date thread for seed"); + let up_to_date_token = match up_to_date_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected seed claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + up_to_date_thread_id, + up_to_date_token.as_str(), + up_to_date.updated_at.timestamp(), + "raw", + "summary", + None, + ) + .await + .expect("mark up-to-date thread succeeded"), + "seed stage1 success should complete for up-to-date thread" + ); + + let mut stale = + test_thread_metadata(&codex_home, stale_thread_id, codex_home.join("stale")); + stale.created_at = eligible_older_at; + stale.updated_at = eligible_older_at; + runtime + .upsert_thread(&stale) + .await + .expect("upsert stale thread"); + + let allowed_sources = vec!["cli".to_string()]; + let claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 1, + max_claimed: 1, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3600, + }, + ) + .await + .expect("claim stage1 startup jobs"); + assert_eq!(claims.len(), 1); + assert_eq!(claims[0].thread.id, stale_thread_id); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn claim_stage1_jobs_skips_threads_with_disabled_memory_mode() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let now = Utc::now(); + let eligible_at = now - Duration::hours(13); + + let current_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); + let disabled_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("disabled thread id"); + let enabled_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("enabled thread id"); + + let mut current = + test_thread_metadata(&codex_home, current_thread_id, codex_home.join("current")); + current.created_at = now; + current.updated_at = now; + runtime + .upsert_thread(¤t) + .await + .expect("upsert current thread"); + + let mut disabled = + test_thread_metadata(&codex_home, disabled_thread_id, codex_home.join("disabled")); + disabled.created_at = eligible_at; + disabled.updated_at = eligible_at; + runtime + .upsert_thread(&disabled) + .await + .expect("upsert disabled thread"); + sqlx::query("UPDATE threads SET memory_mode = 'disabled' WHERE id = ?") + .bind(disabled_thread_id.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("disable thread memory mode"); + + let mut enabled = + test_thread_metadata(&codex_home, enabled_thread_id, codex_home.join("enabled")); + enabled.created_at = eligible_at; + enabled.updated_at = eligible_at; + runtime + .upsert_thread(&enabled) + .await + .expect("upsert enabled thread"); + + let allowed_sources = vec!["cli".to_string()]; + let claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 10, + max_claimed: 10, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3600, + }, + ) + .await + .expect("claim stage1 startup jobs"); + + assert_eq!(claims.len(), 1); + assert_eq!(claims[0].thread.id, enabled_thread_id); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn claim_stage1_jobs_enforces_global_running_cap() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let current_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + current_thread_id, + codex_home.join("current"), + )) + .await + .expect("upsert current"); + + let now = Utc::now(); + let started_at = now.timestamp(); + let lease_until = started_at + 3600; + let eligible_at = now - Duration::hours(13); + let existing_running = 10usize; + let total_candidates = 80usize; + + for idx in 0..total_candidates { + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let mut metadata = test_thread_metadata( + &codex_home, + thread_id, + codex_home.join(format!("thread-{idx}")), + ); + metadata.created_at = eligible_at - Duration::seconds(idx as i64); + metadata.updated_at = eligible_at - Duration::seconds(idx as i64); + runtime + .upsert_thread(&metadata) + .await + .expect("upsert thread"); + + if idx < existing_running { + sqlx::query( + r#" +INSERT INTO jobs ( + kind, + job_key, + status, + worker_id, + ownership_token, + started_at, + finished_at, + lease_until, + retry_at, + retry_remaining, + last_error, + input_watermark, + last_success_watermark +) VALUES (?, ?, 'running', ?, ?, ?, NULL, ?, NULL, ?, NULL, ?, NULL) + "#, + ) + .bind("memory_stage1") + .bind(thread_id.to_string()) + .bind(current_thread_id.to_string()) + .bind(Uuid::new_v4().to_string()) + .bind(started_at) + .bind(lease_until) + .bind(3) + .bind(metadata.updated_at.timestamp()) + .execute(runtime.pool.as_ref()) + .await + .expect("seed running stage1 job"); + } + } + + let allowed_sources = vec!["cli".to_string()]; + let claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 200, + max_claimed: 64, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3600, + }, + ) + .await + .expect("claim stage1 jobs"); + assert_eq!(claims.len(), 54); + + let running_count = sqlx::query( + r#" +SELECT COUNT(*) AS count +FROM jobs +WHERE kind = 'memory_stage1' + AND status = 'running' + AND lease_until IS NOT NULL + AND lease_until > ? + "#, + ) + .bind(Utc::now().timestamp()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("count running stage1 jobs") + .try_get::("count") + .expect("running count value"); + assert_eq!(running_count, 64); + + let more_claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 200, + max_claimed: 64, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3600, + }, + ) + .await + .expect("claim stage1 jobs with cap reached"); + assert_eq!(more_claims.len(), 0); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn claim_stage1_jobs_processes_two_full_batches_across_startup_passes() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let current_thread_id = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("current thread id"); + let mut current = + test_thread_metadata(&codex_home, current_thread_id, codex_home.join("current")); + current.created_at = Utc::now(); + current.updated_at = Utc::now(); + runtime + .upsert_thread(¤t) + .await + .expect("upsert current"); + + let eligible_at = Utc::now() - Duration::hours(13); + for idx in 0..200 { + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let mut metadata = test_thread_metadata( + &codex_home, + thread_id, + codex_home.join(format!("thread-{idx}")), + ); + metadata.created_at = eligible_at - Duration::seconds(idx as i64); + metadata.updated_at = eligible_at - Duration::seconds(idx as i64); + runtime + .upsert_thread(&metadata) + .await + .expect("upsert eligible thread"); + } + + let allowed_sources = vec!["cli".to_string()]; + let first_claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 5_000, + max_claimed: 64, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3_600, + }, + ) + .await + .expect("first stage1 startup claim"); + assert_eq!(first_claims.len(), 64); + + for claim in first_claims { + assert!( + runtime + .mark_stage1_job_succeeded( + claim.thread.id, + claim.ownership_token.as_str(), + claim.thread.updated_at.timestamp(), + "raw", + "summary", + None, + ) + .await + .expect("mark first-batch stage1 success"), + "first batch stage1 completion should succeed" + ); + } + + let second_claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 5_000, + max_claimed: 64, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3_600, + }, + ) + .await + .expect("second stage1 startup claim"); + assert_eq!(second_claims.len(), 64); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn stage1_output_cascades_on_thread_delete() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let cwd = codex_home.join("workspace"); + runtime + .upsert_thread(&test_thread_metadata(&codex_home, thread_id, cwd)) + .await + .expect("upsert thread"); + + let claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) + .await + .expect("claim stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + 100, + "raw", + "sum", + None, + ) + .await + .expect("mark stage1 succeeded"), + "mark stage1 succeeded should write stage1_outputs" + ); + + let count_before = + sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("count before delete") + .try_get::("count") + .expect("count value"); + assert_eq!(count_before, 1); + + sqlx::query("DELETE FROM threads WHERE id = ?") + .bind(thread_id.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("delete thread"); + + let count_after = + sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("count after delete") + .try_get::("count") + .expect("count value"); + assert_eq!(count_after, 0); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn mark_stage1_job_succeeded_no_output_skips_phase2_when_output_was_already_absent() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + let claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) + .await + .expect("claim stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded_no_output(thread_id, ownership_token.as_str()) + .await + .expect("mark stage1 succeeded without output"), + "stage1 no-output success should complete the job" + ); + + let output_row_count = + sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load stage1 output count") + .try_get::("count") + .expect("stage1 output count"); + assert_eq!( + output_row_count, 0, + "stage1 no-output success should not persist empty stage1 outputs" + ); + + let up_to_date = runtime + .try_claim_stage1_job(thread_id, owner_b, 100, 3600, 64) + .await + .expect("claim stage1 up-to-date"); + assert_eq!(up_to_date, Stage1JobClaimOutcome::SkippedUpToDate); + + let global_job_row_count = sqlx::query("SELECT COUNT(*) AS count FROM jobs WHERE kind = ?") + .bind("memory_consolidate_global") + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load phase2 job row count") + .try_get::("count") + .expect("phase2 job row count"); + assert_eq!( + global_job_row_count, 0, + "no-output without an existing stage1 output should not enqueue phase2" + ); + + let claim_phase2 = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2"); + assert_eq!( + claim_phase2, + Phase2JobClaimOutcome::SkippedNotDirty, + "phase2 should remain clean when no-output deleted nothing" + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn mark_stage1_job_succeeded_no_output_enqueues_phase2_when_deleting_output() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + let first_claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) + .await + .expect("claim initial stage1"); + let first_token = match first_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected initial stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded(thread_id, first_token.as_str(), 100, "raw", "sum", None) + .await + .expect("mark initial stage1 succeeded"), + "initial stage1 success should create stage1 output" + ); + + let phase2_claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2 after initial output"); + let (phase2_token, phase2_input_watermark) = match phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim after initial output: {other:?}"), + }; + assert_eq!(phase2_input_watermark, 100); + assert!( + runtime + .mark_global_phase2_job_succeeded( + phase2_token.as_str(), + phase2_input_watermark, + &[], + ) + .await + .expect("mark initial phase2 succeeded"), + "initial phase2 success should clear global dirty state" + ); + + let no_output_claim = runtime + .try_claim_stage1_job(thread_id, owner_b, 101, 3600, 64) + .await + .expect("claim stage1 for no-output delete"); + let no_output_token = match no_output_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected no-output stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded_no_output(thread_id, no_output_token.as_str()) + .await + .expect("mark stage1 no-output after existing output"), + "no-output should succeed when deleting an existing stage1 output" + ); + + let output_row_count = + sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load stage1 output count after delete") + .try_get::("count") + .expect("stage1 output count"); + assert_eq!(output_row_count, 0); + + let claim_phase2 = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2 after no-output deletion"); + let (phase2_token, phase2_input_watermark) = match claim_phase2 { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim after no-output deletion: {other:?}"), + }; + assert_eq!(phase2_input_watermark, 101); + assert!( + runtime + .mark_global_phase2_job_succeeded( + phase2_token.as_str(), + phase2_input_watermark, + &[], + ) + .await + .expect("mark phase2 succeeded after no-output delete") + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn stage1_retry_exhaustion_does_not_block_newer_watermark() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + for attempt in 0..3 { + let claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3_600, 64) + .await + .expect("claim stage1 for retry exhaustion"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!( + "attempt {} should claim stage1 before retries are exhausted: {other:?}", + attempt + 1 + ), + }; + assert!( + runtime + .mark_stage1_job_failed(thread_id, ownership_token.as_str(), "boom", 0) + .await + .expect("mark stage1 failed"), + "attempt {} should decrement retry budget", + attempt + 1 + ); + } + + let exhausted_claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3_600, 64) + .await + .expect("claim stage1 after retry exhaustion"); + assert_eq!( + exhausted_claim, + Stage1JobClaimOutcome::SkippedRetryExhausted + ); + + let newer_source_claim = runtime + .try_claim_stage1_job(thread_id, owner, 101, 3_600, 64) + .await + .expect("claim stage1 with newer source watermark"); + assert!( + matches!(newer_source_claim, Stage1JobClaimOutcome::Claimed { .. }), + "newer source watermark should reset retry budget and be claimable" + ); + + let job_row = sqlx::query( + "SELECT retry_remaining, input_watermark FROM jobs WHERE kind = ? AND job_key = ?", + ) + .bind("memory_stage1") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load stage1 job row after newer-source claim"); + assert_eq!( + job_row + .try_get::("retry_remaining") + .expect("retry_remaining"), + 3 + ); + assert_eq!( + job_row + .try_get::("input_watermark") + .expect("input_watermark"), + 101 + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn phase2_global_consolidation_reruns_when_watermark_advances() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + + runtime + .enqueue_global_consolidation(100) + .await + .expect("enqueue global consolidation"); + + let claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2"); + let (ownership_token, input_watermark) = match claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_global_phase2_job_succeeded(ownership_token.as_str(), input_watermark, &[],) + .await + .expect("mark phase2 succeeded"), + "phase2 success should finalize for current token" + ); + + let claim_up_to_date = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2 up-to-date"); + assert_eq!(claim_up_to_date, Phase2JobClaimOutcome::SkippedNotDirty); + + runtime + .enqueue_global_consolidation(101) + .await + .expect("enqueue global consolidation again"); + + let claim_rerun = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2 rerun"); + assert!( + matches!(claim_rerun, Phase2JobClaimOutcome::Claimed { .. }), + "advanced watermark should be claimable" + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn list_stage1_outputs_for_global_returns_latest_outputs() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let thread_id_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id_a, + codex_home.join("workspace-a"), + )) + .await + .expect("upsert thread a"); + let mut metadata_b = + test_thread_metadata(&codex_home, thread_id_b, codex_home.join("workspace-b")); + metadata_b.git_branch = Some("feature/stage1-b".to_string()); + runtime + .upsert_thread(&metadata_b) + .await + .expect("upsert thread b"); + + let claim = runtime + .try_claim_stage1_job(thread_id_a, owner, 100, 3600, 64) + .await + .expect("claim stage1 a"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id_a, + ownership_token.as_str(), + 100, + "raw memory a", + "summary a", + None, + ) + .await + .expect("mark stage1 succeeded a"), + "stage1 success should persist output a" + ); + + let claim = runtime + .try_claim_stage1_job(thread_id_b, owner, 101, 3600, 64) + .await + .expect("claim stage1 b"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id_b, + ownership_token.as_str(), + 101, + "raw memory b", + "summary b", + Some("rollout-b"), + ) + .await + .expect("mark stage1 succeeded b"), + "stage1 success should persist output b" + ); + + let outputs = runtime + .list_stage1_outputs_for_global(10) + .await + .expect("list stage1 outputs for global"); + assert_eq!(outputs.len(), 2); + assert_eq!(outputs[0].thread_id, thread_id_b); + assert_eq!(outputs[0].rollout_summary, "summary b"); + assert_eq!(outputs[0].rollout_slug.as_deref(), Some("rollout-b")); + assert_eq!(outputs[0].cwd, codex_home.join("workspace-b")); + assert_eq!(outputs[0].git_branch.as_deref(), Some("feature/stage1-b")); + assert_eq!(outputs[1].thread_id, thread_id_a); + assert_eq!(outputs[1].rollout_summary, "summary a"); + assert_eq!(outputs[1].rollout_slug, None); + assert_eq!(outputs[1].cwd, codex_home.join("workspace-a")); + assert_eq!(outputs[1].git_branch, None); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn list_stage1_outputs_for_global_skips_empty_payloads() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id_non_empty = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let thread_id_empty = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id_non_empty, + codex_home.join("workspace-non-empty"), + )) + .await + .expect("upsert non-empty thread"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id_empty, + codex_home.join("workspace-empty"), + )) + .await + .expect("upsert empty thread"); + + sqlx::query( + r#" +INSERT INTO stage1_outputs (thread_id, source_updated_at, raw_memory, rollout_summary, generated_at) +VALUES (?, ?, ?, ?, ?) + "#, + ) + .bind(thread_id_non_empty.to_string()) + .bind(100_i64) + .bind("raw memory") + .bind("summary") + .bind(100_i64) + .execute(runtime.pool.as_ref()) + .await + .expect("insert non-empty stage1 output"); + sqlx::query( + r#" +INSERT INTO stage1_outputs (thread_id, source_updated_at, raw_memory, rollout_summary, generated_at) +VALUES (?, ?, ?, ?, ?) + "#, + ) + .bind(thread_id_empty.to_string()) + .bind(101_i64) + .bind("") + .bind("") + .bind(101_i64) + .execute(runtime.pool.as_ref()) + .await + .expect("insert empty stage1 output"); + + let outputs = runtime + .list_stage1_outputs_for_global(1) + .await + .expect("list stage1 outputs for global"); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0].thread_id, thread_id_non_empty); + assert_eq!(outputs[0].rollout_summary, "summary"); + assert_eq!(outputs[0].cwd, codex_home.join("workspace-non-empty")); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn get_phase2_input_selection_reports_added_retained_and_removed_rows() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let thread_id_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let thread_id_c = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + + for (thread_id, workspace) in [ + (thread_id_a, "workspace-a"), + (thread_id_b, "workspace-b"), + (thread_id_c, "workspace-c"), + ] { + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join(workspace), + )) + .await + .expect("upsert thread"); + } + + for (thread_id, updated_at, slug) in [ + (thread_id_a, 100, Some("rollout-a")), + (thread_id_b, 101, Some("rollout-b")), + (thread_id_c, 102, Some("rollout-c")), + ] { + let claim = runtime + .try_claim_stage1_job(thread_id, owner, updated_at, 3600, 64) + .await + .expect("claim stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + updated_at, + &format!("raw-{updated_at}"), + &format!("summary-{updated_at}"), + slug, + ) + .await + .expect("mark stage1 succeeded"), + "stage1 success should persist output" + ); + } + + let claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2"); + let (ownership_token, input_watermark) = match claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim outcome: {other:?}"), + }; + assert_eq!(input_watermark, 102); + let selected_outputs = runtime + .list_stage1_outputs_for_global(10) + .await + .expect("list stage1 outputs for global") + .into_iter() + .filter(|output| output.thread_id == thread_id_c || output.thread_id == thread_id_a) + .collect::>(); + assert!( + runtime + .mark_global_phase2_job_succeeded( + ownership_token.as_str(), + input_watermark, + &selected_outputs, + ) + .await + .expect("mark phase2 success with selection"), + "phase2 success should persist selected rows" + ); + + let selection = runtime + .get_phase2_input_selection(2, 36_500) + .await + .expect("load phase2 input selection"); + + assert_eq!(selection.selected.len(), 2); + assert_eq!(selection.previous_selected.len(), 2); + assert_eq!(selection.selected[0].thread_id, thread_id_c); + assert_eq!( + selection.selected[0].rollout_path, + codex_home.join(format!("rollout-{thread_id_c}.jsonl")) + ); + assert_eq!(selection.selected[1].thread_id, thread_id_b); + assert_eq!(selection.retained_thread_ids, vec![thread_id_c]); + + assert_eq!(selection.removed.len(), 1); + assert_eq!(selection.removed[0].thread_id, thread_id_a); + assert_eq!( + selection.removed[0].rollout_slug.as_deref(), + Some("rollout-a") + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn get_phase2_input_selection_treats_regenerated_selected_rows_as_added() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + let first_claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) + .await + .expect("claim initial stage1"); + let first_token = match first_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + first_token.as_str(), + 100, + "raw-100", + "summary-100", + Some("rollout-100"), + ) + .await + .expect("mark initial stage1 success"), + "initial stage1 success should persist output" + ); + + let phase2_claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2"); + let (phase2_token, input_watermark) = match phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim outcome: {other:?}"), + }; + let selected_outputs = runtime + .list_stage1_outputs_for_global(1) + .await + .expect("list selected outputs"); + assert!( + runtime + .mark_global_phase2_job_succeeded( + phase2_token.as_str(), + input_watermark, + &selected_outputs, + ) + .await + .expect("mark phase2 success"), + "phase2 success should persist selected rows" + ); + + let refreshed_claim = runtime + .try_claim_stage1_job(thread_id, owner, 101, 3600, 64) + .await + .expect("claim refreshed stage1"); + let refreshed_token = match refreshed_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + refreshed_token.as_str(), + 101, + "raw-101", + "summary-101", + Some("rollout-101"), + ) + .await + .expect("mark refreshed stage1 success"), + "refreshed stage1 success should persist output" + ); + + let selection = runtime + .get_phase2_input_selection(1, 36_500) + .await + .expect("load phase2 input selection"); + assert_eq!(selection.selected.len(), 1); + assert_eq!(selection.previous_selected.len(), 1); + assert_eq!(selection.selected[0].thread_id, thread_id); + assert_eq!(selection.selected[0].source_updated_at.timestamp(), 101); + assert!(selection.retained_thread_ids.is_empty()); + assert!(selection.removed.is_empty()); + + let (selected_for_phase2, selected_for_phase2_source_updated_at) = + sqlx::query_as::<_, (i64, Option)>( + "SELECT selected_for_phase2, selected_for_phase2_source_updated_at FROM stage1_outputs WHERE thread_id = ?", + ) + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load selected_for_phase2"); + assert_eq!(selected_for_phase2, 1); + assert_eq!(selected_for_phase2_source_updated_at, Some(100)); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn get_phase2_input_selection_reports_regenerated_previous_selection_as_removed() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread a"); + let thread_id_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread b"); + let thread_id_c = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread c"); + let thread_id_d = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread d"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + + for (thread_id, workspace) in [ + (thread_id_a, "workspace-a"), + (thread_id_b, "workspace-b"), + (thread_id_c, "workspace-c"), + (thread_id_d, "workspace-d"), + ] { + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join(workspace), + )) + .await + .expect("upsert thread"); + } + + for (thread_id, updated_at, slug) in [ + (thread_id_a, 100, Some("rollout-a-100")), + (thread_id_b, 101, Some("rollout-b-101")), + (thread_id_c, 99, Some("rollout-c-99")), + (thread_id_d, 98, Some("rollout-d-98")), + ] { + let claim = runtime + .try_claim_stage1_job(thread_id, owner, updated_at, 3600, 64) + .await + .expect("claim initial stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + updated_at, + &format!("raw-{updated_at}"), + &format!("summary-{updated_at}"), + slug, + ) + .await + .expect("mark stage1 succeeded"), + "stage1 success should persist output" + ); + } + + let phase2_claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2"); + let (phase2_token, input_watermark) = match phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim outcome: {other:?}"), + }; + let selected_outputs = runtime + .list_stage1_outputs_for_global(2) + .await + .expect("list selected outputs"); + assert_eq!( + selected_outputs + .iter() + .map(|output| output.thread_id) + .collect::>(), + vec![thread_id_b, thread_id_a] + ); + assert!( + runtime + .mark_global_phase2_job_succeeded( + phase2_token.as_str(), + input_watermark, + &selected_outputs, + ) + .await + .expect("mark phase2 success"), + "phase2 success should persist selected rows" + ); + + for (thread_id, updated_at, slug) in [ + (thread_id_a, 102, Some("rollout-a-102")), + (thread_id_c, 103, Some("rollout-c-103")), + (thread_id_d, 104, Some("rollout-d-104")), + ] { + let claim = runtime + .try_claim_stage1_job(thread_id, owner, updated_at, 3600, 64) + .await + .expect("claim refreshed stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + updated_at, + &format!("raw-{updated_at}"), + &format!("summary-{updated_at}"), + slug, + ) + .await + .expect("mark refreshed stage1 success"), + "refreshed stage1 success should persist output" + ); + } + + let selection = runtime + .get_phase2_input_selection(2, 36_500) + .await + .expect("load phase2 input selection"); + assert_eq!( + selection + .selected + .iter() + .map(|output| output.thread_id) + .collect::>(), + vec![thread_id_d, thread_id_c] + ); + assert_eq!( + selection + .previous_selected + .iter() + .map(|output| output.thread_id) + .collect::>(), + vec![thread_id_a, thread_id_b] + ); + assert!(selection.retained_thread_ids.is_empty()); + assert_eq!( + selection + .removed + .iter() + .map(|output| (output.thread_id, output.source_updated_at.timestamp())) + .collect::>(), + vec![(thread_id_a, 102), (thread_id_b, 101)] + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn mark_global_phase2_job_succeeded_updates_selected_snapshot_timestamp() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + let initial_claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) + .await + .expect("claim initial stage1"); + let initial_token = match initial_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + initial_token.as_str(), + 100, + "raw-100", + "summary-100", + Some("rollout-100"), + ) + .await + .expect("mark initial stage1 success"), + "initial stage1 success should persist output" + ); + + let first_phase2_claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim first phase2"); + let (first_phase2_token, first_input_watermark) = match first_phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected first phase2 claim outcome: {other:?}"), + }; + let first_selected_outputs = runtime + .list_stage1_outputs_for_global(1) + .await + .expect("list first selected outputs"); + assert!( + runtime + .mark_global_phase2_job_succeeded( + first_phase2_token.as_str(), + first_input_watermark, + &first_selected_outputs, + ) + .await + .expect("mark first phase2 success"), + "first phase2 success should persist selected rows" + ); + + let refreshed_claim = runtime + .try_claim_stage1_job(thread_id, owner, 101, 3600, 64) + .await + .expect("claim refreshed stage1"); + let refreshed_token = match refreshed_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected refreshed stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + refreshed_token.as_str(), + 101, + "raw-101", + "summary-101", + Some("rollout-101"), + ) + .await + .expect("mark refreshed stage1 success"), + "refreshed stage1 success should persist output" + ); + + let second_phase2_claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim second phase2"); + let (second_phase2_token, second_input_watermark) = match second_phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected second phase2 claim outcome: {other:?}"), + }; + let second_selected_outputs = runtime + .list_stage1_outputs_for_global(1) + .await + .expect("list second selected outputs"); + assert_eq!( + second_selected_outputs[0].source_updated_at.timestamp(), + 101 + ); + assert!( + runtime + .mark_global_phase2_job_succeeded( + second_phase2_token.as_str(), + second_input_watermark, + &second_selected_outputs, + ) + .await + .expect("mark second phase2 success"), + "second phase2 success should persist selected rows" + ); + + let selection = runtime + .get_phase2_input_selection(1, 36_500) + .await + .expect("load phase2 input selection after refresh"); + assert_eq!(selection.retained_thread_ids, vec![thread_id]); + + let (selected_for_phase2, selected_for_phase2_source_updated_at) = + sqlx::query_as::<_, (i64, Option)>( + "SELECT selected_for_phase2, selected_for_phase2_source_updated_at FROM stage1_outputs WHERE thread_id = ?", + ) + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load selected snapshot after phase2"); + assert_eq!(selected_for_phase2, 1); + assert_eq!(selected_for_phase2_source_updated_at, Some(101)); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn mark_global_phase2_job_succeeded_only_marks_exact_selected_snapshots() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + let initial_claim = runtime + .try_claim_stage1_job(thread_id, owner, 100, 3600, 64) + .await + .expect("claim initial stage1"); + let initial_token = match initial_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + initial_token.as_str(), + 100, + "raw-100", + "summary-100", + Some("rollout-100"), + ) + .await + .expect("mark initial stage1 success"), + "initial stage1 success should persist output" + ); + + let phase2_claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim phase2"); + let (phase2_token, input_watermark) = match phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim outcome: {other:?}"), + }; + let selected_outputs = runtime + .list_stage1_outputs_for_global(1) + .await + .expect("list selected outputs"); + assert_eq!(selected_outputs[0].source_updated_at.timestamp(), 100); + + let refreshed_claim = runtime + .try_claim_stage1_job(thread_id, owner, 101, 3600, 64) + .await + .expect("claim refreshed stage1"); + let refreshed_token = match refreshed_claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + refreshed_token.as_str(), + 101, + "raw-101", + "summary-101", + Some("rollout-101"), + ) + .await + .expect("mark refreshed stage1 success"), + "refreshed stage1 success should persist output" + ); + + assert!( + runtime + .mark_global_phase2_job_succeeded( + phase2_token.as_str(), + input_watermark, + &selected_outputs, + ) + .await + .expect("mark phase2 success"), + "phase2 success should still complete" + ); + + let (selected_for_phase2, selected_for_phase2_source_updated_at) = + sqlx::query_as::<_, (i64, Option)>( + "SELECT selected_for_phase2, selected_for_phase2_source_updated_at FROM stage1_outputs WHERE thread_id = ?", + ) + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load selected_for_phase2"); + assert_eq!(selected_for_phase2, 0); + assert_eq!(selected_for_phase2_source_updated_at, None); + + let selection = runtime + .get_phase2_input_selection(1, 36_500) + .await + .expect("load phase2 input selection"); + assert_eq!(selection.selected.len(), 1); + assert_eq!(selection.selected[0].source_updated_at.timestamp(), 101); + assert!(selection.retained_thread_ids.is_empty()); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn record_stage1_output_usage_updates_usage_metadata() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id a"); + let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id b"); + let missing = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("missing id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_a, + codex_home.join("workspace-a"), + )) + .await + .expect("upsert thread a"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_b, + codex_home.join("workspace-b"), + )) + .await + .expect("upsert thread b"); + + let claim_a = runtime + .try_claim_stage1_job(thread_a, owner, 100, 3600, 64) + .await + .expect("claim stage1 a"); + let token_a = match claim_a { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome for a: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded(thread_a, token_a.as_str(), 100, "raw a", "sum a", None) + .await + .expect("mark stage1 succeeded a") + ); + + let claim_b = runtime + .try_claim_stage1_job(thread_b, owner, 101, 3600, 64) + .await + .expect("claim stage1 b"); + let token_b = match claim_b { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome for b: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded(thread_b, token_b.as_str(), 101, "raw b", "sum b", None) + .await + .expect("mark stage1 succeeded b") + ); + + let updated_rows = runtime + .record_stage1_output_usage(&[thread_a, thread_a, thread_b, missing]) + .await + .expect("record stage1 output usage"); + assert_eq!(updated_rows, 3); + + let row_a = + sqlx::query("SELECT usage_count, last_usage FROM stage1_outputs WHERE thread_id = ?") + .bind(thread_a.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load stage1 usage row a"); + let row_b = + sqlx::query("SELECT usage_count, last_usage FROM stage1_outputs WHERE thread_id = ?") + .bind(thread_b.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("load stage1 usage row b"); + + assert_eq!( + row_a + .try_get::("usage_count") + .expect("usage_count a"), + 2 + ); + assert_eq!( + row_b + .try_get::("usage_count") + .expect("usage_count b"), + 1 + ); + + let last_usage_a = row_a.try_get::("last_usage").expect("last_usage a"); + let last_usage_b = row_b.try_get::("last_usage").expect("last_usage b"); + assert_eq!(last_usage_a, last_usage_b); + assert!(last_usage_a > 0); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn get_phase2_input_selection_prioritizes_usage_count_then_recent_usage() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let now = Utc::now(); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id a"); + let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id b"); + let thread_c = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id c"); + + for (thread_id, workspace) in [ + (thread_a, "workspace-a"), + (thread_b, "workspace-b"), + (thread_c, "workspace-c"), + ] { + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join(workspace), + )) + .await + .expect("upsert thread"); + } + + for (thread_id, generated_at, summary) in [ + (thread_a, now - Duration::days(3), "summary-a"), + (thread_b, now - Duration::days(2), "summary-b"), + (thread_c, now - Duration::days(1), "summary-c"), + ] { + let source_updated_at = generated_at.timestamp(); + let claim = runtime + .try_claim_stage1_job(thread_id, owner, source_updated_at, 3600, 64) + .await + .expect("claim stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + source_updated_at, + &format!("raw-{summary}"), + summary, + None, + ) + .await + .expect("mark stage1 success"), + "stage1 success should persist output" + ); + } + + for (thread_id, usage_count, last_usage) in [ + (thread_a, 5_i64, now - Duration::days(10)), + (thread_b, 5_i64, now - Duration::days(1)), + (thread_c, 1_i64, now - Duration::hours(1)), + ] { + sqlx::query( + "UPDATE stage1_outputs SET usage_count = ?, last_usage = ? WHERE thread_id = ?", + ) + .bind(usage_count) + .bind(last_usage.timestamp()) + .bind(thread_id.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("update usage metadata"); + } + + let selection = runtime + .get_phase2_input_selection(3, 30) + .await + .expect("load phase2 input selection"); + + assert_eq!( + selection + .selected + .iter() + .map(|output| output.thread_id) + .collect::>(), + vec![thread_b, thread_a, thread_c] + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn get_phase2_input_selection_excludes_stale_used_memories_but_keeps_fresh_never_used() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let now = Utc::now(); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id a"); + let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id b"); + let thread_c = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id c"); + + for (thread_id, workspace) in [ + (thread_a, "workspace-a"), + (thread_b, "workspace-b"), + (thread_c, "workspace-c"), + ] { + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join(workspace), + )) + .await + .expect("upsert thread"); + } + + for (thread_id, generated_at, summary) in [ + (thread_a, now - Duration::days(40), "summary-a"), + (thread_b, now - Duration::days(2), "summary-b"), + (thread_c, now - Duration::days(50), "summary-c"), + ] { + let source_updated_at = generated_at.timestamp(); + let claim = runtime + .try_claim_stage1_job(thread_id, owner, source_updated_at, 3600, 64) + .await + .expect("claim stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + source_updated_at, + &format!("raw-{summary}"), + summary, + None, + ) + .await + .expect("mark stage1 success"), + "stage1 success should persist output" + ); + } + + for (thread_id, usage_count, last_usage) in [ + (thread_a, Some(9_i64), Some(now - Duration::days(31))), + (thread_b, None, None), + (thread_c, Some(1_i64), Some(now - Duration::days(1))), + ] { + sqlx::query( + "UPDATE stage1_outputs SET usage_count = ?, last_usage = ? WHERE thread_id = ?", + ) + .bind(usage_count) + .bind(last_usage.map(|value| value.timestamp())) + .bind(thread_id.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("update usage metadata"); + } + + let selection = runtime + .get_phase2_input_selection(3, 30) + .await + .expect("load phase2 input selection"); + + assert_eq!( + selection + .selected + .iter() + .map(|output| output.thread_id) + .collect::>(), + vec![thread_c, thread_b] + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn get_phase2_input_selection_prefers_recent_thread_updates_over_recent_generation() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + let older_thread = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("older thread id"); + let newer_thread = + ThreadId::from_string(&Uuid::new_v4().to_string()).expect("newer thread id"); + + for (thread_id, workspace) in [ + (older_thread, "workspace-older"), + (newer_thread, "workspace-newer"), + ] { + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join(workspace), + )) + .await + .expect("upsert thread"); + } + + for (thread_id, source_updated_at, summary) in [ + (older_thread, 100_i64, "summary-older"), + (newer_thread, 200_i64, "summary-newer"), + ] { + let claim = runtime + .try_claim_stage1_job(thread_id, owner, source_updated_at, 3600, 64) + .await + .expect("claim stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + source_updated_at, + &format!("raw-{summary}"), + summary, + None, + ) + .await + .expect("mark stage1 success"), + "stage1 success should persist output" + ); + } + + sqlx::query("UPDATE stage1_outputs SET generated_at = ? WHERE thread_id = ?") + .bind(300_i64) + .bind(older_thread.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("update older generated_at"); + sqlx::query("UPDATE stage1_outputs SET generated_at = ? WHERE thread_id = ?") + .bind(150_i64) + .bind(newer_thread.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("update newer generated_at"); + + let selection = runtime + .get_phase2_input_selection(1, 36_500) + .await + .expect("load phase2 input selection"); + + assert_eq!(selection.selected.len(), 1); + assert_eq!(selection.selected[0].thread_id, newer_thread); + assert_eq!(selection.selected[0].source_updated_at.timestamp(), 200); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn mark_stage1_job_succeeded_enqueues_global_consolidation() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + let thread_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id a"); + let thread_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id b"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_a, + codex_home.join("workspace-a"), + )) + .await + .expect("upsert thread a"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_b, + codex_home.join("workspace-b"), + )) + .await + .expect("upsert thread b"); + + let claim_a = runtime + .try_claim_stage1_job(thread_a, owner, 100, 3600, 64) + .await + .expect("claim stage1 a"); + let token_a = match claim_a { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome for thread a: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_a, + token_a.as_str(), + 100, + "raw-a", + "summary-a", + None, + ) + .await + .expect("mark stage1 succeeded a"), + "stage1 success should persist output for thread a" + ); + + let claim_b = runtime + .try_claim_stage1_job(thread_b, owner, 101, 3600, 64) + .await + .expect("claim stage1 b"); + let token_b = match claim_b { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome for thread b: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_b, + token_b.as_str(), + 101, + "raw-b", + "summary-b", + None, + ) + .await + .expect("mark stage1 succeeded b"), + "stage1 success should persist output for thread b" + ); + + let claim = runtime + .try_claim_global_phase2_job(owner, 3600) + .await + .expect("claim global consolidation"); + let input_watermark = match claim { + Phase2JobClaimOutcome::Claimed { + input_watermark, .. + } => input_watermark, + other => panic!("unexpected global consolidation claim outcome: {other:?}"), + }; + assert_eq!(input_watermark, 101); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn phase2_global_lock_allows_only_one_fresh_runner() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + runtime + .enqueue_global_consolidation(200) + .await + .expect("enqueue global consolidation"); + + let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner a"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner b"); + + let running_claim = runtime + .try_claim_global_phase2_job(owner_a, 3600) + .await + .expect("claim global lock"); + assert!( + matches!(running_claim, Phase2JobClaimOutcome::Claimed { .. }), + "first owner should claim global lock" + ); + + let second_claim = runtime + .try_claim_global_phase2_job(owner_b, 3600) + .await + .expect("claim global lock from second owner"); + assert_eq!(second_claim, Phase2JobClaimOutcome::SkippedRunning); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn phase2_global_lock_stale_lease_allows_takeover() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + runtime + .enqueue_global_consolidation(300) + .await + .expect("enqueue global consolidation"); + + let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner a"); + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner b"); + + let initial_claim = runtime + .try_claim_global_phase2_job(owner_a, 3600) + .await + .expect("claim initial global lock"); + let token_a = match initial_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, .. + } => ownership_token, + other => panic!("unexpected initial claim outcome: {other:?}"), + }; + + sqlx::query("UPDATE jobs SET lease_until = ? WHERE kind = ? AND job_key = ?") + .bind(Utc::now().timestamp() - 1) + .bind("memory_consolidate_global") + .bind("global") + .execute(runtime.pool.as_ref()) + .await + .expect("expire global consolidation lease"); + + let takeover_claim = runtime + .try_claim_global_phase2_job(owner_b, 3600) + .await + .expect("claim stale global lock"); + let (token_b, input_watermark) = match takeover_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected takeover claim outcome: {other:?}"), + }; + assert_ne!(token_a, token_b); + assert_eq!(input_watermark, 300); + + assert_eq!( + runtime + .mark_global_phase2_job_succeeded(token_a.as_str(), 300, &[]) + .await + .expect("mark stale owner success result"), + false, + "stale owner should lose finalization ownership after takeover" + ); + assert!( + runtime + .mark_global_phase2_job_succeeded(token_b.as_str(), 300, &[]) + .await + .expect("mark takeover owner success"), + "takeover owner should finalize consolidation" + ); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn phase2_backfilled_inputs_below_last_success_still_become_dirty() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + runtime + .enqueue_global_consolidation(500) + .await + .expect("enqueue initial consolidation"); + let owner_a = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner a"); + let claim_a = runtime + .try_claim_global_phase2_job(owner_a, 3_600) + .await + .expect("claim initial consolidation"); + let token_a = match claim_a { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => { + assert_eq!(input_watermark, 500); + ownership_token + } + other => panic!("unexpected initial phase2 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_global_phase2_job_succeeded(token_a.as_str(), 500, &[]) + .await + .expect("mark initial phase2 success"), + "initial phase2 success should finalize" + ); + + runtime + .enqueue_global_consolidation(400) + .await + .expect("enqueue backfilled consolidation"); + + let owner_b = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner b"); + let claim_b = runtime + .try_claim_global_phase2_job(owner_b, 3_600) + .await + .expect("claim backfilled consolidation"); + match claim_b { + Phase2JobClaimOutcome::Claimed { + input_watermark, .. + } => { + assert!( + input_watermark > 500, + "backfilled enqueue should advance dirty watermark beyond last success" + ); + } + other => panic!("unexpected backfilled phase2 claim outcome: {other:?}"), + } + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + + #[tokio::test] + async fn phase2_failure_fallback_updates_unowned_running_job() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("initialize runtime"); + + runtime + .enqueue_global_consolidation(400) + .await + .expect("enqueue global consolidation"); + + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner"); + let claim = runtime + .try_claim_global_phase2_job(owner, 3_600) + .await + .expect("claim global consolidation"); + let ownership_token = match claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, .. + } => ownership_token, + other => panic!("unexpected claim outcome: {other:?}"), + }; + + sqlx::query("UPDATE jobs SET ownership_token = NULL WHERE kind = ? AND job_key = ?") + .bind("memory_consolidate_global") + .bind("global") + .execute(runtime.pool.as_ref()) + .await + .expect("clear ownership token"); + + assert_eq!( + runtime + .mark_global_phase2_job_failed(ownership_token.as_str(), "lost", 3_600) + .await + .expect("mark phase2 failed with strict ownership"), + false, + "strict failure update should not match unowned running job" + ); + assert!( + runtime + .mark_global_phase2_job_failed_if_unowned(ownership_token.as_str(), "lost", 3_600) + .await + .expect("fallback failure update should match unowned running job"), + "fallback failure update should transition the unowned running job" + ); + + let claim = runtime + .try_claim_global_phase2_job(ThreadId::new(), 3_600) + .await + .expect("claim after fallback failure"); + assert_eq!(claim, Phase2JobClaimOutcome::SkippedNotDirty); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } +} diff --git a/codex-rs/state/src/runtime/test_support.rs b/codex-rs/state/src/runtime/test_support.rs new file mode 100644 index 0000000000..d749fe2bfb --- /dev/null +++ b/codex-rs/state/src/runtime/test_support.rs @@ -0,0 +1,64 @@ +#[cfg(test)] +use chrono::DateTime; +#[cfg(test)] +use chrono::Utc; +#[cfg(test)] +use codex_protocol::ThreadId; +#[cfg(test)] +use codex_protocol::protocol::AskForApproval; +#[cfg(test)] +use codex_protocol::protocol::SandboxPolicy; +#[cfg(test)] +use std::path::Path; +#[cfg(test)] +use std::path::PathBuf; +#[cfg(test)] +use std::time::SystemTime; +#[cfg(test)] +use std::time::UNIX_EPOCH; +#[cfg(test)] +use uuid::Uuid; + +#[cfg(test)] +use crate::ThreadMetadata; + +#[cfg(test)] +pub(super) fn unique_temp_dir() -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_nanos()); + std::env::temp_dir().join(format!( + "codex-state-runtime-test-{nanos}-{}", + Uuid::new_v4() + )) +} + +#[cfg(test)] +pub(super) fn test_thread_metadata( + codex_home: &Path, + thread_id: ThreadId, + cwd: PathBuf, +) -> ThreadMetadata { + let now = DateTime::::from_timestamp(1_700_000_000, 0).expect("timestamp"); + ThreadMetadata { + id: thread_id, + rollout_path: codex_home.join(format!("rollout-{thread_id}.jsonl")), + created_at: now, + updated_at: now, + source: "cli".to_string(), + agent_nickname: None, + agent_role: None, + model_provider: "test-provider".to_string(), + cwd, + cli_version: "0.0.0".to_string(), + title: String::new(), + sandbox_policy: crate::extract::enum_to_string(&SandboxPolicy::new_read_only_policy()), + approval_mode: crate::extract::enum_to_string(&AskForApproval::OnRequest), + tokens_used: 0, + first_user_message: Some("hello".to_string()), + archived_at: None, + git_sha: None, + git_branch: None, + git_origin_url: None, + } +} diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs new file mode 100644 index 0000000000..e02cfb3e4a --- /dev/null +++ b/codex-rs/state/src/runtime/threads.rs @@ -0,0 +1,560 @@ +use super::*; + +impl StateRuntime { + pub async fn get_thread(&self, id: ThreadId) -> anyhow::Result> { + let row = sqlx::query( + r#" +SELECT + id, + rollout_path, + created_at, + updated_at, + source, + agent_nickname, + agent_role, + model_provider, + cwd, + cli_version, + title, + sandbox_policy, + approval_mode, + tokens_used, + first_user_message, + archived_at, + git_sha, + git_branch, + git_origin_url +FROM threads +WHERE id = ? + "#, + ) + .bind(id.to_string()) + .fetch_optional(self.pool.as_ref()) + .await?; + row.map(|row| ThreadRow::try_from_row(&row).and_then(ThreadMetadata::try_from)) + .transpose() + } + + /// Get dynamic tools for a thread, if present. + pub async fn get_dynamic_tools( + &self, + thread_id: ThreadId, + ) -> anyhow::Result>> { + let rows = sqlx::query( + r#" +SELECT name, description, input_schema +FROM thread_dynamic_tools +WHERE thread_id = ? +ORDER BY position ASC + "#, + ) + .bind(thread_id.to_string()) + .fetch_all(self.pool.as_ref()) + .await?; + if rows.is_empty() { + return Ok(None); + } + let mut tools = Vec::with_capacity(rows.len()); + for row in rows { + let input_schema: String = row.try_get("input_schema")?; + let input_schema = serde_json::from_str::(input_schema.as_str())?; + tools.push(DynamicToolSpec { + name: row.try_get("name")?, + description: row.try_get("description")?, + input_schema, + }); + } + Ok(Some(tools)) + } + + /// Find a rollout path by thread id using the underlying database. + pub async fn find_rollout_path_by_id( + &self, + id: ThreadId, + archived_only: Option, + ) -> anyhow::Result> { + let mut builder = + QueryBuilder::::new("SELECT rollout_path FROM threads WHERE id = "); + builder.push_bind(id.to_string()); + match archived_only { + Some(true) => { + builder.push(" AND archived = 1"); + } + Some(false) => { + builder.push(" AND archived = 0"); + } + None => {} + } + let row = builder.build().fetch_optional(self.pool.as_ref()).await?; + Ok(row + .and_then(|r| r.try_get::("rollout_path").ok()) + .map(PathBuf::from)) + } + + /// List threads using the underlying database. + #[allow(clippy::too_many_arguments)] + pub async fn list_threads( + &self, + page_size: usize, + anchor: Option<&crate::Anchor>, + sort_key: crate::SortKey, + allowed_sources: &[String], + model_providers: Option<&[String]>, + archived_only: bool, + search_term: Option<&str>, + ) -> anyhow::Result { + let limit = page_size.saturating_add(1); + + let mut builder = QueryBuilder::::new( + r#" +SELECT + id, + rollout_path, + created_at, + updated_at, + source, + agent_nickname, + agent_role, + model_provider, + cwd, + cli_version, + title, + sandbox_policy, + approval_mode, + tokens_used, + first_user_message, + archived_at, + git_sha, + git_branch, + git_origin_url +FROM threads + "#, + ); + push_thread_filters( + &mut builder, + archived_only, + allowed_sources, + model_providers, + anchor, + sort_key, + search_term, + ); + push_thread_order_and_limit(&mut builder, sort_key, limit); + + let rows = builder.build().fetch_all(self.pool.as_ref()).await?; + let mut items = rows + .into_iter() + .map(|row| ThreadRow::try_from_row(&row).and_then(ThreadMetadata::try_from)) + .collect::, _>>()?; + let num_scanned_rows = items.len(); + let next_anchor = if items.len() > page_size { + items.pop(); + items + .last() + .and_then(|item| anchor_from_item(item, sort_key)) + } else { + None + }; + Ok(ThreadsPage { + items, + next_anchor, + num_scanned_rows, + }) + } + + /// List thread ids using the underlying database (no rollout scanning). + pub async fn list_thread_ids( + &self, + limit: usize, + anchor: Option<&crate::Anchor>, + sort_key: crate::SortKey, + allowed_sources: &[String], + model_providers: Option<&[String]>, + archived_only: bool, + ) -> anyhow::Result> { + let mut builder = QueryBuilder::::new("SELECT id FROM threads"); + push_thread_filters( + &mut builder, + archived_only, + allowed_sources, + model_providers, + anchor, + sort_key, + None, + ); + push_thread_order_and_limit(&mut builder, sort_key, limit); + + let rows = builder.build().fetch_all(self.pool.as_ref()).await?; + rows.into_iter() + .map(|row| { + let id: String = row.try_get("id")?; + Ok(ThreadId::try_from(id)?) + }) + .collect() + } + + /// Insert or replace thread metadata directly. + pub async fn upsert_thread(&self, metadata: &crate::ThreadMetadata) -> anyhow::Result<()> { + self.upsert_thread_with_creation_memory_mode(metadata, None) + .await + } + + async fn upsert_thread_with_creation_memory_mode( + &self, + metadata: &crate::ThreadMetadata, + creation_memory_mode: Option<&str>, + ) -> anyhow::Result<()> { + sqlx::query( + r#" +INSERT INTO threads ( + id, + rollout_path, + created_at, + updated_at, + source, + agent_nickname, + agent_role, + model_provider, + cwd, + cli_version, + title, + sandbox_policy, + approval_mode, + tokens_used, + first_user_message, + archived, + archived_at, + git_sha, + git_branch, + git_origin_url, + memory_mode +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(id) DO UPDATE SET + rollout_path = excluded.rollout_path, + created_at = excluded.created_at, + updated_at = excluded.updated_at, + source = excluded.source, + agent_nickname = excluded.agent_nickname, + agent_role = excluded.agent_role, + model_provider = excluded.model_provider, + cwd = excluded.cwd, + cli_version = excluded.cli_version, + title = excluded.title, + sandbox_policy = excluded.sandbox_policy, + approval_mode = excluded.approval_mode, + tokens_used = excluded.tokens_used, + first_user_message = excluded.first_user_message, + archived = excluded.archived, + archived_at = excluded.archived_at, + git_sha = excluded.git_sha, + git_branch = excluded.git_branch, + git_origin_url = excluded.git_origin_url + "#, + ) + .bind(metadata.id.to_string()) + .bind(metadata.rollout_path.display().to_string()) + .bind(datetime_to_epoch_seconds(metadata.created_at)) + .bind(datetime_to_epoch_seconds(metadata.updated_at)) + .bind(metadata.source.as_str()) + .bind(metadata.agent_nickname.as_deref()) + .bind(metadata.agent_role.as_deref()) + .bind(metadata.model_provider.as_str()) + .bind(metadata.cwd.display().to_string()) + .bind(metadata.cli_version.as_str()) + .bind(metadata.title.as_str()) + .bind(metadata.sandbox_policy.as_str()) + .bind(metadata.approval_mode.as_str()) + .bind(metadata.tokens_used) + .bind(metadata.first_user_message.as_deref().unwrap_or_default()) + .bind(metadata.archived_at.is_some()) + .bind(metadata.archived_at.map(datetime_to_epoch_seconds)) + .bind(metadata.git_sha.as_deref()) + .bind(metadata.git_branch.as_deref()) + .bind(metadata.git_origin_url.as_deref()) + .bind(creation_memory_mode.unwrap_or("enabled")) + .execute(self.pool.as_ref()) + .await?; + Ok(()) + } + + /// Persist dynamic tools for a thread if none have been stored yet. + /// + /// Dynamic tools are defined at thread start and should not change afterward. + /// This only writes the first time we see tools for a given thread. + pub async fn persist_dynamic_tools( + &self, + thread_id: ThreadId, + tools: Option<&[DynamicToolSpec]>, + ) -> anyhow::Result<()> { + let Some(tools) = tools else { + return Ok(()); + }; + if tools.is_empty() { + return Ok(()); + } + let thread_id = thread_id.to_string(); + let mut tx = self.pool.begin().await?; + for (idx, tool) in tools.iter().enumerate() { + let position = i64::try_from(idx).unwrap_or(i64::MAX); + let input_schema = serde_json::to_string(&tool.input_schema)?; + sqlx::query( + r#" +INSERT INTO thread_dynamic_tools ( + thread_id, + position, + name, + description, + input_schema +) VALUES (?, ?, ?, ?, ?) +ON CONFLICT(thread_id, position) DO NOTHING + "#, + ) + .bind(thread_id.as_str()) + .bind(position) + .bind(tool.name.as_str()) + .bind(tool.description.as_str()) + .bind(input_schema) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + /// Apply rollout items incrementally using the underlying database. + pub async fn apply_rollout_items( + &self, + builder: &ThreadMetadataBuilder, + items: &[RolloutItem], + otel: Option<&OtelManager>, + new_thread_memory_mode: Option<&str>, + ) -> anyhow::Result<()> { + if items.is_empty() { + return Ok(()); + } + let existing_metadata = self.get_thread(builder.id).await?; + let mut metadata = existing_metadata + .clone() + .unwrap_or_else(|| builder.build(&self.default_provider)); + metadata.rollout_path = builder.rollout_path.clone(); + for item in items { + apply_rollout_item(&mut metadata, item, &self.default_provider); + } + if let Some(updated_at) = file_modified_time_utc(builder.rollout_path.as_path()).await { + metadata.updated_at = updated_at; + } + // Keep the thread upsert before dynamic tools to satisfy the foreign key constraint: + // thread_dynamic_tools.thread_id -> threads.id. + let upsert_result = if existing_metadata.is_none() { + self.upsert_thread_with_creation_memory_mode(&metadata, new_thread_memory_mode) + .await + } else { + self.upsert_thread(&metadata).await + }; + if let Err(err) = upsert_result { + if let Some(otel) = otel { + otel.counter(DB_ERROR_METRIC, 1, &[("stage", "apply_rollout_items")]); + } + return Err(err); + } + let dynamic_tools = extract_dynamic_tools(items); + if let Some(dynamic_tools) = dynamic_tools + && let Err(err) = self + .persist_dynamic_tools(builder.id, dynamic_tools.as_deref()) + .await + { + if let Some(otel) = otel { + otel.counter(DB_ERROR_METRIC, 1, &[("stage", "persist_dynamic_tools")]); + } + return Err(err); + } + Ok(()) + } + + /// Mark a thread as archived using the underlying database. + pub async fn mark_archived( + &self, + thread_id: ThreadId, + rollout_path: &Path, + archived_at: DateTime, + ) -> anyhow::Result<()> { + let Some(mut metadata) = self.get_thread(thread_id).await? else { + return Ok(()); + }; + metadata.archived_at = Some(archived_at); + metadata.rollout_path = rollout_path.to_path_buf(); + if let Some(updated_at) = file_modified_time_utc(rollout_path).await { + metadata.updated_at = updated_at; + } + if metadata.id != thread_id { + warn!( + "thread id mismatch during archive: expected {thread_id}, got {}", + metadata.id + ); + } + self.upsert_thread(&metadata).await + } + + /// Mark a thread as unarchived using the underlying database. + pub async fn mark_unarchived( + &self, + thread_id: ThreadId, + rollout_path: &Path, + ) -> anyhow::Result<()> { + let Some(mut metadata) = self.get_thread(thread_id).await? else { + return Ok(()); + }; + metadata.archived_at = None; + metadata.rollout_path = rollout_path.to_path_buf(); + if let Some(updated_at) = file_modified_time_utc(rollout_path).await { + metadata.updated_at = updated_at; + } + if metadata.id != thread_id { + warn!( + "thread id mismatch during unarchive: expected {thread_id}, got {}", + metadata.id + ); + } + self.upsert_thread(&metadata).await + } + + /// Delete a thread metadata row by id. + pub async fn delete_thread(&self, thread_id: ThreadId) -> anyhow::Result { + let result = sqlx::query("DELETE FROM threads WHERE id = ?") + .bind(thread_id.to_string()) + .execute(self.pool.as_ref()) + .await?; + Ok(result.rows_affected()) + } +} + +pub(super) fn extract_dynamic_tools(items: &[RolloutItem]) -> Option>> { + items.iter().find_map(|item| match item { + RolloutItem::SessionMeta(meta_line) => Some(meta_line.meta.dynamic_tools.clone()), + RolloutItem::ResponseItem(_) + | RolloutItem::Compacted(_) + | RolloutItem::TurnContext(_) + | RolloutItem::EventMsg(_) => None, + }) +} + +pub(super) fn push_thread_filters<'a>( + builder: &mut QueryBuilder<'a, Sqlite>, + archived_only: bool, + allowed_sources: &'a [String], + model_providers: Option<&'a [String]>, + anchor: Option<&crate::Anchor>, + sort_key: SortKey, + search_term: Option<&'a str>, +) { + builder.push(" WHERE 1 = 1"); + if archived_only { + builder.push(" AND archived = 1"); + } else { + builder.push(" AND archived = 0"); + } + builder.push(" AND first_user_message <> ''"); + if !allowed_sources.is_empty() { + builder.push(" AND source IN ("); + let mut separated = builder.separated(", "); + for source in allowed_sources { + separated.push_bind(source); + } + separated.push_unseparated(")"); + } + if let Some(model_providers) = model_providers + && !model_providers.is_empty() + { + builder.push(" AND model_provider IN ("); + let mut separated = builder.separated(", "); + for provider in model_providers { + separated.push_bind(provider); + } + separated.push_unseparated(")"); + } + if let Some(search_term) = search_term { + builder.push(" AND instr(title, "); + builder.push_bind(search_term); + builder.push(") > 0"); + } + if let Some(anchor) = anchor { + let anchor_ts = datetime_to_epoch_seconds(anchor.ts); + let column = match sort_key { + SortKey::CreatedAt => "created_at", + SortKey::UpdatedAt => "updated_at", + }; + builder.push(" AND ("); + builder.push(column); + builder.push(" < "); + builder.push_bind(anchor_ts); + builder.push(" OR ("); + builder.push(column); + builder.push(" = "); + builder.push_bind(anchor_ts); + builder.push(" AND id < "); + builder.push_bind(anchor.id.to_string()); + builder.push("))"); + } +} + +pub(super) fn push_thread_order_and_limit( + builder: &mut QueryBuilder<'_, Sqlite>, + sort_key: SortKey, + limit: usize, +) { + let order_column = match sort_key { + SortKey::CreatedAt => "created_at", + SortKey::UpdatedAt => "updated_at", + }; + builder.push(" ORDER BY "); + builder.push(order_column); + builder.push(" DESC, id DESC"); + builder.push(" LIMIT "); + builder.push_bind(limit as i64); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runtime::test_support::test_thread_metadata; + use crate::runtime::test_support::unique_temp_dir; + use pretty_assertions::assert_eq; + + #[tokio::test] + async fn upsert_thread_keeps_creation_memory_mode_for_existing_rows() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string(), None) + .await + .expect("state db should initialize"); + let thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000123").expect("valid thread id"); + let mut metadata = test_thread_metadata(&codex_home, thread_id, codex_home.clone()); + + runtime + .upsert_thread_with_creation_memory_mode(&metadata, Some("disabled")) + .await + .expect("initial insert should succeed"); + + let memory_mode: String = + sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("memory mode should be readable"); + assert_eq!(memory_mode, "disabled"); + + metadata.title = "updated title".to_string(); + runtime + .upsert_thread(&metadata) + .await + .expect("upsert should succeed"); + + let memory_mode: String = + sqlx::query_scalar("SELECT memory_mode FROM threads WHERE id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await + .expect("memory mode should remain readable"); + assert_eq!(memory_mode, "disabled"); + } +} diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 1a87314e98..594acb33d4 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -50,6 +50,7 @@ codex-utils-fuzzy-match = { workspace = true } codex-utils-oss = { workspace = true } codex-utils-sandbox-summary = { workspace = true } codex-utils-sleep-inhibitor = { workspace = true } +codex-utils-string = { workspace = true } color-eyre = { workspace = true } crossterm = { workspace = true, features = ["bracketed-paste", "event-stream"] } derive_more = { workspace = true, features = ["is_variant"] } diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 62d459fc6f..e32a76da0f 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -1,6 +1,7 @@ use crate::app_backtrack::BacktrackState; use crate::app_event::AppEvent; use crate::app_event::ExitMode; +use crate::app_event::RealtimeAudioDeviceKind; #[cfg(target_os = "windows")] use crate::app_event::WindowsSandboxEnableMode; use crate::app_event_sender::AppEventSender; @@ -1414,12 +1415,16 @@ impl App { }; ChatWidget::new(init, thread_manager.clone()) } - SessionSelection::Resume(path) => { + SessionSelection::Resume(target_session) => { let resumed = thread_manager - .resume_thread_from_rollout(config.clone(), path.clone(), auth_manager.clone()) + .resume_thread_from_rollout( + config.clone(), + target_session.path.clone(), + auth_manager.clone(), + ) .await .wrap_err_with(|| { - let path_display = path.display(); + let path_display = target_session.path.display(); format!("Failed to resume session from {path_display}") })?; let init = crate::chatwidget::ChatWidgetInit { @@ -1444,13 +1449,18 @@ impl App { }; ChatWidget::new_from_existing(init, resumed.thread, resumed.session_configured) } - SessionSelection::Fork(path) => { + SessionSelection::Fork(target_session) => { otel_manager.counter("codex.thread.fork", 1, &[("source", "cli_subcommand")]); let forked = thread_manager - .fork_thread(usize::MAX, config.clone(), path.clone(), false) + .fork_thread( + usize::MAX, + config.clone(), + target_session.path.clone(), + false, + ) .await .wrap_err_with(|| { - let path_display = path.display(); + let path_display = target_session.path.display(); format!("Failed to fork session from {path_display}") })?; let init = crate::chatwidget::ChatWidgetInit { @@ -1713,12 +1723,14 @@ impl App { } AppEvent::OpenResumePicker => { match crate::resume_picker::run_resume_picker(tui, &self.config, false).await? { - SessionSelection::Resume(path) => { + SessionSelection::Resume(target_session) => { let current_cwd = self.config.cwd.clone(); let resume_cwd = match crate::resolve_cwd_for_resume_or_fork( tui, + &self.config, ¤t_cwd, - &path, + target_session.thread_id, + &target_session.path, CwdPromptAction::Resume, true, ) @@ -1754,7 +1766,7 @@ impl App { .server .resume_thread_from_rollout( resume_config.clone(), - path.clone(), + target_session.path.clone(), self.auth_manager.clone(), ) .await @@ -1788,7 +1800,7 @@ impl App { } } Err(err) => { - let path_display = path.display(); + let path_display = target_session.path.display(); self.chat_widget.add_error_message(format!( "Failed to resume session from {path_display}: {err}" )); @@ -1925,19 +1937,9 @@ impl App { AppEvent::CodexEvent(event) => { self.enqueue_primary_event(event).await?; } - AppEvent::Exit(mode) => match mode { - ExitMode::ShutdownFirst => { - // Mark the thread we are explicitly shutting down for exit so - // its shutdown completion does not trigger agent failover. - self.pending_shutdown_exit_thread_id = - self.active_thread_id.or(self.chat_widget.thread_id()); - self.chat_widget.submit_op(Op::Shutdown); - } - ExitMode::Immediate => { - self.pending_shutdown_exit_thread_id = None; - return Ok(AppRunControl::Exit(ExitReason::UserRequested)); - } - }, + AppEvent::Exit(mode) => { + return Ok(self.handle_exit_mode(mode)); + } AppEvent::FatalExitRequest(message) => { return Ok(AppRunControl::Exit(ExitReason::Fatal(message))); } @@ -2019,6 +2021,9 @@ impl App { AppEvent::UpdatePersonality(personality) => { self.on_update_personality(personality); } + AppEvent::OpenRealtimeAudioDeviceSelection { kind } => { + self.chat_widget.open_realtime_audio_device_selection(kind); + } AppEvent::OpenReasoningPopup { model } => { self.chat_widget.open_reasoning_popup(model); } @@ -2444,6 +2449,56 @@ impl App { } } } + AppEvent::PersistRealtimeAudioDeviceSelection { kind, name } => { + let builder = match kind { + RealtimeAudioDeviceKind::Microphone => { + ConfigEditsBuilder::new(&self.config.codex_home) + .set_realtime_microphone(name.as_deref()) + } + RealtimeAudioDeviceKind::Speaker => { + ConfigEditsBuilder::new(&self.config.codex_home) + .set_realtime_speaker(name.as_deref()) + } + }; + + match builder.apply().await { + Ok(()) => { + match kind { + RealtimeAudioDeviceKind::Microphone => { + self.config.realtime_audio.microphone = name.clone(); + } + RealtimeAudioDeviceKind::Speaker => { + self.config.realtime_audio.speaker = name.clone(); + } + } + self.chat_widget + .set_realtime_audio_device(kind, name.clone()); + + if self.chat_widget.realtime_conversation_is_live() { + self.chat_widget.open_realtime_audio_restart_prompt(kind); + } else { + let selection = name.unwrap_or_else(|| "System default".to_string()); + self.chat_widget.add_info_message( + format!("Realtime {} set to {selection}", kind.noun()), + None, + ); + } + } + Err(err) => { + tracing::error!( + error = %err, + "failed to persist realtime audio selection" + ); + self.chat_widget.add_error_message(format!( + "Failed to save realtime {}: {err}", + kind.noun() + )); + } + } + } + AppEvent::RestartRealtimeAudioDevice { kind } => { + self.chat_widget.restart_realtime_audio_device(kind); + } AppEvent::UpdateAskForApprovalPolicy(policy) => { self.runtime_approval_policy_override = Some(policy); if let Err(err) = self.config.permissions.approval_policy.set(policy) { @@ -2914,6 +2969,27 @@ impl App { Ok(AppRunControl::Continue) } + fn handle_exit_mode(&mut self, mode: ExitMode) -> AppRunControl { + match mode { + ExitMode::ShutdownFirst => { + // Mark the thread we are explicitly shutting down for exit so + // its shutdown completion does not trigger agent failover. + self.pending_shutdown_exit_thread_id = + self.active_thread_id.or(self.chat_widget.thread_id()); + if self.chat_widget.submit_op(Op::Shutdown) { + AppRunControl::Continue + } else { + self.pending_shutdown_exit_thread_id = None; + AppRunControl::Exit(ExitReason::UserRequested) + } + } + ExitMode::Immediate => { + self.pending_shutdown_exit_thread_id = None; + AppRunControl::Exit(ExitReason::UserRequested) + } + } + } + fn handle_codex_event_now(&mut self, event: Event) { let needs_refresh = matches!( event.msg, @@ -3389,15 +3465,21 @@ mod tests { true ); assert_eq!( - App::should_wait_for_initial_session(&SessionSelection::Resume(PathBuf::from( - "/tmp/restore" - ))), + App::should_wait_for_initial_session(&SessionSelection::Resume( + crate::resume_picker::SessionTarget { + path: PathBuf::from("/tmp/restore"), + thread_id: ThreadId::new(), + } + )), false ); assert_eq!( - App::should_wait_for_initial_session(&SessionSelection::Fork(PathBuf::from( - "/tmp/fork" - ))), + App::should_wait_for_initial_session(&SessionSelection::Fork( + crate::resume_picker::SessionTarget { + path: PathBuf::from("/tmp/fork"), + thread_id: ThreadId::new(), + } + )), false ); } @@ -3433,14 +3515,20 @@ mod tests { #[test] fn startup_waiting_gate_not_applied_for_resume_or_fork_session_selection() { let wait_for_resume = App::should_wait_for_initial_session(&SessionSelection::Resume( - PathBuf::from("/tmp/restore"), + crate::resume_picker::SessionTarget { + path: PathBuf::from("/tmp/restore"), + thread_id: ThreadId::new(), + }, )); assert_eq!( App::should_handle_active_thread_events(wait_for_resume, true), true ); let wait_for_fork = App::should_wait_for_initial_session(&SessionSelection::Fork( - PathBuf::from("/tmp/fork"), + crate::resume_picker::SessionTarget { + path: PathBuf::from("/tmp/fork"), + thread_id: ThreadId::new(), + }, )); assert_eq!( App::should_handle_active_thread_events(wait_for_fork, true), @@ -3569,6 +3657,7 @@ mod tests { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: Vec::new(), }, ), @@ -4702,6 +4791,34 @@ mod tests { } } + #[tokio::test] + async fn shutdown_first_exit_returns_immediate_exit_when_shutdown_submit_fails() { + let mut app = make_test_app().await; + let thread_id = ThreadId::new(); + app.active_thread_id = Some(thread_id); + + let control = app.handle_exit_mode(ExitMode::ShutdownFirst); + + assert_eq!(app.pending_shutdown_exit_thread_id, None); + assert!(matches!( + control, + AppRunControl::Exit(ExitReason::UserRequested) + )); + } + + #[tokio::test] + async fn shutdown_first_exit_waits_for_shutdown_when_submit_succeeds() { + let (mut app, _app_event_rx, mut op_rx) = make_test_app_with_channels().await; + let thread_id = ThreadId::new(); + app.active_thread_id = Some(thread_id); + + let control = app.handle_exit_mode(ExitMode::ShutdownFirst); + + assert_eq!(app.pending_shutdown_exit_thread_id, Some(thread_id)); + assert!(matches!(control, AppRunControl::Continue)); + assert_eq!(op_rx.try_recv(), Ok(Op::Shutdown)); + } + #[tokio::test] async fn clear_only_ui_reset_preserves_chat_session_state() { let mut app = make_test_app().await; diff --git a/codex-rs/tui/src/app/pending_interactive_replay.rs b/codex-rs/tui/src/app/pending_interactive_replay.rs index 3d30914781..d82d014827 100644 --- a/codex-rs/tui/src/app/pending_interactive_replay.rs +++ b/codex-rs/tui/src/app/pending_interactive_replay.rs @@ -402,6 +402,7 @@ mod tests { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: Vec::new(), }, ), @@ -544,6 +545,7 @@ mod tests { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: Vec::new(), }, ), @@ -622,6 +624,7 @@ mod tests { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: Vec::new(), }, ), diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 0767696e28..104895ad9e 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -29,6 +29,28 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RealtimeAudioDeviceKind { + Microphone, + Speaker, +} + +impl RealtimeAudioDeviceKind { + pub(crate) fn title(self) -> &'static str { + match self { + Self::Microphone => "Microphone", + Self::Speaker => "Speaker", + } + } + + pub(crate) fn noun(self) -> &'static str { + match self { + Self::Microphone => "microphone", + Self::Speaker => "speaker", + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg_attr(not(target_os = "windows"), allow(dead_code))] pub(crate) enum WindowsSandboxEnableMode { @@ -166,6 +188,26 @@ pub(crate) enum AppEvent { personality: Personality, }, + /// Open the device picker for a realtime microphone or speaker. + OpenRealtimeAudioDeviceSelection { + kind: RealtimeAudioDeviceKind, + }, + + /// Persist the selected realtime microphone or speaker to top-level config. + #[cfg_attr( + any(target_os = "linux", not(feature = "voice-input")), + allow(dead_code) + )] + PersistRealtimeAudioDeviceSelection { + kind: RealtimeAudioDeviceKind, + name: Option, + }, + + /// Restart the selected realtime microphone or speaker locally. + RestartRealtimeAudioDevice { + kind: RealtimeAudioDeviceKind, + }, + /// Open the reasoning selection popup after picking a model. OpenReasoningPopup { model: ModelPreset, diff --git a/codex-rs/tui/src/audio_device.rs b/codex-rs/tui/src/audio_device.rs new file mode 100644 index 0000000000..467693e7fe --- /dev/null +++ b/codex-rs/tui/src/audio_device.rs @@ -0,0 +1,129 @@ +use codex_core::config::Config; +use cpal::traits::DeviceTrait; +use cpal::traits::HostTrait; +use tracing::warn; + +use crate::app_event::RealtimeAudioDeviceKind; + +pub(crate) fn list_realtime_audio_device_names( + kind: RealtimeAudioDeviceKind, +) -> Result, String> { + let host = cpal::default_host(); + let mut device_names = Vec::new(); + for device in devices(&host, kind)? { + let Ok(name) = device.name() else { + continue; + }; + if !device_names.contains(&name) { + device_names.push(name); + } + } + Ok(device_names) +} + +pub(crate) fn select_configured_input_device_and_config( + config: &Config, +) -> Result<(cpal::Device, cpal::SupportedStreamConfig), String> { + select_device_and_config(RealtimeAudioDeviceKind::Microphone, config) +} + +pub(crate) fn select_configured_output_device_and_config( + config: &Config, +) -> Result<(cpal::Device, cpal::SupportedStreamConfig), String> { + select_device_and_config(RealtimeAudioDeviceKind::Speaker, config) +} + +fn select_device_and_config( + kind: RealtimeAudioDeviceKind, + config: &Config, +) -> Result<(cpal::Device, cpal::SupportedStreamConfig), String> { + let host = cpal::default_host(); + let configured_name = configured_name(kind, config); + let selected = configured_name + .and_then(|name| find_device_by_name(&host, kind, name)) + .or_else(|| { + let default_device = default_device(&host, kind); + if let Some(name) = configured_name && default_device.is_some() { + warn!( + "configured {} audio device `{name}` was unavailable; falling back to system default", + kind.noun() + ); + } + default_device + }) + .ok_or_else(|| missing_device_error(kind, configured_name))?; + + let stream_config = default_config(&selected, kind)?; + Ok((selected, stream_config)) +} + +fn configured_name(kind: RealtimeAudioDeviceKind, config: &Config) -> Option<&str> { + match kind { + RealtimeAudioDeviceKind::Microphone => config.realtime_audio.microphone.as_deref(), + RealtimeAudioDeviceKind::Speaker => config.realtime_audio.speaker.as_deref(), + } +} + +fn find_device_by_name( + host: &cpal::Host, + kind: RealtimeAudioDeviceKind, + name: &str, +) -> Option { + let devices = devices(host, kind).ok()?; + devices + .into_iter() + .find(|device| device.name().ok().as_deref() == Some(name)) +} + +fn devices(host: &cpal::Host, kind: RealtimeAudioDeviceKind) -> Result, String> { + match kind { + RealtimeAudioDeviceKind::Microphone => host + .input_devices() + .map(|devices| devices.collect()) + .map_err(|err| format!("failed to enumerate input audio devices: {err}")), + RealtimeAudioDeviceKind::Speaker => host + .output_devices() + .map(|devices| devices.collect()) + .map_err(|err| format!("failed to enumerate output audio devices: {err}")), + } +} + +fn default_device(host: &cpal::Host, kind: RealtimeAudioDeviceKind) -> Option { + match kind { + RealtimeAudioDeviceKind::Microphone => host.default_input_device(), + RealtimeAudioDeviceKind::Speaker => host.default_output_device(), + } +} + +fn default_config( + device: &cpal::Device, + kind: RealtimeAudioDeviceKind, +) -> Result { + match kind { + RealtimeAudioDeviceKind::Microphone => device + .default_input_config() + .map_err(|err| format!("failed to get default input config: {err}")), + RealtimeAudioDeviceKind::Speaker => device + .default_output_config() + .map_err(|err| format!("failed to get default output config: {err}")), + } +} + +fn missing_device_error(kind: RealtimeAudioDeviceKind, configured_name: Option<&str>) -> String { + match (kind, configured_name) { + (RealtimeAudioDeviceKind::Microphone, Some(name)) => { + format!( + "configured microphone `{name}` was unavailable and no default input audio device was found" + ) + } + (RealtimeAudioDeviceKind::Speaker, Some(name)) => { + format!( + "configured speaker `{name}` was unavailable and no default output audio device was found" + ) + } + (RealtimeAudioDeviceKind::Microphone, None) => { + "no input audio device available".to_string() + } + (RealtimeAudioDeviceKind::Speaker, None) => "no output audio device available".to_string(), + } +} diff --git a/codex-rs/tui/src/bottom_pane/approval_overlay.rs b/codex-rs/tui/src/bottom_pane/approval_overlay.rs index fd58baef46..2ea3c10e9d 100644 --- a/codex-rs/tui/src/bottom_pane/approval_overlay.rs +++ b/codex-rs/tui/src/bottom_pane/approval_overlay.rs @@ -20,10 +20,8 @@ use codex_core::features::Features; use codex_protocol::mcp::RequestId; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::ElicitationAction; -use codex_protocol::protocol::ExecPolicyAmendment; use codex_protocol::protocol::FileChange; use codex_protocol::protocol::NetworkApprovalContext; -use codex_protocol::protocol::NetworkPolicyAmendment; use codex_protocol::protocol::NetworkPolicyRuleAction; use codex_protocol::protocol::Op; use codex_protocol::protocol::ReviewDecision; @@ -46,9 +44,8 @@ pub(crate) enum ApprovalRequest { id: String, command: Vec, reason: Option, + available_decisions: Vec, network_approval_context: Option, - proposed_execpolicy_amendment: Option, - proposed_network_policy_amendments: Option>, additional_permissions: Option, }, ApplyPatch { @@ -67,7 +64,6 @@ pub(crate) enum ApprovalRequest { /// Modal overlay asking the user to approve or deny one or more requests. pub(crate) struct ApprovalOverlay { current_request: Option, - current_variant: Option, queue: Vec, app_event_tx: AppEventSender, list: ListSelectionView, @@ -81,7 +77,6 @@ impl ApprovalOverlay { pub fn new(request: ApprovalRequest, app_event_tx: AppEventSender, features: Features) -> Self { let mut view = Self { current_request: None, - current_variant: None, queue: Vec::new(), app_event_tx: app_event_tx.clone(), list: ListSelectionView::new(Default::default(), app_event_tx), @@ -99,31 +94,28 @@ impl ApprovalOverlay { } fn set_current(&mut self, request: ApprovalRequest) { - self.current_request = Some(request.clone()); - let ApprovalRequestState { variant, header } = ApprovalRequestState::from(request); - self.current_variant = Some(variant.clone()); self.current_complete = false; - let (options, params) = Self::build_options(variant, header, &self.features); + let header = build_header(&request); + let (options, params) = Self::build_options(&request, header, &self.features); + self.current_request = Some(request); self.options = options; self.list = ListSelectionView::new(params, self.app_event_tx.clone()); } fn build_options( - variant: ApprovalVariant, + request: &ApprovalRequest, header: Box, _features: &Features, ) -> (Vec, SelectionViewParams) { - let (options, title) = match &variant { - ApprovalVariant::Exec { + let (options, title) = match request { + ApprovalRequest::Exec { + available_decisions, network_approval_context, - proposed_execpolicy_amendment, - proposed_network_policy_amendments, additional_permissions, .. } => ( exec_options( - proposed_execpolicy_amendment.clone(), - proposed_network_policy_amendments.clone(), + available_decisions, network_approval_context.as_ref(), additional_permissions.as_ref(), ), @@ -137,11 +129,11 @@ impl ApprovalOverlay { }, ), ), - ApprovalVariant::ApplyPatch { .. } => ( + ApprovalRequest::ApplyPatch { .. } => ( patch_options(), "Would you like to make the following edits?".to_string(), ), - ApprovalVariant::McpElicitation { server_name, .. } => ( + ApprovalRequest::McpElicitation { server_name, .. } => ( elicitation_options(), format!("{server_name} needs your approval."), ), @@ -188,18 +180,19 @@ impl ApprovalOverlay { let Some(option) = self.options.get(actual_idx) else { return; }; - if let Some(variant) = self.current_variant.as_ref() { - match (variant, &option.decision) { - (ApprovalVariant::Exec { id, command, .. }, ApprovalDecision::Review(decision)) => { + if let Some(request) = self.current_request.as_ref() { + match (request, &option.decision) { + (ApprovalRequest::Exec { id, command, .. }, ApprovalDecision::Review(decision)) => { self.handle_exec_decision(id, command, decision.clone()); } - (ApprovalVariant::ApplyPatch { id, .. }, ApprovalDecision::Review(decision)) => { + (ApprovalRequest::ApplyPatch { id, .. }, ApprovalDecision::Review(decision)) => { self.handle_patch_decision(id, decision.clone()); } ( - ApprovalVariant::McpElicitation { + ApprovalRequest::McpElicitation { server_name, request_id, + .. }, ApprovalDecision::McpElicitation(decision), ) => { @@ -300,18 +293,19 @@ impl BottomPaneView for ApprovalOverlay { return CancellationEvent::Handled; } if !self.current_complete - && let Some(variant) = self.current_variant.as_ref() + && let Some(request) = self.current_request.as_ref() { - match &variant { - ApprovalVariant::Exec { id, command, .. } => { + match request { + ApprovalRequest::Exec { id, command, .. } => { self.handle_exec_decision(id, command, ReviewDecision::Abort); } - ApprovalVariant::ApplyPatch { id, .. } => { + ApprovalRequest::ApplyPatch { id, .. } => { self.handle_patch_decision(id, ReviewDecision::Abort); } - ApprovalVariant::McpElicitation { + ApprovalRequest::McpElicitation { server_name, request_id, + .. } => { self.handle_elicitation_decision( server_name, @@ -353,122 +347,77 @@ impl Renderable for ApprovalOverlay { } } -struct ApprovalRequestState { - variant: ApprovalVariant, - header: Box, -} - -impl From for ApprovalRequestState { - fn from(value: ApprovalRequest) -> Self { - match value { - ApprovalRequest::Exec { - id, - command, - reason, - network_approval_context, - proposed_execpolicy_amendment, - proposed_network_policy_amendments, - additional_permissions, - } => { - let mut header: Vec> = Vec::new(); - if let Some(reason) = reason { - header.push(Line::from(vec!["Reason: ".into(), reason.italic()])); - header.push(Line::from("")); - } - if let Some(ref additional_permissions) = additional_permissions - && let Some(rule_line) = - format_additional_permissions_rule(additional_permissions) - { - header.push(Line::from(vec![ - "Permission rule: ".into(), - rule_line.cyan(), - ])); - header.push(Line::from("")); - } - let full_cmd = strip_bash_lc_and_escape(&command); - let mut full_cmd_lines = highlight_bash_to_lines(&full_cmd); - if let Some(first) = full_cmd_lines.first_mut() { - first.spans.insert(0, Span::from("$ ")); - } - if network_approval_context.is_none() { - header.extend(full_cmd_lines); - } - Self { - variant: ApprovalVariant::Exec { - id, - command, - network_approval_context, - proposed_execpolicy_amendment, - proposed_network_policy_amendments, - additional_permissions, - }, - header: Box::new(Paragraph::new(header).wrap(Wrap { trim: false })), - } +fn build_header(request: &ApprovalRequest) -> Box { + match request { + ApprovalRequest::Exec { + reason, + command, + network_approval_context, + additional_permissions, + .. + } => { + let mut header: Vec> = Vec::new(); + if let Some(reason) = reason { + header.push(Line::from(vec!["Reason: ".into(), reason.clone().italic()])); + header.push(Line::from("")); } - ApprovalRequest::ApplyPatch { - id, - reason, - cwd, - changes, - } => { - let mut header: Vec> = Vec::new(); - if let Some(reason) = reason - && !reason.is_empty() - { - header.push(Box::new( - Paragraph::new(Line::from_iter(["Reason: ".into(), reason.italic()])) - .wrap(Wrap { trim: false }), - )); - header.push(Box::new(Line::from(""))); - } - header.push(DiffSummary::new(changes, cwd).into()); - Self { - variant: ApprovalVariant::ApplyPatch { id }, - header: Box::new(ColumnRenderable::with(header)), - } + if let Some(additional_permissions) = additional_permissions + && let Some(rule_line) = format_additional_permissions_rule(additional_permissions) + { + header.push(Line::from(vec![ + "Permission rule: ".into(), + rule_line.cyan(), + ])); + header.push(Line::from("")); } - ApprovalRequest::McpElicitation { - server_name, - request_id, - message, - } => { - let header = Paragraph::new(vec![ - Line::from(vec!["Server: ".into(), server_name.clone().bold()]), - Line::from(""), - Line::from(message), - ]) - .wrap(Wrap { trim: false }); - Self { - variant: ApprovalVariant::McpElicitation { - server_name, - request_id, - }, - header: Box::new(header), - } + let full_cmd = strip_bash_lc_and_escape(command); + let mut full_cmd_lines = highlight_bash_to_lines(&full_cmd); + if let Some(first) = full_cmd_lines.first_mut() { + first.spans.insert(0, Span::from("$ ")); } + if network_approval_context.is_none() { + header.extend(full_cmd_lines); + } + Box::new(Paragraph::new(header).wrap(Wrap { trim: false })) + } + ApprovalRequest::ApplyPatch { + reason, + cwd, + changes, + .. + } => { + let mut header: Vec> = Vec::new(); + if let Some(reason) = reason + && !reason.is_empty() + { + header.push(Box::new( + Paragraph::new(Line::from_iter([ + "Reason: ".into(), + reason.clone().italic(), + ])) + .wrap(Wrap { trim: false }), + )); + header.push(Box::new(Line::from(""))); + } + header.push(DiffSummary::new(changes.clone(), cwd.clone()).into()); + Box::new(ColumnRenderable::with(header)) + } + ApprovalRequest::McpElicitation { + server_name, + message, + .. + } => { + let header = Paragraph::new(vec![ + Line::from(vec!["Server: ".into(), server_name.clone().bold()]), + Line::from(""), + Line::from(message.clone()), + ]) + .wrap(Wrap { trim: false }); + Box::new(header) } } } -#[derive(Clone)] -enum ApprovalVariant { - Exec { - id: String, - command: Vec, - network_approval_context: Option, - proposed_execpolicy_amendment: Option, - proposed_network_policy_amendments: Option>, - additional_permissions: Option, - }, - ApplyPatch { - id: String, - }, - McpElicitation { - server_name: String, - request_id: RequestId, - }, -} - #[derive(Clone)] enum ApprovalDecision { Review(ReviewDecision), @@ -492,100 +441,93 @@ impl ApprovalOption { } fn exec_options( - proposed_execpolicy_amendment: Option, - proposed_network_policy_amendments: Option>, + available_decisions: &[ReviewDecision], network_approval_context: Option<&NetworkApprovalContext>, additional_permissions: Option<&PermissionProfile>, ) -> Vec { - if network_approval_context.is_some() { - let mut options = vec![ - ApprovalOption { - label: "Yes, just this once".to_string(), + available_decisions + .iter() + .filter_map(|decision| match decision { + ReviewDecision::Approved => Some(ApprovalOption { + label: if network_approval_context.is_some() { + "Yes, just this once".to_string() + } else { + "Yes, proceed".to_string() + }, decision: ApprovalDecision::Review(ReviewDecision::Approved), display_shortcut: None, additional_shortcuts: vec![key_hint::plain(KeyCode::Char('y'))], - }, - ApprovalOption { - label: "Yes, and allow this host for this conversation".to_string(), + }), + ReviewDecision::ApprovedExecpolicyAmendment { + proposed_execpolicy_amendment, + } => { + let rendered_prefix = + strip_bash_lc_and_escape(proposed_execpolicy_amendment.command()); + if rendered_prefix.contains('\n') || rendered_prefix.contains('\r') { + return None; + } + + Some(ApprovalOption { + label: format!( + "Yes, and don't ask again for commands that start with `{rendered_prefix}`" + ), + decision: ApprovalDecision::Review( + ReviewDecision::ApprovedExecpolicyAmendment { + proposed_execpolicy_amendment: proposed_execpolicy_amendment.clone(), + }, + ), + display_shortcut: None, + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('p'))], + }) + } + ReviewDecision::ApprovedForSession => Some(ApprovalOption { + label: if network_approval_context.is_some() { + "Yes, and allow this host for this conversation".to_string() + } else if additional_permissions.is_some() { + "Yes, and allow these permissions for this session".to_string() + } else { + "Yes, and don't ask again for this command in this session".to_string() + }, decision: ApprovalDecision::Review(ReviewDecision::ApprovedForSession), display_shortcut: None, additional_shortcuts: vec![key_hint::plain(KeyCode::Char('a'))], - }, - ]; - for amendment in proposed_network_policy_amendments.unwrap_or_default() { - let (label, shortcut) = match amendment.action { - NetworkPolicyRuleAction::Allow => ( - "Yes, and allow this host in the future".to_string(), - KeyCode::Char('p'), - ), - NetworkPolicyRuleAction::Deny => continue, - }; - options.push(ApprovalOption { - label, - decision: ApprovalDecision::Review(ReviewDecision::NetworkPolicyAmendment { - network_policy_amendment: amendment, - }), + }), + ReviewDecision::NetworkPolicyAmendment { + network_policy_amendment, + } => { + let (label, shortcut) = match network_policy_amendment.action { + NetworkPolicyRuleAction::Allow => ( + "Yes, and allow this host in the future".to_string(), + KeyCode::Char('p'), + ), + NetworkPolicyRuleAction::Deny => ( + "No, and block this host in the future".to_string(), + KeyCode::Char('d'), + ), + }; + Some(ApprovalOption { + label, + decision: ApprovalDecision::Review(ReviewDecision::NetworkPolicyAmendment { + network_policy_amendment: network_policy_amendment.clone(), + }), + display_shortcut: None, + additional_shortcuts: vec![key_hint::plain(shortcut)], + }) + } + ReviewDecision::Denied => Some(ApprovalOption { + label: "No, continue without running it".to_string(), + decision: ApprovalDecision::Review(ReviewDecision::Denied), display_shortcut: None, - additional_shortcuts: vec![key_hint::plain(shortcut)], - }); - } - options.push(ApprovalOption { - label: "No, and tell Codex what to do differently".to_string(), - decision: ApprovalDecision::Review(ReviewDecision::Abort), - display_shortcut: Some(key_hint::plain(KeyCode::Esc)), - additional_shortcuts: vec![key_hint::plain(KeyCode::Char('n'))], - }); - return options; - } - - if additional_permissions.is_some() { - return vec![ - ApprovalOption { - label: "Yes, proceed".to_string(), - decision: ApprovalDecision::Review(ReviewDecision::Approved), - display_shortcut: None, - additional_shortcuts: vec![key_hint::plain(KeyCode::Char('y'))], - }, - ApprovalOption { + additional_shortcuts: vec![key_hint::plain(KeyCode::Char('d'))], + }), + ReviewDecision::Abort => Some(ApprovalOption { label: "No, and tell Codex what to do differently".to_string(), decision: ApprovalDecision::Review(ReviewDecision::Abort), display_shortcut: Some(key_hint::plain(KeyCode::Esc)), additional_shortcuts: vec![key_hint::plain(KeyCode::Char('n'))], - }, - ]; - } - - vec![ApprovalOption { - label: "Yes, proceed".to_string(), - decision: ApprovalDecision::Review(ReviewDecision::Approved), - display_shortcut: None, - additional_shortcuts: vec![key_hint::plain(KeyCode::Char('y'))], - }] - .into_iter() - .chain(proposed_execpolicy_amendment.and_then(|prefix| { - let rendered_prefix = strip_bash_lc_and_escape(prefix.command()); - if rendered_prefix.contains('\n') || rendered_prefix.contains('\r') { - return None; - } - - Some(ApprovalOption { - label: format!( - "Yes, and don't ask again for commands that start with `{rendered_prefix}`" - ), - decision: ApprovalDecision::Review(ReviewDecision::ApprovedExecpolicyAmendment { - proposed_execpolicy_amendment: prefix, }), - display_shortcut: None, - additional_shortcuts: vec![key_hint::plain(KeyCode::Char('p'))], }) - })) - .chain([ApprovalOption { - label: "No, and tell Codex what to do differently".to_string(), - decision: ApprovalDecision::Review(ReviewDecision::Abort), - display_shortcut: Some(key_hint::plain(KeyCode::Esc)), - additional_shortcuts: vec![key_hint::plain(KeyCode::Char('n'))], - }]) - .collect() + .collect() } fn format_additional_permissions_rule( @@ -669,7 +611,9 @@ mod tests { use super::*; use crate::app_event::AppEvent; use codex_protocol::models::FileSystemPermissions; + use codex_protocol::protocol::ExecPolicyAmendment; use codex_protocol::protocol::NetworkApprovalProtocol; + use codex_protocol::protocol::NetworkPolicyAmendment; use insta::assert_snapshot; use pretty_assertions::assert_eq; use tokio::sync::mpsc::unbounded_channel; @@ -695,9 +639,8 @@ mod tests { id: "test".to_string(), command: vec!["echo".to_string(), "hi".to_string()], reason: Some("reason".to_string()), + available_decisions: vec![ReviewDecision::Approved, ReviewDecision::Abort], network_approval_context: None, - proposed_execpolicy_amendment: None, - proposed_network_policy_amendments: None, additional_permissions: None, } } @@ -740,11 +683,16 @@ mod tests { id: "test".to_string(), command: vec!["echo".to_string()], reason: None, + available_decisions: vec![ + ReviewDecision::Approved, + ReviewDecision::ApprovedExecpolicyAmendment { + proposed_execpolicy_amendment: ExecPolicyAmendment::new(vec![ + "echo".to_string(), + ]), + }, + ReviewDecision::Abort, + ], network_approval_context: None, - proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(vec![ - "echo".to_string(), - ])), - proposed_network_policy_amendments: None, additional_permissions: None, }, tx, @@ -781,21 +729,21 @@ mod tests { id: "test".to_string(), command: vec!["curl".to_string(), "https://example.com".to_string()], reason: None, + available_decisions: vec![ + ReviewDecision::Approved, + ReviewDecision::ApprovedForSession, + ReviewDecision::NetworkPolicyAmendment { + network_policy_amendment: NetworkPolicyAmendment { + host: "example.com".to_string(), + action: NetworkPolicyRuleAction::Allow, + }, + }, + ReviewDecision::Abort, + ], network_approval_context: Some(NetworkApprovalContext { host: "example.com".to_string(), protocol: NetworkApprovalProtocol::Https, }), - proposed_execpolicy_amendment: None, - proposed_network_policy_amendments: Some(vec![ - NetworkPolicyAmendment { - host: "example.com".to_string(), - action: NetworkPolicyRuleAction::Allow, - }, - NetworkPolicyAmendment { - host: "example.com".to_string(), - action: NetworkPolicyRuleAction::Deny, - }, - ]), additional_permissions: None, }, tx, @@ -818,9 +766,8 @@ mod tests { id: "test".into(), command, reason: None, + available_decisions: vec![ReviewDecision::Approved, ReviewDecision::Abort], network_approval_context: None, - proposed_execpolicy_amendment: None, - proposed_network_policy_amendments: None, additional_permissions: None, }; @@ -850,17 +797,17 @@ mod tests { protocol: NetworkApprovalProtocol::Https, }; let options = exec_options( - Some(ExecPolicyAmendment::new(vec!["curl".to_string()])), - Some(vec![ - NetworkPolicyAmendment { - host: "example.com".to_string(), - action: NetworkPolicyRuleAction::Allow, + &[ + ReviewDecision::Approved, + ReviewDecision::ApprovedForSession, + ReviewDecision::NetworkPolicyAmendment { + network_policy_amendment: NetworkPolicyAmendment { + host: "example.com".to_string(), + action: NetworkPolicyRuleAction::Allow, + }, }, - NetworkPolicyAmendment { - host: "example.com".to_string(), - action: NetworkPolicyRuleAction::Deny, - }, - ]), + ReviewDecision::Abort, + ], Some(&network_context), None, ); @@ -877,6 +824,29 @@ mod tests { ); } + #[test] + fn generic_exec_options_can_offer_allow_for_session() { + let options = exec_options( + &[ + ReviewDecision::Approved, + ReviewDecision::ApprovedForSession, + ReviewDecision::Abort, + ], + None, + None, + ); + + let labels: Vec = options.into_iter().map(|option| option.label).collect(); + assert_eq!( + labels, + vec![ + "Yes, proceed".to_string(), + "Yes, and don't ask again for this command in this session".to_string(), + "No, and tell Codex what to do differently".to_string(), + ] + ); + } + #[test] fn additional_permissions_exec_options_hide_execpolicy_amendment() { let additional_permissions = PermissionProfile { @@ -886,7 +856,11 @@ mod tests { }), ..Default::default() }; - let options = exec_options(None, None, None, Some(&additional_permissions)); + let options = exec_options( + &[ReviewDecision::Approved, ReviewDecision::Abort], + None, + Some(&additional_permissions), + ); let labels: Vec = options.into_iter().map(|option| option.label).collect(); assert_eq!( @@ -906,9 +880,8 @@ mod tests { id: "test".into(), command: vec!["cat".into(), "/tmp/readme.txt".into()], reason: None, + available_decisions: vec![ReviewDecision::Approved, ReviewDecision::Abort], network_approval_context: None, - proposed_execpolicy_amendment: None, - proposed_network_policy_amendments: None, additional_permissions: Some(PermissionProfile { file_system: Some(FileSystemPermissions { read: Some(vec![PathBuf::from("/tmp/readme.txt")]), @@ -946,9 +919,8 @@ mod tests { id: "test".into(), command: vec!["cat".into(), "/tmp/readme.txt".into()], reason: Some("need filesystem access".into()), + available_decisions: vec![ReviewDecision::Approved, ReviewDecision::Abort], network_approval_context: None, - proposed_execpolicy_amendment: None, - proposed_network_policy_amendments: None, additional_permissions: Some(PermissionProfile { file_system: Some(FileSystemPermissions { read: Some(vec![PathBuf::from("/tmp/readme.txt")]), @@ -973,21 +945,21 @@ mod tests { id: "test".into(), command: vec!["curl".into(), "https://example.com".into()], reason: Some("network request blocked".into()), + available_decisions: vec![ + ReviewDecision::Approved, + ReviewDecision::ApprovedForSession, + ReviewDecision::NetworkPolicyAmendment { + network_policy_amendment: NetworkPolicyAmendment { + host: "example.com".to_string(), + action: NetworkPolicyRuleAction::Allow, + }, + }, + ReviewDecision::Abort, + ], network_approval_context: Some(NetworkApprovalContext { host: "example.com".to_string(), protocol: NetworkApprovalProtocol::Https, }), - proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(vec!["curl".into()])), - proposed_network_policy_amendments: Some(vec![ - NetworkPolicyAmendment { - host: "example.com".to_string(), - action: NetworkPolicyRuleAction::Allow, - }, - NetworkPolicyAmendment { - host: "example.com".to_string(), - action: NetworkPolicyRuleAction::Deny, - }, - ]), additional_permissions: None, }; diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index e05953d2a9..fe40f4fa7b 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -189,6 +189,7 @@ use codex_protocol::custom_prompts::CustomPrompt; use codex_protocol::custom_prompts::PROMPTS_CMD_PREFIX; use codex_protocol::models::local_image_label_text; use codex_protocol::user_input::ByteRange; +use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use codex_protocol::user_input::TextElement; use codex_utils_fuzzy_match::fuzzy_match; @@ -229,6 +230,12 @@ use tokio::runtime::Handle; /// placeholder in the UI. const LARGE_PASTE_CHAR_THRESHOLD: usize = 1000; +fn user_input_too_large_message(actual_chars: usize) -> String { + format!( + "Message exceeds the maximum length of {MAX_USER_INPUT_TEXT_CHARS} characters ({actual_chars} provided)." + ) +} + /// Result returned when the user interacts with the text area. #[derive(Debug, PartialEq)] pub enum InputResult { @@ -388,6 +395,7 @@ pub(crate) struct ChatComposer { connectors_enabled: bool, personality_command_enabled: bool, realtime_conversation_enabled: bool, + audio_device_selection_enabled: bool, windows_degraded_sandbox_active: bool, status_line_value: Option>, status_line_enabled: bool, @@ -493,6 +501,7 @@ impl ChatComposer { connectors_enabled: false, personality_command_enabled: false, realtime_conversation_enabled: false, + audio_device_selection_enabled: false, windows_degraded_sandbox_active: false, status_line_value: None, status_line_enabled: false, @@ -570,6 +579,13 @@ impl ChatComposer { self.realtime_conversation_enabled = enabled; } + pub fn set_audio_device_selection_enabled(&mut self, enabled: bool) { + self.audio_device_selection_enabled = enabled; + } + + /// Compatibility shim for tests that still toggle the removed steer mode flag. + #[cfg(test)] + pub fn set_steer_enabled(&mut self, _enabled: bool) {} pub fn set_voice_transcription_enabled(&mut self, enabled: bool) { self.voice_state.transcription_enabled = enabled; if !enabled { @@ -2253,6 +2269,7 @@ impl ChatComposer { self.connectors_enabled, self.personality_command_enabled, self.realtime_conversation_enabled, + self.audio_device_selection_enabled, self.windows_degraded_sandbox_active, ) .is_some(); @@ -2309,6 +2326,22 @@ impl ChatComposer { text_elements = expanded.text_elements; } } + let actual_chars = text.chars().count(); + if actual_chars > MAX_USER_INPUT_TEXT_CHARS { + let message = user_input_too_large_message(actual_chars); + self.app_event_tx.send(AppEvent::InsertHistoryCell(Box::new( + history_cell::new_error_event(message), + ))); + self.set_text_content_with_mention_bindings( + original_input.clone(), + original_text_elements, + original_local_image_paths, + original_mention_bindings, + ); + self.pending_pastes.clone_from(&original_pending_pastes); + self.textarea.set_cursor(original_input.len()); + return None; + } // Custom prompt expansion can remove or rewrite image placeholders, so prune any // attachments that no longer have a corresponding placeholder in the expanded text. self.prune_attached_images_for_submission(&text, &text_elements); @@ -2453,6 +2486,7 @@ impl ChatComposer { self.connectors_enabled, self.personality_command_enabled, self.realtime_conversation_enabled, + self.audio_device_selection_enabled, self.windows_degraded_sandbox_active, ) { @@ -2488,6 +2522,7 @@ impl ChatComposer { self.connectors_enabled, self.personality_command_enabled, self.realtime_conversation_enabled, + self.audio_device_selection_enabled, self.windows_degraded_sandbox_active, )?; @@ -3307,6 +3342,7 @@ impl ChatComposer { self.connectors_enabled, self.personality_command_enabled, self.realtime_conversation_enabled, + self.audio_device_selection_enabled, self.windows_degraded_sandbox_active, ) .is_some(); @@ -3369,6 +3405,7 @@ impl ChatComposer { self.connectors_enabled, self.personality_command_enabled, self.realtime_conversation_enabled, + self.audio_device_selection_enabled, self.windows_degraded_sandbox_active, ) { return true; @@ -3423,6 +3460,7 @@ impl ChatComposer { let connectors_enabled = self.connectors_enabled; let personality_command_enabled = self.personality_command_enabled; let realtime_conversation_enabled = self.realtime_conversation_enabled; + let audio_device_selection_enabled = self.audio_device_selection_enabled; let mut command_popup = CommandPopup::new( self.custom_prompts.clone(), CommandPopupFlags { @@ -3430,6 +3468,7 @@ impl ChatComposer { connectors_enabled, personality_command_enabled, realtime_conversation_enabled, + audio_device_selection_enabled, windows_degraded_sandbox_active: self.windows_degraded_sandbox_active, }, ); @@ -5921,6 +5960,118 @@ mod tests { assert!(composer.pending_pastes.is_empty()); } + #[test] + fn submit_at_character_limit_succeeds() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + composer.set_steer_enabled(true); + let input = "x".repeat(MAX_USER_INPUT_TEXT_CHARS); + composer.textarea.set_text_clearing_elements(&input); + + let (result, _needs_redraw) = + composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert!(matches!( + result, + InputResult::Submitted { text, .. } if text == input + )); + } + + #[test] + fn oversized_submit_reports_error_and_restores_draft() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + let (tx, mut rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + composer.set_steer_enabled(true); + let input = "x".repeat(MAX_USER_INPUT_TEXT_CHARS + 1); + composer.textarea.set_text_clearing_elements(&input); + + let (result, _needs_redraw) = + composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(InputResult::None, result); + assert_eq!(composer.textarea.text(), input); + + let mut found_error = false; + while let Ok(event) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = event { + let message = cell + .display_lines(80) + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n"); + assert!(message.contains(&user_input_too_large_message(input.chars().count()))); + found_error = true; + break; + } + } + assert!(found_error, "expected oversized-input error history cell"); + } + + #[test] + fn oversized_queued_submission_reports_error_and_restores_draft() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + let (tx, mut rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + composer.set_steer_enabled(false); + let input = "x".repeat(MAX_USER_INPUT_TEXT_CHARS + 1); + composer.textarea.set_text_clearing_elements(&input); + + let (result, _needs_redraw) = + composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(InputResult::None, result); + assert_eq!(composer.textarea.text(), input); + + let mut found_error = false; + while let Ok(event) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = event { + let message = cell + .display_lines(80) + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n"); + assert!(message.contains(&user_input_too_large_message(input.chars().count()))); + found_error = true; + break; + } + } + assert!(found_error, "expected oversized-input error history cell"); + } + /// Behavior: editing that removes a paste placeholder should also clear the associated /// `pending_pastes` entry so it cannot be submitted accidentally. #[test] @@ -8521,6 +8672,57 @@ mod tests { ); } + #[test] + fn prompt_expansion_over_character_limit_reports_error_and_restores_draft() { + let (tx, mut rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + composer.set_steer_enabled(true); + + composer.set_custom_prompts(vec![CustomPrompt { + name: "my-prompt".to_string(), + path: "/tmp/my-prompt.md".to_string().into(), + content: "Echo: $1".to_string(), + description: None, + argument_hint: None, + }]); + + let oversized_arg = "x".repeat(MAX_USER_INPUT_TEXT_CHARS); + let original_input = format!("/prompts:my-prompt {oversized_arg}"); + composer + .textarea + .set_text_clearing_elements(&original_input); + + let (result, _needs_redraw) = + composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_eq!(InputResult::None, result); + assert_eq!(composer.textarea.text(), original_input); + + let actual_chars = format!("Echo: {oversized_arg}").chars().count(); + let mut found_error = false; + while let Ok(event) = rx.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = event { + let message = cell + .display_lines(80) + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n"); + assert!(message.contains(&user_input_too_large_message(actual_chars))); + found_error = true; + break; + } + } + assert!(found_error, "expected oversized-input error history cell"); + } + #[test] fn selecting_custom_prompt_with_positional_args_submits_numeric_expansion() { let prompt_text = "Header: $1\nArgs: $ARGUMENTS\n"; diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 83b523cc23..62765d9c2c 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -40,6 +40,7 @@ pub(crate) struct CommandPopupFlags { pub(crate) connectors_enabled: bool, pub(crate) personality_command_enabled: bool, pub(crate) realtime_conversation_enabled: bool, + pub(crate) audio_device_selection_enabled: bool, pub(crate) windows_degraded_sandbox_active: bool, } @@ -51,6 +52,7 @@ impl CommandPopup { flags.connectors_enabled, flags.personality_command_enabled, flags.realtime_conversation_enabled, + flags.audio_device_selection_enabled, flags.windows_degraded_sandbox_active, ) .into_iter() @@ -498,6 +500,7 @@ mod tests { connectors_enabled: false, personality_command_enabled: true, realtime_conversation_enabled: false, + audio_device_selection_enabled: false, windows_degraded_sandbox_active: false, }, ); @@ -518,6 +521,7 @@ mod tests { connectors_enabled: false, personality_command_enabled: true, realtime_conversation_enabled: false, + audio_device_selection_enabled: false, windows_degraded_sandbox_active: false, }, ); @@ -538,6 +542,7 @@ mod tests { connectors_enabled: false, personality_command_enabled: false, realtime_conversation_enabled: false, + audio_device_selection_enabled: false, windows_degraded_sandbox_active: false, }, ); @@ -566,6 +571,7 @@ mod tests { connectors_enabled: false, personality_command_enabled: true, realtime_conversation_enabled: false, + audio_device_selection_enabled: false, windows_degraded_sandbox_active: false, }, ); @@ -577,6 +583,36 @@ mod tests { } } + #[test] + fn settings_command_hidden_when_audio_device_selection_is_disabled() { + let mut popup = CommandPopup::new( + Vec::new(), + CommandPopupFlags { + collaboration_modes_enabled: false, + connectors_enabled: false, + personality_command_enabled: true, + realtime_conversation_enabled: true, + audio_device_selection_enabled: false, + windows_degraded_sandbox_active: false, + }, + ); + popup.on_composer_text_change("/aud".to_string()); + + let cmds: Vec<&str> = popup + .filtered_items() + .into_iter() + .filter_map(|item| match item { + CommandItem::Builtin(cmd) => Some(cmd.command()), + CommandItem::UserPrompt(_) => None, + }) + .collect(); + + assert!( + !cmds.contains(&"settings"), + "expected '/settings' to be hidden when audio device selection is disabled, got {cmds:?}" + ); + } + #[test] fn debug_commands_are_hidden_from_popup() { let popup = CommandPopup::new(Vec::new(), CommandPopupFlags::default()); diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 17d885d5f8..dac0069887 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -298,6 +298,11 @@ impl BottomPane { self.request_redraw(); } + pub fn set_audio_device_selection_enabled(&mut self, enabled: bool) { + self.composer.set_audio_device_selection_enabled(enabled); + self.request_redraw(); + } + pub fn set_voice_transcription_enabled(&mut self, enabled: bool) { self.composer.set_voice_transcription_enabled(enabled); self.request_redraw(); @@ -1113,9 +1118,11 @@ mod tests { id: "1".to_string(), command: vec!["echo".into(), "ok".into()], reason: None, + available_decisions: vec![ + codex_protocol::protocol::ReviewDecision::Approved, + codex_protocol::protocol::ReviewDecision::Abort, + ], network_approval_context: None, - proposed_execpolicy_amendment: None, - proposed_network_policy_amendments: None, additional_permissions: None, } } diff --git a/codex-rs/tui/src/bottom_pane/slash_commands.rs b/codex-rs/tui/src/bottom_pane/slash_commands.rs index 86c131e6db..981c61c451 100644 --- a/codex-rs/tui/src/bottom_pane/slash_commands.rs +++ b/codex-rs/tui/src/bottom_pane/slash_commands.rs @@ -14,6 +14,7 @@ pub(crate) fn builtins_for_input( connectors_enabled: bool, personality_command_enabled: bool, realtime_conversation_enabled: bool, + audio_device_selection_enabled: bool, allow_elevate_sandbox: bool, ) -> Vec<(&'static str, SlashCommand)> { built_in_slash_commands() @@ -26,6 +27,7 @@ pub(crate) fn builtins_for_input( .filter(|(_, cmd)| connectors_enabled || *cmd != SlashCommand::Apps) .filter(|(_, cmd)| personality_command_enabled || *cmd != SlashCommand::Personality) .filter(|(_, cmd)| realtime_conversation_enabled || *cmd != SlashCommand::Realtime) + .filter(|(_, cmd)| audio_device_selection_enabled || *cmd != SlashCommand::Settings) .collect() } @@ -36,6 +38,7 @@ pub(crate) fn find_builtin_command( connectors_enabled: bool, personality_command_enabled: bool, realtime_conversation_enabled: bool, + audio_device_selection_enabled: bool, allow_elevate_sandbox: bool, ) -> Option { builtins_for_input( @@ -43,6 +46,7 @@ pub(crate) fn find_builtin_command( connectors_enabled, personality_command_enabled, realtime_conversation_enabled, + audio_device_selection_enabled, allow_elevate_sandbox, ) .into_iter() @@ -57,6 +61,7 @@ pub(crate) fn has_builtin_prefix( connectors_enabled: bool, personality_command_enabled: bool, realtime_conversation_enabled: bool, + audio_device_selection_enabled: bool, allow_elevate_sandbox: bool, ) -> bool { builtins_for_input( @@ -64,6 +69,7 @@ pub(crate) fn has_builtin_prefix( connectors_enabled, personality_command_enabled, realtime_conversation_enabled, + audio_device_selection_enabled, allow_elevate_sandbox, ) .into_iter() @@ -77,14 +83,14 @@ mod tests { #[test] fn debug_command_still_resolves_for_dispatch() { - let cmd = find_builtin_command("debug-config", true, true, true, false, false); + let cmd = find_builtin_command("debug-config", true, true, true, false, false, false); assert_eq!(cmd, Some(SlashCommand::DebugConfig)); } #[test] fn clear_command_resolves_for_dispatch() { assert_eq!( - find_builtin_command("clear", true, true, true, false, false), + find_builtin_command("clear", true, true, true, false, false, false), Some(SlashCommand::Clear) ); } @@ -92,7 +98,23 @@ mod tests { #[test] fn realtime_command_is_hidden_when_realtime_is_disabled() { assert_eq!( - find_builtin_command("realtime", true, true, true, false, false), + find_builtin_command("realtime", true, true, true, false, true, false), + None + ); + } + + #[test] + fn settings_command_is_hidden_when_realtime_is_disabled() { + assert_eq!( + find_builtin_command("settings", true, true, true, false, false, false), + None + ); + } + + #[test] + fn settings_command_is_hidden_when_audio_device_selection_is_disabled() { + assert_eq!( + find_builtin_command("settings", true, true, true, true, false, false), None ); } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index a6c7b1fef2..4031cecaf6 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -37,6 +37,9 @@ use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; +use crate::app_event::RealtimeAudioDeviceKind; +#[cfg(all(not(target_os = "linux"), feature = "voice-input"))] +use crate::audio_device::list_realtime_audio_device_names; use crate::bottom_pane::StatusLineItem; use crate::bottom_pane::StatusLineSetupView; use crate::status::RateLimitWindowDisplay; @@ -855,6 +858,10 @@ impl ChatWidget { && cfg!(not(target_os = "linux")) } + fn realtime_audio_device_selection_enabled(&self) -> bool { + self.realtime_conversation_enabled() && cfg!(feature = "voice-input") + } + /// Synchronize the bottom-pane "task running" indicator with the current lifecycles. /// /// The bottom pane only has one running flag, but this module treats it as a derived state of @@ -2568,13 +2575,13 @@ impl ChatWidget { .unwrap_or_else(|_| ev.command.join(" ")); self.notify(Notification::ExecApprovalRequested { command }); + let available_decisions = ev.effective_available_decisions(); let request = ApprovalRequest::Exec { id: ev.effective_approval_id(), command: ev.command, reason: ev.reason, + available_decisions, network_approval_context: ev.network_approval_context, - proposed_execpolicy_amendment: ev.proposed_execpolicy_amendment, - proposed_network_policy_amendments: ev.proposed_network_policy_amendments, additional_permissions: ev.additional_permissions, }; self.bottom_pane @@ -2882,6 +2889,9 @@ impl ChatWidget { widget .bottom_pane .set_realtime_conversation_enabled(widget.realtime_conversation_enabled()); + widget + .bottom_pane + .set_audio_device_selection_enabled(widget.realtime_audio_device_selection_enabled()); widget .bottom_pane .set_status_line_enabled(!widget.configured_status_line_items().is_empty()); @@ -3056,6 +3066,9 @@ impl ChatWidget { widget .bottom_pane .set_realtime_conversation_enabled(widget.realtime_conversation_enabled()); + widget + .bottom_pane + .set_audio_device_selection_enabled(widget.realtime_audio_device_selection_enabled()); widget .bottom_pane .set_status_line_enabled(!widget.configured_status_line_items().is_empty()); @@ -3219,6 +3232,9 @@ impl ChatWidget { widget .bottom_pane .set_realtime_conversation_enabled(widget.realtime_conversation_enabled()); + widget + .bottom_pane + .set_audio_device_selection_enabled(widget.realtime_audio_device_selection_enabled()); widget .bottom_pane .set_status_line_enabled(!widget.configured_status_line_items().is_empty()); @@ -3531,6 +3547,12 @@ impl ChatWidget { self.start_realtime_conversation(); } } + SlashCommand::Settings => { + if !self.realtime_audio_device_selection_enabled() { + return; + } + self.open_realtime_audio_popup(); + } SlashCommand::Personality => { self.open_personality_popup(); } @@ -4139,7 +4161,7 @@ impl ChatWidget { sandbox_policy: self.config.permissions.sandbox_policy.get().clone(), model: effective_mode.model().to_string(), effort: effective_mode.reasoning_effort(), - summary: self.config.model_reasoning_summary, + summary: None, final_output_json_schema: None, collaboration_mode, personality, @@ -4455,8 +4477,7 @@ impl ChatWidget { | EventMsg::ReasoningContentDelta(_) | EventMsg::ReasoningRawContentDelta(_) | EventMsg::DynamicToolCallRequest(_) - | EventMsg::DynamicToolCallResponse(_) - | EventMsg::SkillRequestApproval(_) => {} + | EventMsg::DynamicToolCallResponse(_) => {} EventMsg::RealtimeConversationStarted(ev) => { if !from_replay { self.on_realtime_conversation_started(ev); @@ -5271,6 +5292,161 @@ impl ChatWidget { }); } + pub(crate) fn open_realtime_audio_popup(&mut self) { + let items = [ + RealtimeAudioDeviceKind::Microphone, + RealtimeAudioDeviceKind::Speaker, + ] + .into_iter() + .map(|kind| { + let description = Some(format!( + "Current: {}", + self.current_realtime_audio_selection_label(kind) + )); + let actions: Vec = vec![Box::new(move |tx| { + tx.send(AppEvent::OpenRealtimeAudioDeviceSelection { kind }); + })]; + SelectionItem { + name: kind.title().to_string(), + description, + actions, + dismiss_on_select: true, + ..Default::default() + } + }) + .collect(); + + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some("Settings".to_string()), + subtitle: Some("Configure settings for Codex.".to_string()), + footer_hint: Some(standard_popup_hint_line()), + items, + ..Default::default() + }); + } + + #[cfg(all(not(target_os = "linux"), feature = "voice-input"))] + pub(crate) fn open_realtime_audio_device_selection(&mut self, kind: RealtimeAudioDeviceKind) { + match list_realtime_audio_device_names(kind) { + Ok(device_names) => { + self.open_realtime_audio_device_selection_with_names(kind, device_names); + } + Err(err) => { + self.add_error_message(format!( + "Failed to load realtime {} devices: {err}", + kind.noun() + )); + } + } + } + + #[cfg(any(target_os = "linux", not(feature = "voice-input")))] + pub(crate) fn open_realtime_audio_device_selection(&mut self, kind: RealtimeAudioDeviceKind) { + let _ = kind; + } + + #[cfg(all(not(target_os = "linux"), feature = "voice-input"))] + fn open_realtime_audio_device_selection_with_names( + &mut self, + kind: RealtimeAudioDeviceKind, + device_names: Vec, + ) { + let current_selection = self.current_realtime_audio_device_name(kind); + let current_available = current_selection + .as_deref() + .is_some_and(|name| device_names.iter().any(|device_name| device_name == name)); + let mut items = vec![SelectionItem { + name: "System default".to_string(), + description: Some("Use your operating system default device.".to_string()), + is_current: current_selection.is_none(), + actions: vec![Box::new(move |tx| { + tx.send(AppEvent::PersistRealtimeAudioDeviceSelection { kind, name: None }); + })], + dismiss_on_select: true, + ..Default::default() + }]; + + if let Some(selection) = current_selection.as_deref() + && !current_available + { + items.push(SelectionItem { + name: format!("Unavailable: {selection}"), + description: Some("Configured device is not currently available.".to_string()), + is_current: true, + is_disabled: true, + disabled_reason: Some("Reconnect the device or choose another one.".to_string()), + ..Default::default() + }); + } + + items.extend(device_names.into_iter().map(|device_name| { + let persisted_name = device_name.clone(); + let actions: Vec = vec![Box::new(move |tx| { + tx.send(AppEvent::PersistRealtimeAudioDeviceSelection { + kind, + name: Some(persisted_name.clone()), + }); + })]; + SelectionItem { + is_current: current_selection.as_deref() == Some(device_name.as_str()), + name: device_name, + actions, + dismiss_on_select: true, + ..Default::default() + } + })); + + let mut header = ColumnRenderable::new(); + header.push(Line::from(format!("Select {}", kind.title()).bold())); + header.push(Line::from( + "Saved devices apply to realtime voice only.".dim(), + )); + + self.bottom_pane.show_selection_view(SelectionViewParams { + header: Box::new(header), + footer_hint: Some(standard_popup_hint_line()), + items, + ..Default::default() + }); + } + + pub(crate) fn open_realtime_audio_restart_prompt(&mut self, kind: RealtimeAudioDeviceKind) { + let restart_actions: Vec = vec![Box::new(move |tx| { + tx.send(AppEvent::RestartRealtimeAudioDevice { kind }); + })]; + let items = vec![ + SelectionItem { + name: "Restart now".to_string(), + description: Some(format!("Restart local {} audio now.", kind.noun())), + actions: restart_actions, + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Apply later".to_string(), + description: Some(format!( + "Keep the current {} until local audio starts again.", + kind.noun() + )), + dismiss_on_select: true, + ..Default::default() + }, + ]; + + let mut header = ColumnRenderable::new(); + header.push(Line::from(format!("Restart {} now?", kind.title()).bold())); + header.push(Line::from( + "Configuration is saved. Restart local audio to use it immediately.".dim(), + )); + + self.bottom_pane.show_selection_view(SelectionViewParams { + header: Box::new(header), + footer_hint: Some(standard_popup_hint_line()), + items, + ..Default::default() + }); + } + fn model_menu_header(&self, title: &str, subtitle: &str) -> Box { let title = title.to_string(); let subtitle = subtitle.to_string(); @@ -6524,6 +6700,8 @@ impl ChatWidget { let realtime_conversation_enabled = self.realtime_conversation_enabled(); self.bottom_pane .set_realtime_conversation_enabled(realtime_conversation_enabled); + self.bottom_pane + .set_audio_device_selection_enabled(self.realtime_audio_device_selection_enabled()); if !realtime_conversation_enabled && self.realtime_conversation.is_live() { self.request_realtime_conversation_close(Some( "Realtime voice mode was closed because the feature was disabled.".to_string(), @@ -6613,6 +6791,17 @@ impl ChatWidget { self.config.personality = Some(personality); } + pub(crate) fn set_realtime_audio_device( + &mut self, + kind: RealtimeAudioDeviceKind, + name: Option, + ) { + match kind { + RealtimeAudioDeviceKind::Microphone => self.config.realtime_audio.microphone = name, + RealtimeAudioDeviceKind::Speaker => self.config.realtime_audio.speaker = name, + } + } + /// Set the syntax theme override in the widget's config copy. pub(crate) fn set_tui_theme(&mut self, theme: Option) { self.config.tui_theme = theme; @@ -6641,6 +6830,22 @@ impl ChatWidget { .unwrap_or_else(|| self.current_collaboration_mode.model()) } + pub(crate) fn realtime_conversation_is_live(&self) -> bool { + self.realtime_conversation.is_active() + } + + fn current_realtime_audio_device_name(&self, kind: RealtimeAudioDeviceKind) -> Option { + match kind { + RealtimeAudioDeviceKind::Microphone => self.config.realtime_audio.microphone.clone(), + RealtimeAudioDeviceKind::Speaker => self.config.realtime_audio.speaker.clone(), + } + } + + fn current_realtime_audio_selection_label(&self, kind: RealtimeAudioDeviceKind) -> String { + self.current_realtime_audio_device_name(kind) + .unwrap_or_else(|| "System default".to_string()) + } + fn sync_personality_command_enabled(&mut self) { self.bottom_pane .set_personality_command_enabled(self.config.features.enabled(Feature::Personality)); diff --git a/codex-rs/tui/src/chatwidget/agent.rs b/codex-rs/tui/src/chatwidget/agent.rs index 63d519a7a5..e14a9e3628 100644 --- a/codex-rs/tui/src/chatwidget/agent.rs +++ b/codex-rs/tui/src/chatwidget/agent.rs @@ -13,6 +13,17 @@ use tokio::sync::mpsc::unbounded_channel; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; +const TUI_NOTIFY_CLIENT: &str = "codex-tui"; + +async fn initialize_app_server_client_name(thread: &CodexThread) { + if let Err(err) = thread + .set_app_server_client_name(Some(TUI_NOTIFY_CLIENT.to_string())) + .await + { + tracing::error!("failed to set app server client name: {err}"); + } +} + /// Spawn the agent bootstrapper and op forwarding loop, returning the /// `UnboundedSender` used by the UI to submit operations. pub(crate) fn spawn_agent( @@ -42,6 +53,7 @@ pub(crate) fn spawn_agent( return; } }; + initialize_app_server_client_name(thread.as_ref()).await; // Forward the captured `SessionConfigured` event so it can be rendered in the UI. let ev = codex_protocol::protocol::Event { @@ -87,6 +99,8 @@ pub(crate) fn spawn_agent_from_existing( let app_event_tx_clone = app_event_tx; tokio::spawn(async move { + initialize_app_server_client_name(thread.as_ref()).await; + // Forward the captured `SessionConfigured` event so it can be rendered in the UI. let ev = codex_protocol::protocol::Event { id: "".to_string(), @@ -123,6 +137,7 @@ pub(crate) fn spawn_op_forwarder(thread: std::sync::Arc) -> Unbound let (codex_op_tx, mut codex_op_rx) = unbounded_channel::(); tokio::spawn(async move { + initialize_app_server_client_name(thread.as_ref()).await; while let Some(op) = codex_op_rx.recv().await { if let Err(e) = thread.submit(op).await { tracing::error!("failed to submit op: {e}"); diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs index 2cf97188b3..98ac76a34f 100644 --- a/codex-rs/tui/src/chatwidget/realtime.rs +++ b/codex-rs/tui/src/chatwidget/realtime.rs @@ -41,6 +41,10 @@ impl RealtimeConversationUiState { | RealtimeConversationPhase::Stopping ) } + + pub(super) fn is_active(&self) -> bool { + matches!(self.phase, RealtimeConversationPhase::Active) + } } #[derive(Clone, Debug, PartialEq)] @@ -207,7 +211,7 @@ impl ChatWidget { { if self.realtime_conversation.audio_player.is_none() { self.realtime_conversation.audio_player = - crate::voice::RealtimeAudioPlayer::start().ok(); + crate::voice::RealtimeAudioPlayer::start(&self.config).ok(); } if let Some(player) = &self.realtime_conversation.audio_player && let Err(err) = player.enqueue_frame(frame) @@ -231,7 +235,10 @@ impl ChatWidget { self.realtime_conversation.meter_placeholder_id = Some(placeholder_id.clone()); self.request_redraw(); - let capture = match crate::voice::VoiceCapture::start_realtime(self.app_event_tx.clone()) { + let capture = match crate::voice::VoiceCapture::start_realtime( + &self.config, + self.app_event_tx.clone(), + ) { Ok(capture) => capture, Err(err) => { self.remove_transcription_placeholder(&placeholder_id); @@ -250,7 +257,7 @@ impl ChatWidget { self.realtime_conversation.capture = Some(capture); if self.realtime_conversation.audio_player.is_none() { self.realtime_conversation.audio_player = - crate::voice::RealtimeAudioPlayer::start().ok(); + crate::voice::RealtimeAudioPlayer::start(&self.config).ok(); } std::thread::spawn(move || { @@ -275,8 +282,50 @@ impl ChatWidget { #[cfg(target_os = "linux")] fn start_realtime_local_audio(&mut self) {} + #[cfg(all(not(target_os = "linux"), feature = "voice-input"))] + pub(crate) fn restart_realtime_audio_device(&mut self, kind: RealtimeAudioDeviceKind) { + if !self.realtime_conversation.is_active() { + return; + } + + match kind { + RealtimeAudioDeviceKind::Microphone => { + self.stop_realtime_microphone(); + self.start_realtime_local_audio(); + } + RealtimeAudioDeviceKind::Speaker => { + self.stop_realtime_speaker(); + match crate::voice::RealtimeAudioPlayer::start(&self.config) { + Ok(player) => { + self.realtime_conversation.audio_player = Some(player); + } + Err(err) => { + self.add_error_message(format!("Failed to start speaker output: {err}")); + } + } + } + } + self.request_redraw(); + } + + #[cfg(any(target_os = "linux", not(feature = "voice-input")))] + pub(crate) fn restart_realtime_audio_device(&mut self, kind: RealtimeAudioDeviceKind) { + let _ = kind; + } + #[cfg(not(target_os = "linux"))] fn stop_realtime_local_audio(&mut self) { + self.stop_realtime_microphone(); + self.stop_realtime_speaker(); + } + + #[cfg(target_os = "linux")] + fn stop_realtime_local_audio(&mut self) { + self.realtime_conversation.meter_placeholder_id = None; + } + + #[cfg(not(target_os = "linux"))] + fn stop_realtime_microphone(&mut self) { if let Some(flag) = self.realtime_conversation.capture_stop_flag.take() { flag.store(true, Ordering::Relaxed); } @@ -286,13 +335,12 @@ impl ChatWidget { if let Some(id) = self.realtime_conversation.meter_placeholder_id.take() { self.remove_transcription_placeholder(&id); } + } + + #[cfg(not(target_os = "linux"))] + fn stop_realtime_speaker(&mut self) { if let Some(player) = self.realtime_conversation.audio_player.take() { player.clear(); } } - - #[cfg(target_os = "linux")] - fn stop_realtime_local_audio(&mut self) { - self.realtime_conversation.meter_placeholder_id = None; - } } diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_audio_selection_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_audio_selection_popup.snap new file mode 100644 index 0000000000..8c60f961f9 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_audio_selection_popup.snap @@ -0,0 +1,11 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: popup +--- + Settings + Configure settings for Codex. + +› 1. Microphone Current: System default + 2. Speaker Current: System default + + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_audio_selection_popup_narrow.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_audio_selection_popup_narrow.snap new file mode 100644 index 0000000000..8c60f961f9 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_audio_selection_popup_narrow.snap @@ -0,0 +1,11 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: popup +--- + Settings + Configure settings for Codex. + +› 1. Microphone Current: System default + 2. Speaker Current: System default + + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_microphone_picker_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_microphone_picker_popup.snap new file mode 100644 index 0000000000..3095e6da97 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__realtime_microphone_picker_popup.snap @@ -0,0 +1,18 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: popup +--- + Select Microphone + Saved devices apply to realtime voice only. + + 1. System default Use your operating system + default device. +› 2. Unavailable: Studio Mic (current) (disabled) Configured device is not + currently available. + (disabled: Reconnect the + device or choose another + one.) + 3. Built-in Mic + 4. USB Mic + + Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 00d4a1df91..b0ba5200a2 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -7,6 +7,8 @@ use super::*; use crate::app_event::AppEvent; use crate::app_event::ExitMode; +#[cfg(all(not(target_os = "linux"), feature = "voice-input"))] +use crate::app_event::RealtimeAudioDeviceKind; use crate::app_event_sender::AppEventSender; use crate::bottom_pane::FeedbackAudience; use crate::bottom_pane::LocalImageAttachment; @@ -751,8 +753,8 @@ async fn enter_with_only_remote_images_submits_user_turn() { chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); - let items = match next_submit_op(&mut op_rx) { - Op::UserTurn { items, .. } => items, + let (items, summary) = match next_submit_op(&mut op_rx) { + Op::UserTurn { items, summary, .. } => (items, summary), other => panic!("expected Op::UserTurn, got {other:?}"), }; assert_eq!( @@ -761,6 +763,7 @@ async fn enter_with_only_remote_images_submits_user_turn() { image_url: remote_url.clone(), }] ); + assert_eq!(summary, None); assert!(chat.remote_image_urls().is_empty()); let mut user_cell = None; @@ -2790,6 +2793,7 @@ async fn exec_approval_emits_proposed_command_and_decision_history() { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }; chat.handle_codex_event(Event { @@ -2839,6 +2843,7 @@ async fn exec_approval_uses_approval_id_when_present() { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }), }); @@ -2875,6 +2880,7 @@ async fn exec_approval_decision_truncates_multiline_and_long_commands() { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }; chat.handle_codex_event(Event { @@ -2929,6 +2935,7 @@ async fn exec_approval_decision_truncates_multiline_and_long_commands() { proposed_execpolicy_amendment: None, proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }; chat.handle_codex_event(Event { @@ -5996,6 +6003,62 @@ async fn personality_selection_popup_snapshot() { assert_snapshot!("personality_selection_popup", popup); } +#[cfg(all(not(target_os = "linux"), feature = "voice-input"))] +#[tokio::test] +async fn realtime_audio_selection_popup_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.2-codex")).await; + chat.open_realtime_audio_popup(); + + let popup = render_bottom_popup(&chat, 80); + assert_snapshot!("realtime_audio_selection_popup", popup); +} + +#[cfg(all(not(target_os = "linux"), feature = "voice-input"))] +#[tokio::test] +async fn realtime_audio_selection_popup_narrow_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.2-codex")).await; + chat.open_realtime_audio_popup(); + + let popup = render_bottom_popup(&chat, 56); + assert_snapshot!("realtime_audio_selection_popup_narrow", popup); +} + +#[cfg(all(not(target_os = "linux"), feature = "voice-input"))] +#[tokio::test] +async fn realtime_microphone_picker_popup_snapshot() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.2-codex")).await; + chat.config.realtime_audio.microphone = Some("Studio Mic".to_string()); + chat.open_realtime_audio_device_selection_with_names( + RealtimeAudioDeviceKind::Microphone, + vec!["Built-in Mic".to_string(), "USB Mic".to_string()], + ); + + let popup = render_bottom_popup(&chat, 80); + assert_snapshot!("realtime_microphone_picker_popup", popup); +} + +#[cfg(all(not(target_os = "linux"), feature = "voice-input"))] +#[tokio::test] +async fn realtime_audio_picker_emits_persist_event() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.2-codex")).await; + chat.open_realtime_audio_device_selection_with_names( + RealtimeAudioDeviceKind::Speaker, + vec!["Desk Speakers".to_string(), "Headphones".to_string()], + ); + + chat.handle_key_event(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); + chat.handle_key_event(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::PersistRealtimeAudioDeviceSelection { + kind: RealtimeAudioDeviceKind::Speaker, + name: Some(name), + }) if name == "Headphones" + ); +} + #[tokio::test] async fn model_picker_hides_show_in_picker_false_models_from_cache() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("test-visible-model")).await; @@ -6014,6 +6077,7 @@ async fn model_picker_hides_show_in_picker_false_models_from_cache() { is_default: false, upgrade: None, show_in_picker, + availability_nux: None, supported_in_api: true, input_modalities: default_input_modalities(), }; @@ -6282,6 +6346,7 @@ async fn single_reasoning_option_skips_selection() { is_default: false, upgrade: None, show_in_picker: true, + availability_nux: None, supported_in_api: true, input_modalities: default_input_modalities(), }; @@ -6748,6 +6813,7 @@ async fn approval_modal_exec_snapshot() -> anyhow::Result<()> { ])), proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }; chat.handle_codex_event(Event { @@ -6808,6 +6874,7 @@ async fn approval_modal_exec_without_reason_snapshot() -> anyhow::Result<()> { ])), proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }; chat.handle_codex_event(Event { @@ -6855,6 +6922,7 @@ async fn approval_modal_exec_multiline_prefix_hides_execpolicy_option_snapshot() proposed_execpolicy_amendment: Some(ExecPolicyAmendment::new(command)), proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }; chat.handle_codex_event(Event { @@ -7221,6 +7289,7 @@ async fn status_widget_and_approval_modal_snapshot() { ])), proposed_network_policy_amendments: None, additional_permissions: None, + available_decisions: None, parsed_cmd: vec![], }; chat.handle_codex_event(Event { diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 78312db27d..624d47d915 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -2602,6 +2602,16 @@ mod tests { insta::assert_snapshot!(rendered); } + #[test] + fn error_event_oversized_input_snapshot() { + let cell = new_error_event( + "Message exceeds the maximum length of 1048576 characters (1048577 provided)." + .to_string(), + ); + let rendered = render_lines(&cell.display_lines(120)).join("\n"); + insta::assert_snapshot!(rendered); + } + #[tokio::test] async fn mcp_tools_output_masks_sensitive_values() { let mut config = test_config().await; @@ -2623,6 +2633,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }; let mut servers = config.mcp_servers.get().clone(); servers.insert("docs".to_string(), stdio_config); @@ -2646,6 +2657,7 @@ mod tests { enabled_tools: None, disabled_tools: None, scopes: None, + oauth_resource: None, }; servers.insert("http".to_string(), http_config); config diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index d51f42b2ff..8dfdbcd871 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -31,8 +31,10 @@ use codex_core::find_thread_path_by_name_str; use codex_core::format_exec_policy_error_with_source; use codex_core::path_utils; use codex_core::read_session_meta_line; +use codex_core::state_db::get_state_db; use codex_core::terminal::Multiplexer; use codex_core::windows_sandbox::WindowsSandboxLevelExt; +use codex_protocol::ThreadId; use codex_protocol::config_types::AltScreenMode; use codex_protocol::config_types::SandboxMode; use codex_protocol::config_types::WindowsSandboxLevel; @@ -61,6 +63,8 @@ mod app_backtrack; mod app_event; mod app_event_sender; mod ascii_animation; +#[cfg(all(not(target_os = "linux"), feature = "voice-input"))] +mod audio_device; mod bottom_pane; mod chatwidget; mod cli; @@ -121,6 +125,7 @@ mod voice; mod voice { use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; + use codex_core::config::Config; use codex_protocol::protocol::RealtimeAudioFrame; use std::sync::Arc; use std::sync::Mutex; @@ -144,7 +149,7 @@ mod voice { Err("voice input is unavailable in this build".to_string()) } - pub fn start_realtime(_tx: AppEventSender) -> Result { + pub fn start_realtime(_config: &Config, _tx: AppEventSender) -> Result { Err("voice input is unavailable in this build".to_string()) } @@ -184,7 +189,7 @@ mod voice { } impl RealtimeAudioPlayer { - pub(crate) fn start() -> Result { + pub(crate) fn start(_config: &Config) -> Result { Err("voice output is unavailable in this build".to_string()) } @@ -665,7 +670,19 @@ async fn run_ratatui_app( find_thread_path_by_name_str(&config.codex_home, id_str).await? }; match path { - Some(path) => resume_picker::SessionSelection::Fork(path), + Some(path) => { + let thread_id = + match resolve_session_thread_id(path.as_path(), is_uuid.then_some(id_str)) + .await + { + Some(thread_id) => thread_id, + None => return missing_session_exit(id_str, "fork"), + }; + resume_picker::SessionSelection::Fork(resume_picker::SessionTarget { + path, + thread_id, + }) + } None => return missing_session_exit(id_str, "fork"), } } else if cli.fork_last { @@ -682,11 +699,37 @@ async fn run_ratatui_app( ) .await { - Ok(page) => page - .items - .first() - .map(|it| resume_picker::SessionSelection::Fork(it.path.clone())) - .unwrap_or(resume_picker::SessionSelection::StartFresh), + Ok(page) => match page.items.first() { + Some(item) => { + match resolve_session_thread_id(item.path.as_path(), None).await { + Some(thread_id) => resume_picker::SessionSelection::Fork( + resume_picker::SessionTarget { + path: item.path.clone(), + thread_id, + }, + ), + None => { + let rollout_path = item.path.display(); + error!( + "Error reading session metadata from latest rollout: {rollout_path}" + ); + restore(); + session_log::log_session_end(); + let _ = tui.terminal.clear(); + return Ok(AppExitInfo { + token_usage: codex_protocol::protocol::TokenUsage::default(), + thread_id: None, + thread_name: None, + update_action: None, + exit_reason: ExitReason::Fatal(format!( + "Found latest saved session at {rollout_path}, but failed to read its metadata. Run `codex fork` to choose from existing sessions." + )), + }); + } + } + } + None => resume_picker::SessionSelection::StartFresh, + }, Err(_) => resume_picker::SessionSelection::StartFresh, } } else if cli.fork_picker { @@ -715,7 +758,21 @@ async fn run_ratatui_app( find_thread_path_by_name_str(&config.codex_home, id_str).await? }; match path { - Some(path) => resume_picker::SessionSelection::Resume(path), + Some(path) => { + let thread_id = match resolve_session_thread_id( + path.as_path(), + is_uuid.then_some(id_str), + ) + .await + { + Some(thread_id) => thread_id, + None => return missing_session_exit(id_str, "resume"), + }; + resume_picker::SessionSelection::Resume(resume_picker::SessionTarget { + path, + thread_id, + }) + } None => return missing_session_exit(id_str, "resume"), } } else if cli.resume_last { @@ -737,7 +794,30 @@ async fn run_ratatui_app( ) .await { - Ok(Some(path)) => resume_picker::SessionSelection::Resume(path), + Ok(Some(path)) => match resolve_session_thread_id(path.as_path(), None).await { + Some(thread_id) => { + resume_picker::SessionSelection::Resume(resume_picker::SessionTarget { + path, + thread_id, + }) + } + None => { + let rollout_path = path.display(); + error!("Error reading session metadata from latest rollout: {rollout_path}"); + restore(); + session_log::log_session_end(); + let _ = tui.terminal.clear(); + return Ok(AppExitInfo { + token_usage: codex_protocol::protocol::TokenUsage::default(), + thread_id: None, + thread_name: None, + update_action: None, + exit_reason: ExitReason::Fatal(format!( + "Found latest saved session at {rollout_path}, but failed to read its metadata. Run `codex resume` to choose from existing sessions." + )), + }); + } + }, _ => resume_picker::SessionSelection::StartFresh, } } else if cli.resume_picker { @@ -761,15 +841,27 @@ async fn run_ratatui_app( let current_cwd = config.cwd.clone(); let allow_prompt = cli.cwd.is_none(); - let action_and_path_if_resume_or_fork = match &session_selection { - resume_picker::SessionSelection::Resume(path) => Some((CwdPromptAction::Resume, path)), - resume_picker::SessionSelection::Fork(path) => Some((CwdPromptAction::Fork, path)), + let action_and_target_session_if_resume_or_fork = match &session_selection { + resume_picker::SessionSelection::Resume(target_session) => { + Some((CwdPromptAction::Resume, target_session)) + } + resume_picker::SessionSelection::Fork(target_session) => { + Some((CwdPromptAction::Fork, target_session)) + } _ => None, }; - let fallback_cwd = match action_and_path_if_resume_or_fork { - Some((action, path)) => { - match resolve_cwd_for_resume_or_fork(&mut tui, ¤t_cwd, path, action, allow_prompt) - .await? + let fallback_cwd = match action_and_target_session_if_resume_or_fork { + Some((action, target_session)) => { + match resolve_cwd_for_resume_or_fork( + &mut tui, + &config, + ¤t_cwd, + target_session.thread_id, + &target_session.path, + action, + allow_prompt, + ) + .await? { ResolveCwdOutcome::Continue(cwd) => cwd, ResolveCwdOutcome::Exit => { @@ -851,12 +943,35 @@ async fn run_ratatui_app( app_result } -pub(crate) async fn read_session_cwd(path: &Path) -> Option { +pub(crate) async fn resolve_session_thread_id( + path: &Path, + id_str_if_uuid: Option<&str>, +) -> Option { + match id_str_if_uuid { + Some(id_str) => ThreadId::from_string(id_str).ok(), + None => read_session_meta_line(path) + .await + .ok() + .map(|meta_line| meta_line.meta.id), + } +} + +pub(crate) async fn read_session_cwd( + config: &Config, + thread_id: ThreadId, + path: &Path, +) -> Option { + if let Some(state_db_ctx) = get_state_db(config, None).await + && let Ok(Some(metadata)) = state_db_ctx.get_thread(thread_id).await + { + return Some(metadata.cwd); + } + // Prefer the latest TurnContext cwd so resume/fork reflects the most recent - // session directory (for the changed-cwd prompt). The alternative would be - // mutating the SessionMeta line when the session cwd changes, but the rollout - // is an append-only JSONL log and rewriting the head would be error-prone. - // When rollouts move to SQLite, we can drop this scan. + // session directory (for the changed-cwd prompt) when DB data is unavailable. + // The alternative would be mutating the SessionMeta line when the session cwd + // changes, but the rollout is an append-only JSONL log and rewriting the head + // would be error-prone. if let Some(cwd) = parse_latest_turn_context_cwd(path).await { return Some(cwd); } @@ -908,12 +1023,14 @@ pub(crate) enum ResolveCwdOutcome { pub(crate) async fn resolve_cwd_for_resume_or_fork( tui: &mut Tui, + config: &Config, current_cwd: &Path, + thread_id: ThreadId, path: &Path, action: CwdPromptAction, allow_prompt: bool, ) -> color_eyre::Result { - let Some(history_cwd) = read_session_cwd(path).await else { + let Some(history_cwd) = read_session_cwd(config, thread_id, path).await else { return Ok(ResolveCwdOutcome::Continue(None)); }; if allow_prompt && cwds_differ(current_cwd, &history_cwd) { @@ -1071,11 +1188,14 @@ mod tests { use codex_core::config::ConfigBuilder; use codex_core::config::ConfigOverrides; use codex_core::config::ProjectConfig; + use codex_core::features::Feature; + use codex_protocol::ThreadId; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; + use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::TurnContextItem; use serial_test::serial; use tempfile::TempDir; @@ -1152,6 +1272,8 @@ mod tests { TurnContextItem { turn_id: None, cwd, + current_date: None, + timezone: None, approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, @@ -1159,7 +1281,9 @@ mod tests { personality: None, collaboration_mode: None, effort: config.model_reasoning_effort, - summary: config.model_reasoning_summary, + summary: config + .model_reasoning_summary + .unwrap_or(codex_protocol::config_types::ReasoningSummary::Auto), user_instructions: None, developer_instructions: None, final_output_json_schema: None, @@ -1194,7 +1318,9 @@ mod tests { } std::fs::write(&rollout_path, text)?; - let cwd = read_session_cwd(&rollout_path).await.expect("expected cwd"); + let cwd = read_session_cwd(&config, ThreadId::new(), &rollout_path) + .await + .expect("expected cwd"); assert_eq!(cwd, second); Ok(()) } @@ -1234,7 +1360,9 @@ mod tests { } std::fs::write(&rollout_path, text)?; - let session_cwd = read_session_cwd(&rollout_path).await.expect("expected cwd"); + let session_cwd = read_session_cwd(&config, ThreadId::new(), &rollout_path) + .await + .expect("expected cwd"); assert_eq!(session_cwd, latest); assert!(cwds_differ(¤t, &session_cwd)); Ok(()) @@ -1339,7 +1467,7 @@ trust_level = "untrusted" #[tokio::test] async fn read_session_cwd_falls_back_to_session_meta() -> std::io::Result<()> { let temp_dir = TempDir::new()?; - let _config = build_config(&temp_dir).await?; + let config = build_config(&temp_dir).await?; let session_cwd = temp_dir.path().join("session"); std::fs::create_dir_all(&session_cwd)?; @@ -1361,8 +1489,67 @@ trust_level = "untrusted" ); std::fs::write(&rollout_path, text)?; - let cwd = read_session_cwd(&rollout_path).await.expect("expected cwd"); + let cwd = read_session_cwd(&config, ThreadId::new(), &rollout_path) + .await + .expect("expected cwd"); assert_eq!(cwd, session_cwd); Ok(()) } + + #[tokio::test] + async fn read_session_cwd_prefers_sqlite_when_thread_id_present() -> std::io::Result<()> { + let temp_dir = TempDir::new()?; + let mut config = build_config(&temp_dir).await?; + config.features.enable(Feature::Sqlite); + + let thread_id = ThreadId::new(); + let rollout_cwd = temp_dir.path().join("rollout-cwd"); + let sqlite_cwd = temp_dir.path().join("sqlite-cwd"); + std::fs::create_dir_all(&rollout_cwd)?; + std::fs::create_dir_all(&sqlite_cwd)?; + + let rollout_path = temp_dir.path().join("rollout.jsonl"); + let rollout_line = RolloutLine { + timestamp: "t0".to_string(), + item: RolloutItem::TurnContext(build_turn_context(&config, rollout_cwd)), + }; + std::fs::write( + &rollout_path, + format!( + "{}\n", + serde_json::to_string(&rollout_line).expect("serialize rollout") + ), + )?; + + let runtime = codex_state::StateRuntime::init( + config.codex_home.clone(), + config.model_provider_id.clone(), + None, + ) + .await + .map_err(std::io::Error::other)?; + runtime + .mark_backfill_complete(None) + .await + .map_err(std::io::Error::other)?; + + let mut builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + rollout_path.clone(), + chrono::Utc::now(), + SessionSource::Cli, + ); + builder.cwd = sqlite_cwd.clone(); + let metadata = builder.build(config.model_provider_id.as_str()); + runtime + .upsert_thread(&metadata) + .await + .map_err(std::io::Error::other)?; + + let cwd = read_session_cwd(&config, thread_id, &rollout_path) + .await + .expect("expected cwd"); + assert_eq!(cwd, sqlite_cwd); + Ok(()) + } } diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs index ba8b47d5ed..2bbe19b6f7 100644 --- a/codex-rs/tui/src/markdown_render.rs +++ b/codex-rs/tui/src/markdown_render.rs @@ -2,6 +2,7 @@ use crate::render::highlight::highlight_code_to_lines; use crate::render::line_utils::line_to_static; use crate::wrapping::RtOptions; use crate::wrapping::adaptive_wrap_line; +use codex_utils_string::normalize_markdown_hash_location_suffix; use pulldown_cmark::CodeBlockKind; use pulldown_cmark::CowStr; use pulldown_cmark::Event; @@ -14,6 +15,8 @@ use ratatui::style::Style; use ratatui::text::Line; use ratatui::text::Span; use ratatui::text::Text; +use regex_lite::Regex; +use std::sync::LazyLock; struct MarkdownStyles { h1: Style, @@ -85,6 +88,48 @@ pub(crate) fn render_markdown_text_with_width(input: &str, width: Option) w.text } +#[derive(Clone, Debug)] +struct LinkState { + destination: String, + show_destination: bool, + hidden_location_suffix: Option, + label_start_span_idx: usize, + label_styled: bool, +} + +fn should_render_link_destination(dest_url: &str) -> bool { + !is_local_path_like_link(dest_url) +} + +static COLON_LOCATION_SUFFIX_RE: LazyLock = + LazyLock::new( + || match Regex::new(r":\d+(?::\d+)?(?:[-–]\d+(?::\d+)?)?$") { + Ok(regex) => regex, + Err(error) => panic!("invalid location suffix regex: {error}"), + }, + ); + +// Covered by load_location_suffix_regexes. +static HASH_LOCATION_SUFFIX_RE: LazyLock = + LazyLock::new(|| match Regex::new(r"^L\d+(?:C\d+)?(?:-L\d+(?:C\d+)?)?$") { + Ok(regex) => regex, + Err(error) => panic!("invalid hash location regex: {error}"), + }); + +fn is_local_path_like_link(dest_url: &str) -> bool { + dest_url.starts_with("file://") + || dest_url.starts_with('/') + || dest_url.starts_with("~/") + || dest_url.starts_with("./") + || dest_url.starts_with("../") + || dest_url.starts_with("\\\\") + || matches!( + dest_url.as_bytes(), + [drive, b':', separator, ..] + if drive.is_ascii_alphabetic() && matches!(separator, b'/' | b'\\') + ) +} + struct Writer<'a, I> where I: Iterator>, @@ -95,7 +140,7 @@ where inline_styles: Vec