Enable reasoning effort overrides by default (#56163)

## What changed

Promote `reasoning_effort_override` to stable and enable it by default, allowing supported models to receive reasoning effort changes through configuration updates in conversation history.

Update request-history snapshots to include the configuration updates, and remove explicit feature enablement from the reasoning effort override test builder so the suite exercises the default.

GitOrigin-RevId: 6ce16d7f985c33f008e5230fc9bae09cdaf141d5
This commit is contained in:
felixxia-oai
2026-10-22 12:46:24 +00:00
committed by copyberry
parent 94350a9101
commit 2f3fb111be
32 changed files with 201 additions and 175 deletions
@@ -47,10 +47,6 @@ fn override_builder() -> TestCodexBuilder {
model.supports_reasoning_effort_updates = true;
})
.with_config(|config| {
config
.features
.enable(Feature::ReasoningEffortOverride)
.expect("enable reasoning effort overrides");
config.model_reasoning_effort = Some(ReasoningEffort::Medium);
})
}
@@ -46,15 +46,16 @@ Scenario: Permission discovery preserves legacy output, exposes reply-only acces
<ENVIRONMENT_CONTEXT>
06:message/user:
Check board permissions, try publishing, then ask a question in the existing thread.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:function_call/collaboration.get_channels:{"query":"legacy"}
08:function_call_output:{"has_more":false,"n_returned":1,"next_cursor":null,"results":[{"channel_name":"policy","created_at":"20...ot","last_message_id":"<UUID 1>","message_count":1}]} [hash=555caa1996907423]
08:function_call/collaboration.get_channels:{"query":"legacy"}
09:function_call_output:{"has_more":false,"n_returned":1,"next_cursor":null,"results":[{"channel_name":"policy","created_at":"20...ot","last_message_id":"<UUID 1>","message_count":1}]} [hash=555caa1996907423]
-- request 3 (request) --
09:function_call/collaboration.get_channels:{"query":"policy"}
10:function_call_output:{"board_permissions":{"create_channel":false,"edit_metadata":false,"manage_permissions":false},"has_more...data":false,"post":false,"read":true,"reply":true}}]} [hash=961b427452007daf]
10:function_call/collaboration.get_channels:{"query":"policy"}
11:function_call_output:{"board_permissions":{"create_channel":false,"edit_metadata":false,"manage_permissions":false},"has_more...data":false,"post":false,"read":true,"reply":true}}]} [hash=961b427452007daf]
-- request 4 (request) --
11:function_call/collaboration.post:{"channel_name":"policy","text":"New policy."}
12:function_call_output:{"error":{"action":"channel.post","code":"permission_denied","message":"You can reply, but cannot start ... policy.","resource":"board:example/channel:policy"}} [hash=890040530da635c0]
12:function_call/collaboration.post:{"channel_name":"policy","text":"New policy."}
13:function_call_output:{"error":{"action":"channel.post","code":"permission_denied","message":"You can reply, but cannot start ... policy.","resource":"board:example/channel:policy"}} [hash=890040530da635c0]
-- request 5 (request) --
13:function_call/collaboration.post:{"text":"A clarification?","thread_id":"<UUID 1>"}
14:function_call_output:{"author":"/root","channel_name":"policy","created_at":"2026-09-18T12:00:00Z","message_id":"<UUID 2>","thread_id":"<UUID 1>"}
14:function_call/collaboration.post:{"text":"A clarification?","thread_id":"<UUID 1>"}
15:function_call_output:{"author":"/root","channel_name":"policy","created_at":"2026-09-18T12:00:00Z","message_id":"<UUID 2>","thread_id":"<UUID 1>"}
@@ -103,10 +103,11 @@ Scenario: Astra starts in one workspace, switches for the next step of the activ
</environment_context>
06:message/user:
Ask before continuing in the other workspace.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:function_call/functions.request_user_input:{"questions":[{"header":"Continue","id":"continue","options":[{"description":"Continue working.","label"...o"}],"question":"Continue in the other workspace?"}]} [hash=028e565351844d96]
08:function_call_output:{"answers":{"continue":{"answers":["Yes"]}}}
09:message/user:
08:function_call/functions.request_user_input:{"questions":[{"header":"Continue","id":"continue","options":[{"description":"Continue working.","label"...o"}],"question":"Continue in the other workspace?"}]} [hash=028e565351844d96]
09:function_call_output:{"answers":{"continue":{"answers":["Yes"]}}}
10:message/user:
<environment_context>
<cwd><CWD>/other-workspace</cwd>
<shell><HOST_SHELL></shell>
@@ -115,9 +116,9 @@ Scenario: Astra starts in one workspace, switches for the next step of the activ
<filesystem><workspace_roots><root><CWD>/other-workspace</root></workspace_roots><permission_profile t...ntry></file_system></permission_profile></filesystem> [hash=0be8191a0451377a]
</environment_context>
-- request 3 (turn) --
10:message/assistant:
11:message/assistant:
I continued in the other workspace.
11:message/user:
12:message/user:
<environment_context>
<cwd><CWD></cwd>
<shell><HOST_SHELL></shell>
@@ -125,5 +126,5 @@ Scenario: Astra starts in one workspace, switches for the next step of the activ
<timezone><HOST_TIMEZONE></timezone>
<filesystem><workspace_roots><root><CWD></root></workspace_roots><permission_profile type="managed"><f...ntry></file_system></permission_profile></filesystem> [hash=e8c1de60307d0a3d]
</environment_context>
12:message/user:
13:message/user:
Which workspace are you using now?
@@ -34,12 +34,13 @@ Scenario: Astra compacts a completed launch-notes turn in the background. The us
<ENVIRONMENT_CONTEXT>
06:message/user:
Draft the launch notes.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:message/assistant:
08:message/assistant:
I drafted the launch notes.
08:compaction_trigger
09:compaction_trigger
## Window 2 (after request 2: input diverged at item 08)
## Window 2 (after request 2: input diverged at item 09)
-- request 3 (request) --
00:additional_tools/developer (3; hash=bf2e30605ff5dc17):
- namespace/functions; hash=fada4a476a3754dd
@@ -69,11 +70,12 @@ Scenario: Astra compacts a completed launch-notes turn in the background. The us
<ENVIRONMENT_CONTEXT>
06:message/user:
Draft the launch notes.
07:message/assistant:
07:configuration_update:reasoning={"effort":"low"}
08:message/assistant:
I drafted the launch notes.
08:message/developer:
09:message/developer:
<COLLABORATION_MODE>
09:message/user:
10:message/user:
Check who can finish the notes.
## Window 3 (after request 3: input diverged at item 02)
@@ -1,6 +1,5 @@
---
source: core/tests/suite/scenarios.rs
assertion_line: 375
expression: "context_snapshot::format_context_snapshot(\"Astra uses the catalog question schema to ask who a launch update is for, keeps working, and receives the user's answer in the active turn.\",\n&entries, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
---
Scenario: Astra uses the catalog question schema to ask who a launch update is for, keeps working, and receives the user's answer in the active turn.
@@ -35,22 +34,23 @@ Scenario: Astra uses the catalog question schema to ask who a launch update is f
<ENVIRONMENT_CONTEXT>
06:message/user:
Draft a short launch update. Ask me who it is for and keep working while I answer.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:function_call/functions.request_user_input_async:{"questions":[{"options":["Internal team","Customers"],"title":"Who should receive the launch update?"}]}
08:function_call_output:{"accepted":true}
08:function_call/functions.request_user_input_async:{"questions":[{"options":["Internal team","Customers"],"title":"Who should receive the launch update?"}]}
09:function_call_output:{"accepted":true}
-- request 3 (request) --
09:message/assistant:
10:message/assistant:
I drafted a short launch update.
10:message/user:
11:message/user:
<send_user_message_question_reply>
[{"answer":"Customers","question":"Who should receive the launch update?","questionItemId":"[\\"request_user_input_async\\",\\"audience-question\\",0]"}]
</send_user_message_question_reply>
-- request 4 (request) --
11:message/assistant:
12:message/assistant:
Here is the launch update for customers.
12:message/developer:
13:message/developer:
<PERMISSIONS_INSTRUCTIONS>
13:message/user:
<ENVIRONMENT_CONTEXT>
14:message/user:
<ENVIRONMENT_CONTEXT>
15:message/user:
Now give it an email subject.
@@ -90,15 +90,16 @@ Scenario: Astra receives host and handler timings on completed and failed code-m
</environment_context>
06:message/user:
Run a code cell, then inspect its timing and a failed wait.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:custom_tool_call/exec:text('ready');
08:custom_tool_call_output:Script completed
08:custom_tool_call/exec:text('ready');
09:custom_tool_call_output:Script completed
Wall time <DURATION> seconds (code-mode <DURATION> seconds; overhead <DURATION> seconds)
Output:
| ready
-- request 3 (turn) --
09:function_call/functions.wait:{"cell_id":"missing"}
10:function_call_output:Script failed
10:function_call/functions.wait:{"cell_id":"missing"}
11:function_call_output:Script failed
Wall time <DURATION> seconds (code-mode <DURATION> seconds; overhead <DURATION> seconds)
Output:
| Script error:
@@ -122,10 +122,11 @@ Settings:
</environment_context>
06:message/user:
Which workflow skills are available?
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:function_call/skills.list:{"authority":{"kind":"executor"}}
08:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"workspace-skills","kind":"executor"},"description":"Us.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=b72a3cc66e4d3c51]
09:message/developer:
08:function_call/skills.list:{"authority":{"kind":"executor"}}
09:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"workspace-skills","kind":"executor"},"description":"Us.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=b72a3cc66e4d3c51]
10:message/developer:
<skills_instructions>
## Skills
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...locators can be expanded using the skill roots table. [hash=f00d6a4d3e192f06]
@@ -124,10 +124,11 @@ Settings:
</environment_context>
06:message/user:
Which workflow skills are available?
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:function_call/skills.list:{"authority":{"kind":"executor"}}
08:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"plugin-skills","kind":"executor"},"description":"Use a.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=a640b563d0b76db9]
09:message/developer:
08:function_call/skills.list:{"authority":{"kind":"executor"}}
09:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"plugin-skills","kind":"executor"},"description":"Use a.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=a640b563d0b76db9]
10:message/developer:
<skills_instructions>
## Skills
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...locators can be expanded using the skill roots table. [hash=f00d6a4d3e192f06]
@@ -34,8 +34,9 @@ Scenario: Astra receives new user input after its unfinished response is interru
<ENVIRONMENT_CONTEXT>
06:message/user:
Draft a launch update.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:message/assistant:
08:message/assistant:
I will draft the update.
08:message/user:
09:message/user:
Make it a customer update.
@@ -103,18 +103,19 @@ Scenario: Astra continues after new user input yields a running code cell, then
</environment_context>
06:message/user:
Check the release status.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:custom_tool_call/exec:// @exec: {"yield_time_ms": 60000}
08:custom_tool_call/exec:// @exec: {"yield_time_ms": 60000}
text(await tools.check_release({}));
08:custom_tool_call_output:Script running with cell ID <CELL_ID>
09:custom_tool_call_output:Script running with cell ID <CELL_ID>
Wall time <DURATION> seconds
Output:
09:message/user:
10:message/user:
Please also highlight the customer impact.
-- request 3 (turn) --
10:function_call/functions.wait:{"cell_id":"<CELL_ID>","yield_time_ms":60000}
11:function_call_output:Script completed
11:function_call/functions.wait:{"cell_id":"<CELL_ID>","yield_time_ms":60000}
12:function_call_output:Script completed
Wall time <DURATION> seconds
Output:
| Release ready
@@ -158,12 +158,13 @@ Settings:
Capabilities from the `calendar` plugin:
- Skills from this plugin are prefixed with `calendar:`.
Use these plugin-associated capabilities to help solve the task.
10:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
10:message/assistant:
11:message/assistant:
Agenda drafted for the kickoff.
11:message/user:
Summarize the kickoff notes with $notes:summarize.
12:message/user:
Summarize the kickoff notes with $notes:summarize.
13:message/user:
<skill>
<name>notes:summarize</name>
<path><PLUGINS_CACHE>/test/notes/local/skills/summarize/SKILL.md</path>
@@ -175,14 +176,14 @@ Settings:
Extract decisions and action items.
</skill>
13:message/developer:
14:message/developer:
Capabilities from the `notes` plugin:
- Skills from this plugin are prefixed with `notes:`.
Use these plugin-associated capabilities to help solve the task.
-- request 3 (compaction) --
14:message/assistant:
15:message/assistant:
The team chose Friday and assigned owners.
15:compaction_trigger
16:compaction_trigger
## Window 2 (after request 3: input diverged at item 02)
Settings: same as window 1
@@ -111,3 +111,4 @@ Settings:
</environment_context>
06:message/user:
Plan a team kickoff. Which planning skills are available?
07:configuration_update:reasoning={"effort":"low"}
@@ -100,16 +100,17 @@ Settings:
</environment_context>
06:message/user:
Check whether the Notes plugin echo tool is available.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
08:custom_tool_call_output:Script completed
08:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
09:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| []
-- request 3 (turn) --
09:message/assistant:
10:message/assistant:
The Notes tool is not installed yet.
10:message/developer:
11:message/developer:
<skills_instructions>
## Skills
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list ...ed into an absolute path using the skill roots table. [hash=ca27c3957f42502b]
@@ -118,9 +119,9 @@ Settings:
### Available skills
- notes:summarize: Summarize team notes (file: r0/notes/local/skills/summarize/SKILL.md)
</skills_instructions>
11:message/user:
I installed Notes. Use $notes:summarize and the Notes tool to check that Mira owns the kickoff.
12:message/user:
I installed Notes. Use $notes:summarize and the Notes tool to check that Mira owns the kickoff.
13:message/user:
<skill>
<name>notes:summarize</name>
<path><PLUGINS_CACHE>/test/notes/local/skills/summarize/SKILL.md</path>
@@ -132,31 +133,31 @@ Settings:
State the owner and date from the notes.
</skill>
13:message/developer:
14:message/developer:
Capabilities from the `notes` plugin:
- Skills from this plugin are prefixed with `notes:`.
- MCP servers from this plugin available in this session: `notes`.
Use these plugin-associated capabilities to help solve the task.
-- request 4 (turn) --
14:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
15:custom_tool_call_output:Script completed
15:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
16:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| ["mcp__notes__echo"]
-- request 5 (turn) --
16:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "Mira owns the kickoff" }); text(result.structuredContent?.echo);
17:custom_tool_call_output:Script completed
17:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "Mira owns the kickoff" }); text(result.structuredContent?.echo);
18:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| ECHOING: Mira owns the kickoff
-- request 6 (turn) --
18:message/assistant:
19:message/assistant:
The Notes plugin confirms Mira owns the kickoff.
19:message/user:
20:message/user:
Use the Notes tool again to check that the kickoff is Friday.
-- request 7 (turn) --
20:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "The kickoff is Friday" }); text(result.structuredContent?.echo);
21:custom_tool_call_output:Script completed
21:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "The kickoff is Friday" }); text(result.structuredContent?.echo);
22:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| ECHOING: The kickoff is Friday
@@ -100,17 +100,18 @@ Settings:
</environment_context>
06:message/user:
Check the Settings release. Read release/diagnostics.log, inspect release/settings.png, confirm the loca...ell me whether another agent is working on this task. [hash=aa221f7aaa101eb2]
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:function_call/collaboration.list_agents:{}
08:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
08:function_call/collaboration.list_agents:{}
09:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
-- request 3 (turn) --
09:custom_tool_call/exec:const [log, ping] = await Promise.all([
10:custom_tool_call/exec:const [log, ping] = await Promise.all([
tools.exec_command({ cmd: "cat release/diagnostics.log", login: false, max_output_tokens: 4000 }),
tools.mcp__rmcp__echo({ message: "settings-release-check" }),
]);
text(`diagnostics:\\n${log.output}`);
text(`MCP: ${ping.structuredContent?.echo ?? "missing"}`);
10:custom_tool_call_output:Script completed
11:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| diagnostics:
@@ -131,13 +132,13 @@ Settings:
diagnostic 120: rendering settings panel
UI-42: observed Settings button label: Save | MCP: ECHOING: settings-release-check
-- request 4 (turn) --
11:custom_tool_call/exec:image(await tools.view_image({ path: "release/settings.png", detail: "original" }));
12:custom_tool_call_output:Script completed
12:custom_tool_call/exec:image(await tools.view_image({ path: "release/settings.png", detail: "original" }));
13:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| <input_image>
-- request 5 (turn) --
13:custom_tool_call/exec:text(await tools.apply_patch(`*** Begin Patch
14:custom_tool_call/exec:text(await tools.apply_patch(`*** Begin Patch
*** Update File: release/status.md
@@
-Status: pending
@@ -146,13 +147,13 @@ Settings:
+MCP: reachable
*** End Patch
`));
14:custom_tool_call_output:Script completed
15:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| {}
-- request 6 (turn) --
15:custom_tool_call/exec:text((await tools.exec_command({ cmd: "cat release/status.md", login: false })).output);
16:custom_tool_call_output:Script completed
16:custom_tool_call/exec:text((await tools.exec_command({ cmd: "cat release/status.md", login: false })).output);
17:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| Status: blocked
@@ -42,13 +42,14 @@ Settings:
<MULTI_AGENT_MODE>
05:message/user:
Ask a worker to check the release, then finish your turn.
06:configuration_update:reasoning={"effort":"medium"}
-- request 2 (turn) --
06:function_call/collaboration.spawn_agent:{"fork_turns":"none","message":"Check whether the release is ready.","task_name":"worker"}
07:function_call_output:{"task_name":"/root/worker"}
07:function_call/collaboration.spawn_agent:{"fork_turns":"none","message":"Check whether the release is ready.","task_name":"worker"}
08:function_call_output:{"task_name":"/root/worker"}
-- request 3 (turn) --
08:message/assistant:
09:message/assistant:
I have delegated the release check.
09:agent_message
10:agent_message
## Window 2 (after request 3: input diverged at item 00)
Settings: same as window 1
@@ -77,9 +78,10 @@ Settings: same as window 1
04:message/developer:
<MULTI_AGENT_MODE>
05:agent_message
06:configuration_update:reasoning={"effort":"medium"}
-- request 5 (turn) --
06:function_call/check_release:{}
07:function_call_output:All release checks passed.
07:function_call/check_release:{}
08:function_call_output:All release checks passed.
Woken parent mailbox:
[
@@ -1,6 +1,6 @@
---
source: core/tests/suite/scenarios.rs
expression: "context_snapshot::format_request_history_snapshot(\"Catalog Code Mode instructions and wait parameter descriptions accompany a yielded cell through completion.\",\n&requests, &ContextSnapshotOptions::default().include_request_settings(),)"
expression: "context_snapshot::format_request_history_snapshot(scenario, &requests,\n&ContextSnapshotOptions::default().include_request_settings(),)"
---
Scenario: Catalog Code Mode instructions and wait parameter descriptions accompany a yielded cell through completion.
@@ -100,15 +100,16 @@ Settings:
</environment_context>
05:message/user:
Start a code cell, yield its initial output, then wait for its completion.
06:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
06:custom_tool_call/exec:text('started'); yield_control(); text('finished');
07:custom_tool_call_output:Script running with cell ID 1
07:custom_tool_call/exec:text('started'); yield_control(); text('finished');
08:custom_tool_call_output:Script running with cell ID 1
Wall time <DURATION> seconds
Output:
| started
-- request 3 (turn) --
08:function_call/functions.wait:{"cell_id":"1"}
09:function_call_output:Script completed
09:function_call/functions.wait:{"cell_id":"1"}
10:function_call_output:Script completed
Wall time <DURATION> seconds
Output:
| finished
@@ -100,12 +100,13 @@ Settings:
</environment_context>
05:message/user:
Find calendar timezone option 99, call it with UTC, then yield and wait for completion.
06:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
06:custom_tool_call/exec:const result = await tools.tool_search({query: "calendar_timezone_option_99", limit: 1});
07:custom_tool_call/exec:const result = await tools.tool_search({query: "calendar_timezone_option_99", limit: 1});
store("calendarToolName", result.tools[0].name);
text(result.tools[0].name);
text(result.tools[0].description);
07:custom_tool_call_output:Script completed
08:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| mcp__codex_apps__calendar_timezone_option_99 | Plan events and manage your calendar.
@@ -117,16 +118,16 @@ Settings:
declare const tools: { mcp__codex_apps__calendar_timezone_option_99(args: { timezone?: string; }): Promise<CallToolResult>; };
```
-- request 3 (turn) --
08:custom_tool_call/exec:const name = load("calendarToolName");
09:custom_tool_call/exec:const name = load("calendarToolName");
text((await tools[name]({ timezone: "UTC" })).content[0].text);
text('started'); yield_control(); text('finished');
09:custom_tool_call_output:Script running with cell ID 2
10:custom_tool_call_output:Script running with cell ID 2
Wall time <DURATION> seconds
Output:
| called calendar_timezone_option_99 for at with | started
-- request 4 (turn) --
10:function_call/functions.wait:{"cell_id":"2"}
11:function_call_output:Script completed
11:function_call/functions.wait:{"cell_id":"2"}
12:function_call_output:Script completed
Wall time <DURATION> seconds
Output:
| finished
@@ -34,10 +34,11 @@ Scenario: Settlement helpers deliver fulfilled and rejected records with input i
<ENVIRONMENT_CONTEXT>
06:message/user:
Collect each completed result, including failures, using the settlement helpers.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:custom_tool_call/exec:for await (const result of as_settled([Promise.resolve("ready")])) text(result);
08:custom_tool_call/exec:for await (const result of as_settled([Promise.resolve("ready")])) text(result);
await stream_settled(new Map([["failed", Promise.reject("unavailable")], ["completed", "done"]]), text);
08:custom_tool_call_output:Script completed
09:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| {"index":0,"status":"fulfilled","value":"ready"} | {"index":"failed","status":"rejected","reason":"unavailable"} | {"index":"completed","status":"fulfilled","value":"done"}
@@ -100,10 +100,11 @@ Settings:
</environment_context>
06:message/user:
Plan the release.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (compaction; Remote /compact request) --
07:message/assistant:
08:message/assistant:
The release plan has three steps.
08:compaction_trigger
09:compaction_trigger
## Window 2 (after request 2: input diverged at item 07)
Settings: unavailable (items only)
@@ -1,6 +1,5 @@
---
source: core/tests/suite/scenarios_frozen_cloud_skills.rs
assertion_line: 142
expression: "context_snapshot::format_context_snapshot(\"A cloud parent lists and reads a frozen skill alongside a named executor after the snapshot file has been removed.\",\n&bodies.iter().map(SnapshotEntry::body).collect::<Vec<_>>(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
---
Scenario: A cloud parent lists and reads a frozen skill alongside a named executor after the snapshot file has been removed.
@@ -111,9 +110,10 @@ Settings:
</environment_context>
06:message/user:
List the cloud skills, then read the work skill.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:function_call/skills.list:{"authority":{"kind":"cloud"}}
08:function_call_output:{"next_cursor":null,"skills":[{"authority":{"kind":"cloud"},"description":"Inspect and explain the reque...:work","package":"skill://demo/work"}],"warnings":[]} [hash=0a735291ec9e0ec7]
08:function_call/skills.list:{"authority":{"kind":"cloud"}}
09:function_call_output:{"next_cursor":null,"skills":[{"authority":{"kind":"cloud"},"description":"Inspect and explain the reque...:work","package":"skill://demo/work"}],"warnings":[]} [hash=0a735291ec9e0ec7]
-- request 3 (request) --
09:function_call/skills.read:{"package":"c0/work","source":{"kind":"cloud"}}
10:function_call_output:{"contents":"# Work\\nInspect the inputs before explaining the result.\\n","installations":[{"package":"sk...cursor":null,"resource":"skill://demo/work/SKILL.md"} [hash=adc0897fa3bc41cf]
10:function_call/skills.read:{"package":"c0/work","source":{"kind":"cloud"}}
11:function_call_output:{"contents":"# Work\\nInspect the inputs before explaining the result.\\n","installations":[{"package":"sk...cursor":null,"resource":"skill://demo/work/SKILL.md"} [hash=adc0897fa3bc41cf]
@@ -1,6 +1,5 @@
---
source: core/tests/suite/scenarios_mcp_ema.rs
assertion_line: 125
expression: "histories[1]"
---
Scenario: Ordinary MCP keeps the same tool schemas, calls, and history when enterprise authentication fails.
@@ -55,8 +54,9 @@ Settings:
<ENVIRONMENT_CONTEXT>
05:message/user:
List my calendar events.
06:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
06:function_call/mcp__ordinary.calendar_list_events:{}
07:function_call_output:Wall time: <DURATION> seconds
07:function_call/mcp__ordinary.calendar_list_events:{}
08:function_call_output:Wall time: <DURATION> seconds
Output:
{"_codex_apps":null}
@@ -1,6 +1,5 @@
---
source: core/tests/suite/scenarios_mcp_ema.rs
assertion_line: 257
expression: "context_snapshot::format_request_history_snapshot(\"Enterprise access disappears after revocation. Prior calls remain in history and ordinary tools still work.\",\n&requests,\n&ContextSnapshotOptions::default().rewrite_known_segments().include_request_settings(),)"
---
Scenario: Enterprise access disappears after revocation. Prior calls remain in history and ordinary tools still work.
@@ -59,9 +58,10 @@ Settings:
<ENVIRONMENT_CONTEXT>
05:message/user:
List my work events.
06:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
06:function_call/mcp__enterprise.calendar_list_events:{}
07:function_call_output:Wall time: <DURATION> seconds
07:function_call/mcp__enterprise.calendar_list_events:{}
08:function_call_output:Wall time: <DURATION> seconds
Output:
{"_codex_apps":null}
@@ -83,16 +83,17 @@ Settings: relative to window 1
<ENVIRONMENT_CONTEXT>
05:message/user:
List my work events.
06:function_call/mcp__enterprise.calendar_list_events:{}
07:function_call_output:Wall time: <DURATION> seconds
06:configuration_update:reasoning={"effort":"low"}
07:function_call/mcp__enterprise.calendar_list_events:{}
08:function_call_output:Wall time: <DURATION> seconds
Output:
{"_codex_apps":null}
08:message/assistant:
09:message/assistant:
Here are your work events.
09:message/user:
10:message/user:
List my other calendar events.
-- request 4 (turn) --
10:function_call/mcp__ordinary.calendar_list_events:{}
11:function_call_output:Wall time: <DURATION> seconds
11:function_call/mcp__ordinary.calendar_list_events:{}
12:function_call_output:Wall time: <DURATION> seconds
Output:
{"_codex_apps":null}
@@ -100,11 +100,12 @@ Settings:
</environment_context>
05:message/user:
List the available resources and templates, then read the example note.
06:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
06:custom_tool_call/exec:text(await tools.list_mcp_resources({"server":"resources"}));
07:custom_tool_call/exec:text(await tools.list_mcp_resources({"server":"resources"}));
text(await tools.list_mcp_resource_templates({"server":"resources"}));
text(await tools.read_mcp_resource({"server":"resources","uri":"memo://codex/example-note"}));
07:custom_tool_call_output:Script completed
08:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| {"server":"resources","resources":[{"server":"resources","uri":"memo://codex/example-note","name":"examp...ed for integration tests.","mimeType":"text/plain"}]} [hash=75d187638a7af2d4] | {"server":"resources","resourceTemplates":[{"server":"resources","uriTemplate":"memo://codex/{slug}","na... resources used in tests.","mimeType":"text/plain"}]} [hash=b1c7082eb114fc9b] | {"server":"resources","uri":"memo://codex/example-note","contents":[{"uri":"memo://codex/example-note","...mple MCP resource served by the rmcp test server."}]} [hash=45385fc57640b555]
@@ -1,6 +1,6 @@
---
source: core/tests/suite/scenarios.rs
expression: "context_snapshot::format_request_history_snapshot(\"Astra calls list_agents using the selected catalog parameter schema.\",\n&mock.requests(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
expression: "context_snapshot::format_request_history_snapshot(description,\n&mock.requests(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
---
Scenario: Astra calls list_agents using the selected catalog parameter schema.
@@ -100,6 +100,7 @@ Settings:
</environment_context>
06:message/user:
Check which agents are working under /root before delegating more work.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:function_call/collaboration.list_agents:{"path_prefix":"/root"}
08:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
08:function_call/collaboration.list_agents:{"path_prefix":"/root"}
09:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
@@ -1,6 +1,5 @@
---
source: core/tests/suite/scenarios.rs
assertion_line: 1272
expression: "context_snapshot::format_request_history_snapshot(description,\n&mock.requests(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
---
Scenario: Native tools are unavailable while the V2 runtime and its instructions remain enabled.
@@ -94,6 +93,7 @@ Settings:
</environment_context>
06:message/user:
Check which agents are working under /root before delegating more work.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:function_call/collaboration.list_agents:{"path_prefix":"/root"}
08:function_call_output:unsupported call: collaborationlist_agents
08:function_call/collaboration.list_agents:{"path_prefix":"/root"}
09:function_call_output:unsupported call: collaborationlist_agents
@@ -109,13 +109,14 @@ Settings:
</environment_context>
07:message/user:
Prepare the release.
08:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
08:message/assistant:
09:message/assistant:
The release is ready for validation.
09:message/developer:
10:message/developer:
Your context window is nearly exhausted (only 0 tokens remaining) and will be automatically reset for yo... and history items will be persistent across windows. [hash=23016acc05795bc3]
10:message/user:
11:message/user:
Suggest my next message.
-- request 3 (turn) --
11:function_call/functions.get_context_remaining:{}
12:function_call_output:You have 0 tokens left in this context window.
12:function_call/functions.get_context_remaining:{}
13:function_call_output:You have 0 tokens left in this context window.
@@ -1,6 +1,6 @@
---
source: core/tests/suite/scenarios_retained_tool_outputs.rs
expression: "context_snapshot::format_request_history_snapshot(if enabled\n{\n \"Retained thread instructions stop emergency tool-output trimming. Compaction shares the 64K retention budget with oversized task details, and a cold resume preserves the retained tool content.\"\n} else\n{\n \"With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.\"\n}, &requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
expression: "context_snapshot::format_request_history_snapshot(if enabled\n{\n \"Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.\"\n} else\n{\n \"With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.\"\n}, &requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
---
Scenario: With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.
@@ -33,16 +33,17 @@ Scenario: With retention disabled, the client can still send retain=true. Tool o
05:message/user:
<ENVIRONMENT_CONTEXT>
06:function_call_output/codex_app.send_message_to_thread:Investigate the failing tests; report your findings without changing files.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:message/assistant:
08:message/assistant:
I will investigate the tests.
08:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
09:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
-- request 3 (turn) --
09:message/assistant:
10:message/assistant:
I will use those task details.
10:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
11:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
## Window 2 (after request 3: input diverged at item 10)
## Window 2 (after request 3: input diverged at item 11)
-- request 4 (compaction) --
00:additional_tools/developer (3; hash=bf2e30605ff5dc17):
- namespace/functions; hash=fada4a476a3754dd
@@ -71,14 +72,15 @@ Scenario: With retention disabled, the client can still send retain=true. Tool o
05:message/user:
<ENVIRONMENT_CONTEXT>
06:function_call_output/codex_app.send_message_to_thread:Investigate the failing tests; report your findings without changing files.
07:message/assistant:
07:configuration_update:reasoning={"effort":"low"}
08:message/assistant:
I will investigate the tests.
08:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
09:message/assistant:
09:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
10:message/assistant:
I will use those task details.
10:function_call_output/codex_app.send_message_to_thread:Output exceeded the available model context and was truncated
11:function_call_output/example.progress:Output exceeded the available model context and was truncated
12:compaction_trigger
11:function_call_output/codex_app.send_message_to_thread:Output exceeded the available model context and was truncated
12:function_call_output/example.progress:Output exceeded the available model context and was truncated
13:compaction_trigger
## Window 3 (after request 4: input diverged at item 06)
-- request 5 (turn) --
@@ -113,3 +115,4 @@ Scenario: With retention disabled, the client can still send retain=true. Tool o
<ENVIRONMENT_CONTEXT>
08:message/user:
What request are you working on?
09:configuration_update:reasoning={"effort":"low"}
@@ -1,6 +1,6 @@
---
source: core/tests/suite/scenarios_retained_tool_outputs.rs
expression: "context_snapshot::format_request_history_snapshot(\"Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.\",\n&requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
expression: "context_snapshot::format_request_history_snapshot(if enabled\n{\n \"Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.\"\n} else\n{\n \"With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.\"\n}, &requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
---
Scenario: Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.
@@ -33,17 +33,18 @@ Scenario: Retained thread instructions stop emergency tool-output trimming. Comp
05:message/user:
<ENVIRONMENT_CONTEXT>
06:function_call_output/codex_app.send_message_to_thread:Investigate the failing tests; report your findings without changing files.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:message/assistant:
08:message/assistant:
I will investigate the tests.
08:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
09:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
-- request 3 (turn) --
09:message/assistant:
10:message/assistant:
I will use those task details.
10:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
11:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
-- request 4 (compaction) --
11:function_call_output/example.progress:Output exceeded the available model context and was truncated
12:compaction_trigger
12:function_call_output/example.progress:Output exceeded the available model context and was truncated
13:compaction_trigger
## Window 2 (after request 4: input diverged at item 06)
-- request 5 (turn) --
@@ -79,3 +80,4 @@ Scenario: Retained thread instructions stop emergency tool-output trimming. Comp
<ENVIRONMENT_CONTEXT>
09:message/user:
What request are you working on?
10:configuration_update:reasoning={"effort":"low"}
@@ -117,8 +117,9 @@ Settings:
</environment_context>
06:message/user:
Which skills are available?
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn) --
07:message/assistant:
08:message/assistant:
Skills are available.
08:message/user:
09:message/user:
Are those same skills still available?
@@ -1,6 +1,5 @@
---
source: core/tests/suite/scenarios.rs
assertion_line: 1456
expression: "context_snapshot::format_context_snapshot(\"Astra reads Cloud instructions, discovers an omitted deployment installation, and reads its instructions from env-04.\",\n&bodies.iter().map(SnapshotEntry::body).collect::<Vec<_>>(),\n&ContextSnapshotOptions::default().rewrite_known_segments(),)"
---
Scenario: Astra reads Cloud instructions, discovers an omitted deployment installation, and reads its instructions from env-04.
@@ -36,9 +35,10 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
<ENVIRONMENT_CONTEXT>
06:message/user:
Read the review and deployment instructions, find all deployment installations, then read the instructions from env-04.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (request) --
07:function_call/skills.read:{"package":"skill://executor-00/review","source":{"kind":"cloud"}}
08:function_call_output:{
08:function_call/skills.read:{"package":"skill://executor-00/review","source":{"kind":"cloud"}}
09:function_call_output:{
"contents": "# Deployment\\nRun scripts/check.py from an available installation.",
"installations": [
{
@@ -58,8 +58,8 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
"resource": "skill://deployment/review/SKILL.md"
}
-- request 3 (request) --
09:function_call/skills.read:{"package":"skill://deployment/deploy"}
10:function_call_output:{
10:function_call/skills.read:{"package":"skill://deployment/deploy"}
11:function_call_output:{
"contents": "# Deployment\\nRun scripts/check.py from an available installation.",
"installations": [
{
@@ -77,8 +77,8 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
"resource": "skill://deployment/deploy/SKILL.md"
}
-- request 4 (request) --
11:function_call/skills.installations:{"package":"skill://deployment/deploy"}
12:function_call_output:{
12:function_call/skills.installations:{"package":"skill://deployment/deploy"}
13:function_call_output:{
"installations": [
{
"package": "skill://deployment/deploy",
@@ -96,8 +96,8 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
"next_cursor": null
}
-- request 5 (request) --
13:function_call/skills.read:{"package":"skill://deployment/deploy","source":{"environment_id":"env-04","kind":"executor"}}
14:function_call_output:{
14:function_call/skills.read:{"package":"skill://deployment/deploy","source":{"environment_id":"env-04","kind":"executor"}}
15:function_call_output:{
"contents": "# Deployment in env-04\\nRun scripts/check.py from this installation.",
"installations": [
{
@@ -44,60 +44,61 @@ Settings:
<ENVIRONMENT_CONTEXT>
06:message/user:
List and call the tools provided by the environment MCP server.
07:configuration_update:reasoning={"effort":"low"}
-- request 2 (turn; turn 1: exec discovery and call results) --
07:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
08:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
const results = [];
for (const t of available) {
const result = await tools[t.name]({});
results.push({name: t.name, result: result.content[0].text});
}
text(JSON.stringify(results));
08:custom_tool_call_output:Script completed
09:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| []
-- request 3 (turn; turn 2: register MCP with lookup; omit-all, direct-only and excluded) --
09:message/user:
10:message/user:
List and call the tools provided by the environment MCP server.
-- request 4 (turn; turn 2: exec discovery and call results) --
10:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
11:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
const results = [];
for (const t of available) {
const result = await tools[t.name]({});
results.push({name: t.name, result: result.content[0].text});
}
text(JSON.stringify(results));
11:custom_tool_call_output:Script completed
12:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| [{"name":"mcp__environment__lookup","result":"called lookup for at with "}]
-- request 5 (turn; turn 3: same MCP name, new endpoint; replace lookup with resolve and search) --
12:message/user:
13:message/user:
List and call the tools provided by the environment MCP server.
-- request 6 (turn; turn 3: exec discovery and call results) --
13:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
14:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
const results = [];
for (const t of available) {
const result = await tools[t.name]({});
results.push({name: t.name, result: result.content[0].text});
}
text(JSON.stringify(results));
14:custom_tool_call_output:Script completed
15:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| [{"name":"mcp__environment__resolve","result":"called resolve for at with "},{"name":"mcp__environment__search","result":"called search for at with "}]
-- request 7 (turn; turn 4: remove MCP server from config) --
15:message/user:
16:message/user:
List and call the tools provided by the environment MCP server.
-- request 8 (turn; turn 4: exec discovery and call results) --
16:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
17:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
const results = [];
for (const t of available) {
const result = await tools[t.name]({});
results.push({name: t.name, result: result.content[0].text});
}
text(JSON.stringify(results));
17:custom_tool_call_output:Script completed
18:custom_tool_call_output:Script completed
Wall time <DURATION> seconds
Output:
| []
+2 -2
View File
@@ -1997,8 +1997,8 @@ pub const FEATURES: &[FeatureSpec] = &[
FeatureSpec {
id: Feature::ReasoningEffortOverride,
key: "reasoning_effort_override",
stage: Stage::UnderDevelopment,
default_enabled: false,
stage: Stage::Stable,
default_enabled: true,
},
FeatureSpec {
id: Feature::CurrentTimeReminder,