mirror of
https://github.com/openai/codex.git
synced 2026-10-22 17:58:46 +00:00
Enable reasoning effort overrides by default (#56163)
## What changed Promote `reasoning_effort_override` to stable and enable it by default, allowing supported models to receive reasoning effort changes through configuration updates in conversation history. Update request-history snapshots to include the configuration updates, and remove explicit feature enablement from the reasoning effort override test builder so the suite exercises the default. GitOrigin-RevId: 6ce16d7f985c33f008e5230fc9bae09cdaf141d5
This commit is contained in:
@@ -47,10 +47,6 @@ fn override_builder() -> TestCodexBuilder {
|
||||
model.supports_reasoning_effort_updates = true;
|
||||
})
|
||||
.with_config(|config| {
|
||||
config
|
||||
.features
|
||||
.enable(Feature::ReasoningEffortOverride)
|
||||
.expect("enable reasoning effort overrides");
|
||||
config.model_reasoning_effort = Some(ReasoningEffort::Medium);
|
||||
})
|
||||
}
|
||||
|
||||
+9
-8
@@ -46,15 +46,16 @@ Scenario: Permission discovery preserves legacy output, exposes reply-only acces
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
Check board permissions, try publishing, then ask a question in the existing thread.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:function_call/collaboration.get_channels:{"query":"legacy"}
|
||||
08:function_call_output:{"has_more":false,"n_returned":1,"next_cursor":null,"results":[{"channel_name":"policy","created_at":"20...ot","last_message_id":"<UUID 1>","message_count":1}]} [hash=555caa1996907423]
|
||||
08:function_call/collaboration.get_channels:{"query":"legacy"}
|
||||
09:function_call_output:{"has_more":false,"n_returned":1,"next_cursor":null,"results":[{"channel_name":"policy","created_at":"20...ot","last_message_id":"<UUID 1>","message_count":1}]} [hash=555caa1996907423]
|
||||
-- request 3 (request) --
|
||||
09:function_call/collaboration.get_channels:{"query":"policy"}
|
||||
10:function_call_output:{"board_permissions":{"create_channel":false,"edit_metadata":false,"manage_permissions":false},"has_more...data":false,"post":false,"read":true,"reply":true}}]} [hash=961b427452007daf]
|
||||
10:function_call/collaboration.get_channels:{"query":"policy"}
|
||||
11:function_call_output:{"board_permissions":{"create_channel":false,"edit_metadata":false,"manage_permissions":false},"has_more...data":false,"post":false,"read":true,"reply":true}}]} [hash=961b427452007daf]
|
||||
-- request 4 (request) --
|
||||
11:function_call/collaboration.post:{"channel_name":"policy","text":"New policy."}
|
||||
12:function_call_output:{"error":{"action":"channel.post","code":"permission_denied","message":"You can reply, but cannot start ... policy.","resource":"board:example/channel:policy"}} [hash=890040530da635c0]
|
||||
12:function_call/collaboration.post:{"channel_name":"policy","text":"New policy."}
|
||||
13:function_call_output:{"error":{"action":"channel.post","code":"permission_denied","message":"You can reply, but cannot start ... policy.","resource":"board:example/channel:policy"}} [hash=890040530da635c0]
|
||||
-- request 5 (request) --
|
||||
13:function_call/collaboration.post:{"text":"A clarification?","thread_id":"<UUID 1>"}
|
||||
14:function_call_output:{"author":"/root","channel_name":"policy","created_at":"2026-09-18T12:00:00Z","message_id":"<UUID 2>","thread_id":"<UUID 1>"}
|
||||
14:function_call/collaboration.post:{"text":"A clarification?","thread_id":"<UUID 1>"}
|
||||
15:function_call_output:{"author":"/root","channel_name":"policy","created_at":"2026-09-18T12:00:00Z","message_id":"<UUID 2>","thread_id":"<UUID 1>"}
|
||||
|
||||
+7
-6
@@ -103,10 +103,11 @@ Scenario: Astra starts in one workspace, switches for the next step of the activ
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Ask before continuing in the other workspace.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:function_call/functions.request_user_input:{"questions":[{"header":"Continue","id":"continue","options":[{"description":"Continue working.","label"...o"}],"question":"Continue in the other workspace?"}]} [hash=028e565351844d96]
|
||||
08:function_call_output:{"answers":{"continue":{"answers":["Yes"]}}}
|
||||
09:message/user:
|
||||
08:function_call/functions.request_user_input:{"questions":[{"header":"Continue","id":"continue","options":[{"description":"Continue working.","label"...o"}],"question":"Continue in the other workspace?"}]} [hash=028e565351844d96]
|
||||
09:function_call_output:{"answers":{"continue":{"answers":["Yes"]}}}
|
||||
10:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD>/other-workspace</cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
@@ -115,9 +116,9 @@ Scenario: Astra starts in one workspace, switches for the next step of the activ
|
||||
<filesystem><workspace_roots><root><CWD>/other-workspace</root></workspace_roots><permission_profile t...ntry></file_system></permission_profile></filesystem> [hash=0be8191a0451377a]
|
||||
</environment_context>
|
||||
-- request 3 (turn) --
|
||||
10:message/assistant:
|
||||
11:message/assistant:
|
||||
I continued in the other workspace.
|
||||
11:message/user:
|
||||
12:message/user:
|
||||
<environment_context>
|
||||
<cwd><CWD></cwd>
|
||||
<shell><HOST_SHELL></shell>
|
||||
@@ -125,5 +126,5 @@ Scenario: Astra starts in one workspace, switches for the next step of the activ
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
<filesystem><workspace_roots><root><CWD></root></workspace_roots><permission_profile type="managed"><f...ntry></file_system></permission_profile></filesystem> [hash=e8c1de60307d0a3d]
|
||||
</environment_context>
|
||||
12:message/user:
|
||||
13:message/user:
|
||||
Which workspace are you using now?
|
||||
|
||||
+8
-6
@@ -34,12 +34,13 @@ Scenario: Astra compacts a completed launch-notes turn in the background. The us
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
Draft the launch notes.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:message/assistant:
|
||||
08:message/assistant:
|
||||
I drafted the launch notes.
|
||||
08:compaction_trigger
|
||||
09:compaction_trigger
|
||||
|
||||
## Window 2 (after request 2: input diverged at item 08)
|
||||
## Window 2 (after request 2: input diverged at item 09)
|
||||
-- request 3 (request) --
|
||||
00:additional_tools/developer (3; hash=bf2e30605ff5dc17):
|
||||
- namespace/functions; hash=fada4a476a3754dd
|
||||
@@ -69,11 +70,12 @@ Scenario: Astra compacts a completed launch-notes turn in the background. The us
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
Draft the launch notes.
|
||||
07:message/assistant:
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
08:message/assistant:
|
||||
I drafted the launch notes.
|
||||
08:message/developer:
|
||||
09:message/developer:
|
||||
<COLLABORATION_MODE>
|
||||
09:message/user:
|
||||
10:message/user:
|
||||
Check who can finish the notes.
|
||||
|
||||
## Window 3 (after request 3: input diverged at item 02)
|
||||
|
||||
+9
-9
@@ -1,6 +1,5 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios.rs
|
||||
assertion_line: 375
|
||||
expression: "context_snapshot::format_context_snapshot(\"Astra uses the catalog question schema to ask who a launch update is for, keeps working, and receives the user's answer in the active turn.\",\n&entries, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
|
||||
---
|
||||
Scenario: Astra uses the catalog question schema to ask who a launch update is for, keeps working, and receives the user's answer in the active turn.
|
||||
@@ -35,22 +34,23 @@ Scenario: Astra uses the catalog question schema to ask who a launch update is f
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
Draft a short launch update. Ask me who it is for and keep working while I answer.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:function_call/functions.request_user_input_async:{"questions":[{"options":["Internal team","Customers"],"title":"Who should receive the launch update?"}]}
|
||||
08:function_call_output:{"accepted":true}
|
||||
08:function_call/functions.request_user_input_async:{"questions":[{"options":["Internal team","Customers"],"title":"Who should receive the launch update?"}]}
|
||||
09:function_call_output:{"accepted":true}
|
||||
-- request 3 (request) --
|
||||
09:message/assistant:
|
||||
10:message/assistant:
|
||||
I drafted a short launch update.
|
||||
10:message/user:
|
||||
11:message/user:
|
||||
<send_user_message_question_reply>
|
||||
[{"answer":"Customers","question":"Who should receive the launch update?","questionItemId":"[\\"request_user_input_async\\",\\"audience-question\\",0]"}]
|
||||
</send_user_message_question_reply>
|
||||
-- request 4 (request) --
|
||||
11:message/assistant:
|
||||
12:message/assistant:
|
||||
Here is the launch update for customers.
|
||||
12:message/developer:
|
||||
13:message/developer:
|
||||
<PERMISSIONS_INSTRUCTIONS>
|
||||
13:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
14:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
15:message/user:
|
||||
Now give it an email subject.
|
||||
|
||||
+5
-4
@@ -90,15 +90,16 @@ Scenario: Astra receives host and handler timings on completed and failed code-m
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Run a code cell, then inspect its timing and a failed wait.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:custom_tool_call/exec:text('ready');
|
||||
08:custom_tool_call_output:Script completed
|
||||
08:custom_tool_call/exec:text('ready');
|
||||
09:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds (code-mode <DURATION> seconds; overhead <DURATION> seconds)
|
||||
Output:
|
||||
| ready
|
||||
-- request 3 (turn) --
|
||||
09:function_call/functions.wait:{"cell_id":"missing"}
|
||||
10:function_call_output:Script failed
|
||||
10:function_call/functions.wait:{"cell_id":"missing"}
|
||||
11:function_call_output:Script failed
|
||||
Wall time <DURATION> seconds (code-mode <DURATION> seconds; overhead <DURATION> seconds)
|
||||
Output:
|
||||
| Script error:
|
||||
|
||||
+4
-3
@@ -122,10 +122,11 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Which workflow skills are available?
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:function_call/skills.list:{"authority":{"kind":"executor"}}
|
||||
08:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"workspace-skills","kind":"executor"},"description":"Us.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=b72a3cc66e4d3c51]
|
||||
09:message/developer:
|
||||
08:function_call/skills.list:{"authority":{"kind":"executor"}}
|
||||
09:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"workspace-skills","kind":"executor"},"description":"Us.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=b72a3cc66e4d3c51]
|
||||
10:message/developer:
|
||||
<skills_instructions>
|
||||
## Skills
|
||||
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...locators can be expanded using the skill roots table. [hash=f00d6a4d3e192f06]
|
||||
|
||||
+4
-3
@@ -124,10 +124,11 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Which workflow skills are available?
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:function_call/skills.list:{"authority":{"kind":"executor"}}
|
||||
08:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"plugin-skills","kind":"executor"},"description":"Use a.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=a640b563d0b76db9]
|
||||
09:message/developer:
|
||||
08:function_call/skills.list:{"authority":{"kind":"executor"}}
|
||||
09:function_call_output:{"next_cursor":null,"skills":[{"authority":{"id":"plugin-skills","kind":"executor"},"description":"Use a.../<SKILLS_ROOT>/skills/active-helper"}],"warnings":[]} [hash=a640b563d0b76db9]
|
||||
10:message/developer:
|
||||
<skills_instructions>
|
||||
## Skills
|
||||
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...locators can be expanded using the skill roots table. [hash=f00d6a4d3e192f06]
|
||||
|
||||
+3
-2
@@ -34,8 +34,9 @@ Scenario: Astra receives new user input after its unfinished response is interru
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
Draft a launch update.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:message/assistant:
|
||||
08:message/assistant:
|
||||
I will draft the update.
|
||||
08:message/user:
|
||||
09:message/user:
|
||||
Make it a customer update.
|
||||
|
||||
+6
-5
@@ -103,18 +103,19 @@ Scenario: Astra continues after new user input yields a running code cell, then
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Check the release status.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:custom_tool_call/exec:// @exec: {"yield_time_ms": 60000}
|
||||
08:custom_tool_call/exec:// @exec: {"yield_time_ms": 60000}
|
||||
text(await tools.check_release({}));
|
||||
08:custom_tool_call_output:Script running with cell ID <CELL_ID>
|
||||
09:custom_tool_call_output:Script running with cell ID <CELL_ID>
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
|
||||
09:message/user:
|
||||
10:message/user:
|
||||
Please also highlight the customer impact.
|
||||
-- request 3 (turn) --
|
||||
10:function_call/functions.wait:{"cell_id":"<CELL_ID>","yield_time_ms":60000}
|
||||
11:function_call_output:Script completed
|
||||
11:function_call/functions.wait:{"cell_id":"<CELL_ID>","yield_time_ms":60000}
|
||||
12:function_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| Release ready
|
||||
|
||||
+7
-6
@@ -158,12 +158,13 @@ Settings:
|
||||
Capabilities from the `calendar` plugin:
|
||||
- Skills from this plugin are prefixed with `calendar:`.
|
||||
Use these plugin-associated capabilities to help solve the task.
|
||||
10:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
10:message/assistant:
|
||||
11:message/assistant:
|
||||
Agenda drafted for the kickoff.
|
||||
11:message/user:
|
||||
Summarize the kickoff notes with $notes:summarize.
|
||||
12:message/user:
|
||||
Summarize the kickoff notes with $notes:summarize.
|
||||
13:message/user:
|
||||
<skill>
|
||||
<name>notes:summarize</name>
|
||||
<path><PLUGINS_CACHE>/test/notes/local/skills/summarize/SKILL.md</path>
|
||||
@@ -175,14 +176,14 @@ Settings:
|
||||
Extract decisions and action items.
|
||||
|
||||
</skill>
|
||||
13:message/developer:
|
||||
14:message/developer:
|
||||
Capabilities from the `notes` plugin:
|
||||
- Skills from this plugin are prefixed with `notes:`.
|
||||
Use these plugin-associated capabilities to help solve the task.
|
||||
-- request 3 (compaction) --
|
||||
14:message/assistant:
|
||||
15:message/assistant:
|
||||
The team chose Friday and assigned owners.
|
||||
15:compaction_trigger
|
||||
16:compaction_trigger
|
||||
|
||||
## Window 2 (after request 3: input diverged at item 02)
|
||||
Settings: same as window 1
|
||||
|
||||
+1
@@ -111,3 +111,4 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Plan a team kickoff. Which planning skills are available?
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
|
||||
+16
-15
@@ -100,16 +100,17 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Check whether the Notes plugin echo tool is available.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
|
||||
08:custom_tool_call_output:Script completed
|
||||
08:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
|
||||
09:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| []
|
||||
-- request 3 (turn) --
|
||||
09:message/assistant:
|
||||
10:message/assistant:
|
||||
The Notes tool is not installed yet.
|
||||
10:message/developer:
|
||||
11:message/developer:
|
||||
<skills_instructions>
|
||||
## Skills
|
||||
A skill is a set of local instructions to follow that is stored in a `SKILL.md` file. Below is the list ...ed into an absolute path using the skill roots table. [hash=ca27c3957f42502b]
|
||||
@@ -118,9 +119,9 @@ Settings:
|
||||
### Available skills
|
||||
- notes:summarize: Summarize team notes (file: r0/notes/local/skills/summarize/SKILL.md)
|
||||
</skills_instructions>
|
||||
11:message/user:
|
||||
I installed Notes. Use $notes:summarize and the Notes tool to check that Mira owns the kickoff.
|
||||
12:message/user:
|
||||
I installed Notes. Use $notes:summarize and the Notes tool to check that Mira owns the kickoff.
|
||||
13:message/user:
|
||||
<skill>
|
||||
<name>notes:summarize</name>
|
||||
<path><PLUGINS_CACHE>/test/notes/local/skills/summarize/SKILL.md</path>
|
||||
@@ -132,31 +133,31 @@ Settings:
|
||||
State the owner and date from the notes.
|
||||
|
||||
</skill>
|
||||
13:message/developer:
|
||||
14:message/developer:
|
||||
Capabilities from the `notes` plugin:
|
||||
- Skills from this plugin are prefixed with `notes:`.
|
||||
- MCP servers from this plugin available in this session: `notes`.
|
||||
Use these plugin-associated capabilities to help solve the task.
|
||||
-- request 4 (turn) --
|
||||
14:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
|
||||
15:custom_tool_call_output:Script completed
|
||||
15:custom_tool_call/exec:text(ALL_TOOLS.filter(({ name }) => name === "mcp__notes__echo").map(({ name }) => name));
|
||||
16:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| ["mcp__notes__echo"]
|
||||
-- request 5 (turn) --
|
||||
16:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "Mira owns the kickoff" }); text(result.structuredContent?.echo);
|
||||
17:custom_tool_call_output:Script completed
|
||||
17:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "Mira owns the kickoff" }); text(result.structuredContent?.echo);
|
||||
18:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| ECHOING: Mira owns the kickoff
|
||||
-- request 6 (turn) --
|
||||
18:message/assistant:
|
||||
19:message/assistant:
|
||||
The Notes plugin confirms Mira owns the kickoff.
|
||||
19:message/user:
|
||||
20:message/user:
|
||||
Use the Notes tool again to check that the kickoff is Friday.
|
||||
-- request 7 (turn) --
|
||||
20:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "The kickoff is Friday" }); text(result.structuredContent?.echo);
|
||||
21:custom_tool_call_output:Script completed
|
||||
21:custom_tool_call/exec:const result = await tools.mcp__notes__echo({ message: "The kickoff is Friday" }); text(result.structuredContent?.echo);
|
||||
22:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| ECHOING: The kickoff is Friday
|
||||
|
||||
+11
-10
@@ -100,17 +100,18 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Check the Settings release. Read release/diagnostics.log, inspect release/settings.png, confirm the loca...ell me whether another agent is working on this task. [hash=aa221f7aaa101eb2]
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:function_call/collaboration.list_agents:{}
|
||||
08:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
|
||||
08:function_call/collaboration.list_agents:{}
|
||||
09:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
|
||||
-- request 3 (turn) --
|
||||
09:custom_tool_call/exec:const [log, ping] = await Promise.all([
|
||||
10:custom_tool_call/exec:const [log, ping] = await Promise.all([
|
||||
tools.exec_command({ cmd: "cat release/diagnostics.log", login: false, max_output_tokens: 4000 }),
|
||||
tools.mcp__rmcp__echo({ message: "settings-release-check" }),
|
||||
]);
|
||||
text(`diagnostics:\\n${log.output}`);
|
||||
text(`MCP: ${ping.structuredContent?.echo ?? "missing"}`);
|
||||
10:custom_tool_call_output:Script completed
|
||||
11:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| diagnostics:
|
||||
@@ -131,13 +132,13 @@ Settings:
|
||||
diagnostic 120: rendering settings panel
|
||||
UI-42: observed Settings button label: Save | MCP: ECHOING: settings-release-check
|
||||
-- request 4 (turn) --
|
||||
11:custom_tool_call/exec:image(await tools.view_image({ path: "release/settings.png", detail: "original" }));
|
||||
12:custom_tool_call_output:Script completed
|
||||
12:custom_tool_call/exec:image(await tools.view_image({ path: "release/settings.png", detail: "original" }));
|
||||
13:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| <input_image>
|
||||
-- request 5 (turn) --
|
||||
13:custom_tool_call/exec:text(await tools.apply_patch(`*** Begin Patch
|
||||
14:custom_tool_call/exec:text(await tools.apply_patch(`*** Begin Patch
|
||||
*** Update File: release/status.md
|
||||
@@
|
||||
-Status: pending
|
||||
@@ -146,13 +147,13 @@ Settings:
|
||||
+MCP: reachable
|
||||
*** End Patch
|
||||
`));
|
||||
14:custom_tool_call_output:Script completed
|
||||
15:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| {}
|
||||
-- request 6 (turn) --
|
||||
15:custom_tool_call/exec:text((await tools.exec_command({ cmd: "cat release/status.md", login: false })).output);
|
||||
16:custom_tool_call_output:Script completed
|
||||
16:custom_tool_call/exec:text((await tools.exec_command({ cmd: "cat release/status.md", login: false })).output);
|
||||
17:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| Status: blocked
|
||||
|
||||
+8
-6
@@ -42,13 +42,14 @@ Settings:
|
||||
<MULTI_AGENT_MODE>
|
||||
05:message/user:
|
||||
Ask a worker to check the release, then finish your turn.
|
||||
06:configuration_update:reasoning={"effort":"medium"}
|
||||
-- request 2 (turn) --
|
||||
06:function_call/collaboration.spawn_agent:{"fork_turns":"none","message":"Check whether the release is ready.","task_name":"worker"}
|
||||
07:function_call_output:{"task_name":"/root/worker"}
|
||||
07:function_call/collaboration.spawn_agent:{"fork_turns":"none","message":"Check whether the release is ready.","task_name":"worker"}
|
||||
08:function_call_output:{"task_name":"/root/worker"}
|
||||
-- request 3 (turn) --
|
||||
08:message/assistant:
|
||||
09:message/assistant:
|
||||
I have delegated the release check.
|
||||
09:agent_message
|
||||
10:agent_message
|
||||
|
||||
## Window 2 (after request 3: input diverged at item 00)
|
||||
Settings: same as window 1
|
||||
@@ -77,9 +78,10 @@ Settings: same as window 1
|
||||
04:message/developer:
|
||||
<MULTI_AGENT_MODE>
|
||||
05:agent_message
|
||||
06:configuration_update:reasoning={"effort":"medium"}
|
||||
-- request 5 (turn) --
|
||||
06:function_call/check_release:{}
|
||||
07:function_call_output:All release checks passed.
|
||||
07:function_call/check_release:{}
|
||||
08:function_call_output:All release checks passed.
|
||||
|
||||
Woken parent mailbox:
|
||||
[
|
||||
|
||||
+6
-5
@@ -1,6 +1,6 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios.rs
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Catalog Code Mode instructions and wait parameter descriptions accompany a yielded cell through completion.\",\n&requests, &ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
expression: "context_snapshot::format_request_history_snapshot(scenario, &requests,\n&ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
---
|
||||
Scenario: Catalog Code Mode instructions and wait parameter descriptions accompany a yielded cell through completion.
|
||||
|
||||
@@ -100,15 +100,16 @@ Settings:
|
||||
</environment_context>
|
||||
05:message/user:
|
||||
Start a code cell, yield its initial output, then wait for its completion.
|
||||
06:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
06:custom_tool_call/exec:text('started'); yield_control(); text('finished');
|
||||
07:custom_tool_call_output:Script running with cell ID 1
|
||||
07:custom_tool_call/exec:text('started'); yield_control(); text('finished');
|
||||
08:custom_tool_call_output:Script running with cell ID 1
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| started
|
||||
-- request 3 (turn) --
|
||||
08:function_call/functions.wait:{"cell_id":"1"}
|
||||
09:function_call_output:Script completed
|
||||
09:function_call/functions.wait:{"cell_id":"1"}
|
||||
10:function_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| finished
|
||||
|
||||
+7
-6
@@ -100,12 +100,13 @@ Settings:
|
||||
</environment_context>
|
||||
05:message/user:
|
||||
Find calendar timezone option 99, call it with UTC, then yield and wait for completion.
|
||||
06:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
06:custom_tool_call/exec:const result = await tools.tool_search({query: "calendar_timezone_option_99", limit: 1});
|
||||
07:custom_tool_call/exec:const result = await tools.tool_search({query: "calendar_timezone_option_99", limit: 1});
|
||||
store("calendarToolName", result.tools[0].name);
|
||||
text(result.tools[0].name);
|
||||
text(result.tools[0].description);
|
||||
07:custom_tool_call_output:Script completed
|
||||
08:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| mcp__codex_apps__calendar_timezone_option_99 | Plan events and manage your calendar.
|
||||
@@ -117,16 +118,16 @@ Settings:
|
||||
declare const tools: { mcp__codex_apps__calendar_timezone_option_99(args: { timezone?: string; }): Promise<CallToolResult>; };
|
||||
```
|
||||
-- request 3 (turn) --
|
||||
08:custom_tool_call/exec:const name = load("calendarToolName");
|
||||
09:custom_tool_call/exec:const name = load("calendarToolName");
|
||||
text((await tools[name]({ timezone: "UTC" })).content[0].text);
|
||||
text('started'); yield_control(); text('finished');
|
||||
09:custom_tool_call_output:Script running with cell ID 2
|
||||
10:custom_tool_call_output:Script running with cell ID 2
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| called calendar_timezone_option_99 for at with | started
|
||||
-- request 4 (turn) --
|
||||
10:function_call/functions.wait:{"cell_id":"2"}
|
||||
11:function_call_output:Script completed
|
||||
11:function_call/functions.wait:{"cell_id":"2"}
|
||||
12:function_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| finished
|
||||
|
||||
+3
-2
@@ -34,10 +34,11 @@ Scenario: Settlement helpers deliver fulfilled and rejected records with input i
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
Collect each completed result, including failures, using the settlement helpers.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:custom_tool_call/exec:for await (const result of as_settled([Promise.resolve("ready")])) text(result);
|
||||
08:custom_tool_call/exec:for await (const result of as_settled([Promise.resolve("ready")])) text(result);
|
||||
await stream_settled(new Map([["failed", Promise.reject("unavailable")], ["completed", "done"]]), text);
|
||||
08:custom_tool_call_output:Script completed
|
||||
09:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| {"index":0,"status":"fulfilled","value":"ready"} | {"index":"failed","status":"rejected","reason":"unavailable"} | {"index":"completed","status":"fulfilled","value":"done"}
|
||||
|
||||
+3
-2
@@ -100,10 +100,11 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Plan the release.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (compaction; Remote /compact request) --
|
||||
07:message/assistant:
|
||||
08:message/assistant:
|
||||
The release plan has three steps.
|
||||
08:compaction_trigger
|
||||
09:compaction_trigger
|
||||
|
||||
## Window 2 (after request 2: input diverged at item 07)
|
||||
Settings: unavailable (items only)
|
||||
|
||||
+5
-5
@@ -1,6 +1,5 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_frozen_cloud_skills.rs
|
||||
assertion_line: 142
|
||||
expression: "context_snapshot::format_context_snapshot(\"A cloud parent lists and reads a frozen skill alongside a named executor after the snapshot file has been removed.\",\n&bodies.iter().map(SnapshotEntry::body).collect::<Vec<_>>(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
---
|
||||
Scenario: A cloud parent lists and reads a frozen skill alongside a named executor after the snapshot file has been removed.
|
||||
@@ -111,9 +110,10 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
List the cloud skills, then read the work skill.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:function_call/skills.list:{"authority":{"kind":"cloud"}}
|
||||
08:function_call_output:{"next_cursor":null,"skills":[{"authority":{"kind":"cloud"},"description":"Inspect and explain the reque...:work","package":"skill://demo/work"}],"warnings":[]} [hash=0a735291ec9e0ec7]
|
||||
08:function_call/skills.list:{"authority":{"kind":"cloud"}}
|
||||
09:function_call_output:{"next_cursor":null,"skills":[{"authority":{"kind":"cloud"},"description":"Inspect and explain the reque...:work","package":"skill://demo/work"}],"warnings":[]} [hash=0a735291ec9e0ec7]
|
||||
-- request 3 (request) --
|
||||
09:function_call/skills.read:{"package":"c0/work","source":{"kind":"cloud"}}
|
||||
10:function_call_output:{"contents":"# Work\\nInspect the inputs before explaining the result.\\n","installations":[{"package":"sk...cursor":null,"resource":"skill://demo/work/SKILL.md"} [hash=adc0897fa3bc41cf]
|
||||
10:function_call/skills.read:{"package":"c0/work","source":{"kind":"cloud"}}
|
||||
11:function_call_output:{"contents":"# Work\\nInspect the inputs before explaining the result.\\n","installations":[{"package":"sk...cursor":null,"resource":"skill://demo/work/SKILL.md"} [hash=adc0897fa3bc41cf]
|
||||
|
||||
+3
-3
@@ -1,6 +1,5 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_mcp_ema.rs
|
||||
assertion_line: 125
|
||||
expression: "histories[1]"
|
||||
---
|
||||
Scenario: Ordinary MCP keeps the same tool schemas, calls, and history when enterprise authentication fails.
|
||||
@@ -55,8 +54,9 @@ Settings:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
05:message/user:
|
||||
List my calendar events.
|
||||
06:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
06:function_call/mcp__ordinary.calendar_list_events:{}
|
||||
07:function_call_output:Wall time: <DURATION> seconds
|
||||
07:function_call/mcp__ordinary.calendar_list_events:{}
|
||||
08:function_call_output:Wall time: <DURATION> seconds
|
||||
Output:
|
||||
{"_codex_apps":null}
|
||||
|
||||
+10
-9
@@ -1,6 +1,5 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_mcp_ema.rs
|
||||
assertion_line: 257
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Enterprise access disappears after revocation. Prior calls remain in history and ordinary tools still work.\",\n&requests,\n&ContextSnapshotOptions::default().rewrite_known_segments().include_request_settings(),)"
|
||||
---
|
||||
Scenario: Enterprise access disappears after revocation. Prior calls remain in history and ordinary tools still work.
|
||||
@@ -59,9 +58,10 @@ Settings:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
05:message/user:
|
||||
List my work events.
|
||||
06:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
06:function_call/mcp__enterprise.calendar_list_events:{}
|
||||
07:function_call_output:Wall time: <DURATION> seconds
|
||||
07:function_call/mcp__enterprise.calendar_list_events:{}
|
||||
08:function_call_output:Wall time: <DURATION> seconds
|
||||
Output:
|
||||
{"_codex_apps":null}
|
||||
|
||||
@@ -83,16 +83,17 @@ Settings: relative to window 1
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
05:message/user:
|
||||
List my work events.
|
||||
06:function_call/mcp__enterprise.calendar_list_events:{}
|
||||
07:function_call_output:Wall time: <DURATION> seconds
|
||||
06:configuration_update:reasoning={"effort":"low"}
|
||||
07:function_call/mcp__enterprise.calendar_list_events:{}
|
||||
08:function_call_output:Wall time: <DURATION> seconds
|
||||
Output:
|
||||
{"_codex_apps":null}
|
||||
08:message/assistant:
|
||||
09:message/assistant:
|
||||
Here are your work events.
|
||||
09:message/user:
|
||||
10:message/user:
|
||||
List my other calendar events.
|
||||
-- request 4 (turn) --
|
||||
10:function_call/mcp__ordinary.calendar_list_events:{}
|
||||
11:function_call_output:Wall time: <DURATION> seconds
|
||||
11:function_call/mcp__ordinary.calendar_list_events:{}
|
||||
12:function_call_output:Wall time: <DURATION> seconds
|
||||
Output:
|
||||
{"_codex_apps":null}
|
||||
|
||||
+3
-2
@@ -100,11 +100,12 @@ Settings:
|
||||
</environment_context>
|
||||
05:message/user:
|
||||
List the available resources and templates, then read the example note.
|
||||
06:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
06:custom_tool_call/exec:text(await tools.list_mcp_resources({"server":"resources"}));
|
||||
07:custom_tool_call/exec:text(await tools.list_mcp_resources({"server":"resources"}));
|
||||
text(await tools.list_mcp_resource_templates({"server":"resources"}));
|
||||
text(await tools.read_mcp_resource({"server":"resources","uri":"memo://codex/example-note"}));
|
||||
07:custom_tool_call_output:Script completed
|
||||
08:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| {"server":"resources","resources":[{"server":"resources","uri":"memo://codex/example-note","name":"examp...ed for integration tests.","mimeType":"text/plain"}]} [hash=75d187638a7af2d4] | {"server":"resources","resourceTemplates":[{"server":"resources","uriTemplate":"memo://codex/{slug}","na... resources used in tests.","mimeType":"text/plain"}]} [hash=b1c7082eb114fc9b] | {"server":"resources","uri":"memo://codex/example-note","contents":[{"uri":"memo://codex/example-note","...mple MCP resource served by the rmcp test server."}]} [hash=45385fc57640b555]
|
||||
|
||||
+4
-3
@@ -1,6 +1,6 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios.rs
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Astra calls list_agents using the selected catalog parameter schema.\",\n&mock.requests(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
expression: "context_snapshot::format_request_history_snapshot(description,\n&mock.requests(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
---
|
||||
Scenario: Astra calls list_agents using the selected catalog parameter schema.
|
||||
|
||||
@@ -100,6 +100,7 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Check which agents are working under /root before delegating more work.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:function_call/collaboration.list_agents:{"path_prefix":"/root"}
|
||||
08:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
|
||||
08:function_call/collaboration.list_agents:{"path_prefix":"/root"}
|
||||
09:function_call_output:{"agents":[{"agent_name":"/root","agent_status":"running"}]}
|
||||
|
||||
+3
-3
@@ -1,6 +1,5 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios.rs
|
||||
assertion_line: 1272
|
||||
expression: "context_snapshot::format_request_history_snapshot(description,\n&mock.requests(),\n&ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
---
|
||||
Scenario: Native tools are unavailable while the V2 runtime and its instructions remain enabled.
|
||||
@@ -94,6 +93,7 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Check which agents are working under /root before delegating more work.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:function_call/collaboration.list_agents:{"path_prefix":"/root"}
|
||||
08:function_call_output:unsupported call: collaborationlist_agents
|
||||
08:function_call/collaboration.list_agents:{"path_prefix":"/root"}
|
||||
09:function_call_output:unsupported call: collaborationlist_agents
|
||||
|
||||
+6
-5
@@ -109,13 +109,14 @@ Settings:
|
||||
</environment_context>
|
||||
07:message/user:
|
||||
Prepare the release.
|
||||
08:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
08:message/assistant:
|
||||
09:message/assistant:
|
||||
The release is ready for validation.
|
||||
09:message/developer:
|
||||
10:message/developer:
|
||||
Your context window is nearly exhausted (only 0 tokens remaining) and will be automatically reset for yo... and history items will be persistent across windows. [hash=23016acc05795bc3]
|
||||
10:message/user:
|
||||
11:message/user:
|
||||
Suggest my next message.
|
||||
-- request 3 (turn) --
|
||||
11:function_call/functions.get_context_remaining:{}
|
||||
12:function_call_output:You have 0 tokens left in this context window.
|
||||
12:function_call/functions.get_context_remaining:{}
|
||||
13:function_call_output:You have 0 tokens left in this context window.
|
||||
|
||||
+15
-12
@@ -1,6 +1,6 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_retained_tool_outputs.rs
|
||||
expression: "context_snapshot::format_request_history_snapshot(if enabled\n{\n \"Retained thread instructions stop emergency tool-output trimming. Compaction shares the 64K retention budget with oversized task details, and a cold resume preserves the retained tool content.\"\n} else\n{\n \"With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.\"\n}, &requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
|
||||
expression: "context_snapshot::format_request_history_snapshot(if enabled\n{\n \"Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.\"\n} else\n{\n \"With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.\"\n}, &requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
|
||||
---
|
||||
Scenario: With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.
|
||||
|
||||
@@ -33,16 +33,17 @@ Scenario: With retention disabled, the client can still send retain=true. Tool o
|
||||
05:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:function_call_output/codex_app.send_message_to_thread:Investigate the failing tests; report your findings without changing files.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:message/assistant:
|
||||
08:message/assistant:
|
||||
I will investigate the tests.
|
||||
08:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
|
||||
09:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
|
||||
-- request 3 (turn) --
|
||||
09:message/assistant:
|
||||
10:message/assistant:
|
||||
I will use those task details.
|
||||
10:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
|
||||
11:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
|
||||
|
||||
## Window 2 (after request 3: input diverged at item 10)
|
||||
## Window 2 (after request 3: input diverged at item 11)
|
||||
-- request 4 (compaction) --
|
||||
00:additional_tools/developer (3; hash=bf2e30605ff5dc17):
|
||||
- namespace/functions; hash=fada4a476a3754dd
|
||||
@@ -71,14 +72,15 @@ Scenario: With retention disabled, the client can still send retain=true. Tool o
|
||||
05:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:function_call_output/codex_app.send_message_to_thread:Investigate the failing tests; report your findings without changing files.
|
||||
07:message/assistant:
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
08:message/assistant:
|
||||
I will investigate the tests.
|
||||
08:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
|
||||
09:message/assistant:
|
||||
09:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
|
||||
10:message/assistant:
|
||||
I will use those task details.
|
||||
10:function_call_output/codex_app.send_message_to_thread:Output exceeded the available model context and was truncated
|
||||
11:function_call_output/example.progress:Output exceeded the available model context and was truncated
|
||||
12:compaction_trigger
|
||||
11:function_call_output/codex_app.send_message_to_thread:Output exceeded the available model context and was truncated
|
||||
12:function_call_output/example.progress:Output exceeded the available model context and was truncated
|
||||
13:compaction_trigger
|
||||
|
||||
## Window 3 (after request 4: input diverged at item 06)
|
||||
-- request 5 (turn) --
|
||||
@@ -113,3 +115,4 @@ Scenario: With retention disabled, the client can still send retain=true. Tool o
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
08:message/user:
|
||||
What request are you working on?
|
||||
09:configuration_update:reasoning={"effort":"low"}
|
||||
|
||||
+9
-7
@@ -1,6 +1,6 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_retained_tool_outputs.rs
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.\",\n&requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
|
||||
expression: "context_snapshot::format_request_history_snapshot(if enabled\n{\n \"Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.\"\n} else\n{\n \"With retention disabled, the client can still send retain=true. Tool outputs use ordinary emergency trimming and are not retained after compaction and cold resume.\"\n}, &requests, &ContextSnapshotOptions::default().rewrite_known_segments(),)"
|
||||
---
|
||||
Scenario: Retained thread instructions stop emergency tool-output trimming. Compaction keeps whole outputs within the shared 64K retention budget, dropping oversized task details and older instructions. A cold resume preserves the latest instruction unchanged.
|
||||
|
||||
@@ -33,17 +33,18 @@ Scenario: Retained thread instructions stop emergency tool-output trimming. Comp
|
||||
05:message/user:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:function_call_output/codex_app.send_message_to_thread:Investigate the failing tests; report your findings without changing files.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:message/assistant:
|
||||
08:message/assistant:
|
||||
I will investigate the tests.
|
||||
08:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
|
||||
09:function_call_output/codex_app.task_details:Investigate the failing tests carefully. Investigate the failing tests carefully. Investigate the failin...s carefully. Investigate the failing tests carefully. [hash=3c65017b6f98a38f]
|
||||
-- request 3 (turn) --
|
||||
09:message/assistant:
|
||||
10:message/assistant:
|
||||
I will use those task details.
|
||||
10:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
|
||||
11:function_call_output/codex_app.send_message_to_thread:Report your findings without changing files.
|
||||
-- request 4 (compaction) --
|
||||
11:function_call_output/example.progress:Output exceeded the available model context and was truncated
|
||||
12:compaction_trigger
|
||||
12:function_call_output/example.progress:Output exceeded the available model context and was truncated
|
||||
13:compaction_trigger
|
||||
|
||||
## Window 2 (after request 4: input diverged at item 06)
|
||||
-- request 5 (turn) --
|
||||
@@ -79,3 +80,4 @@ Scenario: Retained thread instructions stop emergency tool-output trimming. Comp
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
09:message/user:
|
||||
What request are you working on?
|
||||
10:configuration_update:reasoning={"effort":"low"}
|
||||
|
||||
+3
-2
@@ -117,8 +117,9 @@ Settings:
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Which skills are available?
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn) --
|
||||
07:message/assistant:
|
||||
08:message/assistant:
|
||||
Skills are available.
|
||||
08:message/user:
|
||||
09:message/user:
|
||||
Are those same skills still available?
|
||||
|
||||
+9
-9
@@ -1,6 +1,5 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios.rs
|
||||
assertion_line: 1456
|
||||
expression: "context_snapshot::format_context_snapshot(\"Astra reads Cloud instructions, discovers an omitted deployment installation, and reads its instructions from env-04.\",\n&bodies.iter().map(SnapshotEntry::body).collect::<Vec<_>>(),\n&ContextSnapshotOptions::default().rewrite_known_segments(),)"
|
||||
---
|
||||
Scenario: Astra reads Cloud instructions, discovers an omitted deployment installation, and reads its instructions from env-04.
|
||||
@@ -36,9 +35,10 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
Read the review and deployment instructions, find all deployment installations, then read the instructions from env-04.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (request) --
|
||||
07:function_call/skills.read:{"package":"skill://executor-00/review","source":{"kind":"cloud"}}
|
||||
08:function_call_output:{
|
||||
08:function_call/skills.read:{"package":"skill://executor-00/review","source":{"kind":"cloud"}}
|
||||
09:function_call_output:{
|
||||
"contents": "# Deployment\\nRun scripts/check.py from an available installation.",
|
||||
"installations": [
|
||||
{
|
||||
@@ -58,8 +58,8 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
|
||||
"resource": "skill://deployment/review/SKILL.md"
|
||||
}
|
||||
-- request 3 (request) --
|
||||
09:function_call/skills.read:{"package":"skill://deployment/deploy"}
|
||||
10:function_call_output:{
|
||||
10:function_call/skills.read:{"package":"skill://deployment/deploy"}
|
||||
11:function_call_output:{
|
||||
"contents": "# Deployment\\nRun scripts/check.py from an available installation.",
|
||||
"installations": [
|
||||
{
|
||||
@@ -77,8 +77,8 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
|
||||
"resource": "skill://deployment/deploy/SKILL.md"
|
||||
}
|
||||
-- request 4 (request) --
|
||||
11:function_call/skills.installations:{"package":"skill://deployment/deploy"}
|
||||
12:function_call_output:{
|
||||
12:function_call/skills.installations:{"package":"skill://deployment/deploy"}
|
||||
13:function_call_output:{
|
||||
"installations": [
|
||||
{
|
||||
"package": "skill://deployment/deploy",
|
||||
@@ -96,8 +96,8 @@ Scenario: Astra reads Cloud instructions, discovers an omitted deployment instal
|
||||
"next_cursor": null
|
||||
}
|
||||
-- request 5 (request) --
|
||||
13:function_call/skills.read:{"package":"skill://deployment/deploy","source":{"environment_id":"env-04","kind":"executor"}}
|
||||
14:function_call_output:{
|
||||
14:function_call/skills.read:{"package":"skill://deployment/deploy","source":{"environment_id":"env-04","kind":"executor"}}
|
||||
15:function_call_output:{
|
||||
"contents": "# Deployment in env-04\\nRun scripts/check.py from this installation.",
|
||||
"installations": [
|
||||
{
|
||||
|
||||
+12
-11
@@ -44,60 +44,61 @@ Settings:
|
||||
<ENVIRONMENT_CONTEXT>
|
||||
06:message/user:
|
||||
List and call the tools provided by the environment MCP server.
|
||||
07:configuration_update:reasoning={"effort":"low"}
|
||||
-- request 2 (turn; turn 1: exec discovery and call results) --
|
||||
07:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
08:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
const results = [];
|
||||
for (const t of available) {
|
||||
const result = await tools[t.name]({});
|
||||
results.push({name: t.name, result: result.content[0].text});
|
||||
}
|
||||
text(JSON.stringify(results));
|
||||
08:custom_tool_call_output:Script completed
|
||||
09:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| []
|
||||
-- request 3 (turn; turn 2: register MCP with lookup; omit-all, direct-only and excluded) --
|
||||
09:message/user:
|
||||
10:message/user:
|
||||
List and call the tools provided by the environment MCP server.
|
||||
-- request 4 (turn; turn 2: exec discovery and call results) --
|
||||
10:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
11:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
const results = [];
|
||||
for (const t of available) {
|
||||
const result = await tools[t.name]({});
|
||||
results.push({name: t.name, result: result.content[0].text});
|
||||
}
|
||||
text(JSON.stringify(results));
|
||||
11:custom_tool_call_output:Script completed
|
||||
12:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| [{"name":"mcp__environment__lookup","result":"called lookup for at with "}]
|
||||
-- request 5 (turn; turn 3: same MCP name, new endpoint; replace lookup with resolve and search) --
|
||||
12:message/user:
|
||||
13:message/user:
|
||||
List and call the tools provided by the environment MCP server.
|
||||
-- request 6 (turn; turn 3: exec discovery and call results) --
|
||||
13:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
14:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
const results = [];
|
||||
for (const t of available) {
|
||||
const result = await tools[t.name]({});
|
||||
results.push({name: t.name, result: result.content[0].text});
|
||||
}
|
||||
text(JSON.stringify(results));
|
||||
14:custom_tool_call_output:Script completed
|
||||
15:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| [{"name":"mcp__environment__resolve","result":"called resolve for at with "},{"name":"mcp__environment__search","result":"called search for at with "}]
|
||||
-- request 7 (turn; turn 4: remove MCP server from config) --
|
||||
15:message/user:
|
||||
16:message/user:
|
||||
List and call the tools provided by the environment MCP server.
|
||||
-- request 8 (turn; turn 4: exec discovery and call results) --
|
||||
16:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
17:custom_tool_call/exec:const available = ALL_TOOLS.filter(t => t.name.startsWith("mcp__environment__")).sort((a, b) => a.name.localeCompare(b.name));
|
||||
const results = [];
|
||||
for (const t of available) {
|
||||
const result = await tools[t.name]({});
|
||||
results.push({name: t.name, result: result.content[0].text});
|
||||
}
|
||||
text(JSON.stringify(results));
|
||||
17:custom_tool_call_output:Script completed
|
||||
18:custom_tool_call_output:Script completed
|
||||
Wall time <DURATION> seconds
|
||||
Output:
|
||||
| []
|
||||
|
||||
@@ -1997,8 +1997,8 @@ pub const FEATURES: &[FeatureSpec] = &[
|
||||
FeatureSpec {
|
||||
id: Feature::ReasoningEffortOverride,
|
||||
key: "reasoning_effort_override",
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
stage: Stage::Stable,
|
||||
default_enabled: true,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::CurrentTimeReminder,
|
||||
|
||||
Reference in New Issue
Block a user