From c24280b26737542399ffd65e021a32f01442c188 Mon Sep 17 00:00:00 2001 From: Bo Xie Date: Thu, 9 Jul 2026 18:17:11 -0700 Subject: [PATCH] core: pad context window blocks with newlines --- .../core/src/context/token_budget_context.rs | 12 ++++++++++ ..._new_context_window_tool_full_context.snap | 4 ++-- codex-rs/core/tests/suite/token_budget.rs | 24 +++++++++++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/codex-rs/core/src/context/token_budget_context.rs b/codex-rs/core/src/context/token_budget_context.rs index f3eef0b734..e8cabf107e 100644 --- a/codex-rs/core/src/context/token_budget_context.rs +++ b/codex-rs/core/src/context/token_budget_context.rs @@ -63,6 +63,11 @@ impl ContextualUserFragment for TokenBudgetContext { } format!("\n{}\n", lines.join("\n")) } + + fn render(&self) -> String { + let body = self.body(); + format!("\n\n{CONTEXT_WINDOW_OPEN_TAG}{body}{CONTEXT_WINDOW_CLOSE_TAG}\n\n") + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -97,6 +102,13 @@ impl ContextualUserFragment for ContextWindowGuidance { fn body(&self) -> String { format!("\n{}\n", self.message) } + + fn render(&self) -> String { + let body = self.body(); + format!( + "\n\n{CONTEXT_WINDOW_GUIDANCE_OPEN_TAG}{body}{CONTEXT_WINDOW_GUIDANCE_CLOSE_TAG}\n\n" + ) + } } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap index 49687d347b..75a2c826ab 100644 --- a/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap +++ b/codex-rs/core/tests/suite/snapshots/all__suite__token_budget__token_budget_new_context_window_tool_full_context.snap @@ -1,6 +1,6 @@ --- source: core/tests/suite/token_budget.rs -assertion_line: 545 +assertion_line: 1239 expression: snapshot --- Scenario: New context window tool installs fresh full context before the next follow-up request. @@ -9,7 +9,7 @@ Scenario: New context window tool installs fresh full context before the next fo 00:message/developer[3]: [01] [02] - [03] \nThread id: \nFirst context window id: \nCurrent context window id: \nPrevious context window id: \n + [03] \n\n\nThread id: \nFirst context window id: \nCurrent context window id: \nPrevious context window id: \n\n\n 01:message/user:> 02:function_call/update_plan 03:function_call_output:Plan updated diff --git a/codex-rs/core/tests/suite/token_budget.rs b/codex-rs/core/tests/suite/token_budget.rs index 3f55dd70be..7f5911130e 100644 --- a/codex-rs/core/tests/suite/token_budget.rs +++ b/codex-rs/core/tests/suite/token_budget.rs @@ -148,7 +148,7 @@ fn token_budget_contexts(request: &ResponsesRequest) -> Vec { request .message_input_texts("developer") .into_iter() - .filter(|text| text.starts_with(&context_window_prefix)) + .filter(|text| text.trim_start().starts_with(&context_window_prefix)) .collect() } @@ -156,6 +156,7 @@ fn token_budget_window_ids( text: &str, thread_id: codex_protocol::ThreadId, ) -> (String, Option, String) { + let text = text.trim(); let captures = assert_regex_match( &format!( r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})(?:\nPrevious context window id: ([0-9a-f-]{{36}}))?\n{CONTEXT_WINDOW_CLOSE_TAG}$" @@ -343,12 +344,27 @@ async fn token_budget_guidance_follows_context_window() -> Result<()> { let developer_texts = response.single_request().message_input_texts("developer"); let context_window_index = developer_texts .iter() - .position(|text| text.starts_with(CONTEXT_WINDOW_OPEN_TAG)) + .position(|text| text.trim_start().starts_with(CONTEXT_WINDOW_OPEN_TAG)) .expect("context-window metadata should be present"); + let developer_text = developer_texts.concat(); + assert!( + developer_text.contains(&format!("\n\n{CONTEXT_WINDOW_OPEN_TAG}")), + "context-window metadata should follow a blank line" + ); + assert!( + developer_text.contains(&format!( + "{CONTEXT_WINDOW_CLOSE_TAG}\n\n\n\n{CONTEXT_WINDOW_GUIDANCE_OPEN_TAG}" + )), + "context-window guidance should be padded from context-window metadata" + ); + assert!( + developer_text.contains(&format!("{CONTEXT_WINDOW_GUIDANCE_CLOSE_TAG}\n\n")), + "context-window guidance should end with a blank line" + ); assert_eq!( developer_texts.get(context_window_index + 1), Some(&format!( - "{CONTEXT_WINDOW_GUIDANCE_OPEN_TAG}\n{guidance_message}\n{CONTEXT_WINDOW_GUIDANCE_CLOSE_TAG}" + "\n\n{CONTEXT_WINDOW_GUIDANCE_OPEN_TAG}\n{guidance_message}\n{CONTEXT_WINDOW_GUIDANCE_CLOSE_TAG}\n\n" )) ); @@ -423,7 +439,7 @@ async fn token_budget_context_injects_plain_thread_hint_text() -> Result<()> { &format!( r"^{CONTEXT_WINDOW_OPEN_TAG}\nThread id: {thread_id}\nFirst context window id: ([0-9a-f-]{{36}})\nCurrent context window id: ([0-9a-f-]{{36}})\nmanual history hint for thread {thread_id}\nunstructured notes/thread_hint fixture result\n{CONTEXT_WINDOW_CLOSE_TAG}$" ), - &token_budgets[0], + token_budgets[0].trim(), ); assert_eq!( captures.get(1).expect("first window id capture").as_str(),