tests: annotate snapshot context items with user anchors

This commit is contained in:
Charles Cunningham
2026-02-11 19:02:03 -08:00
parent 277dd15df3
commit 0fa584baac
12 changed files with 188 additions and 63 deletions
+68 -5
View File
@@ -245,10 +245,72 @@ fn message_text_for_shape(item: &Value) -> String {
.unwrap_or_else(|| "<NO_TEXT>".to_string())
}
fn is_non_real_user_text(text: &str) -> bool {
text == "<AGENTS_MD>"
|| text == "<PERMISSIONS_INSTRUCTIONS>"
|| text == "<SUMMARIZATION_PROMPT>"
|| text.starts_with("<ENVIRONMENT_CONTEXT")
|| text.starts_with("<SUMMARY:")
}
fn is_shape_context_item_text(text: &str) -> bool {
text == "<AGENTS_MD>"
|| text == "<PERMISSIONS_INSTRUCTIONS>"
|| text.starts_with("<ENVIRONMENT_CONTEXT")
}
fn user_message_label_for_shape(item: &Value) -> Option<String> {
if item.get("type").and_then(Value::as_str) != Some("message") {
return None;
}
if item.get("role").and_then(Value::as_str) != Some("user") {
return None;
}
item.get("content")
.and_then(Value::as_array)
.and_then(|content| {
content
.iter()
.filter_map(|entry| entry.get("text").and_then(Value::as_str))
.map(normalize_shape_text)
.find(|text| !is_non_real_user_text(text))
})
}
fn previous_user_label_for_shape(items: &[Value], idx: usize) -> Option<String> {
items[..idx]
.iter()
.rev()
.find_map(user_message_label_for_shape)
}
fn next_user_label_for_shape(items: &[Value], idx: usize) -> Option<String> {
items
.iter()
.skip(idx.saturating_add(1))
.find_map(user_message_label_for_shape)
}
fn context_anchor_suffix(items: &[Value], idx: usize, role: &str, text: &str) -> Option<String> {
let is_context_message = if role == "developer" {
true
} else {
role == "user" && is_shape_context_item_text(text)
};
if !is_context_message {
return None;
}
let prev = previous_user_label_for_shape(items, idx).unwrap_or_else(|| "<none>".to_string());
let next = next_user_label_for_shape(items, idx).unwrap_or_else(|| "<none>".to_string());
Some(format!(" [prev_user={prev};next_user={next}]"))
}
fn request_input_shape(request: &ResponsesRequest) -> String {
request
.input()
.into_iter()
let input = request.input();
input
.iter()
.enumerate()
.map(|(idx, item)| {
let Some(item_type) = item.get("type").and_then(Value::as_str) else {
@@ -257,8 +319,9 @@ fn request_input_shape(request: &ResponsesRequest) -> String {
match item_type {
"message" => {
let role = item.get("role").and_then(Value::as_str).unwrap_or("unknown");
let text = message_text_for_shape(&item);
format!("{idx:02}:message/{role}:{text}")
let text = message_text_for_shape(item);
let suffix = context_anchor_suffix(&input, idx, role, &text).unwrap_or_default();
format!("{idx:02}:message/{role}:{text}{suffix}")
}
"function_call" => {
let name = item.get("name").and_then(Value::as_str).unwrap_or("unknown");
+67 -5
View File
@@ -113,10 +113,71 @@ fn message_text_for_shape(item: &Value) -> String {
.unwrap_or_else(|| "<NO_TEXT>".to_string())
}
fn is_non_real_user_text(text: &str) -> bool {
text == "<AGENTS_MD>"
|| text == "<PERMISSIONS_INSTRUCTIONS>"
|| text.starts_with("<ENVIRONMENT_CONTEXT")
|| text.starts_with("<SUMMARY:")
}
fn is_shape_context_item_text(text: &str) -> bool {
text == "<AGENTS_MD>"
|| text == "<PERMISSIONS_INSTRUCTIONS>"
|| text.starts_with("<ENVIRONMENT_CONTEXT")
}
fn user_message_label_for_shape(item: &Value) -> Option<String> {
if item.get("type").and_then(Value::as_str) != Some("message") {
return None;
}
if item.get("role").and_then(Value::as_str) != Some("user") {
return None;
}
item.get("content")
.and_then(Value::as_array)
.and_then(|content| {
content
.iter()
.filter_map(|entry| entry.get("text").and_then(Value::as_str))
.map(normalize_shape_text)
.find(|text| !is_non_real_user_text(text))
})
}
fn previous_user_label_for_shape(items: &[Value], idx: usize) -> Option<String> {
items[..idx]
.iter()
.rev()
.find_map(user_message_label_for_shape)
}
fn next_user_label_for_shape(items: &[Value], idx: usize) -> Option<String> {
items
.iter()
.skip(idx.saturating_add(1))
.find_map(user_message_label_for_shape)
}
fn context_anchor_suffix(items: &[Value], idx: usize, role: &str, text: &str) -> Option<String> {
let is_context_message = if role == "developer" {
true
} else {
role == "user" && is_shape_context_item_text(text)
};
if !is_context_message {
return None;
}
let prev = previous_user_label_for_shape(items, idx).unwrap_or_else(|| "<none>".to_string());
let next = next_user_label_for_shape(items, idx).unwrap_or_else(|| "<none>".to_string());
Some(format!(" [prev_user={prev};next_user={next}]"))
}
fn request_input_shape(request: &responses::ResponsesRequest) -> String {
request
.input()
.into_iter()
let input = request.input();
input
.iter()
.enumerate()
.map(|(idx, item)| {
let Some(item_type) = item.get("type").and_then(Value::as_str) else {
@@ -125,8 +186,9 @@ fn request_input_shape(request: &responses::ResponsesRequest) -> String {
match item_type {
"message" => {
let role = item.get("role").and_then(Value::as_str).unwrap_or("unknown");
let text = message_text_for_shape(&item);
format!("{idx:02}:message/{role}:{text}")
let text = message_text_for_shape(item);
let suffix = context_anchor_suffix(&input, idx, role, &text).unwrap_or_default();
format!("{idx:02}:message/{role}:{text}{suffix}")
}
"function_call" => {
let name = item.get("name").and_then(Value::as_str).unwrap_or("unknown");
@@ -5,17 +5,17 @@ expression: "sectioned_request_shapes(\"Manual /compact with prior user history
Scenario: Manual /compact with prior user history compacts existing history and the follow-up turn includes the compact summary plus new user message.
## Local Compaction Request
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/assistant:FIRST_REPLY
05:message/user:<SUMMARIZATION_PROMPT>
## Local Post-Compaction History Layout
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/user:<SUMMARY:MANUAL_SUMMARY>
05:message/user:USER_TWO
@@ -5,7 +5,7 @@ expression: "sectioned_request_shapes(\"Manual /compact with no prior user turn
Scenario: Manual /compact with no prior user turn behaves as a no-op and follow-up turn carries only canonical initial context plus the new user message.
## Local Post-Compaction History Layout
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=<none>]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=<none>]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=<none>]
03:message/user:<SUMMARIZATION_PROMPT>
@@ -5,17 +5,17 @@ expression: "sectioned_request_shapes(\"Mid-turn continuation compaction after t
Scenario: Mid-turn continuation compaction after tool output: compact request includes tool artifacts and follow-up request includes the summary.
## Local Compaction Request
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:function_call/<TOOL_CALL>
05:function_call_output:<TOOL_ERROR_OUTPUT>
06:message/user:<SUMMARIZATION_PROMPT>
## Local Post-Compaction History Layout
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/user:<SUMMARY:MID_TURN_SUMMARY>
@@ -5,9 +5,9 @@ expression: "sectioned_request_shapes(\"Pre-turn auto-compaction context-window
Scenario: Pre-turn auto-compaction context-window failure on main: compaction request excludes the incoming user message and the turn errors.
## Local Compaction Request (Incoming User Excluded on main)
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/assistant:FIRST_REPLY
05:message/user:<SUMMARIZATION_PROMPT>
@@ -5,20 +5,20 @@ expression: "sectioned_request_shapes(\"Pre-turn auto-compaction with a context
Scenario: Pre-turn auto-compaction with a context override emits the context diff in the compact request while the incoming user message is still excluded on main.
## Local Compaction Request
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/assistant:FIRST_REPLY
05:message/user:USER_TWO
06:message/assistant:SECOND_REPLY
07:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD>
07:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD> [prev_user=USER_TWO;next_user=<none>]
08:message/user:<SUMMARIZATION_PROMPT>
## Local Post-Compaction History Layout
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/user:USER_TWO
05:message/user:<SUMMARY:PRE_TURN_SUMMARY>
@@ -5,16 +5,16 @@ expression: "sectioned_request_shapes(\"Remote manual /compact with prior user h
Scenario: Remote manual /compact with prior user history compacts existing history and follow-up includes compact summary plus new user message.
## Remote Compaction Request
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/assistant:REMOTE_MANUAL_FIRST_REPLY
## Remote Post-Compaction History Layout
00:message/user:USER_ONE
01:message/developer:<PERMISSIONS_INSTRUCTIONS>
02:message/user:<AGENTS_MD>
03:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
01:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=USER_ONE;next_user=USER_TWO]
02:message/user:<AGENTS_MD> [prev_user=USER_ONE;next_user=USER_TWO]
03:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=USER_ONE;next_user=USER_TWO]
04:message/user:<SUMMARY:REMOTE_MANUAL_WITH_HISTORY_SUMMARY>
05:message/user:USER_TWO
@@ -5,12 +5,12 @@ expression: "sectioned_request_shapes(\"Remote manual /compact with no prior use
Scenario: Remote manual /compact with no prior user turn still issues a compact request on main; follow-up turn carries canonical context and new user message.
## Remote Compaction Request
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=<none>]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=<none>]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=<none>]
## Remote Post-Compaction History Layout
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
@@ -5,16 +5,16 @@ expression: "sectioned_request_shapes(\"Remote mid-turn continuation compaction
Scenario: Remote mid-turn continuation compaction after tool output: compact request includes tool artifacts and follow-up request includes the summary.
## Remote Compaction Request
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:function_call/<TOOL_CALL>
05:function_call_output:<TOOL_ERROR_OUTPUT>
## Remote Post-Compaction History Layout
00:message/user:USER_ONE
01:message/developer:<PERMISSIONS_INSTRUCTIONS>
02:message/user:<AGENTS_MD>
03:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
01:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=USER_ONE;next_user=<none>]
02:message/user:<AGENTS_MD> [prev_user=USER_ONE;next_user=<none>]
03:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=USER_ONE;next_user=<none>]
04:message/user:<SUMMARY:REMOTE_MID_TURN_SUMMARY>
@@ -5,8 +5,8 @@ expression: "sectioned_request_shapes(\"Remote pre-turn auto-compaction parse fa
Scenario: Remote pre-turn auto-compaction parse failure on main: compaction request excludes the incoming user message and the turn stops.
## Remote Compaction Request (Incoming User Excluded on main)
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/assistant:REMOTE_FIRST_REPLY
@@ -5,21 +5,21 @@ expression: "sectioned_request_shapes(\"Remote pre-turn auto-compaction with a c
Scenario: Remote pre-turn auto-compaction with a context override emits the context diff in the compact request while excluding the incoming user message on main.
## Remote Compaction Request
00:message/developer:<PERMISSIONS_INSTRUCTIONS>
01:message/user:<AGENTS_MD>
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>>
00:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=<none>;next_user=USER_ONE]
01:message/user:<AGENTS_MD> [prev_user=<none>;next_user=USER_ONE]
02:message/user:<ENVIRONMENT_CONTEXT:cwd=<CWD>> [prev_user=<none>;next_user=USER_ONE]
03:message/user:USER_ONE
04:message/assistant:REMOTE_FIRST_REPLY
05:message/user:USER_TWO
06:message/assistant:REMOTE_SECOND_REPLY
07:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD>
07:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD> [prev_user=USER_TWO;next_user=<none>]
## Remote Post-Compaction History Layout
00:message/user:USER_ONE
01:message/user:USER_TWO
02:message/user:USER_THREE
03:message/developer:<PERMISSIONS_INSTRUCTIONS>
04:message/user:<AGENTS_MD>
05:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD>
03:message/developer:<PERMISSIONS_INSTRUCTIONS> [prev_user=USER_THREE;next_user=USER_THREE]
04:message/user:<AGENTS_MD> [prev_user=USER_THREE;next_user=USER_THREE]
05:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD> [prev_user=USER_THREE;next_user=USER_THREE]
06:message/user:<SUMMARY:REMOTE_PRE_TURN_SUMMARY>
07:message/user:USER_THREE