openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-18 17:18:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add app-server notification media filtering (#41416)

Browse Source 
## What changed

- Add the disabled-by-default `omit_app_server_notification_media` feature.
- When enabled, remove inline image and audio content from `item/started`,
  `item/completed`, and `rawResponseItem/completed` notifications, including
  tool output and image-generation results.
- Keep the media in model input; the filter only changes app-server
  notifications.

## Testing

- Verify that image content is absent from all three notification types while
  the model request still receives it.

GitOrigin-RevId: 9a4434d18a3dac6d67c2e0c3723ff4595a9b3797
This commit is contained in:
Vivian Fang
2026-08-28 21:06:35 +00:00
committed by copyberry
parent 170da98842
commit 03147407e3
6 changed files with 337 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/app-server/src/bespoke_event_handling.rs
+21 -25
View File
@@ -1,5 +1,6 @@
use crate::error_code::internal_error;
use crate::error_code::invalid_request;
use crate::notification_media::without_notification_media;
use crate::outgoing_message::ClientRequestResult;
use crate::outgoing_message::ThreadScopedOutgoingMessageSender;
use crate::request_processors::populate_thread_turns_from_history;
@@ -91,6 +92,7 @@ use codex_app_server_protocol::guardian_auto_approval_review_notification;
use codex_app_server_protocol::item_event_to_server_notification;
use codex_core::CodexThread;
use codex_core::ThreadManager;
use codex_features::Feature;
use codex_guardian_v2::StrictReviewReason;
use codex_protocol::ThreadId;
use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool;
@@ -1068,11 +1070,14 @@ pub(crate) async fn apply_bespoke_event_handling(
_ => None,
};
if should_emit {
let notification = item_event_to_server_notification(
let mut notification = item_event_to_server_notification(
EventMsg::ItemStarted(event),
&conversation_id.to_string(),
&event_turn_id,
);
if conversation.enabled(Feature::OmitAppServerNotificationMedia) {
notification = without_notification_media(notification);
}
outgoing.send_server_notification(notification).await;
}
if let Some(params) = dynamic_tool_call_params {
@@ -1093,11 +1098,14 @@ pub(crate) async fn apply_bespoke_event_handling(
&event.item,
)
.await;
let notification = item_event_to_server_notification(
let mut notification = item_event_to_server_notification(
EventMsg::ItemCompleted(event),
&conversation_id.to_string(),
&event_turn_id,
);
if conversation.enabled(Feature::OmitAppServerNotificationMedia) {
notification = without_notification_media(notification);
}
outgoing.send_server_notification(notification).await;
}
msg @ (EventMsg::PatchApplyUpdated(_) | EventMsg::TerminalInteraction(_)) => {
@@ -1129,13 +1137,17 @@ pub(crate) async fn apply_bespoke_event_handling(
.await;
}
EventMsg::RawResponseItem(raw_response_item_event) => {
maybe_emit_raw_response_item_completed(
conversation_id,
&event_turn_id,
raw_response_item_event.item,
&outgoing,
)
.await;
let mut notification = ServerNotification::RawResponseItemCompleted(
RawResponseItemCompletedNotification {
thread_id: conversation_id.to_string(),
turn_id: event_turn_id,
item: raw_response_item_event.item,
},
);
if conversation.enabled(Feature::OmitAppServerNotificationMedia) {
notification = without_notification_media(notification);
}
outgoing.send_server_notification(notification).await;
}
EventMsg::RawResponseCompleted(raw_response_completed_event) => {
let notification = RawResponseCompletedNotification {
@@ -1512,22 +1524,6 @@ async fn complete_command_execution_item(
.await;
}
async fn maybe_emit_raw_response_item_completed(
conversation_id: ThreadId,
turn_id: &str,
item: codex_protocol::models::ResponseItem,
outgoing: &ThreadScopedOutgoingMessageSender,
) {
let notification = RawResponseItemCompletedNotification {
thread_id: conversation_id.to_string(),
turn_id: turn_id.to_string(),
item,
};
outgoing
.send_server_notification(ServerNotification::RawResponseItemCompleted(notification))
.await;
}
async fn find_and_remove_turn_summary(
_conversation_id: ThreadId,
thread_state: &Arc<Mutex<ThreadState>>,
codex-rs/app-server/src/lib.rs
+1
View File
@@ -119,6 +119,7 @@ mod mcp_refresh;
mod message_processor;
mod models;
mod models_refresh_worker;
mod notification_media;
mod otel_reloader;
mod outgoing_message;
mod realtime_event_handling;
codex-rs/app-server/src/notification_media.rs
+196
View File
@@ -0,0 +1,196 @@
use codex_app_server_protocol::DynamicToolCallOutputContentItem;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::UserInput;
use codex_protocol::models::ContentItem;
use codex_protocol::models::FunctionCallOutputContentItem;
use codex_protocol::models::ResponseItem;
pub(crate) fn without_notification_media(notification: ServerNotification) -> ServerNotification {
match notification {
ServerNotification::ItemStarted(mut notification) => {
notification.item = without_thread_item_media(notification.item);
ServerNotification::ItemStarted(notification)
}
ServerNotification::ItemCompleted(mut notification) => {
notification.item = without_thread_item_media(notification.item);
ServerNotification::ItemCompleted(notification)
}
ServerNotification::RawResponseItemCompleted(mut notification) => {
match &mut notification.item {
ResponseItem::Message { content, .. } => {
content.retain(|item| match item {
ContentItem::InputImage { .. } | ContentItem::InputAudio { .. } => false,
ContentItem::InputText { .. } | ContentItem::OutputText { .. } => true,
});
}
ResponseItem::FunctionCallOutput { output, .. }
| ResponseItem::CustomToolCallOutput { output, .. } => {
if let Some(items) = output.content_items_mut() {
items.retain(|item| match item {
FunctionCallOutputContentItem::InputImage { .. }
| FunctionCallOutputContentItem::InputAudio { .. } => false,
FunctionCallOutputContentItem::InputText { .. }
| FunctionCallOutputContentItem::EncryptedContent { .. } => true,
});
}
}
ResponseItem::ImageGenerationCall { result, .. } => result.clear(),
ResponseItem::AdditionalTools { .. }
| ResponseItem::AgentMessage { .. }
| ResponseItem::Reasoning { .. }
| ResponseItem::LocalShellCall { .. }
| ResponseItem::FunctionCall { .. }
| ResponseItem::ToolSearchCall { .. }
| ResponseItem::CustomToolCall { .. }
| ResponseItem::ToolSearchOutput { .. }
| ResponseItem::WebSearchCall { .. }
| ResponseItem::Compaction { .. }
| ResponseItem::CompactionTrigger { .. }
| ResponseItem::ContextCompaction { .. }
| ResponseItem::Other => {}
}
ServerNotification::RawResponseItemCompleted(notification)
}
ServerNotification::Error(_)
| ServerNotification::ThreadStarted(_)
| ServerNotification::ThreadStatusChanged(_)
| ServerNotification::ThreadArchived(_)
| ServerNotification::ThreadDeleted(_)
| ServerNotification::ThreadUnarchived(_)
| ServerNotification::ThreadClosed(_)
| ServerNotification::ThreadReverted(_)
| ServerNotification::SkillsChanged(_)
| ServerNotification::ThreadNameUpdated(_)
| ServerNotification::ThreadGoalUpdated(_)
| ServerNotification::ThreadGoalCleared(_)
| ServerNotification::ThreadQueueChanged(_)
| ServerNotification::ProjectChanged(_)
| ServerNotification::ThreadProjectUpdated(_)
| ServerNotification::EnvironmentConnected(_)
| ServerNotification::EnvironmentDisconnected(_)
| ServerNotification::ThreadSettingsUpdated(_)
| ServerNotification::ThreadTokenUsageUpdated(_)
| ServerNotification::TurnStarted(_)
| ServerNotification::HookStarted(_)
| ServerNotification::TurnCompleted(_)
| ServerNotification::HookCompleted(_)
| ServerNotification::TurnDiffUpdated(_)
| ServerNotification::TurnPlanUpdated(_)
| ServerNotification::ItemGuardianApprovalReviewStarted(_)
| ServerNotification::ItemGuardianApprovalReviewCompleted(_)
| ServerNotification::StrictReviewRequired(_)
| ServerNotification::RawResponseCompleted(_)
| ServerNotification::AgentMessageDelta(_)
| ServerNotification::PlanDelta(_)
| ServerNotification::CommandExecOutputDelta(_)
| ServerNotification::ProcessOutputDelta(_)
| ServerNotification::ProcessExited(_)
| ServerNotification::CommandExecutionOutputDelta(_)
| ServerNotification::TerminalInteraction(_)
| ServerNotification::FileChangeOutputDelta(_)
| ServerNotification::FileChangePatchUpdated(_)
| ServerNotification::ServerRequestResolved(_)
| ServerNotification::McpToolCallProgress(_)
| ServerNotification::McpServerOauthLoginCompleted(_)
| ServerNotification::McpServerStatusUpdated(_)
| ServerNotification::McpServerEventStream(_)
| ServerNotification::AccountUpdated(_)
| ServerNotification::AccountRateLimitsUpdated(_)
| ServerNotification::AppListUpdated(_)
| ServerNotification::RemoteControlStatusChanged(_)
| ServerNotification::ExternalAgentConfigImportProgress(_)
| ServerNotification::ExternalAgentConfigImportCompleted(_)
| ServerNotification::FsChanged(_)
| ServerNotification::ReasoningSummaryTextDelta(_)
| ServerNotification::ReasoningSummaryPartAdded(_)
| ServerNotification::ReasoningTextDelta(_)
| ServerNotification::ContextCompacted(_)
| ServerNotification::ModelRerouted(_)
| ServerNotification::ModelVerification(_)
| ServerNotification::TurnModerationMetadata(_)
| ServerNotification::ModelSafetyBufferingUpdated(_)
| ServerNotification::Warning(_)
| ServerNotification::GuardianWarning(_)
| ServerNotification::DeprecationNotice(_)
| ServerNotification::ConfigWarning(_)
| ServerNotification::FuzzyFileSearchSessionUpdated(_)
| ServerNotification::FuzzyFileSearchSessionCompleted(_)
| ServerNotification::ThreadRealtimeStarted(_)
| ServerNotification::ThreadRealtimeItemAdded(_)
| ServerNotification::ThreadRealtimeItemStarted(_)
| ServerNotification::ThreadRealtimeItemTranscriptDelta(_)
| ServerNotification::ThreadRealtimeItemCompleted(_)
| ServerNotification::ThreadRealtimeTranscriptDelta(_)
| ServerNotification::ThreadRealtimeTranscriptDone(_)
| ServerNotification::ThreadRealtimeOutputAudioDelta(_)
| ServerNotification::ThreadRealtimeSdp(_)
| ServerNotification::ThreadRealtimeError(_)
| ServerNotification::ThreadRealtimeClosed(_)
| ServerNotification::WindowsWorldWritableWarning(_)
| ServerNotification::WindowsSandboxSetupCompleted(_)
| ServerNotification::AccountLoginCompleted(_) => notification,
}
}
fn without_thread_item_media(mut item: ThreadItem) -> ThreadItem {
match &mut item {
ThreadItem::UserMessage { content, .. } => {
content.retain(|item| match item {
UserInput::Image { .. } | UserInput::Audio { .. } => false,
UserInput::Text { .. }
| UserInput::LocalImage { .. }
| UserInput::LocalAudio { .. }
| UserInput::Skill { .. }
| UserInput::Mention { .. } => true,
});
}
ThreadItem::DynamicToolCall {
content_items: Some(content_items),
..
} => {
content_items.retain(|item| match item {
DynamicToolCallOutputContentItem::InputImage { .. }
| DynamicToolCallOutputContentItem::InputAudio { .. } => false,
DynamicToolCallOutputContentItem::InputText { .. } => true,
});
}
ThreadItem::McpToolCall {
result: Some(result),
..
} => {
// TODO(ruslan): Handle oversized results that core has already collapsed into a
// truncated text preview, which can contain inline media that this filter retains.
result.content.retain(|item| {
!matches!(
item.get("type").and_then(serde_json::Value::as_str),
Some("image" | "audio")
) && item
.get("resource")
.and_then(|resource| resource.get("blob"))
.is_none()
});
}
ThreadItem::ImageGeneration(item) => item.result.clear(),
ThreadItem::HookPrompt { .. }
| ThreadItem::AgentMessage { .. }
| ThreadItem::Plan { .. }
| ThreadItem::Reasoning { .. }
| ThreadItem::CommandExecution { .. }
| ThreadItem::FileChange { .. }
| ThreadItem::McpToolCall { result: None, .. }
| ThreadItem::DynamicToolCall {
content_items: None,
..
}
| ThreadItem::CollabAgentToolCall { .. }
| ThreadItem::SubAgentActivity { .. }
| ThreadItem::WebSearch(_)
| ThreadItem::ImageView { .. }
| ThreadItem::Sleep(_)
| ThreadItem::EnteredReviewMode { .. }
| ThreadItem::ExitedReviewMode { .. }
| ThreadItem::ContextCompaction { .. } => {}
}
item
}
codex-rs/app-server/tests/suite/v2/turn_start.rs
+105
View File
@@ -213,6 +213,111 @@ async fn received_response_input_images(server: &wiremock::MockServer) -> Result
Ok(input_images)
}
#[tokio::test]
async fn turn_start_omits_notification_media_without_changing_model_input() -> Result<()> {
let responses = vec![create_final_assistant_message_sse_response("Done")?];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let codex_home = TempDir::new()?;
MockResponsesConfig::new(&server.uri())
.enable_feature(Feature::OmitAppServerNotificationMedia)
.write(codex_home.path())?;
let mut mcp = TestAppServer::builder()
.with_codex_home(codex_home.path())
.build_initialized()
.await?;
let ThreadStartResponse { thread, .. } = mcp
.start_thread(ThreadStartParams {
model: Some("mock-model".to_string()),
experimental_raw_events: true,
..Default::default()
})
.await?;
let _: TurnStartResponse = mcp
.request(|request_id| ClientRequest::TurnStart {
request_id,
params: TurnStartParams {
thread_id: thread.id,
input: vec![
V2UserInput::Text {
text: "Describe this image".to_string(),
text_elements: Vec::new(),
},
V2UserInput::Image {
url: TINY_PNG_DATA_URL.to_string(),
detail: None,
},
],
..Default::default()
},
})
.await?;
let mut user_message_notifications = Vec::new();
timeout(DEFAULT_READ_TIMEOUT, async {
loop {
let notification = mcp
.read_stream_until_matching_notification(
"item notification or turn completion",
|notification| {
matches!(
notification.method.as_str(),
"item/started"
| "item/completed"
| "rawResponseItem/completed"
| "turn/completed"
)
},
)
.await?;
if notification.method == "turn/completed" {
return Ok::<(), anyhow::Error>(());
}
let params = notification.params.context("item notification params")?;
let item = &params["item"];
let item_type = item["type"].as_str();
if item_type == Some("userMessage")
|| (item_type == Some("message") && item["role"] == "user")
{
let content = item["content"].as_array().context("user message content")?;
if !content
.iter()
.any(|item| item["text"] == "Describe this image")
{
continue;
}
assert_eq!(content.len(), 1);
assert!(matches!(
content[0]["type"].as_str(),
Some("text" | "input_text")
));
user_message_notifications.push(notification.method);
}
}
})
.await??;
user_message_notifications.sort();
assert_eq!(
user_message_notifications,
vec![
"item/completed",
"item/started",
"rawResponseItem/completed"
]
);
let model_input_images = received_response_input_images(&server).await?;
assert_eq!(model_input_images.len(), 1);
assert_eq!(model_input_images[0]["image_url"], TINY_PNG_DATA_URL);
Ok(())
}
#[tokio::test]
async fn turn_start_with_empty_input_runs_model_request() -> Result<()> {
let responses = vec![create_final_assistant_message_sse_response("Done")?];
codex-rs/core/config.schema.json
+6
View File
@@ -808,6 +808,9 @@
"non_prefixed_mcp_tool_names": {
"$ref": "#/definitions/FeatureToml_for_NonPrefixedMcpToolNamesConfigToml"
},
"omit_app_server_notification_media": {
"type": "boolean"
},
"personality": {
"type": "boolean"
},
@@ -6011,6 +6014,9 @@
"non_prefixed_mcp_tool_names": {
"$ref": "#/definitions/FeatureToml_for_NonPrefixedMcpToolNamesConfigToml"
},
"omit_app_server_notification_media": {
"type": "boolean"
},
"personality": {
"type": "boolean"
},
codex-rs/features/src/lib.rs
+8
View File
@@ -269,6 +269,8 @@ pub enum Feature {
ExternalMigration,
/// Enable extension-backed image generation.
ImageGeneration,
/// Omit inline image and audio content from app-server item notifications.
OmitAppServerNotificationMedia,
/// Tell the model when a prompt image was resized and include its dimensions.
ImageResizeNotice,
/// Apply one shared pixel and token budget to every image, regardless of legacy detail hints.
@@ -1398,6 +1400,12 @@ pub const FEATURES: &[FeatureSpec] = &[
stage: Stage::Stable,
default_enabled: true,
},
FeatureSpec {
id: Feature::OmitAppServerNotificationMedia,
key: "omit_app_server_notification_media",
stage: Stage::UnderDevelopment,
default_enabled: false,
},
FeatureSpec {
id: Feature::ImageResizeNotice,
key: "image_resize_notice",