mirror of
https://github.com/openai/codex.git
synced 2026-10-18 17:18:52 +00:00
Preserve image transparency metadata in app-server items (#37178)
## What changed - Add `transparentBackground` to image-generation items and legacy completion events. - Map Images API backgrounds to `true` for transparent, `false` for opaque, and `null` for automatic or unavailable values. - Preserve the metadata in live notifications and persisted or resumed thread history while continuing to deserialize older items that omit it. ## Testing - Cover transparent, opaque, and automatic backgrounds, including image edits and thread read/resume flows. - Verify the stable JSON shape and optional TypeScript field. GitOrigin-RevId: 08527f37f23431df45fab1ed34eb82b2139b49d7
This commit is contained in:
@@ -4786,6 +4786,7 @@ async fn turn_event_counts_completed_tool_items() {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: None,
|
||||
result: "ok".to_string(),
|
||||
transparent_background: None,
|
||||
saved_path: None,
|
||||
}),
|
||||
];
|
||||
|
||||
@@ -4731,6 +4731,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
+7
@@ -19779,6 +19779,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
+7
@@ -17538,6 +17538,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1241,6 +1241,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1241,6 +1241,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1386,6 +1386,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1935,6 +1935,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1726,6 +1726,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
+7
@@ -1726,6 +1726,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1726,6 +1726,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1935,6 +1935,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1726,6 +1726,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1935,6 +1935,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1726,6 +1726,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1726,6 +1726,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1386,6 +1386,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1386,6 +1386,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
@@ -1386,6 +1386,13 @@
|
||||
"status": {
|
||||
"type": "string"
|
||||
},
|
||||
"transparentBackground": {
|
||||
"default": null,
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"imageGeneration"
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -3,4 +3,4 @@
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { AbsolutePathBuf } from "./AbsolutePathBuf";
|
||||
|
||||
export type ImageGenerationItem = { id: string, status: string, revisedPrompt: string | null, result: string, savedPath?: AbsolutePathBuf, };
|
||||
export type ImageGenerationItem = { id: string, status: string, revisedPrompt: string | null, result: string, transparentBackground?: boolean, savedPath?: AbsolutePathBuf, };
|
||||
|
||||
@@ -862,6 +862,7 @@ impl ThreadHistoryBuilder {
|
||||
status: String::new(),
|
||||
revised_prompt: None,
|
||||
result: String::new(),
|
||||
transparent_background: None,
|
||||
saved_path: None,
|
||||
});
|
||||
self.upsert_item_in_current_turn(item);
|
||||
@@ -873,6 +874,7 @@ impl ThreadHistoryBuilder {
|
||||
status: payload.status.clone(),
|
||||
revised_prompt: payload.revised_prompt.clone(),
|
||||
result: payload.result.clone(),
|
||||
transparent_background: payload.transparent_background,
|
||||
saved_path: payload.saved_path.clone(),
|
||||
});
|
||||
self.upsert_item_in_current_turn(item);
|
||||
@@ -2076,6 +2078,7 @@ mod tests {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("A blue square".to_string()),
|
||||
result: "cG5n".to_string(),
|
||||
transparent_background: Some(true),
|
||||
saved_path: Some(saved_path.clone()),
|
||||
},
|
||||
)),
|
||||
@@ -2106,6 +2109,7 @@ mod tests {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("A blue square".to_string()),
|
||||
result: "cG5n".to_string(),
|
||||
transparent_background: Some(true),
|
||||
saved_path: Some(saved_path),
|
||||
})]
|
||||
);
|
||||
@@ -2368,6 +2372,7 @@ mod tests {
|
||||
status: "completed".into(),
|
||||
revised_prompt: Some("final prompt".into()),
|
||||
result: "Zm9v".into(),
|
||||
transparent_background: Some(true),
|
||||
saved_path: Some(test_path_buf("/tmp/ig_123.png").abs()),
|
||||
})),
|
||||
RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent {
|
||||
@@ -2407,6 +2412,7 @@ mod tests {
|
||||
status: "completed".into(),
|
||||
revised_prompt: Some("final prompt".into()),
|
||||
result: "Zm9v".into(),
|
||||
transparent_background: Some(true),
|
||||
saved_path: Some(test_path_buf("/tmp/ig_123.png").abs()),
|
||||
}),
|
||||
],
|
||||
|
||||
@@ -928,6 +928,7 @@ impl From<CoreTurnItem> for ThreadItem {
|
||||
status: image.status,
|
||||
revised_prompt: image.revised_prompt,
|
||||
result: image.result,
|
||||
transparent_background: None,
|
||||
saved_path: image.saved_path,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1544,6 +1544,7 @@ The app-server streams JSON-RPC notifications while a turn is running. Each turn
|
||||
- `mcpToolCall` — `{id, server, tool, status, arguments, appContext, mcpAppResourceUri?, pluginId, readOnlyHint, result?, error?}` describing MCP calls; `appContext` is `{connectorId, linkId, resourceUri, appName, actionName}` for calls through a trusted MCP app, where `connectorId` identifies the connector that owns the tool, `linkId` identifies the app link, `resourceUri` points to the widget template, `appName` is the connector's display name, and `actionName` is the stable connector `Action.name`. `readOnlyHint` is `true` for read-only tools, `false` for write-capable tools, and `null` when the annotation is unavailable, including older rollout entries. The hint describes tool capability, not whether an invocation succeeded or performed a write; use `status`, `result`, and `error` to determine the execution outcome. `appName` and `actionName` may be null for older rollout entries. The top-level `mcpAppResourceUri` is deprecated and temporarily duplicated for client migration. `tool` identifies the raw MCP tool. `status` is `inProgress`, `completed`, or `failed`.
|
||||
- `collabToolCall` — `{id, tool, status, senderThreadId, receiverThreadId?, newThreadId?, prompt?, agentStatus?}` describing collab tool calls (`spawn_agent`, `send_input`, `resume_agent`, `wait`, `close_agent`); `status` is `inProgress`, `completed`, or `failed`.
|
||||
- `webSearch` — `{id, query, action?, results?}` for a web search request issued by the agent; `action` mirrors the Responses API web_search action payload (`search`, `open_page`, `find_in_page`) and may be omitted until completion. For standalone web search, `results` contains the out-of-band structured result DTOs returned by `/v1/alpha/search`; clients should ignore result types and fields they do not understand.
|
||||
- `imageGeneration` — `{id, status, revisedPrompt, result, transparentBackground, savedPath?}` for a generated image. `transparentBackground` is `true` when the Images API reports a transparent background, `false` when it reports an opaque background, and `null` when the background is automatic, unavailable, or the item has not completed. The field is always present on v2 item payloads, including persisted and resumed items.
|
||||
- `imageView` — `{id, path}` emitted when the agent invokes the image viewer tool.
|
||||
- `sleep` — `{id, durationMs}` emitted while the agent waits for a duration or new input.
|
||||
- `enteredReviewMode` — `{id, review}` sent when the reviewer starts; `review` is a short user-facing label such as `"current changes"` or the requested target description.
|
||||
|
||||
@@ -106,6 +106,7 @@ mod tests {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("revised".to_string()),
|
||||
result: "base64-result".to_string(),
|
||||
transparent_background: None,
|
||||
saved_path: Some(test_path_buf("/tmp/ig-1.png").abs()),
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -10,6 +10,10 @@ use app_test_support::write_chatgpt_auth;
|
||||
use codex_app_server_protocol::ImageGenerationItem;
|
||||
use codex_app_server_protocol::ItemCompletedNotification;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::ThreadReadParams;
|
||||
use codex_app_server_protocol::ThreadReadResponse;
|
||||
use codex_app_server_protocol::ThreadResumeParams;
|
||||
use codex_app_server_protocol::ThreadResumeResponse;
|
||||
use codex_app_server_protocol::ThreadStartParams;
|
||||
use codex_app_server_protocol::ThreadStartResponse;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
@@ -117,6 +121,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
|
||||
status,
|
||||
revised_prompt,
|
||||
result,
|
||||
transparent_background,
|
||||
saved_path: Some(saved_path),
|
||||
..
|
||||
}) = completed.item
|
||||
@@ -126,6 +131,7 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
|
||||
assert_eq!(status, "completed");
|
||||
assert_eq!(revised_prompt.as_deref(), Some("paint a blue whale"));
|
||||
assert_eq!(result, RESULT);
|
||||
assert_eq!(transparent_background, Some(false));
|
||||
assert_eq!(std::fs::read(&saved_path)?, TINY_PNG_BYTES);
|
||||
|
||||
let image_request = server
|
||||
@@ -179,6 +185,179 @@ async fn standalone_image_generation_returns_saved_path_hint_to_model() -> Resul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transparent_image_preserves_output_metadata_and_persisted_history() -> Result<()> {
|
||||
let call_id = "transparent-image-run-1";
|
||||
let server = responses::start_mock_server().await;
|
||||
mount_image_response_with_background(&server, "transparent").await;
|
||||
responses::mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call_with_namespace(
|
||||
call_id,
|
||||
"image_gen",
|
||||
"imagegen",
|
||||
&json!({"prompt": "a blue whale on a transparent background"}).to_string(),
|
||||
),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_assistant_message("msg-1", "Done"),
|
||||
responses::ev_completed("resp-2"),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri(), ImagegenTestMode::Direct)?;
|
||||
write_chatgpt_auth(
|
||||
codex_home.path(),
|
||||
ChatGptAuthFixture::new("access-chatgpt"),
|
||||
AuthCredentialsStoreMode::File,
|
||||
)?;
|
||||
let mut mcp = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.with_env_overrides(&[("OPENAI_API_KEY", None)])
|
||||
.build_initialized_with_timeout(DEFAULT_READ_TIMEOUT)
|
||||
.await?;
|
||||
start_image_generation_turn(&mut mcp, ThreadStartParams::default()).await?;
|
||||
|
||||
let completed = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
wait_for_image_generation_completed(&mut mcp),
|
||||
)
|
||||
.await??;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
let thread_id = completed.thread_id.clone();
|
||||
let ThreadItem::ImageGeneration(ImageGenerationItem {
|
||||
status,
|
||||
result,
|
||||
transparent_background,
|
||||
..
|
||||
}) = completed.item
|
||||
else {
|
||||
panic!("expected completed image-generation item");
|
||||
};
|
||||
assert_eq!(status, "completed");
|
||||
assert_eq!(result, RESULT);
|
||||
assert_eq!(transparent_background, Some(true));
|
||||
|
||||
drop(mcp);
|
||||
let mut resumed = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.with_env_overrides(&[("OPENAI_API_KEY", None)])
|
||||
.build_initialized_with_timeout(DEFAULT_READ_TIMEOUT)
|
||||
.await?;
|
||||
let read_id = resumed
|
||||
.send_thread_read_request(ThreadReadParams {
|
||||
thread_id: thread_id.clone(),
|
||||
include_turns: true,
|
||||
})
|
||||
.await?;
|
||||
let ThreadReadResponse { thread, .. } =
|
||||
timeout(DEFAULT_READ_TIMEOUT, resumed.read_response(read_id)).await??;
|
||||
let persisted_image = thread
|
||||
.turns
|
||||
.iter()
|
||||
.flat_map(|turn| turn.items.iter())
|
||||
.find_map(|item| match item {
|
||||
ThreadItem::ImageGeneration(item) => Some(item),
|
||||
_ => None,
|
||||
})
|
||||
.context("persisted legacy history should contain the generated image")?;
|
||||
assert_eq!(persisted_image.transparent_background, Some(true));
|
||||
|
||||
let resume_id = resumed
|
||||
.send_thread_resume_request(ThreadResumeParams {
|
||||
thread_id,
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let ThreadResumeResponse { thread, .. } =
|
||||
timeout(DEFAULT_READ_TIMEOUT, resumed.read_response(resume_id)).await??;
|
||||
let resumed_image = thread
|
||||
.turns
|
||||
.iter()
|
||||
.flat_map(|turn| turn.items.iter())
|
||||
.find_map(|item| match item {
|
||||
ThreadItem::ImageGeneration(item) => Some(item),
|
||||
_ => None,
|
||||
})
|
||||
.context("resumed legacy history should contain the generated image")?;
|
||||
assert_eq!(resumed_image.transparent_background, Some(true));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_image_background_preserves_unknown_transparency() -> Result<()> {
|
||||
let call_id = "automatic-image-run-1";
|
||||
let server = responses::start_mock_server().await;
|
||||
mount_image_response_with_background(&server, "auto").await;
|
||||
responses::mount_sse_sequence(
|
||||
&server,
|
||||
vec![
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created("resp-1"),
|
||||
responses::ev_function_call_with_namespace(
|
||||
call_id,
|
||||
"image_gen",
|
||||
"imagegen",
|
||||
&json!({"prompt": "paint a blue whale"}).to_string(),
|
||||
),
|
||||
responses::ev_completed("resp-1"),
|
||||
]),
|
||||
responses::sse(vec![
|
||||
responses::ev_assistant_message("msg-1", "Done"),
|
||||
responses::ev_completed("resp-2"),
|
||||
]),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
create_config_toml(codex_home.path(), &server.uri(), ImagegenTestMode::Direct)?;
|
||||
write_chatgpt_auth(
|
||||
codex_home.path(),
|
||||
ChatGptAuthFixture::new("access-chatgpt"),
|
||||
AuthCredentialsStoreMode::File,
|
||||
)?;
|
||||
let mut mcp = TestAppServer::builder()
|
||||
.with_codex_home(codex_home.path())
|
||||
.with_env_overrides(&[("OPENAI_API_KEY", None)])
|
||||
.build_initialized_with_timeout(DEFAULT_READ_TIMEOUT)
|
||||
.await?;
|
||||
start_image_generation_turn(&mut mcp, ThreadStartParams::default()).await?;
|
||||
let completed = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
wait_for_image_generation_completed(&mut mcp),
|
||||
)
|
||||
.await??;
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
let ThreadItem::ImageGeneration(image) = completed.item else {
|
||||
panic!("expected completed image-generation item");
|
||||
};
|
||||
assert_eq!(image.transparent_background, None);
|
||||
let value = serde_json::to_value(&image)?;
|
||||
assert_eq!(
|
||||
value.get("transparentBackground"),
|
||||
Some(&serde_json::Value::Null),
|
||||
"v2 image-generation items must always include nullable transparency metadata"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()> {
|
||||
let call_id = "image-run-failed";
|
||||
@@ -236,6 +415,7 @@ async fn standalone_image_generation_failure_emits_terminal_item() -> Result<()>
|
||||
status: "failed".to_string(),
|
||||
revised_prompt: Some("paint a blue whale".to_string()),
|
||||
result: String::new(),
|
||||
transparent_background: None,
|
||||
saved_path: None,
|
||||
})
|
||||
);
|
||||
@@ -266,7 +446,7 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>
|
||||
"remote executors use different imagegen storage approaches, so host-local image paths are unavailable"
|
||||
);
|
||||
|
||||
let edit_request = run_image_edit_test(|codex_home| {
|
||||
let (edit_request, _) = run_image_edit_test(|codex_home| {
|
||||
let image_path = codex_home.join("attached.png");
|
||||
std::fs::write(&image_path, TINY_PNG_BYTES)?;
|
||||
Ok((
|
||||
@@ -294,9 +474,9 @@ async fn standalone_image_edit_uses_attached_model_visible_image() -> Result<()>
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standalone_image_edit_uses_recent_pathless_image() -> Result<()> {
|
||||
async fn transparent_image_edit_preserves_metadata_and_recent_pathless_image() -> Result<()> {
|
||||
let image_url = TINY_PNG_DATA_URL;
|
||||
let edit_request = run_image_edit_test(|_| {
|
||||
let (edit_request, completed_image) = run_image_edit_test(|_| {
|
||||
Ok((
|
||||
json!({
|
||||
"prompt": "add a red hat",
|
||||
@@ -317,6 +497,7 @@ async fn standalone_image_edit_uses_recent_pathless_image() -> Result<()> {
|
||||
.await?;
|
||||
assert_eq!(edit_request["prompt"], "add a red hat");
|
||||
assert_eq!(edit_request["images"][0]["image_url"], image_url);
|
||||
assert_eq!(completed_image.transparent_background, Some(true));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -461,7 +642,7 @@ async fn start_image_generation_turn(
|
||||
|
||||
async fn run_image_edit_test(
|
||||
input: impl FnOnce(&Path) -> Result<(serde_json::Value, Vec<V2UserInput>)>,
|
||||
) -> Result<serde_json::Value> {
|
||||
) -> Result<(serde_json::Value, ImageGenerationItem)> {
|
||||
let call_id = "image-edit-1";
|
||||
let server = responses::start_mock_server().await;
|
||||
mount_image_edit_response(&server).await;
|
||||
@@ -510,11 +691,14 @@ async fn run_image_edit_test(
|
||||
input,
|
||||
)
|
||||
.await?;
|
||||
timeout(
|
||||
let completed = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
wait_for_image_generation_completed(&mut mcp),
|
||||
)
|
||||
.await??;
|
||||
let ThreadItem::ImageGeneration(completed_image) = completed.item else {
|
||||
panic!("expected completed image-generation item");
|
||||
};
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
@@ -540,7 +724,10 @@ async fn run_image_edit_test(
|
||||
"chatgpt_cca"
|
||||
);
|
||||
assert_image_turn_id_header(image_request, &turn_id)?;
|
||||
Ok(image_request.body_json::<serde_json::Value>()?)
|
||||
Ok((
|
||||
image_request.body_json::<serde_json::Value>()?,
|
||||
completed_image,
|
||||
))
|
||||
}
|
||||
|
||||
fn assert_image_turn_id_header(request: &wiremock::Request, expected_turn_id: &str) -> Result<()> {
|
||||
@@ -592,10 +779,15 @@ async fn wait_for_image_generation_completed(
|
||||
}
|
||||
|
||||
async fn mount_image_response(server: &MockServer) {
|
||||
mount_image_response_with_background(server, "opaque").await;
|
||||
}
|
||||
|
||||
async fn mount_image_response_with_background(server: &MockServer, background: &str) {
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/api/codex/images/generations"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"created": 1,
|
||||
"background": background,
|
||||
"data": [{"b64_json": RESULT}],
|
||||
})))
|
||||
.expect(1)
|
||||
@@ -608,6 +800,7 @@ async fn mount_image_edit_response(server: &MockServer) {
|
||||
.and(path("/api/codex/images/edits"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"created": 1,
|
||||
"background": "transparent",
|
||||
"data": [{"b64_json": RESULT}],
|
||||
})))
|
||||
.expect(1)
|
||||
|
||||
@@ -1681,6 +1681,7 @@ fn append_resume_redaction_history(
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("secret revised prompt".to_string()),
|
||||
result: "base64-image-result".to_string(),
|
||||
transparent_background: None,
|
||||
saved_path: Some(test_absolute_path("/tmp/ig-1.png")),
|
||||
}),
|
||||
]
|
||||
|
||||
@@ -472,6 +472,7 @@ mod tests {
|
||||
status: "in_progress".to_string(),
|
||||
revised_prompt: None,
|
||||
result: String::new(),
|
||||
transparent_background: None,
|
||||
saved_path: None,
|
||||
});
|
||||
let expected_completed_item = ExtensionItem::ImageGeneration(ImageGenerationItem {
|
||||
@@ -479,6 +480,7 @@ mod tests {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("A tiny blue square".to_string()),
|
||||
result: "cG5n".to_string(),
|
||||
transparent_background: Some(true),
|
||||
saved_path: Some(expected_path.clone()),
|
||||
});
|
||||
codex_tools::TurnItemEmitter::emit_started(
|
||||
@@ -500,6 +502,7 @@ mod tests {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("A tiny blue square".to_string()),
|
||||
result: "cG5n".to_string(),
|
||||
transparent_background: Some(true),
|
||||
saved_path: Some(expected_path.clone()),
|
||||
})],
|
||||
},
|
||||
|
||||
@@ -96,6 +96,7 @@ fn legacy_end_event(item: &ImageGenerationItem) -> EventMsg {
|
||||
status: item.status.clone(),
|
||||
revised_prompt: item.revised_prompt.clone(),
|
||||
result: item.result.clone(),
|
||||
transparent_background: item.transparent_background,
|
||||
saved_path: item.saved_path.clone(),
|
||||
})
|
||||
}
|
||||
@@ -142,6 +143,7 @@ impl ImageGenerationTool {
|
||||
status: "in_progress".to_string(),
|
||||
revised_prompt: None,
|
||||
result: String::new(),
|
||||
transparent_background: None,
|
||||
saved_path: None,
|
||||
},
|
||||
EventMsg::ImageGenerationBegin(ImageGenerationBeginEvent {
|
||||
@@ -155,14 +157,19 @@ impl ImageGenerationTool {
|
||||
}
|
||||
.map_err(|err| format!("image generation failed: {err}"))
|
||||
.and_then(|response| {
|
||||
let transparent_background = match response.background {
|
||||
Some(ImageBackground::Transparent) => Some(true),
|
||||
Some(ImageBackground::Opaque) => Some(false),
|
||||
Some(ImageBackground::Auto) | None => None,
|
||||
};
|
||||
response
|
||||
.data
|
||||
.into_iter()
|
||||
.next()
|
||||
.map(|data| data.b64_json)
|
||||
.map(|data| (data.b64_json, transparent_background))
|
||||
.ok_or_else(|| "image generation returned no image data".to_string())
|
||||
});
|
||||
let result = match result {
|
||||
let (result, transparent_background) = match result {
|
||||
Ok(result) => result,
|
||||
Err(message) => {
|
||||
let item = ImageGenerationItem {
|
||||
@@ -170,6 +177,7 @@ impl ImageGenerationTool {
|
||||
status: "failed".to_string(),
|
||||
revised_prompt: Some(args.prompt),
|
||||
result: String::new(),
|
||||
transparent_background: None,
|
||||
saved_path: None,
|
||||
};
|
||||
let legacy_event = legacy_end_event(&item);
|
||||
@@ -209,6 +217,7 @@ impl ImageGenerationTool {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some(args.prompt),
|
||||
result: result.clone(),
|
||||
transparent_background,
|
||||
saved_path: saved_path.clone(),
|
||||
};
|
||||
let legacy_event = legacy_end_event(&item);
|
||||
|
||||
@@ -15,6 +15,9 @@ pub struct ImageGenerationItem {
|
||||
pub status: String,
|
||||
pub revised_prompt: Option<String>,
|
||||
pub result: String,
|
||||
#[serde(default)]
|
||||
#[ts(optional)]
|
||||
pub transparent_background: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
pub saved_path: Option<AbsolutePathBuf>,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use ts_rs::TS;
|
||||
|
||||
use super::ExtensionItem;
|
||||
use super::image_generation::ImageGenerationItem;
|
||||
@@ -13,6 +14,7 @@ fn completed_image_generation_item() -> ExtensionItem {
|
||||
status: "completed".to_string(),
|
||||
revised_prompt: Some("A blue square".to_string()),
|
||||
result: "cG5n".to_string(),
|
||||
transparent_background: None,
|
||||
saved_path: None,
|
||||
})
|
||||
}
|
||||
@@ -30,12 +32,58 @@ fn image_generation_item_preserves_stable_wire_shape() {
|
||||
"status": "completed",
|
||||
"revisedPrompt": "A blue square",
|
||||
"result": "cG5n",
|
||||
"transparentBackground": null,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ExtensionItem>(value).expect("deserialize extension item"),
|
||||
item
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ExtensionItem>(json!({
|
||||
"kind": "image_gen.generation",
|
||||
"id": "image-1",
|
||||
"status": "completed",
|
||||
"revisedPrompt": "A blue square",
|
||||
"result": "cG5n",
|
||||
}))
|
||||
.expect("deserialize legacy image-generation item without transparency metadata"),
|
||||
item
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_item_preserves_authoritative_transparency() {
|
||||
let ExtensionItem::ImageGeneration(mut image) = completed_image_generation_item() else {
|
||||
panic!("expected image-generation item");
|
||||
};
|
||||
image.transparent_background = Some(true);
|
||||
let item = ExtensionItem::ImageGeneration(image);
|
||||
let value = serde_json::to_value(&item).expect("serialize extension item");
|
||||
|
||||
assert_eq!(
|
||||
value,
|
||||
json!({
|
||||
"kind": "image_gen.generation",
|
||||
"id": "image-1",
|
||||
"status": "completed",
|
||||
"revisedPrompt": "A blue square",
|
||||
"result": "cG5n",
|
||||
"transparentBackground": true,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ExtensionItem>(value).expect("deserialize extension item"),
|
||||
item
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_generation_transparency_is_optional_in_typescript() {
|
||||
assert!(
|
||||
ImageGenerationItem::inline().contains("transparentBackground?: boolean"),
|
||||
"image-generation transparency must remain optional for existing TypeScript clients"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -429,6 +429,7 @@ impl ImageGenerationItem {
|
||||
status: self.status.clone(),
|
||||
revised_prompt: self.revised_prompt.clone(),
|
||||
result: self.result.clone(),
|
||||
transparent_background: None,
|
||||
saved_path: self.saved_path.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2544,6 +2544,9 @@ pub struct ImageGenerationEndEvent {
|
||||
#[ts(optional)]
|
||||
pub revised_prompt: Option<String>,
|
||||
pub result: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
pub transparent_background: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional)]
|
||||
pub saved_path: Option<AbsolutePathBuf>,
|
||||
|
||||
@@ -718,6 +718,7 @@ pub(super) fn handle_image_generation_end(
|
||||
status: status.into(),
|
||||
revised_prompt,
|
||||
result: String::new(),
|
||||
transparent_background: None,
|
||||
saved_path,
|
||||
}),
|
||||
}),
|
||||
|
||||
Reference in New Issue
Block a user