From 29dce2db4371b5148d55aaa1d2ee22543024dc51 Mon Sep 17 00:00:00 2001 From: Philipp Spiess Date: Thu, 6 Aug 2026 21:58:24 +0000 Subject: [PATCH] Support content references for inline visualizations (#37341) ## What changed - Recognize structured `visualize` content references alongside the existing inline visualization directive in cached, streaming, and finalized TUI rendering. - Resolve absolute visualization paths only when they belong to the active thread directory, and show the existing unavailable fallback for incomplete, invalid, or out-of-scope references. - Let embedded visualizations inherit host color, typography, and radius tokens while retaining standalone defaults and legacy aliases. Keep Lucide icons at a stable size when model-authored chart styles are present. ## Testing - Cover content-reference fallback, replay, render-cache bypass, and canonical streaming behavior. GitOrigin-RevId: 887d13647df11c23d519e64d0958fdc41ef3f472 --- codex-rs/tui/src/history_cell/messages.rs | 6 +- .../tui/src/history_cell/messages_tests.rs | 14 +- codex-rs/tui/src/inline_visualization.rs | 52 +++++-- .../inline_visualization/assets/visualize.css | 131 ++++++++++++++---- .../tui/src/inline_visualization_tests.rs | 41 +++++- codex-rs/tui/src/streaming/render.rs | 6 +- codex-rs/tui/src/streaming/render_tests.rs | 14 ++ 7 files changed, 209 insertions(+), 55 deletions(-) diff --git a/codex-rs/tui/src/history_cell/messages.rs b/codex-rs/tui/src/history_cell/messages.rs index 6f9ea937bc..7cc099d981 100644 --- a/codex-rs/tui/src/history_cell/messages.rs +++ b/codex-rs/tui/src/history_cell/messages.rs @@ -395,9 +395,9 @@ impl AgentMarkdownCell { crate::inline_visualization::InlineVisualizationContext, >, ) -> Self { - let rendered_lines = (!markdown_source - .contains(crate::inline_visualization::DIRECTIVE_PREFIX)) - .then(MarkdownRenderCache::default); + let rendered_lines = + (!crate::inline_visualization::contains_inline_visualization(&markdown_source)) + .then(MarkdownRenderCache::default); Self { markdown_source, cwd: cwd.to_path_buf(), diff --git a/codex-rs/tui/src/history_cell/messages_tests.rs b/codex-rs/tui/src/history_cell/messages_tests.rs index eae1ade5cf..900d05d4ee 100644 --- a/codex-rs/tui/src/history_cell/messages_tests.rs +++ b/codex-rs/tui/src/history_cell/messages_tests.rs @@ -71,12 +71,14 @@ fn raw_markdown_bypasses_the_rich_render_cache() { #[test] fn visualization_directives_are_not_cached() { - let cell = AgentMarkdownCell::new( - "::codex-inline-vis{file=\"chart.html\"}".to_string(), - Path::new("/tmp"), - ); + for markdown in [ + "::codex-inline-vis{file=\"chart.html\"}", + "\u{e200}visualize\u{e202}{\"path\":\"/tmp/chart.html\"}\u{e201}", + ] { + let cell = AgentMarkdownCell::new(markdown.to_string(), Path::new("/tmp")); - cell.display_lines(/*width*/ 48); + cell.display_lines(/*width*/ 48); - assert!(cell.rendered_lines.is_none()); + assert!(cell.rendered_lines.is_none()); + } } diff --git a/codex-rs/tui/src/inline_visualization.rs b/codex-rs/tui/src/inline_visualization.rs index 7fd4cbc1a0..eb85c5bafe 100644 --- a/codex-rs/tui/src/inline_visualization.rs +++ b/codex-rs/tui/src/inline_visualization.rs @@ -24,7 +24,9 @@ use uuid::Uuid; use self::viewer::materialize_document; -pub(crate) const DIRECTIVE_PREFIX: &str = "::codex-inline-vis{"; +const DIRECTIVE_PREFIX: &str = "::codex-inline-vis{"; +const CONTENT_REFERENCE_PREFIX: &str = "\u{e200}visualize\u{e202}"; +const CONTENT_REFERENCE_SUFFIX: char = '\u{e201}'; const MAX_FRAGMENT_BYTES: u64 = 2 * 1024 * 1024; #[derive(Clone, Debug)] @@ -81,7 +83,12 @@ impl InlineVisualizationContext { } fn link_for(&self, file: &str) -> Option { - let relative = Path::new(file); + let path = Path::new(file); + let relative = if path.is_absolute() { + path.strip_prefix(&self.thread_dir).ok()? + } else { + path + }; if relative .extension() .and_then(|extension| extension.to_str()) @@ -138,11 +145,15 @@ pub(crate) struct TrustedFileLink { pub(crate) markdown_destination_label: String, } +pub(crate) fn contains_inline_visualization(markdown: &str) -> bool { + markdown.contains(DIRECTIVE_PREFIX) || markdown.contains(CONTENT_REFERENCE_PREFIX) +} + pub(crate) fn rewrite_inline_visualizations<'a>( markdown: &'a str, context: Option<&InlineVisualizationContext>, ) -> InlineVisualizationRewrite<'a> { - if !markdown.contains(DIRECTIVE_PREFIX) { + if !contains_inline_visualization(markdown) { return InlineVisualizationRewrite { markdown: Cow::Borrowed(markdown), trusted_file_links: HashMap::new(), @@ -179,12 +190,15 @@ pub(crate) fn rewrite_inline_visualizations<'a>( let is_code = code_block_ranges .iter() .any(|range| range.start < source_offset && line_start < range.end); - if is_code || !trimmed.starts_with(DIRECTIVE_PREFIX) { + if is_code + || (!trimmed.starts_with(DIRECTIVE_PREFIX) + && !trimmed.starts_with(CONTENT_REFERENCE_PREFIX)) + { rewritten.push_str(line); } else if let Some(file) = parse_directive_file(trimmed) { - if let Some(destination) = context.and_then(|context| context.link_for(file)) { + if let Some(destination) = context.and_then(|context| context.link_for(&file)) { let placeholder = link_placeholder(); - let (markdown_label, display_label) = visualization_link_labels(file); + let (markdown_label, display_label) = visualization_link_labels(&file); let markdown_destination_label = escape_markdown_label(destination.as_str()); rewritten.push_str(&format!( "{markdown_label} \n[{markdown_destination_label}]({placeholder})" @@ -201,7 +215,9 @@ pub(crate) fn rewrite_inline_visualizations<'a>( } else { rewritten.push_str("_Visualization unavailable on this device._"); } - } else if trimmed.ends_with('}') { + } else if trimmed.ends_with(CONTENT_REFERENCE_SUFFIX) + || (trimmed.starts_with(DIRECTIVE_PREFIX) && trimmed.ends_with('}')) + { rewritten.push_str("_Visualization unavailable on this device._"); } rewritten.push_str(newline); @@ -243,13 +259,21 @@ fn link_placeholder() -> String { format!("https://codex.invalid/inline-visualization/{token}") } -fn parse_directive_file(directive: &str) -> Option<&str> { - let attributes = directive - .strip_prefix(DIRECTIVE_PREFIX)? - .strip_suffix('}')? - .trim(); - let value = attributes.strip_prefix("file=\"")?.strip_suffix('"')?; - (!value.is_empty() && !value.contains('"')).then_some(value) +fn parse_directive_file(directive: &str) -> Option> { + if let Some(attributes) = directive.strip_prefix(DIRECTIVE_PREFIX) { + let attributes = attributes.strip_suffix('}')?.trim(); + let value = attributes.strip_prefix("file=\"")?.strip_suffix('"')?; + return (!value.is_empty() && !value.contains('"')).then_some(Cow::Borrowed(value)); + } + + let payload = directive + .strip_prefix(CONTENT_REFERENCE_PREFIX)? + .strip_suffix(CONTENT_REFERENCE_SUFFIX)?; + let payload = serde_json::from_str::(payload).ok()?; + let path = payload.get("path")?.as_str()?; + Path::new(path) + .is_absolute() + .then(|| Cow::Owned(path.to_string())) } #[cfg(test)] diff --git a/codex-rs/tui/src/inline_visualization/assets/visualize.css b/codex-rs/tui/src/inline_visualization/assets/visualize.css index d0073ca311..81626e4d7f 100644 --- a/codex-rs/tui/src/inline_visualization/assets/visualize.css +++ b/codex-rs/tui/src/inline_visualization/assets/visualize.css @@ -1,33 +1,87 @@ :root { color-scheme: light dark; - background-color: var(--background) !important; + background-color: var( + --background, + var(--color-background-primary, light-dark(rgb(255 255 255), rgb(24 24 24))) + ) !important; /* Agent-facing contract; keep in sync with SKILL.md. */ - --background: light-dark(rgb(255 255 255), rgb(24 24 24)); - --foreground: light-dark(rgb(26 28 31), rgb(255 255 255)); - --card: color-mix(in oklab, var(--foreground) 5%, transparent); - --card-foreground: var(--foreground); - --popover: light-dark(rgb(255 255 255), rgb(45 45 45)); - --popover-foreground: var(--foreground); - --primary: light-dark(rgb(51 156 255), rgb(131 195 255)); - --primary-foreground: light-dark(rgb(255 255 255), rgb(13 13 13)); - --secondary: light-dark(rgb(255 255 255 / 96%), rgb(54 54 54 / 96%)); - --secondary-foreground: var(--foreground); + --background: var( + --color-background-primary, + light-dark(rgb(255 255 255), rgb(24 24 24)) + ); + --foreground: var( + --color-text-primary, + light-dark(rgb(26 28 31), rgb(255 255 255)) + ); + --card: color-mix(in oklab, var(--foreground) 5%, var(--background)); + --card-foreground: var( + --color-text-primary, + light-dark(rgb(26 28 31), rgb(255 255 255)) + ); + --popover: var( + --color-background-secondary, + light-dark(rgb(255 255 255), rgb(45 45 45)) + ); + --popover-foreground: var( + --color-text-primary, + light-dark(rgb(26 28 31), rgb(255 255 255)) + ); + --primary: var( + --color-text-info, + light-dark(rgb(51 156 255), rgb(131 195 255)) + ); + --primary-foreground: var( + --color-text-inverse, + light-dark(rgb(255 255 255), rgb(13 13 13)) + ); + --secondary: var( + --color-background-secondary, + light-dark(rgb(255 255 255 / 96%), rgb(54 54 54 / 96%)) + ); + --secondary-foreground: var( + --color-text-primary, + light-dark(rgb(26 28 31), rgb(255 255 255)) + ); --muted: color-mix(in srgb, var(--foreground) 10%, transparent); - --muted-foreground: light-dark( - rgb(26 28 31 / 49.4%), - rgb(255 255 255 / 49.8%) + --muted-foreground: var( + --color-text-secondary, + light-dark(rgb(26 28 31 / 49.4%), rgb(255 255 255 / 49.8%)) ); - --accent: light-dark(rgb(229 242 255), rgb(13 39 63)); - --accent-foreground: var(--primary); - --destructive: light-dark(rgb(226 85 7), rgb(255 133 73)); - --border: light-dark(rgb(26 28 31 / 8%), rgb(255 255 255 / 8.2%)); - --input: light-dark( - rgb(26 28 31 / 11.8%), - color-mix(in oklab, rgb(0 0 0) 10%, transparent) + --accent: var( + --color-background-info, + light-dark(rgb(229 242 255), rgb(13 39 63)) ); - --ring: light-dark(rgb(51 156 255), rgb(131 195 255 / 76%)); - --font-size-base: 14px; + --accent-foreground: var( + --color-text-info, + light-dark(rgb(51 156 255), rgb(131 195 255)) + ); + --destructive: var( + --color-text-warning, + light-dark(rgb(226 85 7), rgb(255 133 73)) + ); + --border: var( + --color-border-secondary, + light-dark(rgb(26 28 31 / 8%), rgb(255 255 255 / 8.2%)) + ); + --input: var( + --color-border-primary, + light-dark( + rgb(26 28 31 / 11.8%), + color-mix(in oklab, rgb(0 0 0) 10%, transparent) + ) + ); + --ring: var( + --color-ring-primary, + light-dark(rgb(51 156 255), rgb(131 195 255 / 76%)) + ); + --font-size-base: var(--font-text-md-size, 14px); + --blue: light-dark(rgb(51 156 255), rgb(51 156 255)); + --orange: light-dark(rgb(226 85 7), rgb(251 106 34)); + --green: light-dark(rgb(0 162 64), rgb(64 201 119)); + --red: light-dark(rgb(224 46 42), rgb(255 103 100)); + --purple: light-dark(rgb(146 79 247), rgb(173 123 249)); + --yellow: light-dark(rgb(255 195 0), rgb(255 210 64)); --viz-series-1: var(--primary); --viz-series-2: light-dark(rgb(243 136 59), rgb(245 154 86)); --viz-series-3: light-dark(rgb(93 201 119), rgb(116 213 139)); @@ -51,7 +105,7 @@ --line-height-normal: calc(var(--font-size-normal) * 1.5); --line-height-tooltip: calc(var(--font-size-tooltip) * 1.4285714286); --line-height-small: calc(var(--font-size-small) + 4px); - --radius: 12.5px; + --radius: var(--border-radius-lg, 12.5px); --radius-sm: calc(var(--radius) * 0.6); --radius-md: calc(var(--radius) * 0.8); --radius-lg: var(--radius); @@ -71,9 +125,6 @@ --viz-accent-bg: var(--accent); --viz-font-size: var(--font-size-base); --viz-warning: var(--destructive); - --color-background-primary: var(--background); - --color-text-primary: var(--foreground); - --color-border-secondary: var(--border); } :root[data-theme="light"] { @@ -89,6 +140,20 @@ } html > body { + /* Preserve MCP aliases for older inline renderers without creating :root cycles. */ + --color-background-primary: var( + --background, + light-dark(rgb(255 255 255), rgb(24 24 24)) + ); + --color-text-primary: var( + --foreground, + light-dark(rgb(26 28 31), rgb(255 255 255)) + ); + --color-border-secondary: var( + --border, + light-dark(rgb(26 28 31 / 8%), rgb(255 255 255 / 8.2%)) + ); + margin: 0; padding: 5px; color: var(--foreground); @@ -102,7 +167,7 @@ html > body { a { color: color-mix(in srgb, var(--viz-accent) 80%, var(--viz-text) 20%); cursor: pointer; - font-weight: var(--viz-font-weight-medium); + font-weight: var(--font-weight-medium, 500); text-decoration: none; text-underline-offset: 2px; } @@ -786,6 +851,16 @@ svg { width: 100%; } +/* Model-authored chart rules can otherwise stretch icons after these styles load. */ +svg.lucide { + display: block; + width: 16px !important; + height: 16px !important; + flex: none; + margin: 0 !important; + stroke-width: 1.6; +} + .text-warning, .text-destructive { color: var(--destructive); diff --git a/codex-rs/tui/src/inline_visualization_tests.rs b/codex-rs/tui/src/inline_visualization_tests.rs index 7b11154785..1f252431c8 100644 --- a/codex-rs/tui/src/inline_visualization_tests.rs +++ b/codex-rs/tui/src/inline_visualization_tests.rs @@ -123,6 +123,41 @@ fn hides_incomplete_streaming_directive() { assert!(rewritten.trusted_file_links.is_empty()); } +#[test] +fn hides_incomplete_streaming_content_reference() { + for reference in [ + "Before\n\u{e200}visualize\u{e202}{\"path\":\"/tmp/chart", + "Before\n\u{e200}visualize\u{e202}{\"path\":\"/tmp/chart.html\"}", + ] { + let rewritten = rewrite_inline_visualizations(reference, /*context*/ None); + + assert_eq!(rewritten.markdown, "Before\n"); + assert!(rewritten.trusted_file_links.is_empty()); + } +} + +#[test] +fn unavailable_or_invalid_content_reference_has_explicit_fallback() { + let (_codex_home, context) = context_with_fragment("
chart
"); + let outside = tempfile::tempdir().expect("outside visualization directory"); + let outside_path = outside.path().join("chart.html"); + fs::write(&outside_path, "
outside
").expect("write outside fragment"); + + let references = [ + serde_json::json!({ "path": outside_path }).to_string(), + serde_json::json!({ "path": "chart.html" }).to_string(), + "{\"path\":".to_string(), + ]; + + for payload in references { + let reference = format!("\u{e200}visualize\u{e202}{payload}\u{e201}"); + assert_eq!( + rewrite_inline_visualizations(&reference, Some(&context)).markdown, + "_Visualization unavailable on this device._" + ); + } +} + #[test] fn unavailable_artifact_has_explicit_fallback() { let codex_home = tempfile::tempdir().expect("temp codex home"); @@ -227,8 +262,12 @@ fn viewer_reuses_path_and_refreshes_static_document() { #[test] fn finalized_agent_cell_replays_visualization_link() { let (_codex_home, context) = context_with_fragment("
chart
"); + let fragment_path = context.thread_dir.join("chart.html"); let cell = AgentMarkdownCell::new_with_inline_visualizations( - "Before\n\n::codex-inline-vis{file=\"chart.html\"}\n\nAfter".to_string(), + format!( + "Before\n\n\u{e200}visualize\u{e202}{}\u{e201}\n\nAfter", + serde_json::json!({ "path": fragment_path }) + ), Path::new("/workspace"), Some(context), ); diff --git a/codex-rs/tui/src/streaming/render.rs b/codex-rs/tui/src/streaming/render.rs index 29070c6783..634bab3550 100644 --- a/codex-rs/tui/src/streaming/render.rs +++ b/codex-rs/tui/src/streaming/render.rs @@ -5,8 +5,8 @@ use crate::history_cell::HistoryRenderMode; use crate::history_cell::raw_lines_from_source; -use crate::inline_visualization::DIRECTIVE_PREFIX; use crate::inline_visualization::InlineVisualizationContext; +use crate::inline_visualization::contains_inline_visualization; use crate::markdown::render_markdown_agent_with_links_cwd_and_visualizations; use crate::markdown::render_streaming_markdown_agent_with_links_and_cwd; use crate::terminal_hyperlinks::HyperlinkLine; @@ -61,7 +61,7 @@ impl StreamingRender { render_mode: HistoryRenderMode, inline_visualization_context: Option<&InlineVisualizationContext>, ) { - self.has_inline_visualization_directive = source.contains(DIRECTIVE_PREFIX); + self.has_inline_visualization_directive = contains_inline_visualization(source); self.lines = match (render_mode, inline_visualization_context) { (HistoryRenderMode::Rich, None) if !self.has_inline_visualization_directive => { let rendered = @@ -107,7 +107,7 @@ impl StreamingRender { return; } - self.has_inline_visualization_directive |= committed_source.contains(DIRECTIVE_PREFIX); + self.has_inline_visualization_directive |= contains_inline_visualization(committed_source); if self.has_inline_visualization_directive { self.recompute( raw_source, diff --git a/codex-rs/tui/src/streaming/render_tests.rs b/codex-rs/tui/src/streaming/render_tests.rs index c7b5ecef71..f40adcde62 100644 --- a/codex-rs/tui/src/streaming/render_tests.rs +++ b/codex-rs/tui/src/streaming/render_tests.rs @@ -250,6 +250,20 @@ fn inline_visualizations_without_context_use_canonical_full_render() { ); } +#[test] +fn inline_visualization_content_references_use_canonical_full_render() { + let (_, render) = assert_rich_stream_matches_full_render( + &[ + "Before.\n\n", + "\u{e200}visualize\u{e202}{\"path\":\"/tmp/missing.html\"}\u{e201}\n", + ], + Some(80), + ); + + assert_eq!(render.stable_source_len, 0); + assert!(render.has_inline_visualization_directive); +} + #[test] fn inline_visualization_directive_survives_raw_to_rich_render_mode_switch() { let cwd = test_cwd();