Fix skills catalog budgets and warning delivery

This commit is contained in:
jif-oai
2026-06-18 10:37:36 +02:00
parent 3fea72fda6
commit b3d7b2d45a
23 changed files with 309 additions and 218 deletions
@@ -79,6 +79,7 @@ async fn contribute_prompt(
fragments.extend(
contributor
.contribute(ContextContributionContext {
thread_id: codex_protocol::ThreadId::new(),
session_store,
thread_store,
turn_store: &turn_store,
@@ -1,13 +1,15 @@
use codex_protocol::ThreadId;
use codex_protocol::protocol::Event;
/// Host-provided fire-and-forget sink for extension-generated events.
///
/// Extensions construct protocol events with the correlation id appropriate for
/// the callback they are handling, then leave persistence, ordering, transport
/// fanout, and logging decisions to the host.
/// the callback they are handling and identify the owning thread explicitly,
/// then leave persistence, ordering, transport fanout, and logging decisions to
/// the host.
pub trait ExtensionEventSink: Send + Sync {
/// Queue one protocol event for host-owned delivery.
fn emit(&self, event: Event);
fn emit(&self, thread_id: ThreadId, event: Event);
}
/// Event sink used when the host does not expose extension event emission.
@@ -15,5 +17,5 @@ pub trait ExtensionEventSink: Send + Sync {
pub struct NoopExtensionEventSink;
impl ExtensionEventSink for NoopExtensionEventSink {
fn emit(&self, _event: Event) {}
fn emit(&self, _thread_id: ThreadId, _event: Event) {}
}
@@ -3,6 +3,7 @@ use std::pin::Pin;
use std::sync::Arc;
use codex_context_fragments::ContextualUserFragment;
use codex_protocol::ThreadId;
use codex_protocol::items::TurnItem;
use codex_protocol::protocol::ReviewDecision;
use codex_protocol::protocol::TokenUsageInfo;
@@ -44,6 +45,7 @@ pub type ExtensionFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
/// Host context available while extensions contribute initial model input.
#[derive(Clone, Copy)]
pub struct ContextContributionContext<'a> {
pub thread_id: ThreadId,
pub session_store: &'a ExtensionData,
pub thread_store: &'a ExtensionData,
pub turn_store: &'a ExtensionData,
@@ -1,5 +1,6 @@
use std::path::PathBuf;
use codex_protocol::ThreadId;
use codex_protocol::user_input::UserInput;
/// Host-owned turn environment summary visible to turn-input contributors.
@@ -16,6 +17,8 @@ pub struct TurnInputEnvironment {
/// Turn facts supplied before the host records turn-local model input items.
#[derive(Debug, Clone)]
pub struct TurnInputContext {
/// Thread that owns this turn.
pub thread_id: ThreadId,
/// Stable host-owned turn identifier.
pub turn_id: String,
/// User input submitted for this turn.
+10 -7
View File
@@ -201,6 +201,7 @@ async fn contributors_preserve_registration_order() {
fragments.extend(
contributor
.contribute(ContextContributionContext {
thread_id: codex_protocol::ThreadId::new(),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
@@ -323,7 +324,7 @@ struct RecordingEventSink {
}
impl ExtensionEventSink for RecordingEventSink {
fn emit(&self, event: Event) {
fn emit(&self, _thread_id: codex_protocol::ThreadId, event: Event) {
let EventMsg::Warning(warning) = event.msg else {
panic!("test sink only accepts warning events");
};
@@ -339,13 +340,15 @@ fn custom_event_sink_survives_registry_build() {
let sink = Arc::new(RecordingEventSink::default());
let builder = ExtensionRegistryBuilder::<()>::with_event_sink(sink.clone());
builder
.event_sink()
.emit(warning_event("builder", "before"));
builder.event_sink().emit(
codex_protocol::ThreadId::new(),
warning_event("builder", "before"),
);
let registry = builder.build();
registry
.event_sink()
.emit(warning_event("registry", "after"));
registry.event_sink().emit(
codex_protocol::ThreadId::new(),
warning_event("registry", "after"),
);
assert_eq!(
sink.events
+12 -8
View File
@@ -22,13 +22,17 @@ impl GoalEventEmitter {
turn_id: Option<String>,
goal: ThreadGoal,
) {
self.sink.emit(Event {
id: event_id.into(),
msg: EventMsg::ThreadGoalUpdated(ThreadGoalUpdatedEvent {
thread_id: goal.thread_id,
turn_id,
goal,
}),
});
let thread_id = goal.thread_id;
self.sink.emit(
thread_id,
Event {
id: event_id.into(),
msg: EventMsg::ThreadGoalUpdated(ThreadGoalUpdatedEvent {
thread_id,
turn_id,
goal,
}),
},
);
}
}
@@ -1407,7 +1407,7 @@ impl RecordingEventSink {
}
impl ExtensionEventSink for RecordingEventSink {
fn emit(&self, event: Event) {
fn emit(&self, _thread_id: ThreadId, event: Event) {
self.events().push(event);
}
}
+1
View File
@@ -185,6 +185,7 @@ async fn prompt_contribution_uses_memory_summary_when_enabled() {
let fragments = extension
.contribute(ContextContributionContext {
thread_id: Default::default(),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
+1
View File
@@ -18,6 +18,7 @@ codex-core-skills = { workspace = true }
codex-exec-server = { workspace = true }
codex-extension-api = { workspace = true }
codex-mcp = { workspace = true }
codex-otel = { workspace = true }
codex-protocol = { workspace = true }
codex-tools = { workspace = true }
codex-utils-absolute-path = { workspace = true }
+20 -10
View File
@@ -20,6 +20,7 @@ use codex_extension_api::ToolExecutor;
use codex_extension_api::TurnInputContext;
use codex_extension_api::TurnInputContributor;
use codex_mcp::McpResourceClient;
use codex_otel::SessionTelemetry;
use codex_protocol::capabilities::SelectedCapabilityRoot;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
@@ -134,17 +135,23 @@ where
)
.await;
for warning in &catalog.warnings {
self.emit_warning(thread_store.level_id(), warning.clone());
self.emit_warning(
context.thread_id,
context.turn_store.level_id(),
warning.clone(),
);
}
let session_telemetry = context.turn_store.get::<SessionTelemetry>();
let Some((fragment, warning)) = available_skills_fragment(
&catalog,
host_snapshot.as_deref(),
context.model_context_window,
session_telemetry.as_deref(),
) else {
return Vec::new();
};
if let Some(warning) = warning {
self.emit_warning(thread_store.level_id(), warning);
self.emit_warning(context.thread_id, context.turn_store.level_id(), warning);
}
vec![PromptFragment::developer_capability(fragment.render())]
})
@@ -206,7 +213,7 @@ where
};
let catalog = self.list_skills(query, &thread_state).await;
for warning in &catalog.warnings {
self.emit_warning(&input.turn_id, warning.clone());
self.emit_warning(input.thread_id, &input.turn_id, warning.clone());
}
let selected_entries = collect_explicit_skill_mentions(&input.user_input, &catalog);
@@ -227,7 +234,7 @@ where
"Skill `{}` exceeded the main prompt context limit and was truncated.",
entry.name
);
self.emit_warning(&input.turn_id, warning.clone());
self.emit_warning(input.thread_id, &input.turn_id, warning.clone());
warnings.push(warning);
}
let fragment = SkillInstructions {
@@ -247,7 +254,7 @@ where
}
Err(message) => {
let warning = format!("Failed to load skill `{}`: {message}", entry.name);
self.emit_warning(&input.turn_id, warning.clone());
self.emit_warning(input.thread_id, &input.turn_id, warning.clone());
warnings.push(warning);
}
}
@@ -332,11 +339,14 @@ impl<C> SkillsExtension<C> {
.map_err(|err| err.message)
}
fn emit_warning(&self, turn_id: &str, message: String) {
self.event_sink.emit(Event {
id: turn_id.to_string(),
msg: EventMsg::Warning(WarningEvent { message }),
});
fn emit_warning(&self, thread_id: codex_protocol::ThreadId, turn_id: &str, message: String) {
self.event_sink.emit(
thread_id,
Event {
id: turn_id.to_string(),
msg: EventMsg::Warning(WarningEvent { message }),
},
);
}
}
+72 -33
View File
@@ -9,8 +9,12 @@ use codex_core_skills::build_available_skills;
use codex_core_skills::default_skill_metadata_budget;
use codex_core_skills::render::SkillCatalogMode;
use codex_core_skills::render::SkillRenderSideEffects;
use codex_extension_api::ContextualUserFragment;
use codex_otel::SessionTelemetry;
use codex_utils_string::approx_token_count;
const MAX_AVAILABLE_SKILLS_BYTES: usize = 8_000;
const MAX_AVAILABLE_SKILLS_TOKENS: usize = 10_000;
const MAX_MAIN_PROMPT_BYTES: usize = 8_000;
pub(crate) const MAX_SKILL_NAME_BYTES: usize = 256;
pub(crate) const MAX_SKILL_PATH_BYTES: usize = 1_024;
@@ -19,16 +23,29 @@ pub(crate) fn available_skills_fragment(
catalog: &SkillCatalog,
host_snapshot: Option<&HostSkillsSnapshot>,
model_context_window: Option<i64>,
session_telemetry: Option<&SessionTelemetry>,
) -> Option<(AvailableSkillsInstructions, Option<String>)> {
let visible_entries = catalog
.entries
.iter()
.filter(|entry| entry.enabled && entry.prompt_visible)
.collect::<Vec<_>>();
let side_effects = session_telemetry
.map_or(SkillRenderSideEffects::None, |session_telemetry| {
SkillRenderSideEffects::ThreadStart { session_telemetry }
});
let host_available = host_snapshot.and_then(|host_snapshot| {
build_available_skills(
host_snapshot.outcome(),
default_skill_metadata_budget(model_context_window),
side_effects,
)
});
if visible_entries
.iter()
.any(|entry| entry.authority.kind == SkillSourceKind::Host)
&& let Some(host_snapshot) = host_snapshot
&& let Some(available) = host_available
{
let external_lines = bounded_skill_lines(
visible_entries
@@ -36,30 +53,8 @@ pub(crate) fn available_skills_fragment(
.copied()
.filter(|entry| entry.authority.kind != SkillSourceKind::Host),
);
if let Some(available) = build_available_skills(
host_snapshot.outcome(),
default_skill_metadata_budget(model_context_window),
SkillRenderSideEffects::None,
) {
let warning = available.warning_message.clone();
return Some((
AvailableSkillsInstructions::from_available_skills_with_additional_lines(
available,
external_lines,
),
warning,
));
}
if !external_lines.is_empty() {
return Some((
AvailableSkillsInstructions::from_skill_lines(
external_lines,
SkillCatalogMode::Mixed,
),
None,
));
}
return None;
let warning = available.warning_message.clone();
return Some((bounded_mixed_fragment(available, external_lines), warning));
}
let mode = if visible_entries
@@ -70,7 +65,7 @@ pub(crate) fn available_skills_fragment(
} else {
SkillCatalogMode::HostOnly
};
let skill_lines = bounded_skill_lines(visible_entries);
let skill_lines = bounded_skill_lines(visible_entries).into_lines();
(!skill_lines.is_empty()).then(|| {
(
AvailableSkillsInstructions::from_skill_lines(skill_lines, mode),
@@ -79,9 +74,23 @@ pub(crate) fn available_skills_fragment(
})
}
struct BoundedSkillLines {
lines: Vec<String>,
omitted: usize,
}
impl BoundedSkillLines {
fn into_lines(mut self) -> Vec<String> {
if let Some(line) = omitted_skills_line(self.omitted) {
self.lines.push(line);
}
self.lines
}
}
fn bounded_skill_lines<'a>(
entries: impl IntoIterator<Item = &'a SkillCatalogEntry>,
) -> Vec<String> {
) -> BoundedSkillLines {
let mut total_bytes = 0usize;
let mut omitted = 0usize;
let mut skill_lines = Vec::new();
@@ -101,13 +110,43 @@ fn bounded_skill_lines<'a>(
skill_lines.push(line);
}
if omitted > 0 {
let skill_word = if omitted == 1 { "skill" } else { "skills" };
skill_lines.push(format!(
"- {omitted} additional {skill_word} omitted from this bounded skills list."
));
BoundedSkillLines {
lines: skill_lines,
omitted,
}
skill_lines
}
fn bounded_mixed_fragment(
available: codex_core_skills::AvailableSkills,
mut external: BoundedSkillLines,
) -> AvailableSkillsInstructions {
loop {
let mut additional_lines = external.lines.clone();
if let Some(line) = omitted_skills_line(external.omitted) {
additional_lines.push(line);
}
let fragment = AvailableSkillsInstructions::from_available_skills_with_additional_lines(
available.clone(),
additional_lines,
);
if approx_token_count(&fragment.render()) <= MAX_AVAILABLE_SKILLS_TOKENS {
return fragment;
}
if external.lines.pop().is_none() {
return AvailableSkillsInstructions::from_available_skills_with_additional_lines(
available,
Vec::new(),
);
}
external.omitted = external.omitted.saturating_add(1);
}
}
fn omitted_skills_line(omitted: usize) -> Option<String> {
(omitted > 0).then(|| {
let skill_word = if omitted == 1 { "skill" } else { "skills" };
format!("- {omitted} additional {skill_word} omitted from this bounded skills list.")
})
}
fn render_skill_line(entry: &SkillCatalogEntry, description: &str) -> String {
+66 -102
View File
@@ -15,6 +15,7 @@ use codex_extension_api::ExtensionEventSink;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ThreadStartInput;
use codex_extension_api::TurnInputContext;
use codex_protocol::ThreadId;
use codex_protocol::capabilities::CapabilityRootLocation;
use codex_protocol::capabilities::SelectedCapabilityRoot;
use codex_protocol::protocol::Event;
@@ -43,6 +44,7 @@ use codex_skills_extension::provider::SkillProviderFuture;
use codex_skills_extension::provider::SkillReadRequest;
use codex_skills_extension::provider::SkillSearchRequest;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_string::approx_token_count;
use pretty_assertions::assert_eq;
type TestResult = Result<(), Box<dyn std::error::Error>>;
@@ -114,20 +116,32 @@ async fn installed_extension_catalog_uses_host_service_snapshot() -> TestResult
}
#[tokio::test]
async fn mixed_catalog_preserves_host_budget_and_external_locator() -> TestResult {
let host_path =
AbsolutePathBuf::try_from(std::env::temp_dir().join("host-demo").join("SKILL.md"))?;
let mut outcome = SkillLoadOutcome::default();
outcome.skills.push(SkillMetadata {
name: "host-demo".to_string(),
description: "x".repeat(5_000),
short_description: None,
interface: None,
dependencies: None,
policy: None,
path_to_skills_md: host_path,
scope: SkillScope::User,
plugin_id: None,
async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_input() -> TestResult
{
let read_requests = Arc::new(Mutex::new(Vec::new()));
let mut executor_entries = vec![test_entry(
SkillSourceKind::Executor,
"env-1",
"executor/lint-fix",
"lint-fix/SKILL.md",
)];
executor_entries.extend((0..20).map(|index| {
test_entry(
SkillSourceKind::Executor,
"env-1",
&format!("executor/extra-{index}"),
&format!("extra-{index}/SKILL.md"),
)
.with_short_description(Some("x".repeat(1_000)))
}));
let executor_provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
entries: executor_entries,
warnings: Vec::new(),
},
read_requests: Arc::clone(&read_requests),
list_calls: None,
fail_first_list: false,
});
let providers = SkillProviders::new()
.with_host_provider(Arc::new(StaticSkillProvider {
@@ -144,25 +158,13 @@ async fn mixed_catalog_preserves_host_budget_and_external_locator() -> TestResul
list_calls: None,
fail_first_list: false,
}))
.with_executor_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
entries: vec![test_entry(
SkillSourceKind::Executor,
"env-1",
"executor/lint-fix",
"lint-fix/SKILL.md",
)],
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}));
.with_executor_provider(executor_provider);
let (event_tx, event_rx) = std::sync::mpsc::channel();
let mut builder =
ExtensionRegistryBuilder::with_event_sink(Arc::new(ChannelEventSink(event_tx)));
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
thread_store.insert(vec![SelectedCapabilityRoot {
@@ -184,82 +186,32 @@ async fn mixed_catalog_preserves_host_budget_and_external_locator() -> TestResul
thread_store: &thread_store,
})
.await;
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome)));
let fragments = registry.context_contributors()[0]
.contribute(ContextContributionContext {
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
model_context_window: Some(10_000),
})
.await;
assert_eq!(1, fragments.len());
let text = fragments[0].text();
assert!(text.contains("host-demo"));
assert!(text.contains("environment resource: skill://executor/lint-fix/SKILL.md"));
let EventMsg::Warning(warning) = event_rx.try_recv()?.msg else {
panic!("expected host budget warning");
};
assert!(warning.message.contains("2% skills context budget"));
Ok(())
}
#[tokio::test]
async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_input() -> TestResult
{
let read_requests = Arc::new(Mutex::new(Vec::new()));
let executor_provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
entries: vec![test_entry(
SkillSourceKind::Executor,
"env-1",
"executor/lint-fix",
"lint-fix/SKILL.md",
)],
warnings: Vec::new(),
},
read_requests: Arc::clone(&read_requests),
list_calls: None,
fail_first_list: false,
let initial_turn_store = ExtensionData::new("initial-turn");
let mut outcome = SkillLoadOutcome::default();
outcome.skills.push(SkillMetadata {
name: "host-demo".to_string(),
description: "x".repeat(40_000),
short_description: None,
interface: None,
dependencies: None,
policy: None,
path_to_skills_md: AbsolutePathBuf::try_from(
std::env::temp_dir().join("host-demo").join("SKILL.md"),
)?,
scope: SkillScope::User,
plugin_id: None,
});
let providers = SkillProviders::new().with_executor_provider(executor_provider);
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
thread_store.insert(vec![SelectedCapabilityRoot {
id: "lint-fix".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: "/skills/lint-fix".to_string(),
},
}]);
let session_source = SessionSource::Cli;
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
initial_turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome)));
let prompt_fragments = registry.context_contributors()[0]
.contribute(ContextContributionContext {
thread_id: ThreadId::default(),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &initial_turn_store,
model_context_window: Some(400_000),
})
.await;
let prompt_fragments = registry.context_contributors()[0]
.contribute(prompt_context(
&session_store,
&thread_store,
&ExtensionData::new("initial-turn"),
))
.await;
assert_eq!(1, prompt_fragments.len());
assert!(
prompt_fragments[0]
@@ -267,16 +219,23 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in
.starts_with(SKILLS_INSTRUCTIONS_OPEN_TAG)
);
assert!(prompt_fragments[0].text().contains("lint-fix"));
assert!(prompt_fragments[0].text().contains("host-demo"));
assert!(
prompt_fragments[0]
.text()
.contains("(environment resource: skill://executor/lint-fix/SKILL.md)")
);
assert!(approx_token_count(prompt_fragments[0].text()) <= 10_000);
let EventMsg::Warning(warning) = event_rx.try_recv()?.1.msg else {
panic!("expected host budget warning");
};
assert!(warning.message.contains("2% skills context budget"));
let turn_store = ExtensionData::new("turn-1");
let fragments = registry.turn_input_contributors()[0]
.contribute(
TurnInputContext {
thread_id: ThreadId::default(),
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Text {
text: "$lint-fix please".to_string(),
@@ -312,6 +271,7 @@ async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_in
let next_fragments = registry.turn_input_contributors()[0]
.contribute(
TurnInputContext {
thread_id: ThreadId::default(),
turn_id: "turn-2".to_string(),
user_input: vec![UserInput::Text {
text: "no skill this time".to_string(),
@@ -376,7 +336,7 @@ async fn orchestrator_catalog_snapshot_caches_failure() -> TestResult {
))
.await;
assert!(initial_fragments.is_empty());
let EventMsg::Warning(warning) = event_rx.try_recv()?.msg else {
let EventMsg::Warning(warning) = event_rx.try_recv()?.1.msg else {
panic!("expected warning event");
};
assert_eq!(
@@ -388,6 +348,7 @@ async fn orchestrator_catalog_snapshot_caches_failure() -> TestResult {
let fragments = registry.turn_input_contributors()[0]
.contribute(
TurnInputContext {
thread_id: ThreadId::default(),
turn_id: turn_id.to_string(),
user_input: vec![UserInput::Text {
text: "$first".to_string(),
@@ -467,6 +428,7 @@ async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> Te
let fragments = registry.turn_input_contributors()[0]
.contribute(
TurnInputContext {
thread_id: ThreadId::default(),
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Mention {
name: "lint-fix".to_string(),
@@ -553,6 +515,7 @@ async fn prompt_hidden_skill_can_still_be_invoked() -> TestResult {
let fragments = registry.turn_input_contributors()[0]
.contribute(
TurnInputContext {
thread_id: ThreadId::default(),
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Text {
text: "$hidden-skill".to_string(),
@@ -588,11 +551,11 @@ struct StaticSkillProvider {
fail_first_list: bool,
}
struct ChannelEventSink(std::sync::mpsc::Sender<Event>);
struct ChannelEventSink(std::sync::mpsc::Sender<(ThreadId, Event)>);
impl ExtensionEventSink for ChannelEventSink {
fn emit(&self, event: Event) {
let _ = self.0.send(event);
fn emit(&self, thread_id: ThreadId, event: Event) {
let _ = self.0.send((thread_id, event));
}
}
@@ -700,6 +663,7 @@ fn prompt_context<'a>(
turn_store: &'a ExtensionData,
) -> ContextContributionContext<'a> {
ContextContributionContext {
thread_id: ThreadId::default(),
session_store,
thread_store,
turn_store,