Add opt-in nonfatal handling for clock read failures (#45825)

## Why

Clock provider failures can abort a turn while preparing time context or running clock tools. Allow turns to continue with an explicit indication that the current time is unavailable.

## What changed

- Add `features.nonfatal_clock_read_errors`, disabled by default, to report clock failures to the model without failing the turn.
- Emit a generic `failed to read current time` notice for context reads and tool errors, without exposing provider error details. Deduplicate context notices per turn and compaction window, and remove inherited notices from forked subagent context.
- Omit unavailable environment dates and explicitly clear previously visible dates with `<current_date status="unavailable" />`.
- Return external sleep clock failures to the model when the feature is enabled, preserving sleep item completion notifications.

## Testing

Add coverage for continued inference after clock failures, notice deduplication across compaction, subagent notice filtering, environment date removal and recovery, and sleep failures during initial and polling reads.

GitOrigin-RevId: a39c3723c06a6f786d8ad59d667e8b9f626c973e
This commit is contained in:
Vivian Fang
2026-09-16 01:16:35 +00:00
committed by copyberry
parent fac58c1153
commit 5bf132cd52
14 changed files with 367 additions and 48 deletions
+2
View File
@@ -6,6 +6,7 @@ use crate::codex_thread::CodexThread;
use crate::config::PermissionProfileSnapshot;
use crate::context::ContextualUserFragment;
use crate::context::CurrentTimeReminder;
use crate::context::CurrentTimeUnavailable;
use crate::context::DeveloperInstructions;
use crate::context::GuardianContextMode;
use crate::context::ManagedDeveloperInstructions;
@@ -135,6 +136,7 @@ fn retain_forked_developer_message(
))
|| MultiAgentModeInstructions::matches_text(text)
|| CurrentTimeReminder::matches_text(text)
|| CurrentTimeUnavailable::matches_text(text)
|| usage_hint_texts
.iter()
.any(|usage_hint_text| usage_hint_text == text))
@@ -41,3 +41,31 @@ impl ContextualUserFragment for CurrentTimeReminder {
format!("It is {}.", self.formatted_time())
}
}
pub(crate) struct CurrentTimeUnavailable;
impl CurrentTimeUnavailable {
pub(crate) const MESSAGE: &str = "failed to read current time";
}
impl ContextualUserFragment for CurrentTimeUnavailable {
fn content_kind(&self) -> ContentItemKind {
ContentItemKind("current_time.unavailable".to_string())
}
fn role(&self) -> &'static str {
"developer"
}
fn markers(&self) -> (&'static str, &'static str) {
Self::type_markers()
}
fn type_markers() -> (&'static str, &'static str) {
("<current_time_unavailable>", "</current_time_unavailable>")
}
fn body(&self) -> String {
Self::MESSAGE.to_string()
}
}
+1
View File
@@ -66,6 +66,7 @@ pub(crate) use contextual_user_message::is_contextual_user_fragment;
pub(crate) use contextual_user_message::is_user_authorization_message;
pub(crate) use contextual_user_message::parse_visible_hook_prompt_message;
pub(crate) use current_time_reminder::CurrentTimeReminder;
pub(crate) use current_time_reminder::CurrentTimeUnavailable;
pub(crate) use developer_instructions::DeveloperInstructions;
pub(crate) use environments_instructions::EnvironmentsInstructions;
pub(crate) use guardian_approved_action::GuardianApprovedAction;
@@ -92,6 +92,7 @@ impl EnvironmentsState {
shell_version: self.shell_version.clone(),
shell_version_removed: false,
current_date: self.current_date.clone(),
current_date_removed: false,
timezone: self.timezone.clone(),
network: self.network.clone(),
filesystem: self.filesystem.clone(),
@@ -183,6 +184,8 @@ impl WorldStateSection for EnvironmentsState {
shell_version_removed: self.shell_version.is_none()
&& previous.shell_version.is_some(),
current_date: self.current_date.clone(),
current_date_removed: self.current_date.is_none()
&& previous.current_date.is_some(),
timezone: self.timezone.clone(),
network: self.network.clone(),
filesystem: self.filesystem.clone(),
@@ -221,6 +224,7 @@ struct RenderedEnvironments {
shell_version: Option<String>,
shell_version_removed: bool,
current_date: Option<String>,
current_date_removed: bool,
timezone: Option<String>,
network: Option<NetworkContext>,
filesystem: Option<FileSystemContext>,
@@ -289,7 +293,11 @@ impl ContextualUserFragment for RenderedEnvironments {
let shell_version = self.shell_version.as_deref();
push_optional_element(&mut rendered, "shell_version", shell_version);
}
push_optional_element(&mut rendered, "current_date", self.current_date.as_deref());
if self.current_date_removed {
rendered.push_str(" <current_date status=\"unavailable\" />\n");
} else {
push_optional_element(&mut rendered, "current_date", self.current_date.as_deref());
}
push_optional_element(&mut rendered, "timezone", self.timezone.as_deref());
if let Some(network) = &self.network {
rendered.push_str(" ");
@@ -396,3 +396,31 @@ fn shell_version_diff_clears_previously_visible_version() {
"<environment_context>\n <shell_version status=\"unavailable\" />\n</environment_context>"
);
}
#[test]
fn current_date_diff_clears_once_and_recovers() {
let available = EnvironmentsState {
current_date: Some("2026-06-17".to_string()),
..Default::default()
};
let unavailable = EnvironmentsState::default();
assert_eq!(
unavailable
.render_diff(PreviousSectionState::Known(&available.snapshot()))
.expect("removed current date")
.render(),
"<environment_context>\n <current_date status=\"unavailable\" />\n</environment_context>"
);
assert!(
unavailable
.render_diff(PreviousSectionState::Known(&unavailable.snapshot()))
.is_none()
);
assert_eq!(
available
.render_diff(PreviousSectionState::Known(&unavailable.snapshot()))
.expect("restored current date")
.render(),
"<environment_context>\n <current_date>2026-06-17</current_date>\n</environment_context>"
);
}
+61 -6
View File
@@ -11,6 +11,7 @@ use super::turn_context::TurnContext;
use crate::config::Config;
use crate::config::CurrentTimeReminderConfig;
use crate::context::ContextualUserFragment;
use crate::context::CurrentTimeUnavailable;
use crate::context_manager::is_user_turn_boundary;
pub(super) fn apply_persistent_defaults(config: &mut Config) {
@@ -40,6 +41,7 @@ pub(super) fn apply_persistent_defaults(config: &mut Config) {
pub(crate) struct CurrentTimeReminderState {
last_delivery_time: Option<DateTime<Utc>>,
last_window_id: Option<String>,
last_clock_failure: Option<(String, uuid::Uuid)>,
pending_user_or_tool_output_boundary: bool,
}
@@ -94,6 +96,59 @@ impl CurrentTimeReminderState {
}
}
impl Session {
pub(super) async fn read_clock_for_context(
&self,
turn_context: &TurnContext,
clock_read: &'static str,
) -> CodexResult<Option<DateTime<Utc>>> {
let error = match self
.services
.time_provider
.current_time(self.thread_id)
.await
{
Ok(time) => return Ok(Some(time)),
Err(error) => error,
};
if !turn_context
.config
.features
.enabled(Feature::NonfatalClockReadErrors)
{
return Err(CodexErr::Fatal(format!(
"failed to read current time: {error:#}"
)));
}
tracing::error!(
clock_read,
thread_id = %self.thread_id,
turn_id = %turn_context.sub_id,
"failed to read current time; the clock provider may be stalled"
);
{
let mut state = self.state.lock().await;
let failure = (
turn_context.sub_id.clone(),
state.auto_compact_window_ids().window_id,
);
// A compacted window may no longer contain the earlier notice.
if state.current_time_reminder.last_clock_failure.as_ref() == Some(&failure) {
return Ok(None);
}
state.current_time_reminder.last_clock_failure = Some(failure);
}
let response_item = ContextualUserFragment::into(CurrentTimeUnavailable);
self.record_conversation_items(
turn_context,
turn_context.model_info(),
std::slice::from_ref(&response_item),
)
.await;
Ok(None)
}
}
pub(super) async fn maybe_record_current_time_reminder(
sess: &Session,
turn_context: &TurnContext,
@@ -110,12 +165,12 @@ pub(super) async fn maybe_record_current_time_reminder(
return Ok(());
};
let current_time = sess
.services
.time_provider
.current_time(sess.thread_id)
.await
.map_err(|err| CodexErr::Fatal(format!("failed to read current time: {err:#}")))?;
let Some(current_time) = sess
.read_clock_for_context(turn_context, "reminder")
.await?
else {
return Ok(());
};
let reminder_is_due = {
let mut state = sess.state.lock().await;
+9 -10
View File
@@ -25,7 +25,6 @@ use crate::context::world_state::WorldState;
use codex_connectors::AppToolPolicyEvaluator;
use codex_extension_api::WorldStateContributionInput;
use codex_features::Feature;
use codex_protocol::error::CodexErr;
use codex_protocol::error::Result as CodexResult;
use codex_protocol::models::BaseInstructionsProvenance;
@@ -199,19 +198,19 @@ impl Session {
}
if turn_context.config.include_environment_context {
let current_date = self
.services
.time_provider
.current_time(self.thread_id())
.await
.map_err(|err| CodexErr::Fatal(format!("failed to read current time: {err:#}")))?
.with_timezone(&chrono::Local)
.format("%Y-%m-%d")
.to_string();
.read_clock_for_context(turn_context, "environment_date")
.await?
.map(|current_time| {
current_time
.with_timezone(&chrono::Local)
.format("%Y-%m-%d")
.to_string()
});
world_state.add_section(
EnvironmentsState::from_turn_context_with_environments(
turn_context,
&step_context.environments,
Some(current_date),
current_date,
)
.await
.with_subagents(environment_subagents),
@@ -1,5 +1,6 @@
use crate::context::ContextualUserFragment;
use crate::context::CurrentTimeReminder;
use crate::context::CurrentTimeUnavailable;
use crate::function_tool::FunctionCallError;
use crate::tools::context::FunctionToolOutput;
use crate::tools::context::ToolInvocation;
@@ -8,6 +9,7 @@ use crate::tools::context::ToolPayload;
use crate::tools::context::boxed_tool_output;
use crate::tools::registry::CoreToolRuntime;
use crate::tools::registry::ToolExecutor;
use codex_features::Feature;
use codex_protocol::models::ResponseInputItem;
use codex_tools::JsonSchema;
use codex_tools::ResponsesApiNamespace;
@@ -98,7 +100,21 @@ impl ToolExecutor<ToolInvocation> for CurrentTimeHandler {
.current_time(invocation.session.thread_id)
.await
.map_err(|err| {
FunctionCallError::Fatal(format!("failed to read current time: {err:#}"))
if invocation
.turn
.config
.features
.enabled(Feature::NonfatalClockReadErrors)
{
tracing::error!(
thread_id = %invocation.session.thread_id,
turn_id = %invocation.turn.sub_id,
"failed to read current time for the clock tool; the clock provider may be stalled"
);
FunctionCallError::RespondToModel(CurrentTimeUnavailable::MESSAGE.to_string())
} else {
FunctionCallError::Fatal(format!("failed to read current time: {err:#}"))
}
})?;
Ok(boxed_tool_output(CurrentTimeOutput(
CurrentTimeReminder::new(current_time),
+17 -13
View File
@@ -1,3 +1,4 @@
use crate::context::CurrentTimeUnavailable;
use crate::function_tool::FunctionCallError;
use crate::tools::context::FunctionToolOutput;
use crate::tools::context::ToolInvocation;
@@ -8,6 +9,7 @@ use crate::tools::registry::CoreToolRuntime;
use crate::tools::registry::ToolExecutor;
use codex_extension_items::ExtensionItem;
use codex_extension_items::sleep::SleepItem;
use codex_features::Feature;
use codex_protocol::items::TurnItem;
use codex_tools::JsonSchema;
use codex_tools::ResponsesApiNamespace;
@@ -111,7 +113,7 @@ impl ToolExecutor<ToolInvocation> for SleepHandler {
.input_queue
.subscribe_activity(turn_state.as_deref())
.await;
let sleep_result: Result<bool, FunctionCallError> = if pending_activity.is_some() {
let sleep_result = if pending_activity.is_some() {
Ok(true)
} else {
let sleep = session
@@ -120,27 +122,29 @@ impl ToolExecutor<ToolInvocation> for SleepHandler {
.sleep(session.thread_id, Duration::from_millis(args.duration_ms));
tokio::pin!(sleep);
tokio::select! {
result = &mut sleep => result
.map(|()| false)
.map_err(|err| {
FunctionCallError::Fatal(format!("failed to sleep: {err:#}"))
}),
result = &mut sleep => result.map(|()| false),
result = activity_rx.changed() => {
if result.is_ok() {
Ok(true)
} else {
sleep
.await
.map(|()| false)
.map_err(|err| {
FunctionCallError::Fatal(format!("failed to sleep: {err:#}"))
})
sleep.await.map(|()| false)
}
}
}
};
session.emit_turn_item_completed(turn.as_ref(), item).await;
let interrupted = sleep_result?;
let interrupted = sleep_result.map_err(|err| {
if turn.config.features.enabled(Feature::NonfatalClockReadErrors) {
tracing::error!(
thread_id = %session.thread_id,
turn_id = %turn.sub_id,
"failed to read current time for the sleep tool; the clock provider may be stalled"
);
FunctionCallError::RespondToModel(CurrentTimeUnavailable::MESSAGE.to_string())
} else {
FunctionCallError::Fatal(format!("failed to sleep: {err:#}"))
}
})?;
let message = if interrupted {
"Sleep interrupted by new input."