Report API spend limit errors by scope (#62329)

## Why

Generic quota errors hide whether the project or organization spend limit
blocked a request. Reporting the scope tells users which limit to check.

## What changed

Recognize API key, project, and organization spend limit errors across
HTTP, SSE, and WebSocket responses. Show a message that names the scope and
treat the error as non-retryable, even when the provider sends
`Retry-After`. Apply the existing usage-limit handling to goals and
compaction.

Expose distinct `CodexErrorInfo` values in the app-server protocol and
generated bindings so clients can identify the limit in live errors and
failed turn history.

## Testing

Extend the transport tests to cover all three scopes and verify that they
stop retries. Extend the app-server test to check that the error survives
`thread/read` and that a subsequent turn can succeed in the same thread.

GitOrigin-RevId: 03341f7a78daa7540e7baa519f1a29ec07e34509
This commit is contained in:
Alex Chang
2026-11-18 18:11:54 +00:00
committed by copyberry
parent 0ba514837d
commit cc7a62e19c
47 changed files with 291 additions and 60 deletions
@@ -880,6 +880,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -9127,6 +9127,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -4881,6 +4881,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -8,6 +8,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -260,6 +260,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -341,6 +341,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -263,6 +263,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -263,6 +263,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -263,6 +263,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -341,6 +341,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -263,6 +263,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -263,6 +263,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -341,6 +341,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -263,6 +263,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -260,6 +260,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -263,6 +263,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -260,6 +260,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -260,6 +260,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -260,6 +260,9 @@
"contextWindowExceeded",
"sessionBudgetExceeded",
"usageLimitExceeded",
"apiKeySpendLimitExceeded",
"projectSpendLimitExceeded",
"organizationSpendLimitExceeded",
"rateLimitExceeded",
"flexUnavailable",
"serverOverloaded",
@@ -9,4 +9,4 @@ import type { NonSteerableTurnKind } from "./NonSteerableTurnKind";
* When an upstream HTTP status is available (for example, from the Responses API or a provider),
* it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant.
*/
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "rateLimitExceeded" | "flexUnavailable" | "serverOverloaded" | "modelNotFound" | "policyCapacityUnavailable" | "inferenceHookDenied" | "cyberPolicy" | "bioPolicy" | "misalignmentPolicyViolation" | "tooManyDenials" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other" | string | { [key: string]: unknown };
export type CodexErrorInfo = "contextWindowExceeded" | "sessionBudgetExceeded" | "usageLimitExceeded" | "apiKeySpendLimitExceeded" | "projectSpendLimitExceeded" | "organizationSpendLimitExceeded" | "rateLimitExceeded" | "flexUnavailable" | "serverOverloaded" | "modelNotFound" | "policyCapacityUnavailable" | "inferenceHookDenied" | "cyberPolicy" | "bioPolicy" | "misalignmentPolicyViolation" | "tooManyDenials" | { "httpConnectionFailed": { httpStatusCode: number | null, } } | { "responseStreamConnectionFailed": { httpStatusCode: number | null, } } | "internalServerError" | "unauthorized" | "badRequest" | "threadRollbackFailed" | "sandboxError" | { "responseStreamDisconnected": { httpStatusCode: number | null, } } | { "responseTooManyFailedAttempts": { httpStatusCode: number | null, } } | { "activeTurnNotSteerable": { turnKind: NonSteerableTurnKind, } } | "other" | string | { [key: string]: unknown };
@@ -77,6 +77,9 @@ pub enum CodexErrorInfo {
ContextWindowExceeded,
SessionBudgetExceeded,
UsageLimitExceeded,
ApiKeySpendLimitExceeded,
ProjectSpendLimitExceeded,
OrganizationSpendLimitExceeded,
RateLimitExceeded,
FlexUnavailable,
ServerOverloaded,
@@ -157,6 +160,15 @@ impl From<CoreCodexErrorInfo> for CodexErrorInfo {
match value {
CoreCodexErrorInfo::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
CoreCodexErrorInfo::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
CoreCodexErrorInfo::ApiKeySpendLimitExceeded => {
CodexErrorInfo::ApiKeySpendLimitExceeded
}
CoreCodexErrorInfo::ProjectSpendLimitExceeded => {
CodexErrorInfo::ProjectSpendLimitExceeded
}
CoreCodexErrorInfo::OrganizationSpendLimitExceeded => {
CodexErrorInfo::OrganizationSpendLimitExceeded
}
CoreCodexErrorInfo::UsageLimitExceeded => CodexErrorInfo::UsageLimitExceeded,
CoreCodexErrorInfo::RateLimitExceeded => CodexErrorInfo::RateLimitExceeded,
CoreCodexErrorInfo::FlexUnavailable => CodexErrorInfo::FlexUnavailable,
@@ -4957,6 +4957,22 @@ fn codex_error_info_serializes_http_status_code_in_camel_case() {
#[test]
fn core_error_info_converts_to_camel_case() {
for (core, expected) in [
(
CoreCodexErrorInfo::ApiKeySpendLimitExceeded,
json!("apiKeySpendLimitExceeded"),
),
(
CoreCodexErrorInfo::ProjectSpendLimitExceeded,
json!("projectSpendLimitExceeded"),
),
(
CoreCodexErrorInfo::OrganizationSpendLimitExceeded,
json!("organizationSpendLimitExceeded"),
),
(
CoreCodexErrorInfo::UsageLimitExceeded,
json!("usageLimitExceeded"),
),
(
CoreCodexErrorInfo::PolicyCapacityUnavailable,
json!("policyCapacityUnavailable"),
@@ -52,6 +52,7 @@ use codex_app_server_protocol::ThreadStatus;
use codex_app_server_protocol::ThreadTurnsListParams;
use codex_app_server_protocol::ThreadTurnsListResponse;
use codex_app_server_protocol::Turn;
use codex_app_server_protocol::TurnCompletedNotification;
use codex_app_server_protocol::TurnItemsView;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
@@ -2865,17 +2866,24 @@ async fn thread_items_read_validates_ids_before_history_access() -> Result<()> {
Ok(())
}
#[test_case::test_case("model_not_found", CodexErrorInfo::ModelNotFound, "Access to this deployment ended."; "model_not_found")]
#[test_case::test_case("inference_hook_denied", CodexErrorInfo::InferenceHookDenied, "Blocked by the workspace hook."; "inference_hook_denied")]
#[test_case::test_case("api_key_spend_limit_exceeded", CodexErrorInfo::ApiKeySpendLimitExceeded, "Request denied", "Your API key has reached its spend limit."; "api_key_spend_limit_exceeded")]
#[test_case::test_case("project_spend_limit_exceeded", CodexErrorInfo::ProjectSpendLimitExceeded, "Request denied", "Your API project has reached its spend limit."; "project_spend_limit_exceeded")]
#[test_case::test_case("organization_spend_limit_exceeded", CodexErrorInfo::OrganizationSpendLimitExceeded, "Request denied", "Your API organization has reached its spend limit."; "organization_spend_limit_exceeded")]
#[test_case::test_case("model_not_found", CodexErrorInfo::ModelNotFound, "Access to this deployment ended.", "Access to this deployment ended."; "model_not_found")]
#[test_case::test_case("inference_hook_denied", CodexErrorInfo::InferenceHookDenied, "Blocked by the workspace hook.", "Blocked by the workspace hook."; "inference_hook_denied")]
#[tokio::test]
async fn thread_read_preserves_provider_error_after_failed_turn(
code: &str,
expected: CodexErrorInfo,
message: &str,
provider_message: &str,
expected_message: &str,
) -> Result<()> {
let server = responses::start_mock_server().await;
let response_mock =
responses::mount_sse_once(&server, responses::sse_failed("resp-1", code, message)).await;
let response_mock = responses::mount_sse_once(
&server,
responses::sse_failed("resp-1", code, provider_message),
)
.await;
let codex_home = TempDir::new()?;
MockResponsesConfig::new(&server.uri()).write(codex_home.path())?;
@@ -2908,12 +2916,14 @@ async fn thread_read_preserves_provider_error_after_failed_turn(
timeout(DEFAULT_READ_TIMEOUT, mcp.read_notification("error")).await??;
assert!(!live.will_retry);
assert_eq!(live.error.codex_error_info, Some(expected));
assert_eq!(live.error.message, message);
timeout(
assert_eq!(live.error.message, expected_message);
let completed: TurnCompletedNotification = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
mcp.read_notification("turn/completed"),
)
.await??;
assert_eq!(completed.turn.status, TurnStatus::Failed);
assert_eq!(completed.turn.error.as_ref(), Some(&live.error));
let read_id = mcp
.send_thread_read_request(ThreadReadParams {
@@ -2931,6 +2941,27 @@ async fn thread_read_preserves_provider_error_after_failed_turn(
);
assert_eq!(response_mock.requests().len(), 1);
let recovered = responses::mount_sse_once(
&server,
responses::sse(vec![
responses::ev_response_created("recovered"),
responses::ev_completed("recovered"),
]),
)
.await;
let next = mcp
.start_turn_and_wait_for_completion(TurnStartParams {
thread_id: thread.id,
input: vec![UserInput::Text {
text: "try again".into(),
text_elements: Vec::new(),
}],
..Default::default()
})
.await?;
assert_ne!(next.turn.id, completed.turn.id);
assert_eq!(next.turn.status, TurnStatus::Completed);
recovered.single_request();
Ok(())
}
+1
View File
@@ -2514,6 +2514,7 @@ fn websocket_error_detail(err: &ApiError) -> String {
ApiError::ContentFilter
| ApiError::WebsocketMessageTooLarge
| ApiError::ContextWindowExceeded
| ApiError::SpendLimitExceeded(_)
| ApiError::QuotaExceeded
| ApiError::UsageNotIncluded
| ApiError::SessionBudgetExceeded
+4 -2
View File
@@ -34,6 +34,7 @@ pub fn map_api_error(err: ApiError) -> CodexErr {
| ApiError::ContentFilter
| ApiError::ResponseIncomplete(_)
| ApiError::ContextWindowExceeded
| ApiError::SpendLimitExceeded(_)
| ApiError::QuotaExceeded
| ApiError::UsageNotIncluded
| ApiError::SessionBudgetExceeded
@@ -60,6 +61,9 @@ pub fn map_api_error(err: ApiError) -> CodexErr {
fn map_api_error_details(err: ApiError) -> CodexErr {
match err {
ApiError::ContextWindowExceeded => CodexErr::ContextWindowExceeded,
ApiError::SpendLimitExceeded(scope) => {
CodexErrorDetails::APISpendLimitExceeded(scope).into()
}
ApiError::QuotaExceeded => CodexErr::QuotaExceeded,
ApiError::UsageNotIncluded => CodexErr::UsageNotIncluded,
ApiError::SessionBudgetExceeded => CodexErr::SessionBudgetExceeded,
@@ -263,8 +267,6 @@ fn map_api_error_details(err: ApiError) -> CodexErr {
Some(
"insufficient_quota"
| "credit_balance_exhausted"
| "organization_spend_limit_exceeded"
| "project_spend_limit_exceeded"
| "organization_usage_limit_exceeded"
)
)
+45 -16
View File
@@ -584,23 +584,52 @@ fn map_api_error_keeps_other_400_errors_generic() {
#[test]
fn map_api_error_distinguishes_http_quota_errors_from_rate_limits() {
for error in [
serde_json::json!({"type": "insufficient_quota"}),
serde_json::json!({"code": "insufficient_quota"}),
serde_json::json!({"code": "credit_balance_exhausted"}),
serde_json::json!({"code": "organization_spend_limit_exceeded"}),
serde_json::json!({"code": "project_spend_limit_exceeded"}),
serde_json::json!({"code": "organization_usage_limit_exceeded"}),
serde_json::json!({"type": "rate_limit_error", "code": "rate_limit_exceeded"}),
serde_json::json!({"type": "rate_limit_error", "code": "slow_down"}),
use CodexErrorInfo::ApiKeySpendLimitExceeded;
use CodexErrorInfo::OrganizationSpendLimitExceeded;
use CodexErrorInfo::ProjectSpendLimitExceeded;
use CodexErrorInfo::UsageLimitExceeded;
let rate_limit = CodexErrorInfo::ResponseTooManyFailedAttempts {
http_status_code: Some(429),
};
for (error, expected) in [
(
serde_json::json!({"type": "insufficient_quota"}),
UsageLimitExceeded,
),
(
serde_json::json!({"code": "insufficient_quota"}),
UsageLimitExceeded,
),
(
serde_json::json!({"code": "credit_balance_exhausted"}),
UsageLimitExceeded,
),
(
serde_json::json!({"type": "insufficient_quota", "code": "api_key_spend_limit_exceeded"}),
ApiKeySpendLimitExceeded,
),
(
serde_json::json!({"type": "insufficient_quota", "code": "organization_spend_limit_exceeded"}),
OrganizationSpendLimitExceeded,
),
(
serde_json::json!({"type": "insufficient_quota", "code": "project_spend_limit_exceeded"}),
ProjectSpendLimitExceeded,
),
(
serde_json::json!({"code": "organization_usage_limit_exceeded"}),
UsageLimitExceeded,
),
(
serde_json::json!({"type": "rate_limit_error", "code": "rate_limit_exceeded"}),
rate_limit.clone(),
),
(
serde_json::json!({"type": "rate_limit_error", "code": "slow_down"}),
rate_limit,
),
] {
let expected = if error["type"] == "rate_limit_error" {
CodexErrorInfo::ResponseTooManyFailedAttempts {
http_status_code: Some(429),
}
} else {
CodexErrorInfo::UsageLimitExceeded
};
let err = map_api_error(ApiError::Transport(TransportError::Http {
retry_after: None,
status: http::StatusCode::TOO_MANY_REQUESTS,
+13
View File
@@ -2,6 +2,7 @@ use crate::rate_limits::RateLimitError;
use codex_client::TransportError;
use codex_http_client::RetryAfter;
use codex_protocol::error::ResponseIncompleteError;
use codex_protocol::error::SpendLimitScope;
use codex_protocol::protocol::MisalignmentErrorDetails;
use http::StatusCode;
use serde_json::Value;
@@ -25,6 +26,8 @@ pub enum ApiError {
ContextWindowExceeded,
#[error("quota exceeded")]
QuotaExceeded,
#[error("spend limit exceeded")]
SpendLimitExceeded(SpendLimitScope),
#[error("usage not included")]
UsageNotIncluded,
#[error("session budget exceeded")]
@@ -100,6 +103,16 @@ pub(crate) fn parse_terminal_provider_error(
.and_then(Value::as_str)
.filter(|message| !message.trim().is_empty());
match (code, status) {
("api_key_spend_limit_exceeded", None | Some(StatusCode::TOO_MANY_REQUESTS)) => {
Some(ApiError::SpendLimitExceeded(SpendLimitScope::ApiKey))
}
("project_spend_limit_exceeded", None | Some(StatusCode::TOO_MANY_REQUESTS)) => {
Some(ApiError::SpendLimitExceeded(SpendLimitScope::Project))
}
("organization_spend_limit_exceeded", None | Some(StatusCode::TOO_MANY_REQUESTS)) => {
Some(ApiError::SpendLimitExceeded(SpendLimitScope::Organization))
}
("inference_hook_denied", None | Some(StatusCode::FORBIDDEN)) => {
Some(ApiError::InferenceHookDenied {
message: message
+1
View File
@@ -710,6 +710,7 @@ async fn process_sse_with_treatment(
| ApiError::ContextWindowExceeded
| ApiError::InvalidRequest { .. }
| ApiError::SessionBudgetExceeded
| ApiError::SpendLimitExceeded(_)
| ApiError::InferenceHookDenied { .. }
| ApiError::AccessProgramNotEnabled { .. }
| ApiError::ContentPolicyViolation { .. }
@@ -45,12 +45,7 @@ pub(super) fn parse_failed_response(response: Option<Value>) -> ApiError {
};
match error.code.as_deref() {
Some(
"insufficient_quota"
| "credit_balance_exhausted"
| "organization_spend_limit_exceeded"
| "project_spend_limit_exceeded",
) => ApiError::QuotaExceeded,
Some("insufficient_quota" | "credit_balance_exhausted") => ApiError::QuotaExceeded,
Some("usage_not_included") => ApiError::UsageNotIncluded,
Some("cyber_policy") => ApiError::CyberPolicy {
message: cyber_policy_message(error.message),
@@ -115,6 +115,18 @@ async fn terminal_failure_with_retry_header_ends_stream_immediately() {
for (code, expected) in [
("flex_unavailable", CodexErrorInfo::FlexUnavailable),
("budget_reached", CodexErrorInfo::SessionBudgetExceeded),
(
"api_key_spend_limit_exceeded",
CodexErrorInfo::ApiKeySpendLimitExceeded,
),
(
"project_spend_limit_exceeded",
CodexErrorInfo::ProjectSpendLimitExceeded,
),
(
"organization_spend_limit_exceeded",
CodexErrorInfo::OrganizationSpendLimitExceeded,
),
("inference_hook_denied", CodexErrorInfo::InferenceHookDenied),
(
"context_length_exceeded",
+1
View File
@@ -38,6 +38,7 @@ pub(crate) fn is_terminal_sampling_outcome(error: &CodexErr) -> bool {
CodexErrorDetails::ResponseIncomplete(_)
| CodexErrorDetails::UsageAccountingFailed(_)
| CodexErrorDetails::SessionBudgetExceeded
| CodexErrorDetails::APISpendLimitExceeded(_)
| CodexErrorDetails::InferenceHookDenied { .. }
)
}
+1 -1
View File
@@ -947,7 +947,7 @@ pub(crate) async fn run_turn(
return Err(err);
}
let error = err.to_codex_protocol_error();
if matches!(error, CodexErrorInfo::UsageLimitExceeded) {
if error.is_usage_limit() {
// Preserve the completed answer while stopping automatic work.
sess.emit_turn_error_lifecycle(
turn_context.as_ref(),
+1 -2
View File
@@ -9,7 +9,6 @@ use crate::session::turn_context::TurnContext;
use crate::state::TaskKind;
use codex_features::Feature;
use codex_model_provider::RemoteCompactionSupport;
use codex_protocol::protocol::CodexErrorInfo;
use codex_user_input::ResolvedUserInput;
use tokio_util::sync::CancellationToken;
@@ -98,7 +97,7 @@ impl SessionTask for CompactTask {
return Err(err);
}
let error = err.to_codex_protocol_error();
if matches!(error, CodexErrorInfo::UsageLimitExceeded) {
if error.is_usage_limit() {
// Compaction already emitted the error; notify extensions without emitting it twice.
session
.emit_turn_error_lifecycle(ctx.as_ref(), error, err.details())
+12 -13
View File
@@ -13,12 +13,17 @@ use core_test_support::wait_for_event;
use pretty_assertions::assert_eq;
use serde_json::json;
#[test_case::test_case("insufficient_quota"; "quota")]
#[test_case::test_case("credit_balance_exhausted"; "credit_balance")]
#[test_case::test_case("organization_spend_limit_exceeded"; "organization_spend_limit")]
#[test_case::test_case("project_spend_limit_exceeded"; "project_spend_limit")]
#[test_case::test_case("insufficient_quota", CodexErrorInfo::UsageLimitExceeded, "Quota exceeded. Check your plan and billing details."; "quota")]
#[test_case::test_case("credit_balance_exhausted", CodexErrorInfo::UsageLimitExceeded, "Quota exceeded. Check your plan and billing details."; "credit_balance")]
#[test_case::test_case("api_key_spend_limit_exceeded", CodexErrorInfo::ApiKeySpendLimitExceeded, "Your API key has reached its spend limit."; "api_key_spend_limit")]
#[test_case::test_case("organization_spend_limit_exceeded", CodexErrorInfo::OrganizationSpendLimitExceeded, "Your API organization has reached its spend limit."; "organization_spend_limit")]
#[test_case::test_case("project_spend_limit_exceeded", CodexErrorInfo::ProjectSpendLimitExceeded, "Your API project has reached its spend limit."; "project_spend_limit")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn quota_exceeded_emits_single_error_event(code: &str) -> Result<()> {
async fn quota_exceeded_emits_single_error_event(
code: &str,
expected: CodexErrorInfo,
expected_message: &str,
) -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
@@ -59,14 +64,8 @@ async fn quota_exceeded_emits_single_error_event(code: &str) -> Result<()> {
match event {
EventMsg::Error(err) => {
error_events += 1;
assert_eq!(
err.message,
"Quota exceeded. Check your plan and billing details."
);
assert_eq!(
err.codex_error_info,
Some(CodexErrorInfo::UsageLimitExceeded)
);
assert_eq!(err.message, expected_message);
assert_eq!(err.codex_error_info, Some(expected.clone()));
}
EventMsg::TurnComplete(_) => break,
_ => {}
+12 -4
View File
@@ -1445,6 +1445,9 @@ async fn assert_terminal_failure(
}
#[test_case::test_case("flex_unavailable", 429, CodexErrorInfo::FlexUnavailable, "Flex capacity unavailable."; "flex")]
#[test_case::test_case("api_key_spend_limit_exceeded", 429, CodexErrorInfo::ApiKeySpendLimitExceeded, "Your API key has reached its spend limit."; "api_key_spend_limit_exceeded")]
#[test_case::test_case("project_spend_limit_exceeded", 429, CodexErrorInfo::ProjectSpendLimitExceeded, "Your API project has reached its spend limit."; "project_spend_limit_exceeded")]
#[test_case::test_case("organization_spend_limit_exceeded", 429, CodexErrorInfo::OrganizationSpendLimitExceeded, "Your API organization has reached its spend limit."; "organization_spend_limit_exceeded")]
#[test_case::test_case("model_not_found", 404, CodexErrorInfo::ModelNotFound, "Access to this deployment ended."; "model")]
#[test_case::test_case("inference_hook_denied", 403, CodexErrorInfo::InferenceHookDenied, "Blocked by the workspace hook."; "inference_hook_denied")]
#[tokio::test(flavor = "current_thread")]
@@ -1457,7 +1460,7 @@ async fn http_and_sse_provider_errors_are_terminal(
skip_if_no_network!(Ok(()));
let error = json!({
"code": code, "message": message
"code": code, "message": message, "headers": {"retry-after": "300"},
});
for response in [
ResponseTemplate::new(status)
@@ -1466,7 +1469,9 @@ async fn http_and_sse_provider_errors_are_terminal(
responses::sse_response(responses::sse(vec![
json!({"type": "error", "error": error}),
])),
responses::sse_response(responses::sse_failed("failed", code, message)),
responses::sse_response(responses::sse(vec![
json!({"type": "response.failed", "response": {"error": error}}),
])),
] {
let server = responses::start_mock_server().await;
let response_mock = responses::mount_response_once(&server, response).await;
@@ -1503,6 +1508,9 @@ async fn http_and_sse_provider_errors_are_terminal(
}
#[test_case::test_case("flex_unavailable", 429, CodexErrorInfo::FlexUnavailable, "Flex capacity unavailable."; "flex")]
#[test_case::test_case("api_key_spend_limit_exceeded", 429, CodexErrorInfo::ApiKeySpendLimitExceeded, "Your API key has reached its spend limit."; "api_key_spend_limit_exceeded")]
#[test_case::test_case("project_spend_limit_exceeded", 429, CodexErrorInfo::ProjectSpendLimitExceeded, "Your API project has reached its spend limit."; "project_spend_limit_exceeded")]
#[test_case::test_case("organization_spend_limit_exceeded", 429, CodexErrorInfo::OrganizationSpendLimitExceeded, "Your API organization has reached its spend limit."; "organization_spend_limit_exceeded")]
#[test_case::test_case("model_not_found", 404, CodexErrorInfo::ModelNotFound, "Access to this deployment ended."; "model")]
#[test_case::test_case("inference_hook_denied", 403, CodexErrorInfo::InferenceHookDenied, "Blocked by the workspace hook."; "inference_hook_denied")]
#[tokio::test(flavor = "current_thread")]
@@ -1515,11 +1523,11 @@ async fn websocket_provider_errors_are_terminal(
skip_if_no_network!(Ok(()));
let error = json!({
"code": code, "message": message
"code": code, "message": message, "headers": {"retry-after": "300"},
});
for event in [
json!({"type": "error", "status": status, "headers": {"retry-after": "300"}, "error": error}),
json!({"type": "error", "error": {"code": code, "message": message, "headers": {"retry-after": "300"}}}),
json!({"type": "error", "error": error}),
json!({"type": "response.failed", "response": {"error": error}}),
] {
let server = responses::start_websocket_server(vec![vec![
+4 -4
View File
@@ -30,7 +30,6 @@ use codex_extension_api::TurnStopInput;
use codex_otel::MetricsClient;
use codex_protocol::ThreadId;
use codex_protocol::items::TurnItem;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadGoalStatus;
@@ -426,13 +425,14 @@ where
return;
};
let reason = match input.error {
CodexErrorInfo::UsageLimitExceeded => ActiveGoalStopReason::UsageLimit,
let reason = if input.error.is_usage_limit() {
ActiveGoalStopReason::UsageLimit
} else {
// The turn has ended because the error was non-retryable or its
// retries were exhausted. Block the goal to prevent automatic
// continuation from looping and consuming tokens, as can happen
// with compaction errors.
_ => ActiveGoalStopReason::TurnError,
ActiveGoalStopReason::TurnError
};
if let Err(err) = runtime
.stop_active_goal_for_turn(input.turn_id, reason)
@@ -197,6 +197,9 @@ fn should_retry_guardian_review(outcome: &GuardianReviewOutcome) -> bool {
CodexErrorInfo::ContextWindowExceeded
| CodexErrorInfo::SessionBudgetExceeded
| CodexErrorInfo::UsageLimitExceeded
| CodexErrorInfo::ApiKeySpendLimitExceeded
| CodexErrorInfo::ProjectSpendLimitExceeded
| CodexErrorInfo::OrganizationSpendLimitExceeded
| CodexErrorInfo::InferenceHookDenied
| CodexErrorInfo::CyberPolicy
| CodexErrorInfo::BioPolicy
@@ -49,7 +49,7 @@ pub(super) fn sampler_failure_reason(error: &LunaSamplerError) -> &'static str {
ApiError::WebsocketMessageTooLarge => "websocket_message_too_large",
ApiError::ResponseIncomplete(_) => "incomplete_response",
ApiError::ContextWindowExceeded => "context_window_exceeded",
ApiError::QuotaExceeded => "quota_exceeded",
ApiError::SpendLimitExceeded(_) | ApiError::QuotaExceeded => "quota_exceeded",
ApiError::UsageNotIncluded => "usage_not_included",
ApiError::SessionBudgetExceeded => "session_budget_exceeded",
ApiError::Retryable { .. } => "retryable_api_error",
@@ -156,6 +156,7 @@ impl SamplingExecution {
)
| ApiError::ContextWindowExceeded
| ApiError::ResponseIncomplete(_)
| ApiError::SpendLimitExceeded(_)
| ApiError::QuotaExceeded
| ApiError::UsageNotIncluded
| ApiError::SessionBudgetExceeded
@@ -20,6 +20,9 @@ enum CodexErrorInfoWire {
ContextWindowExceeded,
SessionBudgetExceeded,
UsageLimitExceeded,
ApiKeySpendLimitExceeded,
ProjectSpendLimitExceeded,
OrganizationSpendLimitExceeded,
RateLimitExceeded,
FlexUnavailable,
ServerOverloaded,
+24
View File
@@ -75,6 +75,14 @@ pub enum SandboxErr {
LandlockRestrict,
}
/// Which customer-configured spend limit prevented inference.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SpendLimitScope {
ApiKey,
Project,
Organization,
}
pub struct CodexErr {
details: CodexErrorDetails,
retry_after: Option<RetryAfter>,
@@ -222,6 +230,12 @@ pub enum CodexErrorDetails {
ConnectionFailed(ConnectionFailedError),
#[error("Quota exceeded. Check your plan and billing details.")]
QuotaExceeded,
#[error("Your API {} has reached its spend limit.", match .0 {
SpendLimitScope::ApiKey => "key",
SpendLimitScope::Project => "project",
SpendLimitScope::Organization => "organization",
})]
APISpendLimitExceeded(SpendLimitScope),
#[error(
"To use Codex with your ChatGPT plan, upgrade to Plus: https://chatgpt.com/explore/plus."
)]
@@ -494,6 +508,7 @@ impl CodexErr {
| CodexErrorDetails::UsageAccountingFailed(_)
| CodexErrorDetails::UsageNotIncluded
| CodexErrorDetails::QuotaExceeded
| CodexErrorDetails::APISpendLimitExceeded(_)
| CodexErrorDetails::InvalidImageRequest()
| CodexErrorDetails::InvalidRequest(_)
| CodexErrorDetails::InvalidAudioInput { .. }
@@ -573,6 +588,15 @@ impl CodexErr {
match &self.details {
CodexErrorDetails::ContextWindowExceeded => CodexErrorInfo::ContextWindowExceeded,
CodexErrorDetails::SessionBudgetExceeded => CodexErrorInfo::SessionBudgetExceeded,
CodexErrorDetails::APISpendLimitExceeded(SpendLimitScope::ApiKey) => {
CodexErrorInfo::ApiKeySpendLimitExceeded
}
CodexErrorDetails::APISpendLimitExceeded(SpendLimitScope::Project) => {
CodexErrorInfo::ProjectSpendLimitExceeded
}
CodexErrorDetails::APISpendLimitExceeded(SpendLimitScope::Organization) => {
CodexErrorInfo::OrganizationSpendLimitExceeded
}
CodexErrorDetails::RateLimitExceeded(_) => CodexErrorInfo::RateLimitExceeded,
CodexErrorDetails::UsageLimitReached(_)
| CodexErrorDetails::QuotaExceeded
+16
View File
@@ -2120,6 +2120,9 @@ pub enum CodexErrorInfo {
ContextWindowExceeded,
SessionBudgetExceeded,
UsageLimitExceeded,
ApiKeySpendLimitExceeded,
ProjectSpendLimitExceeded,
OrganizationSpendLimitExceeded,
RateLimitExceeded,
FlexUnavailable,
ServerOverloaded,
@@ -2163,6 +2166,16 @@ pub enum CodexErrorInfo {
}
impl CodexErrorInfo {
pub fn is_usage_limit(&self) -> bool {
matches!(
self,
Self::UsageLimitExceeded
| Self::ApiKeySpendLimitExceeded
| Self::ProjectSpendLimitExceeded
| Self::OrganizationSpendLimitExceeded
)
}
/// Whether this error should mark the current turn as failed when replaying history.
pub fn affects_turn_status(&self) -> bool {
match self {
@@ -2170,6 +2183,9 @@ impl CodexErrorInfo {
Self::ContextWindowExceeded
| Self::SessionBudgetExceeded
| Self::UsageLimitExceeded
| Self::ApiKeySpendLimitExceeded
| Self::ProjectSpendLimitExceeded
| Self::OrganizationSpendLimitExceeded
| Self::RateLimitExceeded
| Self::FlexUnavailable
| Self::ServerOverloaded
+1 -1
View File
@@ -87,7 +87,7 @@ pub fn telemetry_api_error_message(error: &ApiError) -> String {
}
ApiError::ResponseIncomplete(_) => "incomplete response".to_string(),
ApiError::ContextWindowExceeded => "context window exceeded".to_string(),
ApiError::QuotaExceeded => "quota exceeded".to_string(),
ApiError::SpendLimitExceeded(_) | ApiError::QuotaExceeded => "quota exceeded".to_string(),
ApiError::UsageNotIncluded => "usage not included".to_string(),
ApiError::SessionBudgetExceeded => "session budget exceeded".to_string(),
ApiError::Retryable { .. } => "retryable error".to_string(),
+6 -1
View File
@@ -172,7 +172,12 @@ pub(super) fn app_server_rate_limit_error_kind(
| AppServerCodexErrorInfo::PolicyCapacityUnavailable => {
Some(RateLimitErrorKind::ServerOverloaded)
}
AppServerCodexErrorInfo::UsageLimitExceeded => Some(RateLimitErrorKind::UsageLimit),
AppServerCodexErrorInfo::UsageLimitExceeded
| AppServerCodexErrorInfo::ApiKeySpendLimitExceeded
| AppServerCodexErrorInfo::ProjectSpendLimitExceeded
| AppServerCodexErrorInfo::OrganizationSpendLimitExceeded => {
Some(RateLimitErrorKind::UsageLimit)
}
AppServerCodexErrorInfo::ResponseTooManyFailedAttempts {
http_status_code: Some(429),
} => Some(RateLimitErrorKind::Generic),
@@ -580,6 +580,9 @@ class CodexErrorInfoValue(str, Enum):
context_window_exceeded = "contextWindowExceeded"
session_budget_exceeded = "sessionBudgetExceeded"
usage_limit_exceeded = "usageLimitExceeded"
api_key_spend_limit_exceeded = "apiKeySpendLimitExceeded"
project_spend_limit_exceeded = "projectSpendLimitExceeded"
organization_spend_limit_exceeded = "organizationSpendLimitExceeded"
rate_limit_exceeded = "rateLimitExceeded"
flex_unavailable = "flexUnavailable"
server_overloaded = "serverOverloaded"