Label rollout compression failures by stage and I/O error kind (#45461)

## Why

Rollout compression failure counters report only that an operation failed,
without identifying the failing stage or I/O error kind.

## What changed

- Add `stage` and `error_kind` labels to failure counters for compression runs,
  individual files, materialization for append, and stale temporary file cleanup.
- Record failures at their source, including lock acquisition and task joins,
  and avoid counting file compression failures twice.
- Use static stage labels and a fixed set of error categories, keeping error
  messages, paths, and rollout contents out of metric tags.

GitOrigin-RevId: ac2bfc7ae4cec4e9f60de9557b03345a3986aa53
This commit is contained in:
jif
2026-09-14 16:37:47 +00:00
committed by copyberry
parent 374c4b2d82
commit 3fa9039bd7
2 changed files with 132 additions and 29 deletions
@@ -0,0 +1,53 @@
//! Failure labels for rollout compression. Only static stages and bounded I/O kinds
//! are exported; error messages, paths, and rollout contents never become metric tags.
use std::io;
use super::metrics;
pub(super) enum FailureMetric {
File,
Materialize,
Run,
TempCleanup,
}
impl FailureMetric {
pub(super) fn record(self, stage: &'static str, error: &io::Error) {
let (name, outcome_key) = match self {
Self::File => (metrics::FILE_COUNTER, "outcome"),
Self::Materialize => (metrics::MATERIALIZE_COUNTER, "outcome"),
Self::Run => (metrics::RUN_COUNTER, "status"),
Self::TempCleanup => (metrics::TEMP_CLEANUP_COUNTER, "outcome"),
};
let error_kind = match error.kind() {
io::ErrorKind::NotFound => "not_found",
io::ErrorKind::PermissionDenied => "permission_denied",
io::ErrorKind::AlreadyExists => "already_exists",
io::ErrorKind::InvalidInput => "invalid_input",
io::ErrorKind::InvalidData => "invalid_data",
io::ErrorKind::TimedOut => "timed_out",
io::ErrorKind::WriteZero => "write_zero",
io::ErrorKind::Interrupted => "interrupted",
io::ErrorKind::Unsupported => "unsupported",
io::ErrorKind::UnexpectedEof => "unexpected_eof",
io::ErrorKind::StorageFull => "storage_full",
io::ErrorKind::ReadOnlyFilesystem => "read_only_filesystem",
io::ErrorKind::NotADirectory => "not_a_directory",
io::ErrorKind::IsADirectory => "is_a_directory",
_ => "other",
};
let Some(metrics) = codex_otel::global() else {
return;
};
let _ = metrics.counter(
name,
/*inc*/ 1,
&[
(outcome_key, "failed"),
("stage", stage),
("error_kind", error_kind),
],
);
}
}