Persist thread artifacts in runtime store

This commit is contained in:
Jeremy Rose
2026-05-14 16:44:25 -07:00
parent 76814a24be
commit 4a72c41ed3
85 changed files with 1763 additions and 1451 deletions
+31
View File
@@ -68,6 +68,34 @@ fn apply_session_meta_from_item(metadata: &mut ThreadMetadata, meta_line: &Sessi
metadata.git_branch = git.branch.clone();
metadata.git_origin_url = git.repository_url.clone();
}
for (id, artifact) in &meta_line.artifacts {
match artifact {
Some(artifact) => {
let Some(created_at) =
chrono::DateTime::<chrono::Utc>::from_timestamp(artifact.created_at, 0)
else {
continue;
};
let updated_artifact = crate::ThreadArtifact {
thread_id: metadata.id,
id: id.clone(),
created_at,
artifact_type: artifact.artifact_type.clone(),
payload: artifact.payload.clone(),
};
if let Some(existing) = metadata
.artifacts
.iter_mut()
.find(|existing| existing.id == *id)
{
*existing = updated_artifact;
} else {
metadata.artifacts.push(updated_artifact);
}
}
None => metadata.artifacts.retain(|artifact| artifact.id != *id),
}
}
}
fn apply_turn_context(metadata: &mut ThreadMetadata, turn_ctx: &TurnContextItem) {
@@ -328,6 +356,7 @@ mod tests {
memory_mode: None,
},
git: None,
artifacts: std::collections::HashMap::new(),
}),
"test-provider",
);
@@ -462,6 +491,7 @@ mod tests {
memory_mode: None,
},
git: None,
artifacts: std::collections::HashMap::new(),
}),
"test-provider",
);
@@ -498,6 +528,7 @@ mod tests {
git_sha: None,
git_branch: None,
git_origin_url: None,
artifacts: Vec::new(),
}
}
+1
View File
@@ -43,6 +43,7 @@ pub use model::Stage1JobClaim;
pub use model::Stage1JobClaimOutcome;
pub use model::Stage1Output;
pub use model::Stage1StartupClaimParams;
pub use model::ThreadArtifact;
pub use model::ThreadGoal;
pub use model::ThreadGoalStatus;
pub use model::ThreadMetadata;
+3
View File
@@ -3,6 +3,7 @@ mod backfill_state;
mod graph;
mod log;
mod memories;
mod thread_artifact;
mod thread_goal;
mod thread_metadata;
@@ -24,6 +25,7 @@ pub use memories::Stage1JobClaim;
pub use memories::Stage1JobClaimOutcome;
pub use memories::Stage1Output;
pub use memories::Stage1StartupClaimParams;
pub use thread_artifact::ThreadArtifact;
pub use thread_goal::ThreadGoal;
pub use thread_goal::ThreadGoalStatus;
pub use thread_metadata::Anchor;
@@ -41,6 +43,7 @@ pub(crate) use memories::Stage1OutputRow;
pub(crate) use thread_goal::ThreadGoalRow;
pub(crate) use thread_metadata::ThreadRow;
pub(crate) use thread_metadata::anchor_from_item;
pub(crate) use thread_metadata::artifacts_to_json;
pub(crate) use thread_metadata::datetime_to_epoch_millis;
pub(crate) use thread_metadata::datetime_to_epoch_seconds;
pub(crate) use thread_metadata::epoch_millis_to_datetime;
@@ -0,0 +1,13 @@
use chrono::DateTime;
use chrono::Utc;
use codex_protocol::ThreadId;
use serde_json::Value as JsonValue;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ThreadArtifact {
pub thread_id: ThreadId,
pub id: String,
pub created_at: DateTime<Utc>,
pub artifact_type: String,
pub payload: JsonValue,
}
+57 -1
View File
@@ -7,6 +7,8 @@ use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::ThreadSource;
use serde::Deserialize;
use serde::Serialize;
use sqlx::Row;
use sqlx::sqlite::SqliteRow;
use std::path::PathBuf;
@@ -107,6 +109,8 @@ pub struct ThreadMetadata {
pub git_branch: Option<String>,
/// The git origin URL, if known.
pub git_origin_url: Option<String>,
/// Model-managed artifacts attached to the thread.
pub artifacts: Vec<crate::ThreadArtifact>,
}
/// Builder data required to construct [`ThreadMetadata`] without parsing filenames.
@@ -148,6 +152,8 @@ pub struct ThreadMetadataBuilder {
pub git_branch: Option<String>,
/// The git origin URL, if known.
pub git_origin_url: Option<String>,
/// Model-managed artifacts attached to the thread.
pub artifacts: Vec<crate::ThreadArtifact>,
}
impl ThreadMetadataBuilder {
@@ -177,6 +183,7 @@ impl ThreadMetadataBuilder {
git_sha: None,
git_branch: None,
git_origin_url: None,
artifacts: Vec::new(),
}
}
@@ -221,6 +228,7 @@ impl ThreadMetadataBuilder {
git_sha: self.git_sha.clone(),
git_branch: self.git_branch.clone(),
git_origin_url: self.git_origin_url.clone(),
artifacts: self.artifacts.clone(),
}
}
}
@@ -311,6 +319,9 @@ impl ThreadMetadata {
if self.git_origin_url != other.git_origin_url {
diffs.push("git_origin_url");
}
if self.artifacts != other.artifacts {
diffs.push("artifacts");
}
diffs
}
}
@@ -345,6 +356,7 @@ pub(crate) struct ThreadRow {
git_sha: Option<String>,
git_branch: Option<String>,
git_origin_url: Option<String>,
artifacts_json: String,
}
impl ThreadRow {
@@ -374,6 +386,7 @@ impl ThreadRow {
git_sha: row.try_get("git_sha")?,
git_branch: row.try_get("git_branch")?,
git_origin_url: row.try_get("git_origin_url")?,
artifacts_json: row.try_get("artifacts_json")?,
})
}
}
@@ -407,13 +420,15 @@ impl TryFrom<ThreadRow> for ThreadMetadata {
git_sha,
git_branch,
git_origin_url,
artifacts_json,
} = row;
let thread_source = thread_source
.map(|thread_source| thread_source.parse())
.transpose()
.map_err(anyhow::Error::msg)?;
let thread_id = ThreadId::try_from(id)?;
Ok(Self {
id: ThreadId::try_from(id)?,
id: thread_id,
rollout_path: PathBuf::from(rollout_path),
created_at: epoch_millis_to_datetime(created_at)?,
updated_at: epoch_millis_to_datetime(updated_at)?,
@@ -438,10 +453,49 @@ impl TryFrom<ThreadRow> for ThreadMetadata {
git_sha,
git_branch,
git_origin_url,
artifacts: artifacts_from_json(thread_id, artifacts_json.as_str())?,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct StoredThreadArtifactMetadata {
id: String,
created_at: i64,
artifact_type: String,
payload: serde_json::Value,
}
pub(crate) fn artifacts_to_json(artifacts: &[crate::ThreadArtifact]) -> Result<String> {
serde_json::to_string(
&artifacts
.iter()
.map(|artifact| StoredThreadArtifactMetadata {
id: artifact.id.clone(),
created_at: artifact.created_at.timestamp(),
artifact_type: artifact.artifact_type.clone(),
payload: artifact.payload.clone(),
})
.collect::<Vec<_>>(),
)
.map_err(Into::into)
}
fn artifacts_from_json(thread_id: ThreadId, json: &str) -> Result<Vec<crate::ThreadArtifact>> {
serde_json::from_str::<Vec<StoredThreadArtifactMetadata>>(json)?
.into_iter()
.map(|artifact| {
Ok(crate::ThreadArtifact {
thread_id,
id: artifact.id,
created_at: epoch_seconds_to_datetime(artifact.created_at)?,
artifact_type: artifact.artifact_type,
payload: artifact.payload,
})
})
.collect()
}
pub(crate) fn anchor_from_item(item: &ThreadMetadata, sort_key: SortKey) -> Option<Anchor> {
let ts = match sort_key {
SortKey::CreatedAt => item.created_at,
@@ -524,6 +578,7 @@ mod tests {
git_sha: None,
git_branch: None,
git_origin_url: None,
artifacts_json: "[]".to_string(),
}
}
@@ -554,6 +609,7 @@ mod tests {
git_sha: None,
git_branch: None,
git_origin_url: None,
artifacts: Vec::new(),
}
}
+2 -1
View File
@@ -155,7 +155,8 @@ SELECT
threads.archived_at,
threads.git_sha,
threads.git_branch,
threads.git_origin_url
threads.git_origin_url,
threads.artifacts_json
FROM threads
LEFT JOIN stage1_outputs
ON stage1_outputs.thread_id = threads.id
@@ -67,5 +67,6 @@ pub(super) fn test_thread_metadata(
git_sha: None,
git_branch: None,
git_origin_url: None,
artifacts: Vec::new(),
}
}
+152 -5
View File
@@ -31,7 +31,8 @@ SELECT
threads.archived_at,
threads.git_sha,
threads.git_branch,
threads.git_origin_url
threads.git_origin_url,
threads.artifacts_json
FROM threads
WHERE threads.id = ?
"#,
@@ -479,6 +480,7 @@ ON CONFLICT(child_thread_id) DO NOTHING
) -> anyhow::Result<bool> {
let updated_at = self.allocate_thread_updated_at(metadata.updated_at)?;
let preview = metadata_preview(metadata);
let artifacts_json = crate::model::artifacts_to_json(metadata.artifacts.as_slice())?;
let result = sqlx::query(
r#"
INSERT INTO threads (
@@ -509,8 +511,9 @@ INSERT INTO threads (
git_sha,
git_branch,
git_origin_url,
artifacts_json,
memory_mode
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO NOTHING
"#,
)
@@ -550,6 +553,7 @@ ON CONFLICT(id) DO NOTHING
.bind(metadata.git_sha.as_deref())
.bind(metadata.git_branch.as_deref())
.bind(metadata.git_origin_url.as_deref())
.bind(artifacts_json)
.bind("enabled")
.execute(self.pool.as_ref())
.await?;
@@ -571,6 +575,23 @@ ON CONFLICT(id) DO NOTHING
Ok(result.rows_affected() > 0)
}
pub async fn append_thread_artifact(
&self,
artifact: crate::ThreadArtifact,
) -> anyhow::Result<bool> {
let Some(mut metadata) = self.get_thread(artifact.thread_id).await? else {
return Ok(false);
};
metadata.artifacts.push(artifact);
let artifacts_json = crate::model::artifacts_to_json(metadata.artifacts.as_slice())?;
let result = sqlx::query("UPDATE threads SET artifacts_json = ? WHERE id = ?")
.bind(artifacts_json)
.bind(metadata.id.to_string())
.execute(self.pool.as_ref())
.await?;
Ok(result.rows_affected() > 0)
}
pub async fn update_thread_title(
&self,
thread_id: ThreadId,
@@ -685,6 +706,7 @@ WHERE id = ?
// Backfill/reconcile callers merge existing git info before upserting, but that
// read/modify/write is not atomic. Preserve non-null SQLite git fields here so
// an explicit metadata update cannot be lost if a stale rollout upsert lands later.
let artifacts_json = crate::model::artifacts_to_json(metadata.artifacts.as_slice())?;
sqlx::query(
r#"
INSERT INTO threads (
@@ -715,8 +737,9 @@ INSERT INTO threads (
git_sha,
git_branch,
git_origin_url,
artifacts_json,
memory_mode
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
rollout_path = excluded.rollout_path,
created_at = excluded.created_at,
@@ -743,7 +766,8 @@ ON CONFLICT(id) DO UPDATE SET
archived_at = excluded.archived_at,
git_sha = COALESCE(threads.git_sha, excluded.git_sha),
git_branch = COALESCE(threads.git_branch, excluded.git_branch),
git_origin_url = COALESCE(threads.git_origin_url, excluded.git_origin_url)
git_origin_url = COALESCE(threads.git_origin_url, excluded.git_origin_url),
artifacts_json = excluded.artifacts_json
"#,
)
.bind(metadata.id.to_string())
@@ -782,6 +806,7 @@ ON CONFLICT(id) DO UPDATE SET
.bind(metadata.git_sha.as_deref())
.bind(metadata.git_branch.as_deref())
.bind(metadata.git_origin_url.as_deref())
.bind(artifacts_json)
.bind(creation_memory_mode.unwrap_or("enabled"))
.execute(self.pool.as_ref())
.await?;
@@ -998,7 +1023,8 @@ SELECT
threads.archived_at,
threads.git_sha,
threads.git_branch,
threads.git_origin_url
threads.git_origin_url,
threads.artifacts_json
"#,
);
}
@@ -1169,6 +1195,7 @@ mod tests {
use codex_protocol::protocol::SessionMeta;
use codex_protocol::protocol::SessionMetaLine;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::ThreadArtifact as RolloutThreadArtifact;
use pretty_assertions::assert_eq;
use std::path::PathBuf;
@@ -1210,6 +1237,124 @@ mod tests {
assert_eq!(memory_mode, "disabled");
}
#[tokio::test]
async fn thread_artifacts_restore_from_session_meta_in_creation_order() {
let codex_home = unique_temp_dir();
let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string())
.await
.expect("state db should initialize");
let thread_id =
ThreadId::from_string("00000000-0000-0000-0000-000000000321").expect("valid thread id");
let metadata = test_thread_metadata(&codex_home, thread_id, codex_home.clone());
runtime
.upsert_thread(&metadata)
.await
.expect("thread insert should succeed");
let builder = ThreadMetadataBuilder::new(
thread_id,
metadata.rollout_path.clone(),
metadata.created_at,
SessionSource::Cli,
);
runtime
.apply_rollout_items(
&builder,
&[
RolloutItem::SessionMeta(SessionMetaLine {
meta: SessionMeta {
id: thread_id,
timestamp: metadata.created_at.to_rfc3339(),
source: SessionSource::Cli,
..Default::default()
},
git: None,
artifacts: std::collections::HashMap::from([(
"artifact-1".to_string(),
Some(RolloutThreadArtifact {
created_at: 1_700_000_001,
artifact_type: "github/pull_request".to_string(),
payload: serde_json::json!({ "number": 12 }),
}),
)]),
}),
RolloutItem::SessionMeta(SessionMetaLine {
meta: SessionMeta {
id: thread_id,
timestamp: metadata.created_at.to_rfc3339(),
source: SessionSource::Cli,
..Default::default()
},
git: None,
artifacts: std::collections::HashMap::from([(
"artifact-2".to_string(),
Some(RolloutThreadArtifact {
created_at: 1_700_000_002,
artifact_type: "github/pull_request".to_string(),
payload: serde_json::json!({ "number": 34 }),
}),
)]),
}),
],
/*new_thread_memory_mode*/ None,
/*updated_at_override*/ None,
)
.await
.expect("artifacts should restore from rollout metadata");
assert_eq!(
runtime
.get_thread(thread_id)
.await
.expect("thread should load")
.expect("thread should exist")
.artifacts
.iter()
.map(|artifact| artifact.id.as_str())
.collect::<Vec<_>>(),
vec!["artifact-1", "artifact-2"]
);
runtime
.apply_rollout_items(
&builder,
&[RolloutItem::SessionMeta(SessionMetaLine {
meta: SessionMeta {
id: thread_id,
timestamp: metadata.created_at.to_rfc3339(),
source: SessionSource::Cli,
..Default::default()
},
git: None,
artifacts: std::collections::HashMap::from([
(
"artifact-1".to_string(),
Some(RolloutThreadArtifact {
created_at: 1_700_000_001,
artifact_type: "github/pull_request".to_string(),
payload: serde_json::json!({ "number": 56 }),
}),
),
("artifact-2".to_string(), None),
]),
})],
/*new_thread_memory_mode*/ None,
/*updated_at_override*/ None,
)
.await
.expect("artifact patches should apply from rollout metadata");
let artifacts = runtime
.get_thread(thread_id)
.await
.expect("thread should load")
.expect("thread should exist")
.artifacts;
assert_eq!(artifacts.len(), 1);
assert_eq!(artifacts[0].id, "artifact-1");
assert_eq!(artifacts[0].payload, serde_json::json!({ "number": 56 }));
}
#[tokio::test]
async fn list_threads_updated_after_returns_oldest_changes_first() {
let codex_home = unique_temp_dir();
@@ -1405,6 +1550,7 @@ mod tests {
memory_mode: Some("polluted".to_string()),
},
git: None,
artifacts: std::collections::HashMap::new(),
})];
runtime
@@ -1468,6 +1614,7 @@ mod tests {
branch: Some("rollout-branch".to_string()),
repository_url: Some("git@example.com:openai/codex.git".to_string()),
}),
artifacts: std::collections::HashMap::new(),
})];
runtime