mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
Persist thread artifacts in runtime store
This commit is contained in:
@@ -68,6 +68,34 @@ fn apply_session_meta_from_item(metadata: &mut ThreadMetadata, meta_line: &Sessi
|
||||
metadata.git_branch = git.branch.clone();
|
||||
metadata.git_origin_url = git.repository_url.clone();
|
||||
}
|
||||
for (id, artifact) in &meta_line.artifacts {
|
||||
match artifact {
|
||||
Some(artifact) => {
|
||||
let Some(created_at) =
|
||||
chrono::DateTime::<chrono::Utc>::from_timestamp(artifact.created_at, 0)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let updated_artifact = crate::ThreadArtifact {
|
||||
thread_id: metadata.id,
|
||||
id: id.clone(),
|
||||
created_at,
|
||||
artifact_type: artifact.artifact_type.clone(),
|
||||
payload: artifact.payload.clone(),
|
||||
};
|
||||
if let Some(existing) = metadata
|
||||
.artifacts
|
||||
.iter_mut()
|
||||
.find(|existing| existing.id == *id)
|
||||
{
|
||||
*existing = updated_artifact;
|
||||
} else {
|
||||
metadata.artifacts.push(updated_artifact);
|
||||
}
|
||||
}
|
||||
None => metadata.artifacts.retain(|artifact| artifact.id != *id),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_turn_context(metadata: &mut ThreadMetadata, turn_ctx: &TurnContextItem) {
|
||||
@@ -328,6 +356,7 @@ mod tests {
|
||||
memory_mode: None,
|
||||
},
|
||||
git: None,
|
||||
artifacts: std::collections::HashMap::new(),
|
||||
}),
|
||||
"test-provider",
|
||||
);
|
||||
@@ -462,6 +491,7 @@ mod tests {
|
||||
memory_mode: None,
|
||||
},
|
||||
git: None,
|
||||
artifacts: std::collections::HashMap::new(),
|
||||
}),
|
||||
"test-provider",
|
||||
);
|
||||
@@ -498,6 +528,7 @@ mod tests {
|
||||
git_sha: None,
|
||||
git_branch: None,
|
||||
git_origin_url: None,
|
||||
artifacts: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ pub use model::Stage1JobClaim;
|
||||
pub use model::Stage1JobClaimOutcome;
|
||||
pub use model::Stage1Output;
|
||||
pub use model::Stage1StartupClaimParams;
|
||||
pub use model::ThreadArtifact;
|
||||
pub use model::ThreadGoal;
|
||||
pub use model::ThreadGoalStatus;
|
||||
pub use model::ThreadMetadata;
|
||||
|
||||
@@ -3,6 +3,7 @@ mod backfill_state;
|
||||
mod graph;
|
||||
mod log;
|
||||
mod memories;
|
||||
mod thread_artifact;
|
||||
mod thread_goal;
|
||||
mod thread_metadata;
|
||||
|
||||
@@ -24,6 +25,7 @@ pub use memories::Stage1JobClaim;
|
||||
pub use memories::Stage1JobClaimOutcome;
|
||||
pub use memories::Stage1Output;
|
||||
pub use memories::Stage1StartupClaimParams;
|
||||
pub use thread_artifact::ThreadArtifact;
|
||||
pub use thread_goal::ThreadGoal;
|
||||
pub use thread_goal::ThreadGoalStatus;
|
||||
pub use thread_metadata::Anchor;
|
||||
@@ -41,6 +43,7 @@ pub(crate) use memories::Stage1OutputRow;
|
||||
pub(crate) use thread_goal::ThreadGoalRow;
|
||||
pub(crate) use thread_metadata::ThreadRow;
|
||||
pub(crate) use thread_metadata::anchor_from_item;
|
||||
pub(crate) use thread_metadata::artifacts_to_json;
|
||||
pub(crate) use thread_metadata::datetime_to_epoch_millis;
|
||||
pub(crate) use thread_metadata::datetime_to_epoch_seconds;
|
||||
pub(crate) use thread_metadata::epoch_millis_to_datetime;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
use chrono::DateTime;
|
||||
use chrono::Utc;
|
||||
use codex_protocol::ThreadId;
|
||||
use serde_json::Value as JsonValue;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ThreadArtifact {
|
||||
pub thread_id: ThreadId,
|
||||
pub id: String,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub artifact_type: String,
|
||||
pub payload: JsonValue,
|
||||
}
|
||||
@@ -7,6 +7,8 @@ use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::ThreadSource;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use sqlx::Row;
|
||||
use sqlx::sqlite::SqliteRow;
|
||||
use std::path::PathBuf;
|
||||
@@ -107,6 +109,8 @@ pub struct ThreadMetadata {
|
||||
pub git_branch: Option<String>,
|
||||
/// The git origin URL, if known.
|
||||
pub git_origin_url: Option<String>,
|
||||
/// Model-managed artifacts attached to the thread.
|
||||
pub artifacts: Vec<crate::ThreadArtifact>,
|
||||
}
|
||||
|
||||
/// Builder data required to construct [`ThreadMetadata`] without parsing filenames.
|
||||
@@ -148,6 +152,8 @@ pub struct ThreadMetadataBuilder {
|
||||
pub git_branch: Option<String>,
|
||||
/// The git origin URL, if known.
|
||||
pub git_origin_url: Option<String>,
|
||||
/// Model-managed artifacts attached to the thread.
|
||||
pub artifacts: Vec<crate::ThreadArtifact>,
|
||||
}
|
||||
|
||||
impl ThreadMetadataBuilder {
|
||||
@@ -177,6 +183,7 @@ impl ThreadMetadataBuilder {
|
||||
git_sha: None,
|
||||
git_branch: None,
|
||||
git_origin_url: None,
|
||||
artifacts: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,6 +228,7 @@ impl ThreadMetadataBuilder {
|
||||
git_sha: self.git_sha.clone(),
|
||||
git_branch: self.git_branch.clone(),
|
||||
git_origin_url: self.git_origin_url.clone(),
|
||||
artifacts: self.artifacts.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -311,6 +319,9 @@ impl ThreadMetadata {
|
||||
if self.git_origin_url != other.git_origin_url {
|
||||
diffs.push("git_origin_url");
|
||||
}
|
||||
if self.artifacts != other.artifacts {
|
||||
diffs.push("artifacts");
|
||||
}
|
||||
diffs
|
||||
}
|
||||
}
|
||||
@@ -345,6 +356,7 @@ pub(crate) struct ThreadRow {
|
||||
git_sha: Option<String>,
|
||||
git_branch: Option<String>,
|
||||
git_origin_url: Option<String>,
|
||||
artifacts_json: String,
|
||||
}
|
||||
|
||||
impl ThreadRow {
|
||||
@@ -374,6 +386,7 @@ impl ThreadRow {
|
||||
git_sha: row.try_get("git_sha")?,
|
||||
git_branch: row.try_get("git_branch")?,
|
||||
git_origin_url: row.try_get("git_origin_url")?,
|
||||
artifacts_json: row.try_get("artifacts_json")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -407,13 +420,15 @@ impl TryFrom<ThreadRow> for ThreadMetadata {
|
||||
git_sha,
|
||||
git_branch,
|
||||
git_origin_url,
|
||||
artifacts_json,
|
||||
} = row;
|
||||
let thread_source = thread_source
|
||||
.map(|thread_source| thread_source.parse())
|
||||
.transpose()
|
||||
.map_err(anyhow::Error::msg)?;
|
||||
let thread_id = ThreadId::try_from(id)?;
|
||||
Ok(Self {
|
||||
id: ThreadId::try_from(id)?,
|
||||
id: thread_id,
|
||||
rollout_path: PathBuf::from(rollout_path),
|
||||
created_at: epoch_millis_to_datetime(created_at)?,
|
||||
updated_at: epoch_millis_to_datetime(updated_at)?,
|
||||
@@ -438,10 +453,49 @@ impl TryFrom<ThreadRow> for ThreadMetadata {
|
||||
git_sha,
|
||||
git_branch,
|
||||
git_origin_url,
|
||||
artifacts: artifacts_from_json(thread_id, artifacts_json.as_str())?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
struct StoredThreadArtifactMetadata {
|
||||
id: String,
|
||||
created_at: i64,
|
||||
artifact_type: String,
|
||||
payload: serde_json::Value,
|
||||
}
|
||||
|
||||
pub(crate) fn artifacts_to_json(artifacts: &[crate::ThreadArtifact]) -> Result<String> {
|
||||
serde_json::to_string(
|
||||
&artifacts
|
||||
.iter()
|
||||
.map(|artifact| StoredThreadArtifactMetadata {
|
||||
id: artifact.id.clone(),
|
||||
created_at: artifact.created_at.timestamp(),
|
||||
artifact_type: artifact.artifact_type.clone(),
|
||||
payload: artifact.payload.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn artifacts_from_json(thread_id: ThreadId, json: &str) -> Result<Vec<crate::ThreadArtifact>> {
|
||||
serde_json::from_str::<Vec<StoredThreadArtifactMetadata>>(json)?
|
||||
.into_iter()
|
||||
.map(|artifact| {
|
||||
Ok(crate::ThreadArtifact {
|
||||
thread_id,
|
||||
id: artifact.id,
|
||||
created_at: epoch_seconds_to_datetime(artifact.created_at)?,
|
||||
artifact_type: artifact.artifact_type,
|
||||
payload: artifact.payload,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn anchor_from_item(item: &ThreadMetadata, sort_key: SortKey) -> Option<Anchor> {
|
||||
let ts = match sort_key {
|
||||
SortKey::CreatedAt => item.created_at,
|
||||
@@ -524,6 +578,7 @@ mod tests {
|
||||
git_sha: None,
|
||||
git_branch: None,
|
||||
git_origin_url: None,
|
||||
artifacts_json: "[]".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -554,6 +609,7 @@ mod tests {
|
||||
git_sha: None,
|
||||
git_branch: None,
|
||||
git_origin_url: None,
|
||||
artifacts: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -155,7 +155,8 @@ SELECT
|
||||
threads.archived_at,
|
||||
threads.git_sha,
|
||||
threads.git_branch,
|
||||
threads.git_origin_url
|
||||
threads.git_origin_url,
|
||||
threads.artifacts_json
|
||||
FROM threads
|
||||
LEFT JOIN stage1_outputs
|
||||
ON stage1_outputs.thread_id = threads.id
|
||||
|
||||
@@ -67,5 +67,6 @@ pub(super) fn test_thread_metadata(
|
||||
git_sha: None,
|
||||
git_branch: None,
|
||||
git_origin_url: None,
|
||||
artifacts: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,8 @@ SELECT
|
||||
threads.archived_at,
|
||||
threads.git_sha,
|
||||
threads.git_branch,
|
||||
threads.git_origin_url
|
||||
threads.git_origin_url,
|
||||
threads.artifacts_json
|
||||
FROM threads
|
||||
WHERE threads.id = ?
|
||||
"#,
|
||||
@@ -479,6 +480,7 @@ ON CONFLICT(child_thread_id) DO NOTHING
|
||||
) -> anyhow::Result<bool> {
|
||||
let updated_at = self.allocate_thread_updated_at(metadata.updated_at)?;
|
||||
let preview = metadata_preview(metadata);
|
||||
let artifacts_json = crate::model::artifacts_to_json(metadata.artifacts.as_slice())?;
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO threads (
|
||||
@@ -509,8 +511,9 @@ INSERT INTO threads (
|
||||
git_sha,
|
||||
git_branch,
|
||||
git_origin_url,
|
||||
artifacts_json,
|
||||
memory_mode
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO NOTHING
|
||||
"#,
|
||||
)
|
||||
@@ -550,6 +553,7 @@ ON CONFLICT(id) DO NOTHING
|
||||
.bind(metadata.git_sha.as_deref())
|
||||
.bind(metadata.git_branch.as_deref())
|
||||
.bind(metadata.git_origin_url.as_deref())
|
||||
.bind(artifacts_json)
|
||||
.bind("enabled")
|
||||
.execute(self.pool.as_ref())
|
||||
.await?;
|
||||
@@ -571,6 +575,23 @@ ON CONFLICT(id) DO NOTHING
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
pub async fn append_thread_artifact(
|
||||
&self,
|
||||
artifact: crate::ThreadArtifact,
|
||||
) -> anyhow::Result<bool> {
|
||||
let Some(mut metadata) = self.get_thread(artifact.thread_id).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
metadata.artifacts.push(artifact);
|
||||
let artifacts_json = crate::model::artifacts_to_json(metadata.artifacts.as_slice())?;
|
||||
let result = sqlx::query("UPDATE threads SET artifacts_json = ? WHERE id = ?")
|
||||
.bind(artifacts_json)
|
||||
.bind(metadata.id.to_string())
|
||||
.execute(self.pool.as_ref())
|
||||
.await?;
|
||||
Ok(result.rows_affected() > 0)
|
||||
}
|
||||
|
||||
pub async fn update_thread_title(
|
||||
&self,
|
||||
thread_id: ThreadId,
|
||||
@@ -685,6 +706,7 @@ WHERE id = ?
|
||||
// Backfill/reconcile callers merge existing git info before upserting, but that
|
||||
// read/modify/write is not atomic. Preserve non-null SQLite git fields here so
|
||||
// an explicit metadata update cannot be lost if a stale rollout upsert lands later.
|
||||
let artifacts_json = crate::model::artifacts_to_json(metadata.artifacts.as_slice())?;
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO threads (
|
||||
@@ -715,8 +737,9 @@ INSERT INTO threads (
|
||||
git_sha,
|
||||
git_branch,
|
||||
git_origin_url,
|
||||
artifacts_json,
|
||||
memory_mode
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
rollout_path = excluded.rollout_path,
|
||||
created_at = excluded.created_at,
|
||||
@@ -743,7 +766,8 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
archived_at = excluded.archived_at,
|
||||
git_sha = COALESCE(threads.git_sha, excluded.git_sha),
|
||||
git_branch = COALESCE(threads.git_branch, excluded.git_branch),
|
||||
git_origin_url = COALESCE(threads.git_origin_url, excluded.git_origin_url)
|
||||
git_origin_url = COALESCE(threads.git_origin_url, excluded.git_origin_url),
|
||||
artifacts_json = excluded.artifacts_json
|
||||
"#,
|
||||
)
|
||||
.bind(metadata.id.to_string())
|
||||
@@ -782,6 +806,7 @@ ON CONFLICT(id) DO UPDATE SET
|
||||
.bind(metadata.git_sha.as_deref())
|
||||
.bind(metadata.git_branch.as_deref())
|
||||
.bind(metadata.git_origin_url.as_deref())
|
||||
.bind(artifacts_json)
|
||||
.bind(creation_memory_mode.unwrap_or("enabled"))
|
||||
.execute(self.pool.as_ref())
|
||||
.await?;
|
||||
@@ -998,7 +1023,8 @@ SELECT
|
||||
threads.archived_at,
|
||||
threads.git_sha,
|
||||
threads.git_branch,
|
||||
threads.git_origin_url
|
||||
threads.git_origin_url,
|
||||
threads.artifacts_json
|
||||
"#,
|
||||
);
|
||||
}
|
||||
@@ -1169,6 +1195,7 @@ mod tests {
|
||||
use codex_protocol::protocol::SessionMeta;
|
||||
use codex_protocol::protocol::SessionMetaLine;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::ThreadArtifact as RolloutThreadArtifact;
|
||||
use pretty_assertions::assert_eq;
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -1210,6 +1237,124 @@ mod tests {
|
||||
assert_eq!(memory_mode, "disabled");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn thread_artifacts_restore_from_session_meta_in_creation_order() {
|
||||
let codex_home = unique_temp_dir();
|
||||
let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string())
|
||||
.await
|
||||
.expect("state db should initialize");
|
||||
let thread_id =
|
||||
ThreadId::from_string("00000000-0000-0000-0000-000000000321").expect("valid thread id");
|
||||
let metadata = test_thread_metadata(&codex_home, thread_id, codex_home.clone());
|
||||
runtime
|
||||
.upsert_thread(&metadata)
|
||||
.await
|
||||
.expect("thread insert should succeed");
|
||||
|
||||
let builder = ThreadMetadataBuilder::new(
|
||||
thread_id,
|
||||
metadata.rollout_path.clone(),
|
||||
metadata.created_at,
|
||||
SessionSource::Cli,
|
||||
);
|
||||
runtime
|
||||
.apply_rollout_items(
|
||||
&builder,
|
||||
&[
|
||||
RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
id: thread_id,
|
||||
timestamp: metadata.created_at.to_rfc3339(),
|
||||
source: SessionSource::Cli,
|
||||
..Default::default()
|
||||
},
|
||||
git: None,
|
||||
artifacts: std::collections::HashMap::from([(
|
||||
"artifact-1".to_string(),
|
||||
Some(RolloutThreadArtifact {
|
||||
created_at: 1_700_000_001,
|
||||
artifact_type: "github/pull_request".to_string(),
|
||||
payload: serde_json::json!({ "number": 12 }),
|
||||
}),
|
||||
)]),
|
||||
}),
|
||||
RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
id: thread_id,
|
||||
timestamp: metadata.created_at.to_rfc3339(),
|
||||
source: SessionSource::Cli,
|
||||
..Default::default()
|
||||
},
|
||||
git: None,
|
||||
artifacts: std::collections::HashMap::from([(
|
||||
"artifact-2".to_string(),
|
||||
Some(RolloutThreadArtifact {
|
||||
created_at: 1_700_000_002,
|
||||
artifact_type: "github/pull_request".to_string(),
|
||||
payload: serde_json::json!({ "number": 34 }),
|
||||
}),
|
||||
)]),
|
||||
}),
|
||||
],
|
||||
/*new_thread_memory_mode*/ None,
|
||||
/*updated_at_override*/ None,
|
||||
)
|
||||
.await
|
||||
.expect("artifacts should restore from rollout metadata");
|
||||
|
||||
assert_eq!(
|
||||
runtime
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load")
|
||||
.expect("thread should exist")
|
||||
.artifacts
|
||||
.iter()
|
||||
.map(|artifact| artifact.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["artifact-1", "artifact-2"]
|
||||
);
|
||||
|
||||
runtime
|
||||
.apply_rollout_items(
|
||||
&builder,
|
||||
&[RolloutItem::SessionMeta(SessionMetaLine {
|
||||
meta: SessionMeta {
|
||||
id: thread_id,
|
||||
timestamp: metadata.created_at.to_rfc3339(),
|
||||
source: SessionSource::Cli,
|
||||
..Default::default()
|
||||
},
|
||||
git: None,
|
||||
artifacts: std::collections::HashMap::from([
|
||||
(
|
||||
"artifact-1".to_string(),
|
||||
Some(RolloutThreadArtifact {
|
||||
created_at: 1_700_000_001,
|
||||
artifact_type: "github/pull_request".to_string(),
|
||||
payload: serde_json::json!({ "number": 56 }),
|
||||
}),
|
||||
),
|
||||
("artifact-2".to_string(), None),
|
||||
]),
|
||||
})],
|
||||
/*new_thread_memory_mode*/ None,
|
||||
/*updated_at_override*/ None,
|
||||
)
|
||||
.await
|
||||
.expect("artifact patches should apply from rollout metadata");
|
||||
|
||||
let artifacts = runtime
|
||||
.get_thread(thread_id)
|
||||
.await
|
||||
.expect("thread should load")
|
||||
.expect("thread should exist")
|
||||
.artifacts;
|
||||
assert_eq!(artifacts.len(), 1);
|
||||
assert_eq!(artifacts[0].id, "artifact-1");
|
||||
assert_eq!(artifacts[0].payload, serde_json::json!({ "number": 56 }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_threads_updated_after_returns_oldest_changes_first() {
|
||||
let codex_home = unique_temp_dir();
|
||||
@@ -1405,6 +1550,7 @@ mod tests {
|
||||
memory_mode: Some("polluted".to_string()),
|
||||
},
|
||||
git: None,
|
||||
artifacts: std::collections::HashMap::new(),
|
||||
})];
|
||||
|
||||
runtime
|
||||
@@ -1468,6 +1614,7 @@ mod tests {
|
||||
branch: Some("rollout-branch".to_string()),
|
||||
repository_url: Some("git@example.com:openai/codex.git".to_string()),
|
||||
}),
|
||||
artifacts: std::collections::HashMap::new(),
|
||||
})];
|
||||
|
||||
runtime
|
||||
|
||||
Reference in New Issue
Block a user