Track connectors detected in external agent sessions (#36959)

## What changed

- Return connector candidates found in detected external agent sessions, aggregating session counts by connector name.
- Resolve connector names from both cached plugin manifests and project MCP server metadata.
- Persist detected connector clues in the session import ledger and merge them into completed imports without case-insensitive duplicates.

## Testing

- Add coverage for project MCP metadata detection and for accumulating detected and imported connector candidates.

GitOrigin-RevId: 97a7dfe4a61599686b2222cddd9595acec8b4501
This commit is contained in:
charlesgong-openai
2026-08-04 19:45:56 +00:00
committed by copyberry
parent 4bd5b9fd09
commit 92689b6b7b
8 changed files with 399 additions and 19 deletions
@@ -30,6 +30,7 @@ use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::ServerNotification;
use codex_arg0::Arg0DispatchPaths;
use codex_core::ThreadManager;
use codex_external_agent_migration::DetectedConnectorCandidate;
use codex_external_agent_migration::ExternalAgentConfigDetectOptions;
use codex_external_agent_migration::ExternalAgentConfigImportItemResult as CoreImportItemResult;
use codex_external_agent_migration::ExternalAgentConfigImportOutcome as CoreImportOutcome;
@@ -40,11 +41,13 @@ use codex_external_agent_migration::PluginImportOutcome;
use codex_external_agent_migration::record_import_error;
use codex_external_agent_migration::sessions::ExternalAgentSessionMigration as CoreSessionMigration;
use codex_external_agent_migration::sessions::read_imported_connector_candidates;
use codex_external_agent_migration::sessions::record_detected_session_connectors;
use codex_features::Feature;
use codex_rollout::StateDbHandle;
use codex_state::ExternalAgentConfigImportFailureRecord;
use codex_state::ExternalAgentConfigImportSuccessRecord;
use codex_thread_store::ThreadStore;
use std::collections::BTreeMap;
use std::collections::HashSet;
use std::path::PathBuf;
@@ -146,8 +149,24 @@ impl ExternalAgentConfigRequestProcessor {
.detect(options)
.await
.map_err(|err| internal_error(err.to_string()))?;
let sessions = items
.iter()
.filter_map(|item| item.details.as_ref())
.flat_map(|details| details.sessions.iter().cloned())
.collect::<Vec<_>>();
let (connector_names_by_source_path, connectors) =
detected_session_connectors(&migration_service, &sessions);
record_detected_session_connectors(
self.migration_service.codex_home(),
connector_names_by_source_path,
)
.map_err(|err| {
internal_error(format!(
"failed to record detected connector candidates: {err}"
))
})?;
Ok(detect_response(items))
Ok(detect_response(items, connectors))
}
pub(crate) async fn import(
@@ -251,6 +270,8 @@ impl ExternalAgentConfigRequestProcessor {
});
let pending_plugin_imports = import_outcome.pending_plugin_imports;
tokio::spawn(async move {
let connector_names_by_source_path =
detected_session_connectors(&plugin_migration_service, &pending_session_imports).0;
let session_progress_outgoing = Arc::clone(&outgoing);
let session_import_id = import_id.clone();
let session_imports = async move {
@@ -260,6 +281,7 @@ impl ExternalAgentConfigRequestProcessor {
pending_session_imports,
session_import_result,
session_metadata_mode,
connector_names_by_source_path,
)
.await;
send_import_progress(&session_progress_outgoing, &session_import_id, &item_result)
@@ -688,6 +710,50 @@ async fn record_import_history(
.await
}
fn detected_session_connectors(
migration_service: &ExternalAgentConfigService,
sessions: &[CoreSessionMigration],
) -> (
BTreeMap<PathBuf, Vec<String>>,
Vec<DetectedConnectorCandidate>,
) {
let mut connector_names_by_source_path = BTreeMap::new();
let mut connectors_by_name = BTreeMap::<String, DetectedConnectorCandidate>::new();
for session in sessions {
if !session.path.is_file() {
continue;
}
let session_connectors =
migration_service.detect_session_connectors(std::slice::from_ref(session));
if session_connectors.is_empty() {
continue;
}
if let Ok(source_path) = std::fs::canonicalize(&session.path) {
connector_names_by_source_path.insert(
source_path,
session_connectors
.iter()
.map(|candidate| candidate.name.clone())
.collect(),
);
}
for candidate in session_connectors {
let key = candidate.name.to_lowercase();
let session_count = candidate.session_count;
let connector = connectors_by_name.entry(key).or_insert_with(|| {
let mut connector = candidate;
connector.session_count = 0;
connector
});
connector.session_count = connector.session_count.saturating_add(session_count);
}
}
(
connector_names_by_source_path,
connectors_by_name.into_values().collect(),
)
}
fn apply_plugin_outcome_to_item_result(
item_result: &mut CoreImportItemResult,
plugin_outcome: PluginImportOutcome,
@@ -8,6 +8,8 @@ use codex_app_server_protocol::ExternalAgentConfigImportItemTypeSuccess as Proto
use codex_app_server_protocol::ExternalAgentConfigImportTypeResult as ProtocolImportTypeResult;
use codex_app_server_protocol::ExternalAgentConfigMigrationItem;
use codex_app_server_protocol::ExternalAgentConfigMigrationItemType;
use codex_app_server_protocol::ExternalAgentDetectedConnectorCandidate;
use codex_app_server_protocol::ExternalAgentDetectedConnectorSource;
use codex_app_server_protocol::HookMigration;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::McpServerMigration;
@@ -15,6 +17,8 @@ use codex_app_server_protocol::MigrationDetails;
use codex_app_server_protocol::PluginsMigration;
use codex_app_server_protocol::SkillMigration;
use codex_app_server_protocol::SubagentMigration;
use codex_external_agent_migration::DetectedConnectorCandidate as CoreDetectedConnectorCandidate;
use codex_external_agent_migration::DetectedConnectorSource as CoreDetectedConnectorSource;
use codex_external_agent_migration::ExternalAgentConfigImportItemResult as CoreImportItemResult;
use codex_external_agent_migration::ExternalAgentConfigImportRawError as CoreImportRawError;
use codex_external_agent_migration::ExternalAgentConfigImportSuccess;
@@ -27,10 +31,27 @@ use codex_external_agent_migration::sessions::ExternalAgentSessionMigration;
use codex_state::ExternalAgentConfigImportFailureRecord;
use codex_state::ExternalAgentConfigImportSuccessRecord;
pub(super) fn detect_response(items: Vec<CoreMigrationItem>) -> ExternalAgentConfigDetectResponse {
pub(super) fn detect_response(
items: Vec<CoreMigrationItem>,
connectors: Vec<CoreDetectedConnectorCandidate>,
) -> ExternalAgentConfigDetectResponse {
ExternalAgentConfigDetectResponse {
items: items.into_iter().map(protocol_migration_item).collect(),
connectors: Vec::new(),
connectors: connectors
.into_iter()
.map(|candidate| ExternalAgentDetectedConnectorCandidate {
name: candidate.name,
session_count: candidate.session_count,
source: match candidate.source {
CoreDetectedConnectorSource::RemoteMcpServersConfig => {
ExternalAgentDetectedConnectorSource::RemoteMcpServersConfig
}
CoreDetectedConnectorSource::SessionToolUse => {
ExternalAgentDetectedConnectorSource::SessionToolUse
}
},
})
.collect(),
}
}
@@ -1,3 +1,4 @@
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::path::PathBuf;
use std::sync::Arc;
@@ -18,6 +19,7 @@ use codex_external_agent_migration::sessions::PendingSessionImport;
use codex_external_agent_migration::sessions::SessionImportTarget;
use codex_external_agent_migration::sessions::SessionMetadataMode;
use codex_external_agent_migration::sessions::append_existing_session;
use codex_external_agent_migration::sessions::append_imported_session_connector_names;
use codex_external_agent_migration::sessions::detect_imported_cla_session_connectors;
use codex_external_agent_migration::sessions::prepare_validated_session_import_with_metadata_mode;
use codex_external_agent_migration::sessions::record_completed_session_imports;
@@ -95,6 +97,7 @@ impl ExternalAgentSessionImporter {
sessions: Vec<ExternalAgentSessionMigration>,
mut item_result: ExternalAgentConfigImportItemResult,
metadata_mode: SessionMetadataMode,
mut connector_names_by_source_path: BTreeMap<PathBuf, Vec<String>>,
) -> ExternalAgentConfigImportItemResult {
if sessions.is_empty() {
return item_result;
@@ -122,6 +125,7 @@ impl ExternalAgentSessionImporter {
futures::pin_mut!(import_results);
let mut completed_imports = Vec::new();
let mut appended_connector_names_by_source_path = BTreeMap::new();
while let Some(result) = import_results.next().await {
match result {
Ok(Some(SessionImportOutcome::Created(completed_import))) => {
@@ -142,6 +146,12 @@ impl ExternalAgentSessionImporter {
Some(imported_thread_id.to_string()),
title,
);
if let Some(connector_names) =
connector_names_by_source_path.remove(&source_path)
{
appended_connector_names_by_source_path
.insert(source_path, connector_names);
}
}
Ok(None) => {}
Err(failure) => {
@@ -161,6 +171,18 @@ impl ExternalAgentSessionImporter {
}
}
}
if let Err(err) = append_imported_session_connector_names(
&self.codex_home,
appended_connector_names_by_source_path,
) {
record_import_error(
&mut item_result,
"session_ledger_update",
Some("failed_to_update_session_connector_metadata"),
err.to_string(),
/*source*/ None,
);
}
if completed_imports.is_empty() {
return item_result;
}
@@ -197,6 +219,17 @@ impl ExternalAgentSessionImporter {
.remove(&attribution.session_id)
.unwrap_or_default();
}
for completed_import in &mut completed_imports {
let Some(connector_names) =
connector_names_by_source_path.remove(&completed_import.import.source_path)
else {
continue;
};
completed_import
.import
.connector_names
.extend(connector_names);
}
let completed_imports = completed_imports
.into_iter()
.map(|completed_import| completed_import.import)
@@ -14,6 +14,9 @@ use crate::sessions::ExternalAgentSessionMigration;
const PLUGIN_CACHE_DIR: &str = "plugins/cache";
const PLUGIN_MANIFEST_PATH: &str = ".cursor-plugin/plugin.json";
const MCP_CONFIG_PATH: &str = ".mcp.json";
const PROJECT_MCP_DIR: &str = "mcps";
const PROJECT_MCP_SERVER_METADATA_PATH: &str = "SERVER_METADATA.json";
const AGENT_TRANSCRIPTS_DIR: &str = "agent-transcripts";
const MCP_TOOL_CALL: &str = "CallMcpTool";
#[derive(Deserialize)]
@@ -24,17 +27,23 @@ struct PluginManifest {
mcp_servers: Option<JsonValue>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct ProjectMcpServerMetadata {
server_identifier: String,
server_name: String,
}
pub(crate) fn detect_cur_session_connectors(
sessions: &[ExternalAgentSessionMigration],
external_agent_home: &Path,
) -> Vec<DetectedConnectorCandidate> {
let connector_names_by_server_id = cached_connector_names_by_server_id(external_agent_home);
if connector_names_by_server_id.is_empty() {
return Vec::new();
}
let cached_connector_names_by_server_id =
cached_connector_names_by_server_id(external_agent_home);
let mut candidates = BTreeMap::<String, DetectedConnectorCandidate>::new();
for session in sessions {
let mut connector_names_by_server_id = cached_connector_names_by_server_id.clone();
connector_names_by_server_id.extend(project_connector_names_by_server_id(session));
for (key, name) in session_connector_names(session, &connector_names_by_server_id) {
let candidate = candidates.entry(key).or_insert(DetectedConnectorCandidate {
name,
@@ -47,6 +56,36 @@ pub(crate) fn detect_cur_session_connectors(
candidates.into_values().collect()
}
fn project_connector_names_by_server_id(
session: &ExternalAgentSessionMigration,
) -> BTreeMap<String, String> {
let Some(project_root) = session
.path
.ancestors()
.find(|path| path.file_name().and_then(|name| name.to_str()) == Some(AGENT_TRANSCRIPTS_DIR))
.and_then(Path::parent)
else {
return BTreeMap::new();
};
let mut connector_names = BTreeMap::new();
for server_root in child_directories(&project_root.join(PROJECT_MCP_DIR)) {
let Some(metadata) = read_project_mcp_server_metadata(&server_root) else {
continue;
};
let Some(display_name) =
crate::sessions::normalized_connector_display_name(Some(&metadata.server_name))
else {
continue;
};
let server_identifier = metadata.server_identifier.trim();
if server_identifier.is_empty() {
continue;
}
connector_names.insert(server_identifier.to_lowercase(), display_name);
}
connector_names
}
fn session_connector_names(
session: &ExternalAgentSessionMigration,
connector_names_by_server_id: &BTreeMap<String, String>,
@@ -134,6 +173,11 @@ fn read_plugin_manifest(version_root: &Path) -> Option<PluginManifest> {
serde_json::from_str(&contents).ok()
}
fn read_project_mcp_server_metadata(server_root: &Path) -> Option<ProjectMcpServerMetadata> {
let contents = fs::read_to_string(server_root.join(PROJECT_MCP_SERVER_METADATA_PATH)).ok()?;
serde_json::from_str(&contents).ok()
}
fn cached_mcp_server_names(version_root: &Path, manifest: &PluginManifest) -> Vec<String> {
manifest
.mcp_servers
@@ -83,6 +83,61 @@ fn detects_connector_used_by_session_mcp_tool_call() {
);
}
#[test]
fn detects_connector_from_project_mcp_server_metadata() {
let source_home = TempDir::new().expect("source home");
let project_root = source_home.path().join("projects/project");
let session_path = project_root
.join(AGENT_TRANSCRIPTS_DIR)
.join("session-1")
.join("session-1.jsonl");
fs::create_dir_all(session_path.parent().expect("session parent")).expect("session directory");
fs::write(
&session_path,
serde_json::json!({
"role": "assistant",
"message": {
"content": [{
"type": "tool_use",
"name": "CallMcpTool",
"input": {"server": "plugin-figma-figma", "toolName": "whoami"}
}]
}
})
.to_string(),
)
.expect("session");
let server_root = project_root
.join(PROJECT_MCP_DIR)
.join("plugin-figma-figma");
fs::create_dir_all(&server_root).expect("server directory");
fs::write(
server_root.join(PROJECT_MCP_SERVER_METADATA_PATH),
serde_json::json!({
"serverIdentifier": "plugin-figma-figma",
"serverName": "Figma",
})
.to_string(),
)
.expect("server metadata");
assert_eq!(
detect_cur_session_connectors(
&[ExternalAgentSessionMigration {
path: session_path,
cwd: source_home.path().to_path_buf(),
title: None,
}],
source_home.path(),
),
vec![DetectedConnectorCandidate {
name: "Figma".to_string(),
session_count: 1,
source: DetectedConnectorSource::SessionToolUse,
}]
);
}
fn write_cached_plugin(
source_home: &Path,
plugin_name: &str,
@@ -19,6 +19,14 @@ const SESSION_HASH_BUFFER_SIZE: usize = 64 * 1024;
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct ImportedExternalAgentSessionLedger {
records: Vec<ImportedExternalAgentSessionRecord>,
#[serde(default)]
detected_connector_records: Vec<DetectedExternalAgentSessionConnectorRecord>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct DetectedExternalAgentSessionConnectorRecord {
source_path: PathBuf,
connector_names: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -176,40 +184,106 @@ pub fn record_completed_session_imports(
record.imported_thread_id = import.imported_thread_id;
record.imported_at = imported_at;
record.source_modified_at = source_modified_at.or(record.source_modified_at);
record.connector_names = import.connector_names;
append_connector_names(&mut record.connector_names, import.connector_names);
record.title = import.title;
ledger.records.push(record);
continue;
}
let mut connector_names = Vec::new();
append_connector_names(&mut connector_names, import.connector_names);
ledger.records.push(ImportedExternalAgentSessionRecord {
source_path: import.source_path,
content_sha256: import.source_content_sha256,
imported_thread_id: import.imported_thread_id,
imported_at,
source_modified_at,
connector_names: import.connector_names,
connector_names,
title: import.title,
});
}
save_import_ledger(codex_home, &ledger)
}
pub fn record_detected_session_connectors(
codex_home: &Path,
connector_names_by_source_path: BTreeMap<PathBuf, Vec<String>>,
) -> io::Result<()> {
if connector_names_by_source_path.is_empty() {
return Ok(());
}
let mut ledger = load_import_ledger(codex_home)?;
for (source_path, connector_names) in connector_names_by_source_path {
let source_path = canonical_source_path(&source_path)?;
if let Some(record) = ledger
.detected_connector_records
.iter_mut()
.find(|record| record.source_path == source_path)
{
append_connector_names(&mut record.connector_names, connector_names);
continue;
}
let mut detected_connector_names = Vec::new();
append_connector_names(&mut detected_connector_names, connector_names);
ledger
.detected_connector_records
.push(DetectedExternalAgentSessionConnectorRecord {
source_path,
connector_names: detected_connector_names,
});
}
save_import_ledger(codex_home, &ledger)
}
pub fn append_imported_session_connector_names(
codex_home: &Path,
connector_names_by_source_path: BTreeMap<PathBuf, Vec<String>>,
) -> io::Result<()> {
if connector_names_by_source_path.is_empty() {
return Ok(());
}
let mut ledger = load_import_ledger(codex_home)?;
for (source_path, connector_names) in connector_names_by_source_path {
let source_path = canonical_source_path(&source_path)?;
for record in ledger
.records
.iter_mut()
.filter(|record| record.source_path == source_path)
{
append_connector_names(&mut record.connector_names, connector_names.clone());
}
}
save_import_ledger(codex_home, &ledger)
}
pub fn read_imported_connector_candidates(
codex_home: &Path,
) -> io::Result<Vec<ImportedConnectorCandidate>> {
let ledger = load_import_ledger(codex_home)?;
let mut connector_names_by_source = BTreeMap::new();
let mut connector_names_by_source = BTreeMap::<PathBuf, Vec<String>>::new();
for record in ledger.detected_connector_records {
connector_names_by_source
.entry(record.source_path)
.or_default()
.extend(record.connector_names);
}
for record in ledger.records {
connector_names_by_source.insert(record.source_path, record.connector_names);
connector_names_by_source
.entry(record.source_path)
.or_default()
.extend(record.connector_names);
}
let mut candidates_by_name = BTreeMap::<String, ImportedConnectorCandidate>::new();
for connector_names in connector_names_by_source.into_values() {
let connector_names = connector_names
let mut connector_names_by_key = BTreeMap::new();
for name in connector_names
.into_iter()
.filter_map(|name| super::normalized_connector_display_name(Some(&name)))
.map(|name| (name.to_lowercase(), name))
.collect::<BTreeMap<_, _>>();
for (key, name) in connector_names {
{
connector_names_by_key
.entry(name.to_lowercase())
.or_insert(name);
}
for (key, name) in connector_names_by_key {
let candidate = candidates_by_name
.entry(key)
.or_insert(ImportedConnectorCandidate {
@@ -224,6 +298,18 @@ pub fn read_imported_connector_candidates(
Ok(candidates)
}
fn append_connector_names(connector_names: &mut Vec<String>, additional_names: Vec<String>) {
for name in additional_names {
if connector_names
.iter()
.any(|existing_name| existing_name.eq_ignore_ascii_case(&name))
{
continue;
}
connector_names.push(name);
}
}
impl ImportedExternalAgentSessionLedger {
pub(crate) fn source_states(&self) -> HashMap<&Path, ImportedSourceState> {
let mut states = HashMap::new();
@@ -1,13 +1,16 @@
use super::CompletedExternalAgentSessionImport;
use super::ImportedConnectorCandidate;
use super::ImportedExternalAgentSessionLedger;
use super::append_imported_session_connector_names;
use super::checkpoint_existing_session_import;
use super::read_imported_connector_candidates;
use super::record_completed_session_imports;
use super::record_detected_session_connectors;
use codex_protocol::ThreadId;
use pretty_assertions::assert_eq;
use sha2::Digest;
use sha2::Sha256;
use std::collections::BTreeMap;
use std::path::Path;
use tempfile::TempDir;
@@ -94,7 +97,7 @@ fn completed_import_refreshes_existing_record_metadata() {
assert_eq!(ledger.records[0].source_path, source_path);
assert_eq!(ledger.records[0].imported_thread_id, second_thread_id);
assert!(ledger.records[0].source_modified_at.is_some());
assert_eq!(ledger.records[0].connector_names, vec!["Slack"]);
assert_eq!(ledger.records[0].connector_names, vec!["Gmail", "Slack"]);
assert_eq!(ledger.records[0].title.as_deref(), Some("Second title"));
}
@@ -168,7 +171,7 @@ fn checkpoint_is_compare_and_swap_and_preserves_record_metadata() {
}
#[test]
fn connector_candidates_use_latest_import_for_each_source() {
fn connector_candidates_accumulate_imports_for_each_source() {
let root = TempDir::new().expect("tempdir");
let codex_home = root.path().join("codex-home");
let first_source = root.path().join("first.jsonl");
@@ -207,7 +210,7 @@ fn connector_candidates_use_latest_import_for_each_source() {
vec![
ImportedConnectorCandidate {
name: "Gmail".to_string(),
session_count: 1,
session_count: 2,
},
ImportedConnectorCandidate {
name: "Slack".to_string(),
@@ -217,6 +220,76 @@ fn connector_candidates_use_latest_import_for_each_source() {
);
}
#[test]
fn connector_candidates_accumulate_detected_and_imported_session_clues() {
let root = TempDir::new().expect("tempdir");
let codex_home = root.path().join("codex-home");
let source_path = root.path().join("session.jsonl");
std::fs::write(&source_path, "session contents").expect("source");
let source_path = std::fs::canonicalize(source_path).expect("canonical source");
record_detected_session_connectors(
&codex_home,
BTreeMap::from([(source_path.clone(), vec!["Figma".to_string()])]),
)
.expect("record detected connectors");
assert_eq!(
read_imported_connector_candidates(&codex_home).expect("read detected connectors"),
vec![ImportedConnectorCandidate {
name: "Figma".to_string(),
session_count: 1,
}]
);
record_completed_session_imports(
&codex_home,
vec![CompletedExternalAgentSessionImport {
source_path: source_path.clone(),
source_content_sha256: "content-sha".to_string(),
imported_thread_id: ThreadId::new(),
connector_names: vec!["Slack".to_string()],
title: None,
}],
)
.expect("record completed import");
assert_eq!(
read_imported_connector_candidates(&codex_home).expect("read imported connectors"),
vec![
ImportedConnectorCandidate {
name: "Figma".to_string(),
session_count: 1,
},
ImportedConnectorCandidate {
name: "Slack".to_string(),
session_count: 1,
},
]
);
append_imported_session_connector_names(
&codex_home,
BTreeMap::from([(source_path, vec!["Gmail".to_string(), "FIGMA".to_string()])]),
)
.expect("append imported connector names");
assert_eq!(
read_imported_connector_candidates(&codex_home).expect("read appended connectors"),
vec![
ImportedConnectorCandidate {
name: "Figma".to_string(),
session_count: 1,
},
ImportedConnectorCandidate {
name: "Gmail".to_string(),
session_count: 1,
},
ImportedConnectorCandidate {
name: "Slack".to_string(),
session_count: 1,
},
]
);
}
fn ledger_bytes(codex_home: &Path) -> Vec<u8> {
std::fs::read(super::import_ledger_path(codex_home)).expect("ledger")
}
@@ -24,9 +24,11 @@ pub use append::append_existing_session;
use export::load_session_for_import_with_content_sha256;
pub use ledger::CompletedExternalAgentSessionImport;
pub use ledger::ImportedConnectorCandidate;
pub use ledger::append_imported_session_connector_names;
pub use ledger::has_current_session_been_imported;
pub use ledger::read_imported_connector_candidates;
pub use ledger::record_completed_session_imports;
pub use ledger::record_detected_session_connectors;
pub use records_cla::summarize_session;
const SESSION_TITLE_MAX_LEN: usize = 120;