diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 521f9c1120..095b984b38 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -3587,6 +3587,7 @@ async fn reducer_ingests_skill_invoked_fact() { skill_scope: codex_protocol::protocol::SkillScope::User, skill_path, plugin_id: None, + remote_plugin_id: None, invocation_type: InvocationType::Explicit, }], })), @@ -3605,6 +3606,7 @@ async fn reducer_ingests_skill_invoked_fact() { "product_client_id": TEST_PRODUCT_CLIENT_ID, "skill_scope": "user", "plugin_id": null, + "remote_plugin_id": null, "repo_url": null, "thread_id": "thread-1", "turn_id": "turn-1", @@ -3616,7 +3618,7 @@ async fn reducer_ingests_skill_invoked_fact() { } #[tokio::test] -async fn reducer_includes_plugin_id_for_plugin_skill_invocations() { +async fn reducer_includes_plugin_ids_for_plugin_skill_invocations() { let mut reducer = AnalyticsReducer::default(); let mut events = Vec::new(); let tracking = test_tracking_context("thread-1", "turn-1"); @@ -3632,6 +3634,7 @@ async fn reducer_includes_plugin_id_for_plugin_skill_invocations() { skill_scope: codex_protocol::protocol::SkillScope::User, skill_path, plugin_id: Some("sample@test".to_string()), + remote_plugin_id: Some("plugins~Plugin_sample".to_string()), invocation_type: InvocationType::Explicit, }], })), @@ -3641,8 +3644,11 @@ async fn reducer_includes_plugin_id_for_plugin_skill_invocations() { let payload = serde_json::to_value(&events).expect("serialize events"); assert_eq!( - payload[0]["event_params"]["plugin_id"], - json!("sample@test") + ( + &payload[0]["event_params"]["plugin_id"], + &payload[0]["event_params"]["remote_plugin_id"], + ), + (&json!("sample@test"), &json!("plugins~Plugin_sample")) ); } diff --git a/codex-rs/analytics/src/client_tests.rs b/codex-rs/analytics/src/client_tests.rs index ba76199a69..202a6a0b57 100644 --- a/codex-rs/analytics/src/client_tests.rs +++ b/codex-rs/analytics/src/client_tests.rs @@ -115,6 +115,7 @@ fn sample_skill_track_event(thread_id: &str, plugin_id: Option<&str>) -> TrackEv product_client_id: None, skill_scope: None, plugin_id: plugin_id.map(str::to_string), + remote_plugin_id: None, repo_url: None, thread_id: Some(thread_id.to_string()), turn_id: Some("turn-1".to_string()), diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 2da6194f04..e5ac0cb1d6 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -139,6 +139,7 @@ pub(crate) struct SkillInvocationEventParams { pub(crate) product_client_id: Option, pub(crate) skill_scope: Option, pub(crate) plugin_id: Option, + pub(crate) remote_plugin_id: Option, pub(crate) repo_url: Option, pub(crate) thread_id: Option, pub(crate) turn_id: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 41df723a37..519007b3f5 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -237,6 +237,7 @@ pub struct SkillInvocation { pub skill_scope: SkillScope, pub skill_path: PathBuf, pub plugin_id: Option, + pub remote_plugin_id: Option, pub invocation_type: InvocationType, } diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index b253e6b68d..7777e02397 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -759,6 +759,7 @@ impl AnalyticsReducer { repo_url, skill_scope: Some(skill_scope.to_string()), plugin_id: invocation.plugin_id, + remote_plugin_id: invocation.remote_plugin_id, }, }, )); diff --git a/codex-rs/core-skills/src/injection.rs b/codex-rs/core-skills/src/injection.rs index 74f6898353..2a243b9ee9 100644 --- a/codex-rs/core-skills/src/injection.rs +++ b/codex-rs/core-skills/src/injection.rs @@ -99,6 +99,7 @@ pub async fn build_skill_injections( skill_scope: skill.scope, skill_path: skill.path_to_skills_md.to_path_buf(), plugin_id: skill.plugin_id.clone(), + remote_plugin_id: skill.remote_plugin_id.clone(), invocation_type: InvocationType::Explicit, }); result.items.push(SkillInjection { diff --git a/codex-rs/core/src/skills.rs b/codex-rs/core/src/skills.rs index 628a9885ae..8c68ed5e42 100644 --- a/codex-rs/core/src/skills.rs +++ b/codex-rs/core/src/skills.rs @@ -66,6 +66,7 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation( skill_scope: candidate.scope, skill_path: candidate.path_to_skills_md.to_path_buf(), plugin_id: candidate.plugin_id, + remote_plugin_id: candidate.remote_plugin_id, invocation_type: InvocationType::Implicit, }; let skill_scope = match invocation.skill_scope { diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rs index f104caeff2..73ac51d911 100644 --- a/codex-rs/core/tests/suite/plugins.rs +++ b/codex-rs/core/tests/suite/plugins.rs @@ -15,6 +15,7 @@ use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; use core_test_support::apps_test_server::AppsTestServer; use core_test_support::apps_test_server::SEARCH_CALENDAR_CREATE_TOOL; use core_test_support::responses::ResponseMock; @@ -43,8 +44,10 @@ use tempfile::TempDir; use wiremock::MockServer; const SAMPLE_PLUGIN_CONFIG_NAME: &str = "sample@test"; +const SAMPLE_REMOTE_PLUGIN_CONFIG_NAME: &str = "sample@openai-curated-remote"; const SAMPLE_PLUGIN_DISPLAY_NAME: &str = "sample"; const SAMPLE_PLUGIN_DESCRIPTION: &str = "inspect sample data"; +const SAMPLE_REMOTE_PLUGIN_ID: &str = "plugins~Plugin_sample"; const SAMPLE_PLUGIN_APP_NAMESPACE: &str = "mcp__codex_apps__google_calendar"; const SAMPLE_PLUGIN_MCP_NAMESPACE: &str = "mcp__sample"; const PLUGIN_APP_SEARCH_CALL_ID: &str = "plugin-app-search"; @@ -56,7 +59,18 @@ fn sample_plugin_root(home: &TempDir) -> std::path::PathBuf { } fn write_sample_plugin_manifest_and_config(home: &TempDir) -> std::path::PathBuf { - let plugin_root = sample_plugin_root(home); + write_sample_plugin_manifest_and_config_at_root( + home, + sample_plugin_root(home), + SAMPLE_PLUGIN_CONFIG_NAME, + ) +} + +fn write_sample_plugin_manifest_and_config_at_root( + home: &TempDir, + plugin_root: std::path::PathBuf, + plugin_config_name: &str, +) -> std::path::PathBuf { std::fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); std::fs::write( plugin_root.join(".codex-plugin/plugin.json"), @@ -68,7 +82,7 @@ fn write_sample_plugin_manifest_and_config(home: &TempDir) -> std::path::PathBuf std::fs::write( home.path().join("config.toml"), format!( - "[features]\nplugins = true\n\n[plugins.\"{SAMPLE_PLUGIN_CONFIG_NAME}\"]\nenabled = true\n" + "[features]\nplugins = true\n\n[plugins.\"{plugin_config_name}\"]\nenabled = true\n" ), ) .expect("write config"); @@ -104,7 +118,21 @@ fn write_remote_plugin_script_and_config(home: &TempDir) -> std::path::PathBuf { } fn write_plugin_skill_plugin(home: &TempDir) -> std::path::PathBuf { - let plugin_root = write_sample_plugin_manifest_and_config(home); + write_sample_plugin_skill(write_sample_plugin_manifest_and_config(home)) +} + +fn write_remote_plugin_skill_plugin(home: &TempDir) -> std::path::PathBuf { + let plugin_root = home + .path() + .join("plugins/cache/openai-curated-remote/sample/local"); + write_sample_plugin_skill(write_sample_plugin_manifest_and_config_at_root( + home, + plugin_root, + SAMPLE_REMOTE_PLUGIN_CONFIG_NAME, + )) +} + +fn write_sample_plugin_skill(plugin_root: std::path::PathBuf) -> std::path::PathBuf { let skill_dir = plugin_root.join("skills/sample-search"); std::fs::create_dir_all(skill_dir.as_path()).expect("create plugin skill dir"); std::fs::write( @@ -664,31 +692,7 @@ async fn explicit_plugin_mentions_track_plugin_used_analytics() -> Result<()> { .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; - let deadline = Instant::now() + Duration::from_secs(10); - let plugin_event = loop { - let requests = server.received_requests().await.unwrap_or_default(); - if let Some(event) = requests - .into_iter() - .filter(|request| request.url.path() == "/codex/analytics-events/events") - .find_map(|request| { - let payload: serde_json::Value = serde_json::from_slice(&request.body).ok()?; - payload["events"].as_array().and_then(|events| { - events - .iter() - .find(|event| event["event_type"] == "codex_plugin_used") - .cloned() - }) - }) - { - break event; - } - if Instant::now() >= deadline { - panic!("timed out waiting for plugin analytics request"); - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; - - let event = plugin_event; + let event = wait_for_analytics_event(&server, "codex_plugin_used").await; assert_eq!(event["event_params"]["plugin_id"], "sample@test"); assert_eq!(event["event_params"]["plugin_name"], "sample"); assert_eq!(event["event_params"]["marketplace_name"], "test"); @@ -712,3 +716,136 @@ async fn explicit_plugin_mentions_track_plugin_used_analytics() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn explicit_plugin_skill_invocation_tracks_remote_plugin_id() -> Result<()> { + skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let _resp_mock = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + + let codex_home = Arc::new(TempDir::new()?); + let skill_path = std::fs::canonicalize(write_remote_plugin_skill_plugin(codex_home.as_ref()))?; + persist_sample_remote_plugin_id(codex_home.as_ref()); + let test_codex = build_analytics_plugin_test_codex(&server, codex_home).await?; + let codex = Arc::clone(&test_codex.codex); + + codex + .submit(Op::UserInput { + items: vec![UserInput::Skill { + name: "sample:sample-search".into(), + path: skill_path, + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let event = wait_for_analytics_event(&server, "skill_invocation").await; + assert_eq!( + event["event_params"]["plugin_id"], + SAMPLE_REMOTE_PLUGIN_CONFIG_NAME + ); + assert_eq!( + event["event_params"]["remote_plugin_id"], + SAMPLE_REMOTE_PLUGIN_ID + ); + assert_eq!(event["event_params"]["invoke_type"], "explicit"); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn implicit_plugin_skill_invocation_tracks_remote_plugin_id() -> Result<()> { + skip_if_no_network!(Ok(())); + let server = start_mock_server().await; + let codex_home = Arc::new(TempDir::new()?); + let skill_path = write_remote_plugin_skill_plugin(codex_home.as_ref()); + persist_sample_remote_plugin_id(codex_home.as_ref()); + let command_args = serde_json::json!({ + "command": format!("cat {}", skill_path.display()), + "login": false, + }) + .to_string(); + let _resp_mock = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call("call-1", "shell_command", &command_args), + ev_completed("resp-1"), + ]), + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ], + ) + .await; + let test_codex = build_analytics_plugin_test_codex(&server, codex_home).await?; + let codex = Arc::clone(&test_codex.codex); + + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "inspect the sample skill".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + + let event = wait_for_analytics_event(&server, "skill_invocation").await; + assert_eq!( + event["event_params"]["plugin_id"], + SAMPLE_REMOTE_PLUGIN_CONFIG_NAME + ); + assert_eq!( + event["event_params"]["remote_plugin_id"], + SAMPLE_REMOTE_PLUGIN_ID + ); + assert_eq!(event["event_params"]["invoke_type"], "implicit"); + + Ok(()) +} + +fn persist_sample_remote_plugin_id(home: &TempDir) { + let plugin_id = + PluginId::parse(SAMPLE_REMOTE_PLUGIN_CONFIG_NAME).expect("remote plugin id should parse"); + PluginStore::new(home.path().to_path_buf()) + .write_remote_plugin_id(&plugin_id, SAMPLE_REMOTE_PLUGIN_ID) + .expect("persist remote plugin id"); +} + +async fn wait_for_analytics_event(server: &MockServer, event_type: &str) -> serde_json::Value { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let requests = server.received_requests().await.unwrap_or_default(); + if let Some(event) = requests + .into_iter() + .filter(|request| request.url.path() == "/codex/analytics-events/events") + .find_map(|request| { + let payload: serde_json::Value = serde_json::from_slice(&request.body).ok()?; + payload["events"].as_array().and_then(|events| { + events + .iter() + .find(|event| event["event_type"] == event_type) + .cloned() + }) + }) + { + break event; + } + if Instant::now() >= deadline { + panic!("timed out waiting for {event_type} analytics request"); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +}