diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index fd294385e5fc..82d015fbb6e6 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -66,15 +66,20 @@ use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponsePayload; use codex_app_server_protocol::CodexErrorInfo; +use codex_app_server_protocol::CollabAgentTool; +use codex_app_server_protocol::CollabAgentToolCallStatus; use codex_app_server_protocol::CommandAction; use codex_app_server_protocol::CommandExecutionSource; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::DynamicToolCallStatus; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::McpToolCallStatus; use codex_app_server_protocol::NonSteerableTurnKind; +use codex_app_server_protocol::PatchApplyStatus; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy; use codex_app_server_protocol::ServerNotification; @@ -1501,6 +1506,14 @@ async fn item_lifecycle_notifications_publish_command_execution_event() { let mut events = Vec::new(); ingest_tool_review_prerequisites(&mut reducer, &mut events).await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_started_notification( + "thread-1", "turn-1", + ))), + &mut events, + ) + .await; reducer .ingest( AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted( @@ -1911,6 +1924,15 @@ async fn subagent_tool_items_inherit_parent_connection_metadata() { ) .await; events.clear(); + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_started_notification( + "thread-subagent", + "turn-subagent", + ))), + &mut events, + ) + .await; reducer .ingest( @@ -2785,6 +2807,17 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["num_input_images"], json!(1)); assert_eq!(payload["event_params"]["status"], json!("completed")); assert_eq!(payload["event_params"]["steer_count"], json!(0)); + assert_eq!(payload["event_params"]["total_tool_call_count"], json!(0)); + assert_eq!(payload["event_params"]["shell_command_count"], json!(0)); + assert_eq!(payload["event_params"]["file_change_count"], json!(0)); + assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(0)); + assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(0)); + assert_eq!( + payload["event_params"]["subagent_tool_call_count"], + json!(0) + ); + assert_eq!(payload["event_params"]["web_search_count"], json!(0)); + assert_eq!(payload["event_params"]["image_generation_count"], json!(0)); assert_eq!(payload["event_params"]["started_at"], json!(455)); assert_eq!(payload["event_params"]["completed_at"], json!(456)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); @@ -2798,6 +2831,158 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["total_tokens"], json!(321)); } +#[tokio::test] +async fn turn_event_counts_completed_tool_items() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + /*include_token_usage*/ false, + ) + .await; + + let completed_tool_items = vec![ + sample_command_execution_item(CommandExecutionStatus::Completed, Some(0), Some(1)), + ThreadItem::FileChange { + id: "file-change-1".to_string(), + changes: Vec::new(), + status: PatchApplyStatus::Completed, + }, + ThreadItem::McpToolCall { + id: "mcp-1".to_string(), + server: "server".to_string(), + tool: "search".to_string(), + status: McpToolCallStatus::Completed, + arguments: json!({}), + mcp_app_resource_uri: None, + result: None, + error: None, + duration_ms: Some(2), + }, + ThreadItem::DynamicToolCall { + id: "dynamic-1".to_string(), + namespace: None, + tool: "render".to_string(), + arguments: json!({}), + status: DynamicToolCallStatus::Completed, + content_items: None, + success: Some(true), + duration_ms: Some(3), + }, + ThreadItem::CollabAgentToolCall { + id: "collab-1".to_string(), + tool: CollabAgentTool::SpawnAgent, + status: CollabAgentToolCallStatus::Completed, + sender_thread_id: "thread-2".to_string(), + receiver_thread_ids: vec!["thread-child".to_string()], + prompt: Some("help".to_string()), + model: Some("gpt-5".to_string()), + reasoning_effort: None, + agents_states: Default::default(), + }, + ThreadItem::WebSearch { + id: "web-1".to_string(), + query: "codex".to_string(), + action: None, + }, + ThreadItem::ImageGeneration { + id: "image-1".to_string(), + status: "completed".to_string(), + revised_prompt: None, + result: "ok".to_string(), + saved_path: None, + }, + ]; + + for item in completed_tool_items { + reducer + .ingest( + AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted( + ItemCompletedNotification { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + completed_at_ms: 1_000, + item, + }, + ))), + &mut out, + ) + .await; + } + + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + let turn_event = out + .iter() + .find(|event| matches!(event, TrackEventRequest::TurnEvent(_))) + .expect("turn event should be emitted"); + let payload = serde_json::to_value(turn_event).expect("serialize turn event"); + assert_eq!(payload["event_params"]["total_tool_call_count"], json!(7)); + assert_eq!(payload["event_params"]["shell_command_count"], json!(1)); + assert_eq!(payload["event_params"]["file_change_count"], json!(1)); + assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(1)); + assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(1)); + assert_eq!( + payload["event_params"]["subagent_tool_call_count"], + json!(1) + ); + assert_eq!(payload["event_params"]["web_search_count"], json!(1)); + assert_eq!(payload["event_params"]["image_generation_count"], json!(1)); +} + +#[tokio::test] +async fn item_completed_without_turn_state_does_not_create_turn_state() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + reducer + .ingest( + AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted( + ItemCompletedNotification { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + completed_at_ms: 1_000, + item: sample_command_execution_item( + CommandExecutionStatus::Completed, + Some(0), + Some(1), + ), + }, + ))), + &mut out, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert!(out.is_empty()); +} + #[tokio::test] async fn accepted_steers_increment_turn_steer_count() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 2232c88d3996..74f79552dbd3 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -686,8 +686,6 @@ pub(crate) struct CodexTurnEventParams { pub(crate) status: Option, pub(crate) turn_error: Option, pub(crate) steer_count: Option, - // TODO(rhan-oai): Populate these once tool-call accounting is emitted from - // core; the schema is reserved but these fields are currently always None. pub(crate) total_tool_call_count: Option, pub(crate) shell_command_count: Option, pub(crate) file_change_count: Option, diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 81530444de48..43fdb911310e 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -265,6 +265,7 @@ struct TurnState { token_usage: Option, completed: Option, steer_count: usize, + tool_counts: TurnToolCounts, } #[derive(Hash, Eq, PartialEq)] @@ -274,6 +275,42 @@ struct ToolItemKey { item_id: String, } +#[derive(Default)] +struct TurnToolCounts { + total: usize, + shell_command: usize, + file_change: usize, + mcp_tool_call: usize, + dynamic_tool_call: usize, + subagent_tool_call: usize, + web_search: usize, + image_generation: usize, +} + +impl TurnToolCounts { + fn record(&mut self, item: &ThreadItem) { + match item { + ThreadItem::CommandExecution { .. } => self.shell_command += 1, + ThreadItem::FileChange { .. } => self.file_change += 1, + ThreadItem::McpToolCall { .. } => self.mcp_tool_call += 1, + ThreadItem::DynamicToolCall { .. } => self.dynamic_tool_call += 1, + ThreadItem::CollabAgentToolCall { .. } => self.subagent_tool_call += 1, + ThreadItem::WebSearch { .. } => self.web_search += 1, + ThreadItem::ImageGeneration { .. } => self.image_generation += 1, + ThreadItem::UserMessage { .. } + | ThreadItem::HookPrompt { .. } + | ThreadItem::AgentMessage { .. } + | ThreadItem::Plan { .. } + | ThreadItem::Reasoning { .. } + | ThreadItem::ImageView { .. } + | ThreadItem::EnteredReviewMode { .. } + | ThreadItem::ExitedReviewMode { .. } + | ThreadItem::ContextCompaction { .. } => return, + } + self.total += 1; + } +} + impl AnalyticsReducer { pub(crate) async fn ingest(&mut self, input: AnalyticsFact, out: &mut Vec) { match input { @@ -489,6 +526,7 @@ impl AnalyticsReducer { token_usage: None, completed: None, steer_count: 0, + tool_counts: TurnToolCounts::default(), }); turn_state.thread_id = Some(thread_id); turn_state.num_input_images = Some(num_input_images); @@ -511,6 +549,7 @@ impl AnalyticsReducer { token_usage: None, completed: None, steer_count: 0, + tool_counts: TurnToolCounts::default(), }); turn_state.thread_id = Some(input.thread_id); turn_state.token_usage = Some(input.token_usage); @@ -674,6 +713,7 @@ impl AnalyticsReducer { token_usage: None, completed: None, steer_count: 0, + tool_counts: TurnToolCounts::default(), }); turn_state.connection_id = Some(connection_id); turn_state.thread_id = Some(pending_request.thread_id); @@ -767,6 +807,16 @@ impl AnalyticsReducer { let Some(item_id) = tracked_tool_item_id(¬ification.item) else { return; }; + let Some(turn_state) = self.turns.get_mut(¬ification.turn_id) else { + tracing::warn!( + thread_id = %notification.thread_id, + turn_id = %notification.turn_id, + item_id, + "dropping turn tool count update: missing turn state" + ); + return; + }; + turn_state.tool_counts.record(¬ification.item); let key = ToolItemKey { thread_id: notification.thread_id.clone(), turn_id: notification.turn_id.clone(), @@ -812,6 +862,7 @@ impl AnalyticsReducer { token_usage: None, completed: None, steer_count: 0, + tool_counts: TurnToolCounts::default(), }); turn_state.started_at = notification .turn @@ -831,6 +882,7 @@ impl AnalyticsReducer { token_usage: None, completed: None, steer_count: 0, + tool_counts: TurnToolCounts::default(), }); turn_state.completed = Some(CompletedTurnState { status: analytics_turn_status(notification.turn.status), @@ -1711,14 +1763,14 @@ fn codex_turn_event_params( status: completed.status, turn_error: completed.turn_error, steer_count: Some(turn_state.steer_count), - total_tool_call_count: None, - shell_command_count: None, - file_change_count: None, - mcp_tool_call_count: None, - dynamic_tool_call_count: None, - subagent_tool_call_count: None, - web_search_count: None, - image_generation_count: None, + total_tool_call_count: Some(turn_state.tool_counts.total), + shell_command_count: Some(turn_state.tool_counts.shell_command), + file_change_count: Some(turn_state.tool_counts.file_change), + mcp_tool_call_count: Some(turn_state.tool_counts.mcp_tool_call), + dynamic_tool_call_count: Some(turn_state.tool_counts.dynamic_tool_call), + subagent_tool_call_count: Some(turn_state.tool_counts.subagent_tool_call), + web_search_count: Some(turn_state.tool_counts.web_search), + image_generation_count: Some(turn_state.tool_counts.image_generation), input_tokens: token_usage .as_ref() .map(|token_usage| token_usage.input_tokens),