[codex-analytics] add turn tool counts to turn events

codex-rs/analytics/src/analytics_client_tests.rs

@@ -66,15 +66,20 @@ use codex_app_server_protocol::ClientInfo;
 use codex_app_server_protocol::ClientRequest;
 use codex_app_server_protocol::ClientResponsePayload;
 use codex_app_server_protocol::CodexErrorInfo;
+use codex_app_server_protocol::CollabAgentTool;
+use codex_app_server_protocol::CollabAgentToolCallStatus;
 use codex_app_server_protocol::CommandAction;
 use codex_app_server_protocol::CommandExecutionSource;
 use codex_app_server_protocol::CommandExecutionStatus;
+use codex_app_server_protocol::DynamicToolCallStatus;
 use codex_app_server_protocol::InitializeCapabilities;
 use codex_app_server_protocol::InitializeParams;
 use codex_app_server_protocol::ItemCompletedNotification;
 use codex_app_server_protocol::ItemStartedNotification;
 use codex_app_server_protocol::JSONRPCErrorError;
+use codex_app_server_protocol::McpToolCallStatus;
 use codex_app_server_protocol::NonSteerableTurnKind;
+use codex_app_server_protocol::PatchApplyStatus;
 use codex_app_server_protocol::RequestId;
 use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy;
 use codex_app_server_protocol::ServerNotification;
@@ -1501,6 +1506,14 @@ async fn item_lifecycle_notifications_publish_command_execution_event() {
     let mut events = Vec::new();
 
     ingest_tool_review_prerequisites(&mut reducer, &mut events).await;
+    reducer
+        .ingest(
+            AnalyticsFact::Notification(Box::new(sample_turn_started_notification(
+                "thread-1", "turn-1",
+            ))),
+            &mut events,
+        )
+        .await;
     reducer
         .ingest(
             AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
@@ -1911,6 +1924,15 @@ async fn subagent_tool_items_inherit_parent_connection_metadata() {
         )
         .await;
     events.clear();
+    reducer
+        .ingest(
+            AnalyticsFact::Notification(Box::new(sample_turn_started_notification(
+                "thread-subagent",
+                "turn-subagent",
+            ))),
+            &mut events,
+        )
+        .await;
 
     reducer
         .ingest(
@@ -2785,6 +2807,17 @@ async fn turn_lifecycle_emits_turn_event() {
     assert_eq!(payload["event_params"]["num_input_images"], json!(1));
     assert_eq!(payload["event_params"]["status"], json!("completed"));
     assert_eq!(payload["event_params"]["steer_count"], json!(0));
+    assert_eq!(payload["event_params"]["total_tool_call_count"], json!(0));
+    assert_eq!(payload["event_params"]["shell_command_count"], json!(0));
+    assert_eq!(payload["event_params"]["file_change_count"], json!(0));
+    assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(0));
+    assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(0));
+    assert_eq!(
+        payload["event_params"]["subagent_tool_call_count"],
+        json!(0)
+    );
+    assert_eq!(payload["event_params"]["web_search_count"], json!(0));
+    assert_eq!(payload["event_params"]["image_generation_count"], json!(0));
     assert_eq!(payload["event_params"]["started_at"], json!(455));
     assert_eq!(payload["event_params"]["completed_at"], json!(456));
     assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
@@ -2798,6 +2831,158 @@ async fn turn_lifecycle_emits_turn_event() {
     assert_eq!(payload["event_params"]["total_tokens"], json!(321));
 }
 
+#[tokio::test]
+async fn turn_event_counts_completed_tool_items() {
+    let mut reducer = AnalyticsReducer::default();
+    let mut out = Vec::new();
+
+    ingest_turn_prerequisites(
+        &mut reducer,
+        &mut out,
+        /*include_initialize*/ true,
+        /*include_resolved_config*/ true,
+        /*include_started*/ true,
+        /*include_token_usage*/ false,
+    )
+    .await;
+
+    let completed_tool_items = vec![
+        sample_command_execution_item(CommandExecutionStatus::Completed, Some(0), Some(1)),
+        ThreadItem::FileChange {
+            id: "file-change-1".to_string(),
+            changes: Vec::new(),
+            status: PatchApplyStatus::Completed,
+        },
+        ThreadItem::McpToolCall {
+            id: "mcp-1".to_string(),
+            server: "server".to_string(),
+            tool: "search".to_string(),
+            status: McpToolCallStatus::Completed,
+            arguments: json!({}),
+            mcp_app_resource_uri: None,
+            result: None,
+            error: None,
+            duration_ms: Some(2),
+        },
+        ThreadItem::DynamicToolCall {
+            id: "dynamic-1".to_string(),
+            namespace: None,
+            tool: "render".to_string(),
+            arguments: json!({}),
+            status: DynamicToolCallStatus::Completed,
+            content_items: None,
+            success: Some(true),
+            duration_ms: Some(3),
+        },
+        ThreadItem::CollabAgentToolCall {
+            id: "collab-1".to_string(),
+            tool: CollabAgentTool::SpawnAgent,
+            status: CollabAgentToolCallStatus::Completed,
+            sender_thread_id: "thread-2".to_string(),
+            receiver_thread_ids: vec!["thread-child".to_string()],
+            prompt: Some("help".to_string()),
+            model: Some("gpt-5".to_string()),
+            reasoning_effort: None,
+            agents_states: Default::default(),
+        },
+        ThreadItem::WebSearch {
+            id: "web-1".to_string(),
+            query: "codex".to_string(),
+            action: None,
+        },
+        ThreadItem::ImageGeneration {
+            id: "image-1".to_string(),
+            status: "completed".to_string(),
+            revised_prompt: None,
+            result: "ok".to_string(),
+            saved_path: None,
+        },
+    ];
+
+    for item in completed_tool_items {
+        reducer
+            .ingest(
+                AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
+                    ItemCompletedNotification {
+                        thread_id: "thread-2".to_string(),
+                        turn_id: "turn-2".to_string(),
+                        completed_at_ms: 1_000,
+                        item,
+                    },
+                ))),
+                &mut out,
+            )
+            .await;
+    }
+
+    reducer
+        .ingest(
+            AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
+                "thread-2",
+                "turn-2",
+                AppServerTurnStatus::Completed,
+                /*codex_error_info*/ None,
+            ))),
+            &mut out,
+        )
+        .await;
+
+    let turn_event = out
+        .iter()
+        .find(|event| matches!(event, TrackEventRequest::TurnEvent(_)))
+        .expect("turn event should be emitted");
+    let payload = serde_json::to_value(turn_event).expect("serialize turn event");
+    assert_eq!(payload["event_params"]["total_tool_call_count"], json!(7));
+    assert_eq!(payload["event_params"]["shell_command_count"], json!(1));
+    assert_eq!(payload["event_params"]["file_change_count"], json!(1));
+    assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(1));
+    assert_eq!(payload["event_params"]["dynamic_tool_call_count"], json!(1));
+    assert_eq!(
+        payload["event_params"]["subagent_tool_call_count"],
+        json!(1)
+    );
+    assert_eq!(payload["event_params"]["web_search_count"], json!(1));
+    assert_eq!(payload["event_params"]["image_generation_count"], json!(1));
+}
+
+#[tokio::test]
+async fn item_completed_without_turn_state_does_not_create_turn_state() {
+    let mut reducer = AnalyticsReducer::default();
+    let mut out = Vec::new();
+
+    reducer
+        .ingest(
+            AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
+                ItemCompletedNotification {
+                    thread_id: "thread-2".to_string(),
+                    turn_id: "turn-2".to_string(),
+                    completed_at_ms: 1_000,
+                    item: sample_command_execution_item(
+                        CommandExecutionStatus::Completed,
+                        Some(0),
+                        Some(1),
+                    ),
+                },
+            ))),
+            &mut out,
+        )
+        .await;
+
+    reducer
+        .ingest(
+            AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
+                "thread-2",
+                "turn-2",
+                AppServerTurnStatus::Completed,
+                /*codex_error_info*/ None,
+            ))),
+            &mut out,
+        )
+        .await;
+
+    assert!(out.is_empty());
+}
+
 #[tokio::test]
 async fn accepted_steers_increment_turn_steer_count() {
     let mut reducer = AnalyticsReducer::default();

codex-rs/analytics/src/events.rs

@@ -686,8 +686,6 @@ pub(crate) struct CodexTurnEventParams {
     pub(crate) status: Option<TurnStatus>,
     pub(crate) turn_error: Option<CodexErrorInfo>,
     pub(crate) steer_count: Option<usize>,
-    // TODO(rhan-oai): Populate these once tool-call accounting is emitted from
-    // core; the schema is reserved but these fields are currently always None.
     pub(crate) total_tool_call_count: Option<usize>,
     pub(crate) shell_command_count: Option<usize>,
     pub(crate) file_change_count: Option<usize>,

codex-rs/analytics/src/reducer.rs

@@ -265,6 +265,7 @@ struct TurnState {
     token_usage: Option<TokenUsage>,
     completed: Option<CompletedTurnState>,
     steer_count: usize,
+    tool_counts: TurnToolCounts,
 }
 
 #[derive(Hash, Eq, PartialEq)]
@@ -274,6 +275,42 @@ struct ToolItemKey {
     item_id: String,
 }
 
+#[derive(Default)]
+struct TurnToolCounts {
+    total: usize,
+    shell_command: usize,
+    file_change: usize,
+    mcp_tool_call: usize,
+    dynamic_tool_call: usize,
+    subagent_tool_call: usize,
+    web_search: usize,
+    image_generation: usize,
+}
+
+impl TurnToolCounts {
+    fn record(&mut self, item: &ThreadItem) {
+        match item {
+            ThreadItem::CommandExecution { .. } => self.shell_command += 1,
+            ThreadItem::FileChange { .. } => self.file_change += 1,
+            ThreadItem::McpToolCall { .. } => self.mcp_tool_call += 1,
+            ThreadItem::DynamicToolCall { .. } => self.dynamic_tool_call += 1,
+            ThreadItem::CollabAgentToolCall { .. } => self.subagent_tool_call += 1,
+            ThreadItem::WebSearch { .. } => self.web_search += 1,
+            ThreadItem::ImageGeneration { .. } => self.image_generation += 1,
+            ThreadItem::UserMessage { .. }
+            | ThreadItem::HookPrompt { .. }
+            | ThreadItem::AgentMessage { .. }
+            | ThreadItem::Plan { .. }
+            | ThreadItem::Reasoning { .. }
+            | ThreadItem::ImageView { .. }
+            | ThreadItem::EnteredReviewMode { .. }
+            | ThreadItem::ExitedReviewMode { .. }
+            | ThreadItem::ContextCompaction { .. } => return,
+        }
+        self.total += 1;
+    }
+}
+
 impl AnalyticsReducer {
     pub(crate) async fn ingest(&mut self, input: AnalyticsFact, out: &mut Vec<TrackEventRequest>) {
         match input {
@@ -489,6 +526,7 @@ impl AnalyticsReducer {
             token_usage: None,
             completed: None,
             steer_count: 0,
+            tool_counts: TurnToolCounts::default(),
         });
         turn_state.thread_id = Some(thread_id);
         turn_state.num_input_images = Some(num_input_images);
@@ -511,6 +549,7 @@ impl AnalyticsReducer {
             token_usage: None,
             completed: None,
             steer_count: 0,
+            tool_counts: TurnToolCounts::default(),
         });
         turn_state.thread_id = Some(input.thread_id);
         turn_state.token_usage = Some(input.token_usage);
@@ -674,6 +713,7 @@ impl AnalyticsReducer {
                     token_usage: None,
                     completed: None,
                     steer_count: 0,
+                    tool_counts: TurnToolCounts::default(),
                 });
                 turn_state.connection_id = Some(connection_id);
                 turn_state.thread_id = Some(pending_request.thread_id);
@@ -767,6 +807,16 @@ impl AnalyticsReducer {
                 let Some(item_id) = tracked_tool_item_id(&notification.item) else {
                     return;
                 };
+                let Some(turn_state) = self.turns.get_mut(&notification.turn_id) else {
+                    tracing::warn!(
+                        thread_id = %notification.thread_id,
+                        turn_id = %notification.turn_id,
+                        item_id,
+                        "dropping turn tool count update: missing turn state"
+                    );
+                    return;
+                };
+                turn_state.tool_counts.record(&notification.item);
                 let key = ToolItemKey {
                     thread_id: notification.thread_id.clone(),
                     turn_id: notification.turn_id.clone(),
@@ -812,6 +862,7 @@ impl AnalyticsReducer {
                     token_usage: None,
                     completed: None,
                     steer_count: 0,
+                    tool_counts: TurnToolCounts::default(),
                 });
                 turn_state.started_at = notification
                     .turn
@@ -831,6 +882,7 @@ impl AnalyticsReducer {
                             token_usage: None,
                             completed: None,
                             steer_count: 0,
+                            tool_counts: TurnToolCounts::default(),
                         });
                 turn_state.completed = Some(CompletedTurnState {
                     status: analytics_turn_status(notification.turn.status),
@@ -1711,14 +1763,14 @@ fn codex_turn_event_params(
         status: completed.status,
         turn_error: completed.turn_error,
         steer_count: Some(turn_state.steer_count),
-        total_tool_call_count: None,
-        shell_command_count: None,
-        file_change_count: None,
-        mcp_tool_call_count: None,
-        dynamic_tool_call_count: None,
-        subagent_tool_call_count: None,
-        web_search_count: None,
-        image_generation_count: None,
+        total_tool_call_count: Some(turn_state.tool_counts.total),
+        shell_command_count: Some(turn_state.tool_counts.shell_command),
+        file_change_count: Some(turn_state.tool_counts.file_change),
+        mcp_tool_call_count: Some(turn_state.tool_counts.mcp_tool_call),
+        dynamic_tool_call_count: Some(turn_state.tool_counts.dynamic_tool_call),
+        subagent_tool_call_count: Some(turn_state.tool_counts.subagent_tool_call),
+        web_search_count: Some(turn_state.tool_counts.web_search),
+        image_generation_count: Some(turn_state.tool_counts.image_generation),
         input_tokens: token_usage
             .as_ref()
             .map(|token_usage| token_usage.input_tokens),