Skip to content

[codex-analytics] emit tool item events from item lifecycle #17090

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rhan-oai merged 3 commits into from
May 6, 2026
Merged

[codex-analytics] emit tool item events from item lifecycle #17090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b554eee
[codex-analytics] emit tool item events from item lifecycle
rhan-oai May 5, 2026
91f63ee
[codex-analytics] clean up tool item cache shape
rhan-oai May 6, 2026
41485e3
[codex-analytics] document tool duration semantics
rhan-oai May 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
274 changes: 273 additions & 1 deletion codex-rs/analytics/src/analytics_client_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::CodexToolItemEventBase;
use crate::events::CodexTurnEventRequest;
use crate::events::CommandExecutionSource;
use crate::events::GuardianApprovalRequestSource;
use crate::events::GuardianReviewDecision;
use crate::events::GuardianReviewEventParams;
Expand Down Expand Up @@ -67,8 +66,13 @@ use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponsePayload;
use codex_app_server_protocol::CodexErrorInfo;
use codex_app_server_protocol::CommandAction;
use codex_app_server_protocol::CommandExecutionSource;
use codex_app_server_protocol::CommandExecutionStatus;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::ItemCompletedNotification;
use codex_app_server_protocol::ItemStartedNotification;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::NonSteerableTurnKind;
use codex_app_server_protocol::RequestId;
Expand All @@ -78,6 +82,7 @@ use codex_app_server_protocol::SessionSource as AppServerSessionSource;
use codex_app_server_protocol::Thread;
use codex_app_server_protocol::ThreadArchiveParams;
use codex_app_server_protocol::ThreadArchiveResponse;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadResumeResponse;
use codex_app_server_protocol::ThreadSource as AppServerThreadSource;
use codex_app_server_protocol::ThreadStartResponse;
Expand Down Expand Up @@ -598,6 +603,90 @@ async fn ingest_turn_prerequisites(
}
}

async fn ingest_tool_review_prerequisites(
reducer: &mut AnalyticsReducer,
events: &mut Vec<TrackEventRequest>,
) {
reducer
.ingest(sample_initialize_fact(/*connection_id*/ 7), events)
.await;
reducer
.ingest(
AnalyticsFact::ClientResponse {
connection_id: 7,
request_id: RequestId::Integer(1),
response: Box::new(sample_thread_start_response(
"thread-1", /*ephemeral*/ false, "gpt-5",
)),
},
events,
)
.await;
events.clear();
}

fn sample_initialize_fact(connection_id: u64) -> AnalyticsFact {
AnalyticsFact::Initialize {
connection_id,
params: InitializeParams {
client_info: ClientInfo {
name: "codex-tui".to_string(),
title: None,
version: "1.0.0".to_string(),
},
capabilities: Some(InitializeCapabilities {
experimental_api: false,
opt_out_notification_methods: None,
}),
},
product_client_id: DEFAULT_ORIGINATOR.to_string(),
runtime: CodexRuntimeMetadata {
codex_rs_version: "0.99.0".to_string(),
runtime_os: "linux".to_string(),
runtime_os_version: "24.04".to_string(),
runtime_arch: "x86_64".to_string(),
},
rpc_transport: AppServerRpcTransport::Websocket,
}
}

fn sample_command_execution_item(
status: CommandExecutionStatus,
exit_code: Option<i32>,
duration_ms: Option<i64>,
) -> ThreadItem {
ThreadItem::CommandExecution {
id: "item-1".to_string(),
command: "echo hi".to_string(),
cwd: test_path_buf("/tmp").abs(),
process_id: Some("pid-1".to_string()),
source: CommandExecutionSource::Agent,
status,
command_actions: Vec::new(),
aggregated_output: None,
exit_code,
duration_ms,
}
}

fn sample_command_execution_item_with_actions(
status: CommandExecutionStatus,
exit_code: Option<i32>,
duration_ms: Option<i64>,
command_actions: Vec<CommandAction>,
) -> ThreadItem {
let mut item = sample_command_execution_item(status, exit_code, duration_ms);
let ThreadItem::CommandExecution {
command_actions: item_command_actions,
..
} = &mut item
else {
unreachable!("sample command execution item should be CommandExecution");
};
*item_command_actions = command_actions;
item
}

fn expected_absolute_path(path: &PathBuf) -> String {
std::fs::canonicalize(path)
.unwrap_or_else(|_| path.to_path_buf())
Expand Down Expand Up @@ -930,6 +1019,7 @@ fn command_execution_event_serializes_expected_shape() {
started_at_ms: 123_000,
completed_at_ms: 125_000,
duration_ms: Some(2000),
execution_duration_ms: Some(1900),
review_count: 0,
guardian_review_count: 0,
user_review_count: 0,
Expand Down Expand Up @@ -978,6 +1068,7 @@ fn command_execution_event_serializes_expected_shape() {
"started_at_ms": 123000,
"completed_at_ms": 125000,
"duration_ms": 2000,
"execution_duration_ms": 1900,
"review_count": 0,
"guardian_review_count": 0,
"user_review_count": 0,
Expand Down Expand Up @@ -1404,6 +1495,114 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() {
assert_eq!(payload[0]["event_params"]["review_timeout_ms"], 90_000);
}

#[tokio::test]
async fn item_lifecycle_notifications_publish_command_execution_event() {
let mut reducer = AnalyticsReducer::default();
let mut events = Vec::new();

ingest_tool_review_prerequisites(&mut reducer, &mut events).await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
ItemStartedNotification {
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
started_at_ms: 1_000,
item: sample_command_execution_item(
CommandExecutionStatus::InProgress,
/*exit_code*/ None,
/*duration_ms*/ None,
),
},
))),
&mut events,
)
.await;
assert!(
events.is_empty(),
"tool item event should emit on completion"
);

reducer
.ingest(
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
ItemCompletedNotification {
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
completed_at_ms: 1_045,
item: sample_command_execution_item_with_actions(
CommandExecutionStatus::Completed,
Some(0),
Some(42),
vec![
CommandAction::Read {
command: "cat README.md".to_string(),
name: "README.md".to_string(),
path: test_path_buf("/tmp/README.md").abs(),
},
CommandAction::ListFiles {
command: "ls".to_string(),
path: None,
},
CommandAction::Search {
command: "rg TODO".to_string(),
query: Some("TODO".to_string()),
path: None,
},
CommandAction::Unknown {
command: "cargo test".to_string(),
},
],
),
},
))),
&mut events,
)
.await;

let payload = serde_json::to_value(&events).expect("serialize events");
assert_eq!(payload.as_array().expect("events array").len(), 1);
assert_eq!(payload[0]["event_type"], "codex_command_execution_event");
assert_eq!(payload[0]["event_params"]["thread_id"], "thread-1");
assert_eq!(payload[0]["event_params"]["turn_id"], "turn-1");
assert_eq!(payload[0]["event_params"]["item_id"], "item-1");
assert_eq!(payload[0]["event_params"]["tool_name"], "shell");
assert_eq!(
payload[0]["event_params"]["command_execution_source"],
"agent"
);
assert_eq!(payload[0]["event_params"]["terminal_status"], "completed");
assert_eq!(
payload[0]["event_params"]["final_approval_outcome"],
"unknown"
);
assert_eq!(
payload[0]["event_params"]["failure_kind"],
serde_json::Value::Null
);
assert_eq!(payload[0]["event_params"]["exit_code"], 0);
assert_eq!(payload[0]["event_params"]["command_total_action_count"], 4);
assert_eq!(payload[0]["event_params"]["command_read_action_count"], 1);
assert_eq!(
payload[0]["event_params"]["command_list_files_action_count"],
1
);
assert_eq!(payload[0]["event_params"]["command_search_action_count"], 1);
assert_eq!(
payload[0]["event_params"]["command_unknown_action_count"],
1
);
assert_eq!(payload[0]["event_params"]["started_at_ms"], 1_000);
assert_eq!(payload[0]["event_params"]["completed_at_ms"], 1_045);
assert_eq!(payload[0]["event_params"]["duration_ms"], 45);
assert_eq!(payload[0]["event_params"]["execution_duration_ms"], 42);
assert_eq!(
payload[0]["event_params"]["app_server_client"]["client_name"],
"codex-tui"
);
assert_eq!(payload[0]["event_params"]["thread_source"], "user");
}

#[test]
fn subagent_thread_started_review_serializes_expected_shape() {
let event = TrackEventRequest::ThreadInitialized(subagent_thread_started_event_request(
Expand Down Expand Up @@ -1687,6 +1886,79 @@ async fn subagent_thread_started_inherits_parent_connection_for_new_thread() {
);
}

#[tokio::test]
async fn subagent_tool_items_inherit_parent_connection_metadata() {
let mut reducer = AnalyticsReducer::default();
let mut events = Vec::new();

ingest_tool_review_prerequisites(&mut reducer, &mut events).await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::SubAgentThreadStarted(
SubAgentThreadStartedInput {
thread_id: "thread-subagent".to_string(),
parent_thread_id: Some("thread-1".to_string()),
product_client_id: "codex-tui".to_string(),
client_name: "codex-tui".to_string(),
client_version: "1.0.0".to_string(),
model: "gpt-5".to_string(),
ephemeral: false,
subagent_source: SubAgentSource::Review,
created_at: 128,
},
)),
&mut events,
)
.await;
events.clear();

reducer
.ingest(
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
ItemStartedNotification {
thread_id: "thread-subagent".to_string(),
turn_id: "turn-subagent".to_string(),
started_at_ms: 1_000,
item: sample_command_execution_item(
CommandExecutionStatus::InProgress,
/*exit_code*/ None,
/*duration_ms*/ None,
),
},
))),
&mut events,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(ServerNotification::ItemCompleted(
ItemCompletedNotification {
thread_id: "thread-subagent".to_string(),
turn_id: "turn-subagent".to_string(),
completed_at_ms: 1_042,
item: sample_command_execution_item(
CommandExecutionStatus::Completed,
Some(0),
Some(42),
),
},
))),
&mut events,
)
.await;

let payload = serde_json::to_value(&events).expect("serialize events");
assert_eq!(payload.as_array().expect("events array").len(), 1);
assert_eq!(payload[0]["event_type"], "codex_command_execution_event");
assert_eq!(payload[0]["event_params"]["thread_source"], "subagent");
assert_eq!(payload[0]["event_params"]["subagent_source"], "review");
assert_eq!(payload[0]["event_params"]["parent_thread_id"], "thread-1");
assert_eq!(
payload[0]["event_params"]["app_server_client"]["client_name"],
"codex-tui"
);
}

#[test]
fn plugin_used_event_serializes_expected_shape() {
let tracking = TrackEventsContext {
Expand Down
19 changes: 15 additions & 4 deletions codex-rs/analytics/src/client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,10 +333,6 @@ impl AnalyticsEventsClient {
});
}

pub fn track_notification(&self, notification: ServerNotification) {
self.record_fact(AnalyticsFact::Notification(Box::new(notification)));
}

pub fn track_server_request(&self, connection_id: u64, request: ServerRequest) {
self.record_fact(AnalyticsFact::ServerRequest {
connection_id,
Expand All @@ -349,6 +345,21 @@ impl AnalyticsEventsClient {
response: Box::new(response),
});
}

pub fn track_notification(&self, notification: ServerNotification) {
if !matches!(
notification,
ServerNotification::TurnStarted(_)
| ServerNotification::TurnCompleted(_)
| ServerNotification::ItemStarted(_)
| ServerNotification::ItemCompleted(_)
| ServerNotification::ItemGuardianApprovalReviewStarted(_)
| ServerNotification::ItemGuardianApprovalReviewCompleted(_)
) {
return;
}
self.record_fact(AnalyticsFact::Notification(Box::new(notification)));
}
}

async fn send_track_events(
Expand Down
Loading
Loading