From 00d347c561b650c978b0605ea28ca3df93994123 Mon Sep 17 00:00:00 2001 From: David Reveman Date: Wed, 14 Oct 2026 13:12:01 -0700 Subject: [PATCH 1/2] Add AgentProf workflows as a Perfetto skill overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add six performance analysis runbooks and standalone SQL resources under `ai/skills/dev.agentprof`. Reuse the pinned Perfetto skill, references and packager through a small workflow mapping and router patch. Export the tested reader and prompt-attribution SQL with a separate namespace and no content-bearing tool/prompt fields. Add capture/coverage-aware latency, occupancy, context, delegation and resource queries. Tests: `npm run check`; `npm test` (229 existing tests plus 18 workflow checks); all resources against the four harnesses’ reviewed recordings; standalone packaging; independent review. --- CONTRIBUTING.md | 27 ++ ai/README.md | 25 ++ .../agentprof/analyze_context_pressure.md | 52 +++ .../agentprof/analyze_model_performance.md | 68 +++ .../agentprof/analyze_tools_and_delegation.md | 58 +++ .../workflows/agentprof/compare_agent_runs.md | 57 +++ .../agentprof/correlate_runtime_resources.md | 57 +++ .../agentprof/investigate_agent_run.md | 111 +++++ .../agentprof/scripts/agentprof_setup.sql | 407 ++++++++++++++++++ .../agentprof/scripts/agentprof_turns.sql | 253 +++++++++++ .../agentprof/scripts/child_launches.sql | 15 + .../agentprof/scripts/compactions.sql | 8 + .../agentprof/scripts/context_changes.sql | 7 + .../agentprof/scripts/context_summary.sql | 13 + .../agentprof/scripts/delegation.sql | 16 + .../workflows/agentprof/scripts/inventory.sql | 30 ++ .../agentprof/scripts/model_summary.sql | 31 ++ .../agentprof/scripts/request_headers.sql | 8 + .../agentprof/scripts/run_summary.sql | 51 +++ .../agentprof/scripts/runtime_samples.sql | 7 + .../agentprof/scripts/runtime_summary.sql | 11 + .../agentprof/scripts/slow_responses.sql | 9 + .../agentprof/scripts/slow_tools.sql | 9 + .../agentprof/scripts/slow_turns.sql | 10 + .../agentprof/scripts/tool_summary.sql | 14 + .../agentprof/scripts/unassigned_work.sql | 6 + package.json | 6 +- ...AgentProf-performance-investigations.patch | 45 ++ tools/generate-skills-sql.ts | 73 ++++ tools/perfetto | 41 +- tools/test_skills_overlay.py | 111 +++++ tools/test_skills_sql.py | 231 ++++++++++ 32 files changed, 1854 insertions(+), 13 deletions(-) create mode 100644 ai/README.md create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/analyze_context_pressure.md create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/analyze_model_performance.md create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/analyze_tools_and_delegation.md create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/compare_agent_runs.md create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/correlate_runtime_resources.md create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/investigate_agent_run.md create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql create mode 100644 ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql create mode 100644 third_party/patches/perfetto/0007-ai-Route-AgentProf-performance-investigations.patch create mode 100644 tools/generate-skills-sql.ts create mode 100644 tools/test_skills_overlay.py create mode 100644 tools/test_skills_sql.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6bf9309..a0488bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,6 +18,7 @@ the headers and package metadata. Preserve third-party copyright and license not | `packages/claude-tracing/`, `packages/codex-tracing/`, `packages/muse-tracing/` | Harness recording plugins and tests | | `packages/agent-tracing/` | Shared trace serialization | | `tools/` | Checkout manager and synthetic trace/browser validation | +| `ai/skills/dev.agentprof/` | Portable workflows overlaid onto the upstream Perfetto skill | | `third_party/perfetto.toml` | Upstream Perfetto URL and initial revision pin | | `third_party/patches/perfetto/` | Patches to upstream-owned files | | `third_party/overlays/perfetto/` | Agent Profiler-owned files, mirroring upstream paths | @@ -123,6 +124,32 @@ aside, change the pin, and run setup. Resolve conflicts with `git am --continue` inside the new checkout, then capture the resulting series. Rebuild and run trace/browser checks before landing the update. +## AgentProf AI workflows + +Keep portable workflow pages and SQL under +`ai/skills/dev.agentprof/workflows/agentprof/`. This namespace is an overlay source, +not another installed skill. Setup links it into the pinned Perfetto skill; +changes to the upstream router remain a small ordered patch. Reuse shared +Perfetto references and packaging instead of copying them. + +```bash +npm run build:skills-sql +python3 tools/perfetto setup +PERFETTO_TRACE_PROCESSOR=/path/to/local/trace_processor npm run test:skills +cd third_party/src/perfetto +python3 tools/release/build_ai_agents.py --output ../../../artifacts/ai-agents +``` + +The upstream command only assembles local output without a commit/push option. +The installed skill is `artifacts/ai-agents/plugins/perfetto/skills/perfetto/`. +The SQL exports are checked against the UI reader, but run with only a trace +and Trace Processor. Tests cover packaging, overlay collisions, all four harness +recordings, missing measurements, prompt-level turns, capture scoping and overlap. +No model calls are needed. Run model-based Perfetto evals separately only with +explicit approval; do not attach private traces to eval artifacts. + +See [ai/README.md](ai/README.md) for upstream contribution and overlay import. + ## Useful bug reports Include the harness revision, UI revision, recording settings, expected diff --git a/ai/README.md b/ai/README.md new file mode 100644 index 0000000..eb10f71 --- /dev/null +++ b/ai/README.md @@ -0,0 +1,25 @@ +# AgentProf AI workflow overlay + +`skills/dev.agentprof/` is a source overlay, not an independently installed skill. +It adds the `agentprof` workflow domain to Perfetto's existing `perfetto` skill. +There is no copied router, bootstrap, binary wrapper or package manifest here. + +`python3 tools/perfetto setup` links `skills/dev.agentprof/workflows/` into the +pinned checkout's `ai/skills/perfetto/workflows/`. The small router edit is kept +in the normal Perfetto patch series. The upstream bundler packages the resulting +single skill unchanged; see [CONTRIBUTING.md](../CONTRIBUTING.md). + +The workflows ship standalone SQL. `tools/generate-skills-sql.ts` exports the +shared reader and turn-attribution SQL from AgentProf's tested UI queries, with +an isolated `agentprof_skill_*` namespace and without prompt/tool payloads. This +is an AgentProf maintenance step, not a runtime or upstream packaging dependency. +`npm run build:skills-sql` refreshes the two exports; `npm run check` verifies them. + +For an upstream Perfetto contribution, copy `skills/dev.agentprof/workflows/` +to `ai/skills/perfetto/workflows/` and apply only the skill-router patch. No +AgentProf checkout manager or generated build dependency belongs in that change. + +An environment with its own Perfetto bootstrap can import the same +`skills/dev.agentprof/` namespace as a workflow-only overlay. Keep its environment +override separate. Remove the backport once the pinned upstream source includes +these workflows rather than silently replacing imported upstream files. diff --git a/ai/skills/dev.agentprof/workflows/agentprof/analyze_context_pressure.md b/ai/skills/dev.agentprof/workflows/agentprof/analyze_context_pressure.md new file mode 100644 index 0000000..1df6693 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/analyze_context_pressure.md @@ -0,0 +1,52 @@ +# Analyze AgentProf context pressure + +Use for context growth, model-window pressure, composition changes and compaction. +First initialize and select captures using +[investigate an agent run]($SKILL_ROOT/workflows/agentprof/investigate_agent_run.md). +Reuse its warm session. + +## 1. Establish measurement basis and coverage + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/context_summary.sql +``` + +One row per capture reports its recorded peak/window/share plus the latest +snapshot's ID, timestamp, stage, model, basis, coverage, estimated/reported totals, +configured/effective window, compaction threshold, omitted changes and category +composition. Show the snapshot timestamp: it may be stale relative to the peak. +Do not present latest composition as the composition at peak pressure. + +No snapshots means detailed context composition is unavailable. A recorded context +counter peak can still be useful; NULL is not an empty context. Estimated tokens +may be character-based and are not tokenizer-exact. Provider-reported input usage, +estimated retained context and cumulative input counters are different quantities. +Model window/share is usable only when its window is unambiguous and positive. +Do not sum independent child-agent context windows. + +## 2. Explain changes and compaction + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/context_changes.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/compactions.sql +``` + +The top 50 retained changes report signed deltas, category/change type, basis, +baseline, previous/current estimates and source operation where recorded. Preserve +negative changes and baseline resets. Do not subtract estimates across basis +changes; omitted changes prevent a complete item-level reconciliation. Labels and +content are deliberately excluded. + +Compactions report event IDs/timestamps, incomplete status, measured completed +duration and before/after tokens. NULL or incomplete duration is not zero. Where +both totals use comparable recorded semantics, compute reduction and report the +compaction's time cost. A smaller context does not by itself prove faster inference +or preserved task quality. + +## 3. Report + +State peak pressure and the window definition, snapshot coverage/basis, largest +supported growth/removal categories, and compaction trade-offs. For latency +correlation, inspect model-response IDs in the same capture; sparse snapshots do +not establish the exact context submitted with every request. Follow shared +report/privacy rules. Do not recover omitted content from harness journals. diff --git a/ai/skills/dev.agentprof/workflows/agentprof/analyze_model_performance.md b/ai/skills/dev.agentprof/workflows/agentprof/analyze_model_performance.md new file mode 100644 index 0000000..163c93f --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/analyze_model_performance.md @@ -0,0 +1,68 @@ +# Analyze AgentProf model performance + +Use for response latency, first-content latency, token throughput and model/provider +comparisons. First initialize and select captures using +[investigate an agent run]($SKILL_ROOT/workflows/agentprof/investigate_agent_run.md). +Reuse that warm session and its scope; do not rerun setup unnecessarily. + +## 1. Compare like measurements + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/model_summary.sql +``` + +Rows are grouped by capture, provider, raw model identity and recorded timing +semantics. Columns include response/incomplete counts, duration sample count, +mean/median/p95 duration, first-content mean/sample count, reported usage/sample +counts, and output-token throughput/sample count. Median is the middle value +(or mean of two middle values); p95 is nearest-rank. Small samples are descriptive, +not a reliable estimate of a population tail. + +Only completed measured responses contribute to latency and throughput. Known +zero tokens are retained; unknown tokens are NULL. Throughput is reported output +tokens divided by the sum of eligible response durations, **not wall-clock output +rate** or streaming-only decode speed. Show its denominator and sample count. + +Do not collapse timing groups: + +- Pi generally records native response spans and first content. +- Claude can record request durations including retries; request placement may + come from journal observations or native modifications. +- Codex plugin-only transcript completions report usage without measured model + duration or first content. A timestamped completion is not a latency sample. +- Muse native durations do not imply first-content timestamps. + +First content can include reasoning or tool-call data. It is not necessarily the +first visible text token. Provider input/cache accounting also differs: preserve +reported fields; do not assume a universal cache-hit denominator or sum cached +input into input totals twice. Pricing requires a separately verified rate table. + +## 2. Inspect the slow responses and request phase + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/slow_responses.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/request_headers.sql +``` + +The first query returns at most 50 response IDs, start timestamps, timing labels, +measured duration/first content, reported tokens and incomplete flags. The second +returns up to 50 provider-request spans with recorded phase and timing labels. +Interpret a span as request-to-headers only when its phase establishes that +boundary; otherwise report the recorded request span without inventing its end +semantics. Request-to-headers is neither first content nor full response duration. No rows +means no such spans were recorded. Do not add request spans to response duration +or infer retry counts from a retries-inclusive label. + +`model-call` script lifecycles are not native assistant responses; queueing and +wrapper work may be included. Their durations belong in +[tools and delegation]($SKILL_ROOT/workflows/agentprof/analyze_tools_and_delegation.md), +not this throughput denominator. + +## 3. Report + +State which provider/model/timing cohort dominates measured latency, the sample +counts and missing coverage, and specific response IDs to investigate. Distinguish +high first-content latency from a long overall response only where both are +recorded. Token volume and context may correlate with latency; these recordings +alone do not prove a provider bottleneck or task quality improvement. Follow the +shared report/privacy rules in the entry workflow. diff --git a/ai/skills/dev.agentprof/workflows/agentprof/analyze_tools_and_delegation.md b/ai/skills/dev.agentprof/workflows/agentprof/analyze_tools_and_delegation.md new file mode 100644 index 0000000..2a65608 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/analyze_tools_and_delegation.md @@ -0,0 +1,58 @@ +# Analyze AgentProf tools and delegation + +Use for slow tools, scripted calls, parallel work and child-agent relationships. +First initialize and select captures using +[investigate an agent run]($SKILL_ROOT/workflows/agentprof/investigate_agent_run.md). +Keep that session warm and its selection intact. + +## 1. Find the expensive measured operations + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/tool_summary.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/slow_tools.sql +``` + +The summary groups capture, operation kind/name and timing semantics. It reports +calls, incomplete and duration sample counts, summed completed work, longest +completed call, error and outcome counts. NULL outcomes are not success. +`summed_work_ms` is work volume, **not elapsed occupancy**. Use the shared +`run_summary.sql` for interval-union occupancy, parallel-tool time and peak +concurrency; coincident starts/ends do not inflate concurrency. + +The slow-operation query returns at most 50 event IDs, timestamps, kinds, measured +durations, incomplete/error flags, parent-call IDs and script descendant counts. +It omits prompts, tool arguments, code and outputs. Separate actual execution +from hook/dispatch timing. An incomplete call has no measured completed duration. + +Scripts enclose their nested work. Do not add script duration and descendant tool +durations as elapsed time. `model-call` containers may include queueing and +wrapper lifecycle work; they are not native model-response durations. + +## 2. Inspect delegation without inventing dependencies + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/delegation.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/child_launches.sql +``` + +The hierarchy query returns capture/root/parent IDs, subagent status, harness, +retained bounds and parent-child extent overlap. That is overlap of capture +**extents**, not proof of simultaneous productive work or a causal critical path. +A selected parent also includes its descendants. Parent links use capture-scoped +flows and unambiguous session/time evidence; unresolved or cyclic evidence must +not be forced into a tree. + +The launch query reports recorded launch event IDs, timestamps, measured launch +lifecycle duration, incomplete flags and child session/role metadata. Launch +duration is not child execution duration. Session IDs alone cannot disambiguate +reused captures. Cross-check unavailable-child diagnostics in the inventory; a +missing child recording is not evidence that no child ran. + +## 3. Report + +Name the slow operations and timing semantics, failed/incomplete coverage, serial +versus overlapping measured execution, and resolved/unresolved worker relationships. +Identify a critical path only if recorded dependency edges actually establish it; +overlap or gaps alone do not. Suggestions such as batching or parallelizing are +hypotheses to test on equivalent work, not demonstrated speedups. Follow the +shared report/privacy rules; never dump tool content by default. diff --git a/ai/skills/dev.agentprof/workflows/agentprof/compare_agent_runs.md b/ai/skills/dev.agentprof/workflows/agentprof/compare_agent_runs.md new file mode 100644 index 0000000..a55df76 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/compare_agent_runs.md @@ -0,0 +1,57 @@ +# Compare AgentProf runs + +Use for before/after changes and comparisons across harnesses, models or workers. +Initialize each trace once using +[investigate an agent run]($SKILL_ROOT/workflows/agentprof/investigate_agent_run.md), +then retain one warm session per trace. Capture IDs are local to each trace. + +## 1. Define an honest cohort + +Before ranking, establish equivalent tasks, correctness criteria, model/provider, +effort, tool availability, initial context, cache/warm-up state, harness versions, +hardware and capture boundaries. Use recorded metadata and user-supplied labels; +when content is absent, do not infer task equivalence from session IDs or invent +it from timing. A successful tool call is not task correctness. + +Keep selected capture IDs and trace aliases explicit. Compare complete matching +windows where possible; partial captures, missing child recordings, dropped events +or incompatible timing semantics require a restricted comparison or no ranking. +Do not silently roll child token totals or context into the parent. + +## 2. Collect identical scorecards + +Run these for each selected session/scope: + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/inventory.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/run_summary.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/model_summary.sql +``` + +Compare observed extent, prompt/response/tool counts, measured occupancy and +concurrency, reported token-counter peaks, response usage and measurement sample +counts, and peak context/window. Preserve counter peaks versus response usage and +per-provider timing groups. Codex plugin-only captures cannot enter a measured +model-latency ranking. Missing metrics stay “unavailable”, not zero. + +For repeated equivalent runs, report sample count, median and range per condition; +show per-run values before aggregation. For a single run, report an observed delta, +not a statistically established improvement. Do not combine percentiles from +separate cohorts or present throughput as task quality. + +## 3. Check cross-session time alignment only if needed + +Duration-only comparisons need no shared timeline. For simultaneous recordings, +clock snapshots map BOOTTIME/monotonic event domains to system CLOCK_REALTIME; +they do not correct machine realtime skew. Verify machine identity/clock coverage +and synchronization assumptions before asserting cross-machine ordering or overlap. +Inventory clock errors are trace-wide; zero errors is not proof of synchronized +hosts. Never join unrelated processes by raw PID/TID alone. + +## 4. Report + +Provide a comparison table with trace/capture scope, task-equivalence evidence, +configuration differences, coverage and comparable measured metrics. State which +conclusions are supported, which remain confounded and what controlled recording +would discriminate them. Do not automatically launch that recording or call a +model. Follow the shared report/privacy rules. diff --git a/ai/skills/dev.agentprof/workflows/agentprof/correlate_runtime_resources.md b/ai/skills/dev.agentprof/workflows/agentprof/correlate_runtime_resources.md new file mode 100644 index 0000000..8b01990 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/correlate_runtime_resources.md @@ -0,0 +1,57 @@ +# Correlate AgentProf runtime resources + +Use when recorded process CPU, RSS or JS-heap samples may explain a slow phase. +First initialize and select captures using +[investigate an agent run]($SKILL_ROOT/workflows/agentprof/investigate_agent_run.md). +Reuse the warm trace session. + +## 1. Check actual resource coverage + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/runtime_summary.sql +``` + +Rows identify capture and counter-track IDs, metric, explicit unit, sample count, +first/last sample timestamps, minimum/maximum and sampled CPU time. No rows means +these resources were not recorded. Pi commonly records them; do not claim the +other harnesses do solely because they are supported by AgentProf. + +RSS and JS heap are byte gauges, not allocations or memory attribution. A terminal +zero gauge ends a counter lifetime, not evidence memory was freed to zero. The +summary's gauge extrema use positive samples. CPU is **microseconds consumed +since the previous sample**, not a utilization percentage or cumulative counter. +Its sum covers sampled intervals only; it is not automatically full-run CPU time. +Closing/baseline samples and incomplete coverage must be called out. + +## 2. Align samples with the measured phase + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/runtime_samples.sql +``` + +This returns the latest 200 timestamped samples. If the phase is older, narrow a +validated query over `counter` joined to `agentprof_skill_counter_tracks` by +track ID, using its capture ID and numeric timestamp bounds. Do not infer a peak +from an uninspected truncated sample window. Locate response/tool/turn IDs with +`slow_responses.sql`, `slow_tools.sql` or `slow_turns.sql` and report +sample spacing relative to the phase duration. + +CPU samples describe the preceding sampling interval; use their previous sample +timestamp for overlap, not the sample instant as an execution span. Process-level +resource pressure can coincide with a tool/response without identifying its cause. +An external tool process may not be included in the harness's process samples. + +For OS-thread detail, first discover actual process/thread tracks and available +scheduler or profile tables using +[querying.md]($SKILL_ROOT/infra-references/querying.md). AgentProf OS-thread spans +do not imply `sched`, per-thread CPU, runnable delay, CPU stacks or allocation +profiles were collected. Use existing generic Perfetto analysis only when those +data sources are present; scope by process/thread identity and machine, not PID +alone. Do not launch a new system recording without authorization. + +## 3. Report + +Name the resource peak or sampled CPU interval and the overlapping agent phase, +with capture/track IDs, timestamp range, units and coverage. Treat correlation as +a lead, not proof of CPU-bound execution, a memory leak or attribution to a model. +Follow the shared report/privacy rules. diff --git a/ai/skills/dev.agentprof/workflows/agentprof/investigate_agent_run.md b/ai/skills/dev.agentprof/workflows/agentprof/investigate_agent_run.md new file mode 100644 index 0000000..8d71622 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/investigate_agent_run.md @@ -0,0 +1,111 @@ +# Investigate an AgentProf run + +Use this workflow for “why was the agent slow?” or an unfamiliar AgentProf trace. +It analyzes existing recordings from Pi, Claude Code, Codex and Muse. It does not +launch a harness, call a model, record a session or upload a trace. + +If the trace is not already loaded, follow +[querying.md]($SKILL_ROOT/infra-references/querying.md) first. Keep that session warm. +For every workflow here, read Perfetto's environment setup before invoking its +commands; use the trace session identifier in place of SESSION below. + +## 1. Inventory before interpreting + +Initialize the namespaced reader and run the inventory: + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/agentprof_setup.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/inventory.sql +``` + +Setup is self-contained SQL. No AgentProf checkout, UI or TypeScript is needed. +Run it once per trace session; rerunning resets only the workflow's +`agentprof_skill_*` objects and clears its capture selection. It does not replace +UI-owned `agentprof_*` tables. No prompt text or tool payloads are materialized. + +No inventory rows means this reader found no supported AgentProf events. Stop +rather than invent agent metrics; generic Perfetto analysis may still apply. +The inventory returns at most 100 captures; if necessary inspect +`SELECT COUNT(*) FROM agentprof_skill_capture_runs` and narrow the scope. + +Inspect: + +- `capture_id` identifies a capture in this trace, not a globally stable ID. + Keep captures separate even when session IDs, native IDs or PIDs repeat. +- `harness/provider/model/effort` are recorded configuration. A capture may + contain multiple models; model-response rows give their actual identities. +- `measured_responses`, `first_content_samples`, `output_usage_samples`, + `context_snapshots` and `runtime_metrics` describe observed coverage. + Missing is not zero; do not infer capabilities solely from the harness name. +- Dropped events, lane overflows, incomplete work and unavailable children reduce + confidence. NULL diagnostic counters mean not recorded, not “healthy”. +- `trace_clock_errors` is trace-wide, repeated for convenience. Zero reported + errors does not establish cross-machine clock synchronization. + +For a focused investigation, substitute numeric capture IDs from the inventory: + +```sql +DELETE FROM agentprof_skill_scope; +INSERT INTO agentprof_skill_scope VALUES (CAPTURE_ID); +``` + +Execute that SQL with `trace_processor query --remote SESSION '...'`. +The query files do not substitute variables. An empty scope selects all captures; +multiple inserted IDs select several. Do not interpolate prompt text into SQL. + +## 2. Locate the measured time + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/run_summary.sql +``` + +One row per capture reports `observed_extent_ms`, model-only and tool-only +occupancy, their overlap, script-only occupancy, unattributed time, tool +concurrency, reported token-counter peaks and context. The window covers retained +capture events plus measured response intervals, not the entire session or trace. +Script/model-call containers are not extra model responses or nested tool work. +Counter token peaks are not interchangeable with response-usage sums. + +Occupancy uses interval unions, not summed durations. A large model-only share +means recorded responses dominate the measured activity—not proof of provider +compute time. Gaps are **unattributed**, not inferred thinking, queueing, approval +or waiting. Never extend an incomplete operation to the next prompt or trace end +and call that a measured duration. + +If prompt boundaries are recorded, run the optional turn attribution: + +```bash +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/agentprof_turns.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/slow_turns.sql +trace_processor query --remote SESSION --query-file $SKILL_ROOT/workflows/agentprof/scripts/unassigned_work.sql +``` + +A turn is one user-prompt interaction, not a native model/tool iteration. The top +20 rows retain recorded completed bounds; incomplete turns show only observed +extent. Unknown durations remain unknown. Attribution uses unique native identity, +explicit capture-scoped flows, ancestry, then unambiguous containment. Ambiguous +work stays unassigned; provider-request rows linked to responses are not counted +twice. Report unknown models/tools and unassigned counts alongside occupancy. + +Without prompts, do not manufacture turns from iterations. Analyze recorded +responses and tools instead. + +## 3. Drill into the strongest evidence + +- Response latency/usage: [model performance]($SKILL_ROOT/workflows/agentprof/analyze_model_performance.md). +- Tool serialization, scripts or workers: [tools and delegation]($SKILL_ROOT/workflows/agentprof/analyze_tools_and_delegation.md). +- Context growth or compaction: [context pressure]($SKILL_ROOT/workflows/agentprof/analyze_context_pressure.md). +- Before/after or harness comparisons: [compare runs]($SKILL_ROOT/workflows/agentprof/compare_agent_runs.md). +- Recorded CPU/memory samples: [runtime resources]($SKILL_ROOT/workflows/agentprof/correlate_runtime_resources.md). + +## Reporting and privacy + +Save the evidence report following the shared Perfetto router. Include trace and +capture scope, measured numbers, coverage, event IDs/timestamp ranges, validated +SQL and remaining hypotheses. Separate observations from causal claims. + +Default reports should contain counts and operational categories, not raw prompts, +commands, arguments or outputs. Metadata-only capture is not anonymization: +identifiers, names and paths may remain. Prefer capture-local numeric IDs in a +shareable report. Do not recover omitted content from transcripts or journals, +and do not upload artifacts without the user's authorization. diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql new file mode 100644 index 0000000..72c0f75 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql @@ -0,0 +1,407 @@ +-- SPDX-License-Identifier: Apache-2.0 +-- Exported from AgentProf reader SQL. Standalone: no UI or build dependency. +DROP TABLE IF EXISTS agentprof_skill_turns; +DROP TABLE IF EXISTS agentprof_skill_turn_events; +DROP TABLE IF EXISTS agentprof_skill_turn_prompts; +DROP TABLE IF EXISTS agentprof_skill_turn_activity_totals; +DROP TABLE IF EXISTS agentprof_skill_turn_bounds; +DROP TABLE IF EXISTS agentprof_skill_turn_linked_requests; +DROP TABLE IF EXISTS agentprof_skill_turn_request_matches; +DROP TABLE IF EXISTS agentprof_skill_turn_explicit_requests; +DROP TABLE IF EXISTS agentprof_skill_turn_owners; +DROP TABLE IF EXISTS agentprof_skill_turn_prompt_ancestry; +DROP TABLE IF EXISTS agentprof_skill_turn_parent_lookup; +DROP TABLE IF EXISTS agentprof_skill_turn_native_prompts; +DROP TABLE IF EXISTS agentprof_skill_turn_flows; +DROP TABLE IF EXISTS agentprof_skill_turn_flow_ids; +DROP TABLE IF EXISTS agentprof_skill_turn_work; +DROP TABLE IF EXISTS agentprof_skill_turn_clusters; +DROP TABLE IF EXISTS agentprof_skill_turn_prompt_lookup; + +DROP TABLE IF EXISTS agentprof_skill_track_process; +DROP TABLE IF EXISTS agentprof_skill_counter_tracks; +DROP TABLE IF EXISTS agentprof_skill_capture_markers; +DROP TABLE IF EXISTS agentprof_skill_captures; +DROP TABLE IF EXISTS agentprof_skill_slice_parent_lookup; +DROP TABLE IF EXISTS agentprof_skill_slice_capture; +DROP TABLE IF EXISTS agentprof_skill_slices; +DROP TABLE IF EXISTS agentprof_skill_tool_arguments; +DROP TABLE IF EXISTS agentprof_skill_tool_preflights; +DROP TABLE IF EXISTS agentprof_skill_tool_calls; +DROP TABLE IF EXISTS agentprof_skill_script_children; +DROP TABLE IF EXISTS agentprof_skill_session_captures; +DROP TABLE IF EXISTS agentprof_skill_capture_parents; +DROP TABLE IF EXISTS agentprof_skill_capture_hierarchy; +DROP TABLE IF EXISTS agentprof_skill_configuration; +DROP TABLE IF EXISTS agentprof_skill_session_labels; +DROP TABLE IF EXISTS agentprof_skill_capture_runs; +DROP TABLE IF EXISTS agentprof_skill_messages; +DROP TABLE IF EXISTS agentprof_skill_activity; +DROP TABLE IF EXISTS agentprof_skill_context_snapshots; +DROP TABLE IF EXISTS agentprof_skill_context_changes; + +CREATE PERFETTO TABLE agentprof_skill_track_process AS +WITH RECURSIVE ancestry(track_id, ancestor_id) AS ( + SELECT id, id FROM track + UNION + SELECT a.track_id, t.parent_id FROM ancestry a + JOIN track t ON t.id = a.ancestor_id WHERE t.parent_id IS NOT NULL +) +SELECT a.track_id, MAX(COALESCE(p.upid, c.upid, th.upid)) AS upid, + COALESCE(MAX(CASE WHEN t.name = 'agentprof.capture' THEN t.id END), + -1 - MAX(COALESCE(p.upid, c.upid, th.upid)), -1 - a.track_id) AS capture_id +FROM ancestry a JOIN track t ON t.id = a.ancestor_id +LEFT JOIN process_track p ON p.id = a.ancestor_id +LEFT JOIN process_counter_track c ON c.id = a.ancestor_id +LEFT JOIN thread_track tt ON tt.id = a.ancestor_id +LEFT JOIN thread th ON th.utid = tt.utid +GROUP BY a.track_id; + + +CREATE PERFETTO TABLE agentprof_skill_counter_tracks AS +WITH names(metric, display_name, group_name) AS (VALUES + ('context.system', 'System instructions', 'Context'), + ('context.rules', 'Rules and memory', 'Context'), + ('context.skills', 'Skills', 'Context'), + ('context.tools', 'Tool definitions', 'Context'), + ('context.environment', 'Environment', 'Context'), + ('context.prompts', 'User prompts', 'Context'), + ('context.assistant', 'Assistant history', 'Context'), + ('context.results', 'Tool results', 'Context'), + ('context.summaries', 'Compaction summaries', 'Context'), + ('context.overhead', 'Harness overhead', 'Context'), + ('context.messages', 'Conversation', 'Context'), + ('context.unattributed', 'Unattributed', 'Context'), + ('llm.tokens.input', 'Input tokens', NULL), + ('llm.tokens.output', 'Output tokens', NULL), + ('llm.context.estimated_tokens', 'Context size', NULL), + ('llm.context.window_tokens', 'Context window', NULL), + ('runtime.rss', 'Resident memory', 'Runtime'), + ('runtime.heap', 'JS heap', 'Runtime'), + ('runtime.cpu', 'CPU time (interval)', 'Runtime'), + ('tracing.droppedEvents', 'Dropped events', 'Tracing'), + ('tracing.queueDepth', 'Queue depth', 'Tracing'), + ('tracing.laneOverflows', 'Lane overflows', 'Tracing') +) +SELECT t.id, n.metric AS name, n.display_name, n.group_name, m.capture_id +FROM counter_track t JOIN names n ON t.name IN (n.metric, n.display_name, 'Context: ' || n.display_name) + OR (n.metric = 'llm.context.estimated_tokens' AND t.name = 'Context size (est.)') +JOIN agentprof_skill_track_process m ON m.track_id = t.id; + + +-- OS thread tracks are shared by captures and have no capture parent. Join +-- their capture spans to the generic capture track using its recorded UUID. +CREATE PERFETTO TABLE agentprof_skill_capture_markers AS +SELECT s.id AS slice_id, COALESCE(root.id, m.capture_id) AS capture_id, + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.capture_id'), EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_capture_id')) AS recorded_id, + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.session_id'), EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_session_id')) AS session_id, + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.label'), EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_capture_label')) AS label +FROM slice s JOIN agentprof_skill_track_process m ON m.track_id = s.track_id +LEFT JOIN track root ON root.name = 'agentprof.capture' + AND printf('%x', EXTRACT_ARG(root.source_arg_set_id, 'trace_id')) = + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.capture_id'), EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_capture_id')) +WHERE s.category IN ('pi.metadata', 'claude.metadata', 'codex.metadata', 'muse.metadata', 'agentprof.metadata') AND (s.name IN ('tracing', 'tracing-start') + OR s.name GLOB 'profile ([0-9]*)'); + +CREATE PERFETTO TABLE agentprof_skill_captures AS +SELECT capture_id, MAX(recorded_id) AS recorded_id, MAX(session_id) AS session_id, + MAX(label) AS label FROM agentprof_skill_capture_markers GROUP BY capture_id; + + +CREATE TABLE agentprof_skill_slice_parent_lookup AS +SELECT s.id, s.parent_id FROM slice s WHERE s.parent_id IS NOT NULL + AND s.track_id IN (SELECT DISTINCT marker.track_id FROM slice marker + JOIN agentprof_skill_capture_markers m ON m.slice_id = marker.id); +CREATE INDEX agentprof_skill_slice_parent_idx ON agentprof_skill_slice_parent_lookup(parent_id); +CREATE PERFETTO TABLE agentprof_skill_slice_capture AS +WITH RECURSIVE captured(id, capture_id) AS ( + SELECT slice_id, capture_id FROM agentprof_skill_capture_markers + UNION ALL + SELECT s.id, c.capture_id FROM agentprof_skill_slice_parent_lookup s + JOIN captured c ON s.parent_id = c.id + WHERE s.id NOT IN (SELECT slice_id FROM agentprof_skill_capture_markers) +) +SELECT c.id, c.capture_id FROM captured c JOIN slice s ON s.id = c.id +WHERE s.category GLOB 'pi.*' OR s.category GLOB 'claude.*' OR s.category GLOB 'codex.*' + OR s.category GLOB 'muse.*' OR s.category GLOB 'agentprof.*'; +DROP INDEX agentprof_skill_slice_parent_idx; +DROP TABLE agentprof_skill_slice_parent_lookup; + + +CREATE PERFETTO TABLE agentprof_skill_slices AS +SELECT s.*, COALESCE(t.name, th.name || ' ' || th.tid, 'Thread ' || th.tid) AS track_name, + m.upid, tt.utid, COALESCE(sc.capture_id, m.capture_id) AS capture_id, + COALESCE(cap.label, SUBSTR(cap.recorded_id, 1, 8), 'Legacy capture') AS capture, + COALESCE(cap.session_id, (SELECT SUBSTR(a.string_value, 9) FROM args a + WHERE a.arg_set_id = p.arg_set_id AND a.key GLOB 'chrome.process_label*' + AND a.string_value GLOB 'session:*' LIMIT 1), + 'process ' || COALESCE(CAST(m.upid AS TEXT), 'unknown')) AS session, + COALESCE(CASE WHEN s.name = 'codemode' AND s.category = 'pi.tools' THEN 'script' END, + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.kind'), EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_event_kind')), + CASE + WHEN (',' || s.category || ',') GLOB '*,pi.agent,*' AND s.name = 'prompt' THEN 'prompt' + WHEN s.category = 'pi.agent' AND s.name = 'turn' THEN 'turn' + WHEN s.category = 'pi.llm' AND s.name IN ('request', 'provider-request') THEN 'provider-request' + WHEN s.category = 'pi.tools' AND (t.name IN ('tools', 'Tools') OR t.name GLOB 'tools.lane.*') THEN 'tool-execution' + WHEN s.category = 'pi.llm' AND s.name GLOB 'assistant ttft=*' THEN 'assistant-message' + ELSE 'other' + END) AS kind, + (s.dur < 0 OR COALESCE(COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.incomplete'), EXTRACT_ARG(s.arg_set_id, 'debug.pi_tracing_incomplete')), 0) + OR COALESCE(COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.incomplete'), EXTRACT_ARG(s.arg_set_id, 'debug.pi_tracing_providerIncomplete')), 0)) AS incomplete +FROM slice s JOIN track t ON t.id = s.track_id +LEFT JOIN agentprof_skill_track_process m ON m.track_id = t.id +LEFT JOIN thread_track tt ON tt.id = t.id +LEFT JOIN thread th ON th.utid = tt.utid +LEFT JOIN agentprof_skill_slice_capture sc ON sc.id = s.id +LEFT JOIN process p ON p.upid = m.upid +LEFT JOIN agentprof_skill_captures cap ON cap.capture_id = COALESCE(sc.capture_id, m.capture_id) +WHERE s.category GLOB 'pi.*' OR s.category GLOB 'claude.*' OR s.category GLOB 'codex.*' OR s.category GLOB 'muse.*' OR s.category GLOB 'agentprof.*'; + +CREATE PERFETTO TABLE agentprof_skill_tool_arguments AS +SELECT CAST(NULL AS INTEGER) AS arg_set_id, CAST(NULL AS STRING) AS arguments WHERE 0; + +-- Older recordings kept arguments on preflight rather than execution spans. +CREATE PERFETTO TABLE agentprof_skill_tool_preflights AS +SELECT capture_id, COALESCE(EXTRACT_ARG(arg_set_id, 'debug.call_id'), + EXTRACT_ARG(arg_set_id, 'debug.agentprof_tool_call_id')) AS call_id, MAX(id) AS id +FROM agentprof_skill_slices WHERE name = 'tool-preflight' GROUP BY capture_id, call_id; + +CREATE PERFETTO TABLE agentprof_skill_tool_calls AS +SELECT s.*, COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.call_id'), + EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_tool_call_id')) AS call_id, + EXTRACT_ARG(s.arg_set_id, 'debug.parent_call_id') AS parent_call_id, + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.is_error'), + EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_tool_is_error')) AS is_error, + NULL AS intent, + COALESCE(a.arguments, legacy.arguments) AS arguments, + COALESCE(NULLIF(EXTRACT_ARG(s.arg_set_id, 'debug.language'), ''), + CASE WHEN s.kind = 'script' AND s.name = 'codemode' AND s.category = 'pi.tools' THEN 'JavaScript' END) AS language, + EXTRACT_ARG(s.arg_set_id, 'debug.line_count') AS line_count, + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.args_truncated'), + EXTRACT_ARG(p.arg_set_id, 'debug.truncated'), 0) AS args_truncated +FROM agentprof_skill_slices s LEFT JOIN agentprof_skill_tool_preflights pf + ON pf.capture_id = s.capture_id AND pf.call_id = COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.call_id'), + EXTRACT_ARG(s.arg_set_id, 'debug.agentprof_tool_call_id')) +LEFT JOIN agentprof_skill_slices p ON p.id = pf.id +LEFT JOIN agentprof_skill_tool_arguments a ON a.arg_set_id = s.arg_set_id +LEFT JOIN agentprof_skill_tool_arguments legacy ON legacy.arg_set_id = p.arg_set_id +WHERE s.kind IN ('tool-execution', 'script', 'model-call'); + +CREATE PERFETTO TABLE agentprof_skill_script_children AS +WITH RECURSIVE children(script_id, id, depth) AS ( + SELECT p.id, c.id, 1 FROM agentprof_skill_tool_calls p JOIN agentprof_skill_tool_calls c + ON c.capture_id = p.capture_id AND c.parent_call_id = p.call_id + WHERE p.kind = 'script' AND c.id != p.id + UNION ALL + SELECT c.script_id, next.id, c.depth + 1 FROM children c + JOIN agentprof_skill_tool_calls p ON p.id = c.id + JOIN agentprof_skill_tool_calls next ON next.capture_id = p.capture_id AND next.parent_call_id = p.call_id + WHERE next.id != p.id AND c.depth < 32 +) SELECT * FROM children; + + +CREATE PERFETTO TABLE agentprof_skill_session_captures AS +SELECT capture_id, session, MIN(ts) AS start_ts, MAX(ts + MAX(dur, 0)) AS end_ts, + MAX(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.parent_session'), EXTRACT_ARG(arg_set_id, 'debug.parentSession'))) AS parent_session, + MAX(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.parent_session'), EXTRACT_ARG(arg_set_id, 'debug.parentSession')) IS NOT NULL OR + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.child_role'), EXTRACT_ARG(arg_set_id, 'debug.childRole')) IS NOT NULL) AS is_subagent +FROM agentprof_skill_slices GROUP BY capture_id, session; + +CREATE PERFETTO TABLE agentprof_skill_capture_parents AS +WITH candidates AS ( + SELECT DISTINCT c.capture_id, p.capture_id AS parent_capture_id, 0 AS priority + FROM flow f JOIN agentprof_skill_slices p ON p.id = f.slice_out + JOIN agentprof_skill_slices c ON c.id = f.slice_in + WHERE p.kind = 'tool-execution' AND c.name = 'prompt-input' + AND p.capture_id != c.capture_id + UNION + SELECT c.capture_id, p.capture_id, 1 FROM agentprof_skill_session_captures c + JOIN agentprof_skill_slices launch ON COALESCE(EXTRACT_ARG(launch.arg_set_id, 'debug.child_session'), EXTRACT_ARG(launch.arg_set_id, 'debug.childSession')) = c.session + JOIN agentprof_skill_session_captures p ON p.capture_id = launch.capture_id + WHERE c.capture_id != p.capture_id + AND (c.parent_session IS NULL OR c.parent_session = p.session) + AND c.start_ts BETWEEN p.start_ts AND p.end_ts + UNION + SELECT c.capture_id, p.capture_id, 2 FROM agentprof_skill_session_captures c + JOIN agentprof_skill_session_captures p ON p.session = c.parent_session + WHERE c.capture_id != p.capture_id AND c.start_ts BETWEEN p.start_ts AND p.end_ts +), preferred AS ( + SELECT *, MIN(priority) OVER (PARTITION BY capture_id) AS best FROM candidates +) +SELECT capture_id, MIN(parent_capture_id) AS parent_capture_id +FROM preferred WHERE priority = best GROUP BY capture_id +HAVING COUNT(DISTINCT parent_capture_id) = 1; + +CREATE PERFETTO TABLE agentprof_skill_capture_hierarchy AS +WITH RECURSIVE ancestors(capture_id, ancestor_id) AS ( + SELECT capture_id, capture_id FROM agentprof_skill_session_captures + UNION + SELECT a.capture_id, p.parent_capture_id FROM ancestors a + JOIN agentprof_skill_capture_parents p ON p.capture_id = a.ancestor_id +), roots AS ( + SELECT a.capture_id, MIN(a.ancestor_id) AS root_capture_id FROM ancestors a + LEFT JOIN agentprof_skill_capture_parents p ON p.capture_id = a.ancestor_id + WHERE p.capture_id IS NULL GROUP BY a.capture_id +) +SELECT c.capture_id, COALESCE(r.root_capture_id, c.capture_id) AS root_capture_id, + p.parent_capture_id, + COALESCE(c.is_subagent, 0) OR p.parent_capture_id IS NOT NULL AS is_subagent +FROM agentprof_skill_session_captures c LEFT JOIN roots r USING(capture_id) +LEFT JOIN agentprof_skill_capture_parents p USING(capture_id); + +CREATE PERFETTO TABLE agentprof_skill_configuration AS + SELECT capture_id, arg_set_id, + COALESCE(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.harness'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_harness')), 'Not recorded') AS harness, + COALESCE(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.provider'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_llm_provider')), 'Not recorded') AS provider, + COALESCE(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.model'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_llm_model')), 'Not recorded') AS model, + COALESCE(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.effort'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_llm_effort')), 'Not recorded') AS effort, + EXTRACT_ARG(arg_set_id, 'debug.context_window_tokens') AS context_window_tokens + FROM agentprof_skill_slices + WHERE category IN ('pi.metadata', 'claude.metadata', 'codex.metadata', 'muse.metadata', 'agentprof.metadata') AND (name IN ('tracing', 'tracing-start', 'run-configuration') + OR name GLOB 'profile ([0-9]*)'); +CREATE PERFETTO TABLE agentprof_skill_session_labels AS +SELECT capture_id, JSON_GROUP_ARRAY(label) AS session_labels FROM ( + SELECT DISTINCT c.capture_id, TRIM(a.string_value) AS label + FROM agentprof_skill_configuration c JOIN args a USING(arg_set_id) + WHERE a.flat_key = 'debug.session_labels' AND a.key GLOB 'debug.session_labels[[]*' + AND TRIM(a.string_value) != '' + ORDER BY c.capture_id, label +) GROUP BY capture_id; +CREATE PERFETTO TABLE agentprof_skill_capture_runs AS +WITH configuration AS ( + SELECT capture_id, GROUP_CONCAT(DISTINCT harness) AS harness, + GROUP_CONCAT(DISTINCT provider) AS provider, GROUP_CONCAT(DISTINCT model) AS model, + GROUP_CONCAT(DISTINCT effort) AS effort, + MAX(context_window_tokens) AS context_window_tokens, + COUNT(DISTINCT context_window_tokens) AS context_window_count + FROM agentprof_skill_configuration GROUP BY capture_id + ), per_track AS ( + SELECT p.capture_id, t.id, t.name, MAX(c.value) AS maximum FROM counter c + JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id + JOIN agentprof_skill_track_process p ON p.track_id = t.id + WHERE t.name IN ('llm.tokens.input', 'llm.tokens.output', 'llm.context.estimated_tokens') + GROUP BY p.capture_id, t.id, t.name + ), tokens AS ( + SELECT capture_id, + SUM(CASE WHEN name = 'llm.tokens.input' THEN maximum END) AS input_tokens, + SUM(CASE WHEN name = 'llm.tokens.output' THEN maximum END) AS output_tokens, + MAX(CASE WHEN name = 'llm.context.estimated_tokens' THEN maximum END) AS peak_context + FROM per_track GROUP BY capture_id + ), capture_peak AS ( + SELECT m.capture_id, + MAX(EXTRACT_ARG(s.arg_set_id, 'debug.peak_context_tokens')) AS peak_context + FROM agentprof_skill_capture_markers m JOIN slice s ON s.id = m.slice_id + GROUP BY m.capture_id + ), captures AS ( + SELECT capture_id, session, MIN(ts) AS start_ts, MAX(ts + MAX(dur, 0)) AS end_ts, + SUM(kind = 'prompt') AS turns, SUM(kind = 'assistant-message') AS responses, + SUM(kind = 'tool-execution') AS tools, SUM(incomplete) AS incomplete, + (MAX(ts + MAX(dur, 0)) - MIN(ts)) / 1e6 AS duration_ms + FROM agentprof_skill_slices GROUP BY capture_id, session + ) SELECT s.*, COALESCE(c.label, SUBSTR(c.recorded_id, 1, 8), 'Legacy capture') AS capture, + c.recorded_id, configuration.harness, configuration.provider, configuration.model, labels.session_labels, + configuration.effort, configuration.context_window_tokens, + tokens.input_tokens, tokens.output_tokens, + COALESCE(capture_peak.peak_context, tokens.peak_context) AS peak_context, + CASE WHEN configuration.context_window_count = 1 AND configuration.context_window_tokens > 0 + THEN CAST(COALESCE(capture_peak.peak_context, tokens.peak_context) AS DOUBLE) + / configuration.context_window_tokens END AS context_share + FROM captures s LEFT JOIN agentprof_skill_captures c USING(capture_id) + LEFT JOIN agentprof_skill_session_labels labels USING(capture_id) + LEFT JOIN configuration USING(capture_id) LEFT JOIN tokens USING(capture_id) + LEFT JOIN capture_peak USING(capture_id) + ORDER BY s.start_ts, s.capture_id; + +CREATE PERFETTO TABLE agentprof_skill_messages AS +SELECT *, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.provider'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_llm_provider')) AS provider, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.model'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_llm_model')) AS model, + CASE WHEN dur > 0 THEN dur ELSE + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.duration_ns'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_stream_duration_ns')) END AS message_ns, + CASE WHEN dur > 0 THEN ts ELSE + ts - COALESCE(EXTRACT_ARG(arg_set_id, 'debug.duration_ns'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_stream_duration_ns')) END AS message_start_ts, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.first_content_ns'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_stream_first_content_ns')) AS first_ns, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.input_tokens'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_usage_input')) AS input_tokens, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.output_tokens'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_usage_output')) AS output_tokens, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.cache_read_tokens'), EXTRACT_ARG(arg_set_id, 'debug.agentprof_usage_cacheRead')) AS cache_read_tokens +FROM agentprof_skill_slices WHERE kind = 'assistant-message'; + +-- Sweep interval boundaries together so coincident starts/ends do not create +-- artificial concurrency. Only completed, measured intervals contribute. +CREATE PERFETTO TABLE agentprof_skill_activity AS +WITH intervals AS ( + SELECT ts, ts + dur AS end_ts, 1 AS tools, 0 AS models, 0 AS scripts + FROM agentprof_skill_slices WHERE kind = 'tool-execution' AND NOT incomplete AND dur > 0 + UNION ALL + SELECT message_start_ts, message_start_ts + message_ns, 0, 1, 0 FROM agentprof_skill_messages + WHERE message_ns > 0 AND NOT incomplete + UNION ALL + SELECT ts, ts + dur, 0, 0, 1 FROM agentprof_skill_slices + WHERE kind IN ('script', 'model-call') AND NOT incomplete AND dur > 0 +), boundaries AS ( + SELECT ts, tools, models, scripts FROM intervals + UNION ALL SELECT end_ts, -tools, -models, -scripts FROM intervals +), deltas AS ( + SELECT ts, SUM(tools) AS tools, SUM(models) AS models, SUM(scripts) AS scripts FROM boundaries GROUP BY ts +) +SELECT ts, LEAD(ts) OVER (ORDER BY ts) - ts AS dur, + SUM(tools) OVER (ORDER BY ts) AS tools, + SUM(models) OVER (ORDER BY ts) AS models, + SUM(scripts) OVER (ORDER BY ts) AS scripts +FROM deltas; + +CREATE PERFETTO TABLE agentprof_skill_context_snapshots AS +SELECT id AS event_id, arg_set_id, capture_id, session, + ts + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.context.sample_offset_ns'), 0) AS ts, + EXTRACT_ARG(arg_set_id, 'debug.context.stage') AS stage, + EXTRACT_ARG(arg_set_id, 'debug.context.basis') AS basis, + EXTRACT_ARG(arg_set_id, 'debug.context.coverage') AS coverage, + EXTRACT_ARG(arg_set_id, 'debug.context.baseline') AS baseline, + EXTRACT_ARG(arg_set_id, 'debug.context.model') AS model, + EXTRACT_ARG(arg_set_id, 'debug.context.estimated_tokens') AS estimated_tokens, + EXTRACT_ARG(arg_set_id, 'debug.context.reported_tokens') AS reported_tokens, + EXTRACT_ARG(arg_set_id, 'debug.context.window_tokens') AS window_tokens, + EXTRACT_ARG(arg_set_id, 'debug.context.effective_window_tokens') AS effective_window_tokens, + EXTRACT_ARG(arg_set_id, 'debug.context.compact_threshold_tokens') AS compact_threshold_tokens, + EXTRACT_ARG(arg_set_id, 'debug.context.omitted_changes') AS omitted_changes, + (SELECT JSON_GROUP_OBJECT(SUBSTR(a.key, LENGTH('debug.context.categories.') + 1), a.int_value) + FROM args a WHERE a.arg_set_id = s.arg_set_id AND a.key GLOB 'debug.context.categories.*') AS categories +FROM agentprof_skill_slices s WHERE EXTRACT_ARG(arg_set_id, 'debug.context.version') = 1; + +CREATE PERFETTO TABLE agentprof_skill_context_changes AS +WITH measured AS ( + SELECT s.*, CASE WHEN NOT s.baseline AND s.basis = LAG(s.basis) OVER (PARTITION BY s.capture_id ORDER BY s.ts, s.event_id) + THEN LAG(s.estimated_tokens) OVER (PARTITION BY s.capture_id ORDER BY s.ts, s.event_id) END AS previous_estimated_tokens + FROM agentprof_skill_context_snapshots s +), prefixes AS ( + SELECT a.arg_set_id, SUBSTR(a.key, 1, LENGTH(a.key) - 3) AS prefix + FROM args a WHERE a.key GLOB 'debug.context.changes[[]*].id' +), changes AS ( + SELECT s.event_id AS snapshot_id, s.capture_id, s.session, s.ts, COALESCE(EXTRACT_ARG(p.arg_set_id, 'debug.context.item_stage'), s.stage) AS stage, s.basis, + s.baseline, s.estimated_tokens, s.reported_tokens, + s.previous_estimated_tokens, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.id') AS item_id, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.category') AS category, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.change') AS change, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.tokens') AS tokens, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.delta_tokens') AS delta_tokens, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.chars') AS chars, + NULL AS label, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.source_id') AS source_id, + EXTRACT_ARG(p.arg_set_id, p.prefix || '.source_kind') AS source_kind + FROM prefixes p JOIN measured s USING(arg_set_id) +) +SELECT c.*, COALESCE((SELECT t.id FROM agentprof_skill_tool_calls t + WHERE t.capture_id = c.capture_id AND t.call_id = c.source_id + AND c.source_id != '' ORDER BY t.ts DESC LIMIT 1), c.snapshot_id) AS event_id, + (SELECT t.name FROM agentprof_skill_tool_calls t WHERE t.capture_id = c.capture_id + AND t.call_id = c.source_id AND c.source_id != '' ORDER BY t.ts DESC LIMIT 1) AS tool +FROM changes c; + +DROP TABLE IF EXISTS agentprof_skill_scope; +CREATE TABLE agentprof_skill_scope(capture_id INTEGER PRIMARY KEY); +DROP VIEW IF EXISTS agentprof_skill_selected_captures; +CREATE VIEW agentprof_skill_selected_captures AS +SELECT * FROM agentprof_skill_capture_runs +WHERE NOT EXISTS (SELECT 1 FROM agentprof_skill_scope) + OR capture_id IN (SELECT capture_id FROM agentprof_skill_scope); diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql new file mode 100644 index 0000000..d27606d --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql @@ -0,0 +1,253 @@ +-- SPDX-License-Identifier: Apache-2.0 +-- Exported from AgentProf reader SQL. Standalone: no UI or build dependency. +DROP TABLE IF EXISTS agentprof_skill_turns; +DROP TABLE IF EXISTS agentprof_skill_turn_events; +DROP TABLE IF EXISTS agentprof_skill_turn_prompts; +DROP TABLE IF EXISTS agentprof_skill_turn_activity_totals; +DROP TABLE IF EXISTS agentprof_skill_turn_bounds; +DROP TABLE IF EXISTS agentprof_skill_turn_linked_requests; +DROP TABLE IF EXISTS agentprof_skill_turn_request_matches; +DROP TABLE IF EXISTS agentprof_skill_turn_explicit_requests; +DROP TABLE IF EXISTS agentprof_skill_turn_owners; +DROP TABLE IF EXISTS agentprof_skill_turn_prompt_ancestry; +DROP TABLE IF EXISTS agentprof_skill_turn_parent_lookup; +DROP TABLE IF EXISTS agentprof_skill_turn_native_prompts; +DROP TABLE IF EXISTS agentprof_skill_turn_flows; +DROP TABLE IF EXISTS agentprof_skill_turn_flow_ids; +DROP TABLE IF EXISTS agentprof_skill_turn_work; +DROP TABLE IF EXISTS agentprof_skill_turn_clusters; +DROP TABLE IF EXISTS agentprof_skill_turn_prompt_lookup; + +CREATE PERFETTO TABLE agentprof_skill_turn_prompts AS +WITH prompts AS ( + SELECT s.id AS prompt_id, s.capture_id, s.session, s.capture, s.ts AS start_ts, r.start_ts AS capture_start_ts, + s.incomplete, CASE WHEN s.dur >= 0 THEN s.ts + s.dur ELSE r.end_ts END AS recorded_end_ts, + LEAD(s.ts) OVER (PARTITION BY s.capture_id ORDER BY s.ts, s.id) AS next_start_ts, + ROW_NUMBER() OVER (PARTITION BY s.capture_id ORDER BY s.ts, s.id) AS turn_number, + NULL AS prompt_text, + EXTRACT_ARG(s.arg_set_id, 'debug.turn_id') AS native_turn_id + FROM agentprof_skill_slices s LEFT JOIN agentprof_skill_capture_runs r USING(capture_id) + WHERE s.kind = 'prompt' +) +SELECT *, CASE WHEN incomplete THEN MIN(COALESCE(recorded_end_ts, start_ts), + COALESCE(next_start_ts, recorded_end_ts, start_ts)) + ELSE COALESCE(recorded_end_ts, start_ts) END AS attribution_end_ts +FROM prompts; + +CREATE TABLE agentprof_skill_turn_prompt_lookup AS +WITH previous AS ( + SELECT *, MAX(attribution_end_ts) OVER (PARTITION BY capture_id ORDER BY start_ts, prompt_id + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS previous_end + FROM agentprof_skill_turn_prompts +) +SELECT *, SUM(previous_end IS NULL OR start_ts >= previous_end) OVER ( + PARTITION BY capture_id ORDER BY start_ts, prompt_id ROWS UNBOUNDED PRECEDING) AS cluster_id +FROM previous; +CREATE INDEX agentprof_skill_turn_prompt_id ON agentprof_skill_turn_prompt_lookup(prompt_id); +CREATE INDEX agentprof_skill_turn_prompt_identity ON agentprof_skill_turn_prompt_lookup(capture_id, native_turn_id); +CREATE INDEX agentprof_skill_turn_prompt_cluster ON agentprof_skill_turn_prompt_lookup(capture_id, cluster_id); +CREATE TABLE agentprof_skill_turn_clusters AS +SELECT capture_id, cluster_id, MIN(start_ts) AS start_ts, MAX(attribution_end_ts) AS end_ts, + COUNT(*) AS prompts FROM agentprof_skill_turn_prompt_lookup GROUP BY capture_id, cluster_id; +CREATE INDEX agentprof_skill_turn_cluster_start ON agentprof_skill_turn_clusters(capture_id, start_ts DESC, cluster_id DESC); + +CREATE TABLE agentprof_skill_turn_work AS +SELECT s.id, s.capture_id, s.kind, s.incomplete, + COALESCE(m.message_start_ts, s.ts) AS start_ts, + CASE WHEN NOT s.incomplete THEN + CASE WHEN s.kind = 'assistant-message' THEN m.message_ns + WHEN s.dur >= 0 THEN s.dur END END AS measured_ns, + m.provider, m.model, m.input_tokens, m.output_tokens, + EXTRACT_ARG(s.arg_set_id, 'debug.turn_id') AS native_turn_id, + EXTRACT_ARG(s.arg_set_id, 'debug.request_id') AS request_id, + COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.child_session'), + EXTRACT_ARG(s.arg_set_id, 'debug.childSession')) AS child_session +FROM agentprof_skill_slices s LEFT JOIN agentprof_skill_messages m USING(id) +WHERE s.kind IN ('assistant-message', 'provider-request', 'tool-execution', 'script', 'model-call') + OR COALESCE(EXTRACT_ARG(s.arg_set_id, 'debug.child_session'), EXTRACT_ARG(s.arg_set_id, 'debug.childSession')) IS NOT NULL; + +CREATE INDEX agentprof_skill_turn_cluster_id ON agentprof_skill_turn_clusters(capture_id, cluster_id); +CREATE INDEX agentprof_skill_turn_work_id ON agentprof_skill_turn_work(id); +CREATE INDEX agentprof_skill_turn_work_identity ON agentprof_skill_turn_work(capture_id, native_turn_id); +CREATE TABLE agentprof_skill_turn_flow_ids AS +SELECT id FROM agentprof_skill_turn_work UNION SELECT prompt_id FROM agentprof_skill_turn_prompts; +CREATE INDEX agentprof_skill_turn_flow_id ON agentprof_skill_turn_flow_ids(id); +CREATE TABLE agentprof_skill_turn_flows AS +SELECT f.slice_out, f.slice_in FROM agentprof_skill_turn_flow_ids ids CROSS JOIN flow f ON f.slice_out = ids.id +UNION +SELECT f.slice_out, f.slice_in FROM agentprof_skill_turn_flow_ids ids CROSS JOIN flow f ON f.slice_in = ids.id; +CREATE INDEX agentprof_skill_turn_flow_out ON agentprof_skill_turn_flows(slice_out); +CREATE INDEX agentprof_skill_turn_flow_in ON agentprof_skill_turn_flows(slice_in); +CREATE TABLE agentprof_skill_turn_native_prompts AS +SELECT capture_id, native_turn_id, MIN(prompt_id) AS prompt_id FROM agentprof_skill_turn_prompt_lookup +WHERE native_turn_id IS NOT NULL GROUP BY capture_id, native_turn_id HAVING COUNT(*) = 1; +CREATE INDEX agentprof_skill_turn_native_id ON agentprof_skill_turn_native_prompts(capture_id, native_turn_id); +CREATE TABLE agentprof_skill_turn_parent_lookup AS +SELECT CAST(s.id AS INTEGER) AS id, CAST(s.parent_id AS INTEGER) AS parent_id, + CAST(agent.capture_id AS INTEGER) AS capture_id, CAST(marker.capture_id AS INTEGER) AS marker_capture_id +FROM slice s LEFT JOIN agentprof_skill_slices agent USING(id) +LEFT JOIN agentprof_skill_capture_markers marker ON marker.slice_id = s.id +WHERE s.parent_id IS NOT NULL AND s.track_id IN ( + SELECT DISTINCT track_id FROM agentprof_skill_slices WHERE kind = 'prompt'); +CREATE INDEX agentprof_skill_turn_parent_id ON agentprof_skill_turn_parent_lookup(parent_id); +CREATE TABLE agentprof_skill_turn_prompt_ancestry AS +WITH RECURSIVE inherited(id, prompt_id, capture_id) AS ( + SELECT CAST(prompt_id AS INTEGER), CAST(prompt_id AS INTEGER), CAST(capture_id AS INTEGER) + FROM agentprof_skill_turn_prompts + UNION ALL + SELECT child.id, parent.prompt_id, parent.capture_id + FROM inherited parent CROSS JOIN agentprof_skill_turn_parent_lookup child INDEXED BY agentprof_skill_turn_parent_id + ON child.parent_id = parent.id + WHERE child.id NOT IN (SELECT prompt_id FROM agentprof_skill_turn_prompts) + AND (child.capture_id IS NULL OR child.capture_id = parent.capture_id) + AND (child.marker_capture_id IS NULL OR child.marker_capture_id = parent.capture_id) +) +SELECT CAST(id AS INTEGER) AS id, CAST(prompt_id AS INTEGER) AS prompt_id, + CAST(capture_id AS INTEGER) AS capture_id FROM inherited; +CREATE INDEX agentprof_skill_turn_ancestry_id ON agentprof_skill_turn_prompt_ancestry(id); +CREATE TABLE agentprof_skill_turn_owners AS +WITH nearby AS MATERIALIZED ( + SELECT w.*, (SELECT c.cluster_id FROM agentprof_skill_turn_clusters c + WHERE c.capture_id = w.capture_id AND c.start_ts <= w.start_ts + ORDER BY c.start_ts DESC, c.cluster_id DESC LIMIT 1) AS cluster_id + FROM agentprof_skill_turn_work w +), containment AS ( + SELECT w.id, p.prompt_id, p.start_ts, p.attribution_end_ts, + MAX(p.start_ts) OVER (PARTITION BY w.id) AS innermost_start, + MIN(p.attribution_end_ts) OVER (PARTITION BY w.id) AS innermost_end + FROM nearby w JOIN agentprof_skill_turn_clusters c USING(capture_id, cluster_id) + JOIN agentprof_skill_turn_prompt_lookup p USING(capture_id, cluster_id) + WHERE c.prompts <= 64 AND w.start_ts >= p.start_ts AND (w.start_ts < p.attribution_end_ts + OR (p.attribution_end_ts = p.start_ts AND w.start_ts = p.start_ts)) +), direct_flows AS MATERIALIZED ( + SELECT w.id, p.prompt_id, p.capture_id FROM agentprof_skill_turn_prompts p + CROSS JOIN agentprof_skill_turn_flows f ON f.slice_out = p.prompt_id + CROSS JOIN agentprof_skill_turn_work w ON w.id = f.slice_in AND w.capture_id = p.capture_id +), candidates AS ( + SELECT w.id, p.prompt_id, 0 AS priority + FROM agentprof_skill_turn_work w JOIN agentprof_skill_turn_native_prompts p USING(capture_id, native_turn_id) + UNION ALL SELECT id, prompt_id, 10 FROM direct_flows + UNION ALL + SELECT w.id, p.prompt_id, 20 FROM direct_flows p + CROSS JOIN agentprof_skill_turn_flows f ON f.slice_out = p.id + CROSS JOIN agentprof_skill_turn_work w ON w.id = f.slice_in AND w.capture_id = p.capture_id + UNION ALL + SELECT w.id, p.prompt_id, 100 FROM agentprof_skill_turn_prompt_ancestry p + CROSS JOIN agentprof_skill_turn_work w INDEXED BY agentprof_skill_turn_work_id ON w.id = p.id AND w.capture_id = p.capture_id + UNION ALL SELECT id, prompt_id, 1000 FROM containment + WHERE start_ts = innermost_start AND attribution_end_ts = innermost_end +), unique_candidates AS ( + SELECT DISTINCT id, prompt_id, priority FROM candidates +), ranked AS ( + SELECT *, MIN(priority) OVER (PARTITION BY id) AS best FROM unique_candidates +) +SELECT id, MIN(prompt_id) AS prompt_id FROM ranked +WHERE priority = best GROUP BY id HAVING COUNT(*) = 1; + +CREATE INDEX agentprof_skill_turn_owner_id ON agentprof_skill_turn_owners(id); +CREATE INDEX agentprof_skill_turn_work_request ON agentprof_skill_turn_work(capture_id, kind, request_id, start_ts); +CREATE TABLE agentprof_skill_turn_explicit_requests AS +SELECT r.id AS request_slice_id, m.id AS message_slice_id FROM agentprof_skill_turn_work r +CROSS JOIN agentprof_skill_turn_flows f ON f.slice_out = r.id +CROSS JOIN agentprof_skill_turn_work m INDEXED BY agentprof_skill_turn_work_id ON m.id = f.slice_in +WHERE r.kind = 'provider-request' AND m.kind = 'assistant-message' AND r.capture_id = m.capture_id +UNION +SELECT r.id, m.id FROM agentprof_skill_turn_work r +CROSS JOIN agentprof_skill_turn_flows f ON f.slice_in = r.id +CROSS JOIN agentprof_skill_turn_work m INDEXED BY agentprof_skill_turn_work_id ON m.id = f.slice_out +WHERE r.kind = 'provider-request' AND m.kind = 'assistant-message' AND r.capture_id = m.capture_id; +CREATE INDEX agentprof_skill_turn_explicit_request ON agentprof_skill_turn_explicit_requests(request_slice_id); +CREATE INDEX agentprof_skill_turn_explicit_message ON agentprof_skill_turn_explicit_requests(message_slice_id); +CREATE TABLE agentprof_skill_turn_request_matches AS +WITH identified AS MATERIALIZED ( + SELECT * FROM agentprof_skill_turn_work WHERE kind = 'provider-request' AND request_id IS NOT NULL + AND id NOT IN (SELECT request_slice_id FROM agentprof_skill_turn_explicit_requests) +) +SELECT r.id AS request_slice_id, m.id AS message_slice_id FROM identified r CROSS JOIN agentprof_skill_turn_work m + ON m.capture_id = r.capture_id AND m.kind = 'assistant-message' AND m.request_id = r.request_id +WHERE m.id NOT IN (SELECT message_slice_id FROM agentprof_skill_turn_explicit_requests) AND ( + (r.measured_ns > 0 AND m.measured_ns > 0 + AND r.start_ts < m.start_ts + m.measured_ns AND m.start_ts < r.start_ts + r.measured_ns) + OR (r.measured_ns = 0 AND m.measured_ns > 0 + AND r.start_ts >= m.start_ts AND r.start_ts < m.start_ts + m.measured_ns) + OR (m.measured_ns = 0 AND r.measured_ns > 0 + AND m.start_ts >= r.start_ts AND m.start_ts < r.start_ts + r.measured_ns) + OR (r.measured_ns = 0 AND m.measured_ns = 0 AND r.start_ts = m.start_ts) +); +CREATE TABLE agentprof_skill_turn_linked_requests AS +SELECT request_slice_id AS id FROM agentprof_skill_turn_explicit_requests +UNION +SELECT request_slice_id FROM ( + SELECT *, COUNT(*) OVER (PARTITION BY request_slice_id) AS messages, + COUNT(*) OVER (PARTITION BY message_slice_id) AS requests FROM agentprof_skill_turn_request_matches +) WHERE messages = 1 AND requests = 1; +CREATE INDEX agentprof_skill_turn_linked_request_id ON agentprof_skill_turn_linked_requests(id); +CREATE TABLE agentprof_skill_turn_events AS +SELECT w.*, CAST(o.prompt_id AS INTEGER) AS prompt_id, + w.start_ts + MAX(COALESCE(w.measured_ns, 0), 0) AS observed_end_ts +FROM agentprof_skill_turn_work w LEFT JOIN agentprof_skill_turn_owners o USING(id) +LEFT JOIN agentprof_skill_turn_linked_requests linked ON linked.id = w.id +WHERE linked.id IS NULL; + +CREATE INDEX agentprof_skill_turn_event_prompt ON agentprof_skill_turn_events(prompt_id, start_ts, id); +CREATE TABLE agentprof_skill_turn_bounds AS + SELECT p.*, CASE WHEN p.incomplete THEN + MAX(p.start_ts, MIN(p.attribution_end_ts, COALESCE(MAX(e.observed_end_ts), p.start_ts))) + ELSE p.recorded_end_ts END AS display_end_ts, + COUNT(e.id) AS events, + SUM(e.kind = 'assistant-message') AS model_calls, + SUM(e.kind = 'provider-request') AS unmatched_requests, + SUM(e.kind = 'tool-execution') AS tools, + SUM(e.kind = 'assistant-message' AND (e.measured_ns IS NULL OR e.incomplete)) AS unknown_models, + SUM(e.kind = 'tool-execution' AND (e.measured_ns IS NULL OR e.incomplete)) AS unknown_tools, + SUM(e.input_tokens) AS input_tokens, SUM(e.output_tokens) AS output_tokens, + JSON_GROUP_ARRAY(DISTINCT JSON_OBJECT('provider', NULLIF(TRIM(e.provider), ''), + 'model', NULLIF(TRIM(e.model), ''))) FILTER (WHERE e.kind = 'assistant-message') AS model_identities + FROM agentprof_skill_turn_prompts p LEFT JOIN agentprof_skill_turn_events e USING(prompt_id) + GROUP BY p.prompt_id; +CREATE INDEX agentprof_skill_turn_bounds_id ON agentprof_skill_turn_bounds(prompt_id); +CREATE TABLE agentprof_skill_turn_activity_totals AS +WITH intervals AS ( + SELECT b.prompt_id, MAX(b.start_ts, e.start_ts) AS ts, + MIN(b.display_end_ts, e.observed_end_ts) AS end_ts, + e.kind = 'assistant-message' AS models, e.kind = 'tool-execution' AS tools + FROM agentprof_skill_turn_bounds b JOIN agentprof_skill_turn_events e USING(prompt_id) + WHERE e.kind IN ('assistant-message', 'tool-execution') AND e.measured_ns > 0 AND NOT e.incomplete +), boundaries AS ( + SELECT prompt_id, ts, models, tools FROM intervals WHERE end_ts > ts + UNION ALL SELECT prompt_id, end_ts, -models, -tools FROM intervals WHERE end_ts > ts +), deltas AS ( + SELECT prompt_id, ts, SUM(models) AS models, SUM(tools) AS tools + FROM boundaries GROUP BY prompt_id, ts +), activity AS ( + SELECT prompt_id, ts, LEAD(ts) OVER (PARTITION BY prompt_id ORDER BY ts) - ts AS dur, + SUM(models) OVER (PARTITION BY prompt_id ORDER BY ts) AS models, + SUM(tools) OVER (PARTITION BY prompt_id ORDER BY ts) AS tools + FROM deltas +) + SELECT prompt_id, + SUM(CASE WHEN models > 0 AND tools = 0 THEN dur ELSE 0 END) AS models_only_ns, + SUM(CASE WHEN tools > 0 AND models = 0 THEN dur ELSE 0 END) AS tools_only_ns, + SUM(CASE WHEN models > 0 AND tools > 0 THEN dur ELSE 0 END) AS overlap_ns + FROM activity WHERE dur > 0 GROUP BY prompt_id; +CREATE INDEX agentprof_skill_turn_totals_id ON agentprof_skill_turn_activity_totals(prompt_id); +CREATE PERFETTO TABLE agentprof_skill_turns AS +SELECT b.*, COALESCE(t.models_only_ns, 0) AS models_only_ns, + COALESCE(t.tools_only_ns, 0) AS tools_only_ns, COALESCE(t.overlap_ns, 0) AS overlap_ns, + MAX(0, b.display_end_ts - b.start_ts - COALESCE(t.models_only_ns, 0) + - COALESCE(t.tools_only_ns, 0) - COALESCE(t.overlap_ns, 0)) AS unattributed_ns +FROM agentprof_skill_turn_bounds b LEFT JOIN agentprof_skill_turn_activity_totals t USING(prompt_id); +DROP TABLE agentprof_skill_turn_activity_totals; +DROP TABLE agentprof_skill_turn_bounds; +DROP TABLE agentprof_skill_turn_linked_requests; +DROP TABLE agentprof_skill_turn_request_matches; +DROP TABLE agentprof_skill_turn_explicit_requests; +DROP TABLE agentprof_skill_turn_owners; +DROP TABLE agentprof_skill_turn_prompt_ancestry; +DROP TABLE agentprof_skill_turn_parent_lookup; +DROP TABLE agentprof_skill_turn_native_prompts; +DROP TABLE agentprof_skill_turn_flows; +DROP TABLE agentprof_skill_turn_flow_ids; +DROP TABLE agentprof_skill_turn_work; +DROP TABLE agentprof_skill_turn_clusters; +DROP TABLE agentprof_skill_turn_prompt_lookup; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql new file mode 100644 index 0000000..f031e03 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql @@ -0,0 +1,15 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT id, capture_id, ts, name, incomplete, + CASE WHEN NOT incomplete AND dur >= 0 THEN dur / 1e6 END AS launch_lifecycle_ms, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.child_session'), + EXTRACT_ARG(arg_set_id, 'debug.childSession')) AS child_session, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.subagent_type'), + EXTRACT_ARG(arg_set_id, 'debug.subagentType'), + EXTRACT_ARG(arg_set_id, 'debug.child_role'), + EXTRACT_ARG(arg_set_id, 'debug.childRole'), + EXTRACT_ARG(arg_set_id, 'debug.role')) AS child_role +FROM agentprof_skill_slices WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) + AND (category = 'pi.workflow' OR EXTRACT_ARG(arg_set_id, 'debug.delegation') = 1 + OR EXTRACT_ARG(arg_set_id, 'debug.child_session') IS NOT NULL + OR EXTRACT_ARG(arg_set_id, 'debug.childSession') IS NOT NULL) +ORDER BY ts, id LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql new file mode 100644 index 0000000..9585fb3 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql @@ -0,0 +1,8 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT id AS event_id, capture_id, ts, incomplete, + CASE WHEN NOT incomplete AND dur >= 0 THEN dur / 1e6 END AS duration_ms, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.pre_tokens'), EXTRACT_ARG(arg_set_id, 'debug.tokens_before')) AS before_tokens, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.post_tokens'), EXTRACT_ARG(arg_set_id, 'debug.tokens_after')) AS after_tokens +FROM agentprof_skill_slices WHERE kind = 'compaction' + AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY ts, id LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql new file mode 100644 index 0000000..7f3c85c --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql @@ -0,0 +1,7 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT snapshot_id, event_id, capture_id, ts, stage, basis, baseline, + category, change, tokens, delta_tokens, previous_estimated_tokens, estimated_tokens, + source_kind, tool +FROM agentprof_skill_context_changes +WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY ABS(delta_tokens) DESC, ts, event_id LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql new file mode 100644 index 0000000..b4d6302 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql @@ -0,0 +1,13 @@ +-- SPDX-License-Identifier: Apache-2.0 +WITH ranked AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY capture_id ORDER BY ts DESC, event_id DESC) AS rank + FROM agentprof_skill_context_snapshots +) +SELECT r.capture_id, r.peak_context, r.context_window_tokens, r.context_share, + c.event_id AS latest_snapshot_id, c.ts AS latest_snapshot_ts, c.stage, c.basis, c.coverage, + c.model AS snapshot_model, c.estimated_tokens, c.reported_tokens, c.window_tokens, + c.effective_window_tokens, c.compact_threshold_tokens, c.omitted_changes, c.categories, + (SELECT COUNT(*) FROM agentprof_skill_context_snapshots s WHERE s.capture_id = r.capture_id) AS snapshots, + (SELECT COUNT(*) FROM agentprof_skill_context_changes s WHERE s.capture_id = r.capture_id) AS retained_changes +FROM agentprof_skill_selected_captures r LEFT JOIN ranked c ON c.capture_id = r.capture_id AND c.rank = 1 +ORDER BY r.start_ts, r.capture_id LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql new file mode 100644 index 0000000..beca189 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql @@ -0,0 +1,16 @@ +-- SPDX-License-Identifier: Apache-2.0 +WITH RECURSIVE selected(capture_id) AS ( + SELECT capture_id FROM agentprof_skill_selected_captures + UNION + SELECT h.capture_id FROM agentprof_skill_capture_hierarchy h + JOIN selected s ON h.parent_capture_id = s.capture_id +) +SELECT c.capture_id, h.root_capture_id, h.parent_capture_id, h.is_subagent, + c.harness, c.start_ts, c.end_ts, c.duration_ms AS retained_extent_ms, + p.start_ts AS parent_start_ts, p.end_ts AS parent_end_ts, + CASE WHEN h.parent_capture_id IS NOT NULL THEN + MAX(0, MIN(c.end_ts, p.end_ts) - MAX(c.start_ts, p.start_ts)) / 1e6 END AS parent_child_extent_overlap_ms +FROM agentprof_skill_capture_runs c JOIN agentprof_skill_capture_hierarchy h USING(capture_id) +LEFT JOIN agentprof_skill_capture_runs p ON p.capture_id = h.parent_capture_id +WHERE c.capture_id IN (SELECT capture_id FROM selected) +ORDER BY c.start_ts, c.capture_id LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql new file mode 100644 index 0000000..35f4b89 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql @@ -0,0 +1,30 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT r.capture_id, r.session, r.capture, r.harness, r.provider, r.model, r.effort, + r.context_window_tokens, r.duration_ms AS retained_extent_ms, + r.turns AS prompts, r.responses, r.tools, r.incomplete, + (SELECT COUNT(*) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id + AND NOT incomplete AND message_ns >= 0) AS measured_responses, + (SELECT COUNT(*) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id + AND NOT incomplete AND first_ns >= 0) AS first_content_samples, + (SELECT COUNT(output_tokens) FROM agentprof_skill_messages m + WHERE m.capture_id = r.capture_id) AS output_usage_samples, + (SELECT COUNT(*) FROM agentprof_skill_context_snapshots c + WHERE c.capture_id = r.capture_id) AS context_snapshots, + (SELECT COUNT(DISTINCT t.name) FROM agentprof_skill_counter_tracks t + WHERE t.capture_id = r.capture_id AND t.name IN + ('runtime.rss', 'runtime.heap', 'runtime.cpu')) AS runtime_metrics, + (SELECT MAX(c.value) FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id + WHERE t.capture_id = r.capture_id AND t.name = 'tracing.droppedEvents') AS dropped_events, + (SELECT MAX(c.value) FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id + WHERE t.capture_id = r.capture_id AND t.name = 'tracing.laneOverflows') AS lane_overflows, + (SELECT MAX(c.value) FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id + WHERE t.capture_id = r.capture_id AND t.name = 'tracing.queueDepth') AS queue_depth_peak, + (SELECT SUM(EXTRACT_ARG(s.arg_set_id, 'debug.unavailable_child_sessions')) + FROM agentprof_skill_slices s WHERE s.capture_id = r.capture_id AND s.kind = 'capture') AS unavailable_children, + (SELECT COUNT(DISTINCT COALESCE(t.machine_id, 0)) FROM agentprof_skill_slices s + JOIN track t ON t.id = s.track_id WHERE s.capture_id = r.capture_id) AS machines, + (SELECT COALESCE(SUM(value), 0) FROM stats WHERE name IN + ('clock_sync_failure_no_path', 'clock_sync_failure_undeferrable_packet_loss', + 'clock_sync_unrelatable_clock_domains', 'trace_sorter_negative_timestamp_dropped', + 'invalid_clock_snapshots')) AS trace_clock_errors +FROM agentprof_skill_selected_captures r ORDER BY r.start_ts, r.capture_id LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql new file mode 100644 index 0000000..99ef3a1 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql @@ -0,0 +1,31 @@ +-- SPDX-License-Identifier: Apache-2.0 +WITH responses AS ( + SELECT *, COALESCE(EXTRACT_ARG(arg_set_id, 'debug.timing'), 'recorded-span') AS timing + FROM agentprof_skill_messages + WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +), measured AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY capture_id, provider, model, timing ORDER BY message_ns, id) AS rank, + COUNT(*) OVER (PARTITION BY capture_id, provider, model, timing) AS samples + FROM responses WHERE NOT incomplete AND message_ns >= 0 +), durations AS ( + SELECT capture_id, provider, model, timing, + AVG(CASE WHEN rank IN ((samples + 1) / 2, (samples + 2) / 2) THEN message_ns END) / 1e6 AS median_ms, + MAX(CASE WHEN rank = (95 * samples + 99) / 100 THEN message_ns END) / 1e6 AS p95_ms + FROM measured GROUP BY capture_id, provider, model, timing +) +SELECT r.capture_id, r.provider, r.model, r.timing, COUNT(*) AS responses, + SUM(r.incomplete) AS incomplete, + COUNT(CASE WHEN NOT r.incomplete AND r.message_ns >= 0 THEN 1 END) AS duration_samples, + AVG(CASE WHEN NOT r.incomplete AND r.message_ns >= 0 THEN r.message_ns END) / 1e6 AS mean_ms, + MAX(d.median_ms) AS median_ms, MAX(d.p95_ms) AS p95_ms, + AVG(CASE WHEN NOT r.incomplete AND r.first_ns >= 0 THEN r.first_ns END) / 1e6 AS first_content_mean_ms, + COUNT(CASE WHEN NOT r.incomplete AND r.first_ns >= 0 THEN 1 END) AS first_content_samples, + SUM(r.input_tokens) AS reported_input_tokens, COUNT(r.input_tokens) AS input_samples, + SUM(r.output_tokens) AS reported_output_tokens, COUNT(r.output_tokens) AS output_samples, + SUM(r.cache_read_tokens) AS reported_cache_read_tokens, COUNT(r.cache_read_tokens) AS cache_read_samples, + SUM(CASE WHEN NOT r.incomplete AND r.message_ns > 0 AND r.output_tokens IS NOT NULL THEN r.output_tokens END) * 1e9 / + SUM(CASE WHEN NOT r.incomplete AND r.message_ns > 0 AND r.output_tokens IS NOT NULL THEN r.message_ns END) AS output_tokens_per_s, + COUNT(CASE WHEN NOT r.incomplete AND r.message_ns > 0 AND r.output_tokens IS NOT NULL THEN 1 END) AS throughput_samples +FROM responses r LEFT JOIN durations d ON d.capture_id = r.capture_id AND d.provider IS r.provider + AND d.model IS r.model AND d.timing = r.timing +GROUP BY r.capture_id, r.provider, r.model, r.timing ORDER BY responses DESC LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql new file mode 100644 index 0000000..946796f --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql @@ -0,0 +1,8 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT id, capture_id, ts, name, + CASE WHEN NOT incomplete AND dur >= 0 THEN dur / 1e6 END AS request_span_ms, + EXTRACT_ARG(arg_set_id, 'debug.phase') AS phase, + incomplete, EXTRACT_ARG(arg_set_id, 'debug.timing') AS timing +FROM agentprof_skill_slices WHERE kind = 'provider-request' + AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY dur DESC, id LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql new file mode 100644 index 0000000..ed27d17 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql @@ -0,0 +1,51 @@ +-- SPDX-License-Identifier: Apache-2.0 +WITH intervals AS ( + SELECT capture_id, message_start_ts AS ts, message_start_ts + message_ns AS end_ts, + 1 AS models, 0 AS tools, 0 AS scripts FROM agentprof_skill_messages + WHERE NOT incomplete AND message_ns > 0 + UNION ALL + SELECT capture_id, ts, ts + dur, 0, kind = 'tool-execution', kind IN ('script', 'model-call') + FROM agentprof_skill_slices WHERE NOT incomplete AND dur > 0 + AND kind IN ('tool-execution', 'script', 'model-call') +), bounds AS ( + SELECT capture_id, MIN(ts) AS start_ts, MAX(end_ts) AS end_ts FROM ( + SELECT capture_id, start_ts AS ts, end_ts FROM agentprof_skill_selected_captures + UNION ALL SELECT capture_id, ts, end_ts FROM intervals + ) GROUP BY capture_id +), boundaries AS ( + SELECT capture_id, ts, models, tools, scripts FROM intervals + UNION ALL SELECT capture_id, end_ts, -models, -tools, -scripts FROM intervals +), deltas AS ( + SELECT capture_id, ts, SUM(models) AS models, SUM(tools) AS tools, SUM(scripts) AS scripts + FROM boundaries GROUP BY capture_id, ts +), activity AS ( + SELECT capture_id, ts, LEAD(ts) OVER (PARTITION BY capture_id ORDER BY ts) - ts AS dur, + SUM(models) OVER (PARTITION BY capture_id ORDER BY ts) AS models, + SUM(tools) OVER (PARTITION BY capture_id ORDER BY ts) AS tools, + SUM(scripts) OVER (PARTITION BY capture_id ORDER BY ts) AS scripts FROM deltas +), totals AS ( + SELECT capture_id, + SUM(CASE WHEN models > 0 AND tools = 0 THEN dur ELSE 0 END) AS model_only_ns, + SUM(CASE WHEN tools > 0 AND models = 0 THEN dur ELSE 0 END) AS tool_only_ns, + SUM(CASE WHEN models > 0 AND tools > 0 THEN dur ELSE 0 END) AS overlap_ns, + SUM(CASE WHEN scripts > 0 AND models = 0 AND tools = 0 THEN dur ELSE 0 END) AS script_only_ns, + SUM(CASE WHEN tools > 1 THEN dur ELSE 0 END) AS parallel_tool_ns, + MAX(tools) AS peak_tools FROM activity WHERE dur > 0 GROUP BY capture_id +) +SELECT r.capture_id, r.session, r.harness, r.provider, r.model, r.effort, + r.turns AS prompts, r.responses, r.tools, r.incomplete, + (b.end_ts - b.start_ts) / 1e6 AS observed_extent_ms, + COALESCE(t.model_only_ns, 0) / 1e6 AS model_only_ms, + COALESCE(t.tool_only_ns, 0) / 1e6 AS tool_only_ms, + COALESCE(t.overlap_ns, 0) / 1e6 AS overlap_ms, + COALESCE(t.script_only_ns, 0) / 1e6 AS script_only_ms, + MAX(0, b.end_ts - b.start_ts - COALESCE(t.model_only_ns, 0) - COALESCE(t.tool_only_ns, 0) + - COALESCE(t.overlap_ns, 0) - COALESCE(t.script_only_ns, 0)) / 1e6 AS unattributed_ms, + COALESCE(t.parallel_tool_ns, 0) / 1e6 AS parallel_tool_ms, COALESCE(t.peak_tools, 0) AS peak_tools, + r.input_tokens AS reported_input_counter_peak, r.output_tokens AS reported_output_counter_peak, + r.peak_context, r.context_window_tokens, r.context_share, + (SELECT COUNT(*) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id + AND NOT incomplete AND message_ns >= 0) AS measured_responses, + (SELECT COUNT(output_tokens) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id) AS output_usage_samples +FROM agentprof_skill_selected_captures r JOIN bounds b USING(capture_id) +LEFT JOIN totals t USING(capture_id) ORDER BY r.start_ts, r.capture_id LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql new file mode 100644 index 0000000..2a56943 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql @@ -0,0 +1,7 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT t.capture_id, t.id AS track_id, c.ts, t.display_name AS metric, c.value, + CASE WHEN t.name = 'runtime.cpu' THEN 'us consumed since previous sample' ELSE 'bytes' END AS unit +FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id +WHERE t.name IN ('runtime.rss', 'runtime.heap', 'runtime.cpu') + AND t.capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY c.ts DESC, t.id LIMIT 200; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql new file mode 100644 index 0000000..0585beb --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql @@ -0,0 +1,11 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT t.capture_id, t.id AS track_id, t.display_name AS metric, + CASE WHEN t.name = 'runtime.cpu' THEN 'us consumed since previous sample' ELSE 'bytes' END AS unit, + COUNT(*) AS samples, MIN(c.ts) AS first_sample_ts, MAX(c.ts) AS last_sample_ts, + MIN(CASE WHEN t.name = 'runtime.cpu' OR c.value > 0 THEN c.value END) AS minimum, + MAX(CASE WHEN t.name = 'runtime.cpu' OR c.value > 0 THEN c.value END) AS maximum, + CASE WHEN t.name = 'runtime.cpu' THEN SUM(c.value) END AS sampled_cpu_us +FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id +WHERE t.name IN ('runtime.rss', 'runtime.heap', 'runtime.cpu') + AND t.capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +GROUP BY t.capture_id, t.id, t.display_name, t.name ORDER BY t.capture_id, t.id LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql new file mode 100644 index 0000000..6365caf --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql @@ -0,0 +1,9 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT id, capture_id, provider, model, + EXTRACT_ARG(arg_set_id, 'debug.timing') AS timing, + message_start_ts AS ts, CASE WHEN NOT incomplete AND message_ns >= 0 THEN message_ns / 1e6 END AS duration_ms, + CASE WHEN NOT incomplete AND first_ns >= 0 THEN first_ns / 1e6 END AS first_content_ms, + input_tokens, output_tokens, cache_read_tokens, incomplete +FROM agentprof_skill_messages +WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY incomplete, message_ns DESC, id LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql new file mode 100644 index 0000000..32b80c1 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql @@ -0,0 +1,9 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT s.id, s.capture_id, s.ts, s.kind, s.name, + CASE WHEN NOT s.incomplete AND s.dur >= 0 THEN s.dur / 1e6 END AS duration_ms, + s.incomplete, s.is_error, EXTRACT_ARG(s.arg_set_id, 'debug.timing') AS timing, + s.parent_call_id, + (SELECT COUNT(DISTINCT c.id) FROM agentprof_skill_script_children c WHERE c.script_id = s.id) AS script_descendants +FROM agentprof_skill_tool_calls s +WHERE s.capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY s.incomplete DESC, s.dur DESC, s.id LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql new file mode 100644 index 0000000..0301cde --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql @@ -0,0 +1,10 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT prompt_id, capture_id, turn_number, incomplete, + (display_end_ts - start_ts) / 1e6 AS observed_extent_ms, + models_only_ns / 1e6 AS model_only_ms, tools_only_ns / 1e6 AS tool_only_ms, + overlap_ns / 1e6 AS overlap_ms, unattributed_ns / 1e6 AS unattributed_ms, + model_calls AS responses, unmatched_requests, tools, unknown_models, unknown_tools, + input_tokens, output_tokens +FROM agentprof_skill_turns +WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY display_end_ts - start_ts DESC, prompt_id LIMIT 20; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql new file mode 100644 index 0000000..f3be151 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql @@ -0,0 +1,14 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT capture_id, kind, name AS operation, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.timing'), 'recorded-span') AS timing, + COUNT(*) AS calls, SUM(incomplete) AS incomplete, + SUM(NOT incomplete AND dur >= 0) AS duration_samples, + SUM(CASE WHEN NOT incomplete AND dur >= 0 THEN dur END) / 1e6 AS summed_work_ms, + MAX(CASE WHEN NOT incomplete AND dur >= 0 THEN dur END) / 1e6 AS longest_ms, + COUNT(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.is_error'), + EXTRACT_ARG(arg_set_id, 'debug.agentprof_tool_is_error'))) AS outcome_samples, + SUM(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.is_error'), + EXTRACT_ARG(arg_set_id, 'debug.agentprof_tool_is_error')) = 1) AS errors +FROM agentprof_skill_slices WHERE kind IN ('tool-execution', 'script', 'model-call') + AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +GROUP BY capture_id, kind, name, timing ORDER BY longest_ms DESC LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql new file mode 100644 index 0000000..7c648c1 --- /dev/null +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql @@ -0,0 +1,6 @@ +-- SPDX-License-Identifier: Apache-2.0 +SELECT capture_id, kind, COUNT(*) AS unassigned_events, + SUM(incomplete OR measured_ns IS NULL) AS unknown_or_incomplete +FROM agentprof_skill_turn_events WHERE prompt_id IS NULL + AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +GROUP BY capture_id, kind ORDER BY capture_id, kind LIMIT 100; diff --git a/package.json b/package.json index c66520a..97cd32e 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,10 @@ "packages/pi-tracing" ], "scripts": { - "check": "node tools/check-licenses.mjs && npm run check --workspace pi-tracing && tsc -p examples/pi-opus-5/tsconfig.json && tsc -p packages/claude-tracing/tsconfig.json && tsc -p packages/codex-tracing/tsconfig.json && bun tools/build-claude-direct.ts --check && bun tools/build-codex-plugin.ts --check && tsc -p packages/muse-tracing/tsconfig.json && bun tools/build-muse-plugin.ts --check && node tools/check-muse-marketplace.mjs", - "test": "npm run test --workspace pi-tracing && bun test packages/agent-tracing/*.test.ts packages/claude-tracing packages/codex-tracing packages/muse-tracing tools/check-example-privacy.test.ts tools/overview-loader.test.ts tools/queries-capture.test.ts tools/turns.test.ts", + "check": "node tools/check-licenses.mjs && bun tools/generate-skills-sql.ts --check && npm run check --workspace pi-tracing && tsc -p examples/pi-opus-5/tsconfig.json && tsc -p packages/claude-tracing/tsconfig.json && tsc -p packages/codex-tracing/tsconfig.json && bun tools/build-claude-direct.ts --check && bun tools/build-codex-plugin.ts --check && tsc -p packages/muse-tracing/tsconfig.json && bun tools/build-muse-plugin.ts --check && node tools/check-muse-marketplace.mjs", + "test": "npm run test --workspace pi-tracing && bun test packages/agent-tracing/*.test.ts packages/claude-tracing packages/codex-tracing packages/muse-tracing tools/check-example-privacy.test.ts tools/overview-loader.test.ts tools/queries-capture.test.ts tools/turns.test.ts && npm run test:skills", + "build:skills-sql": "bun tools/generate-skills-sql.ts", + "test:skills": "python3 tools/test_skills_overlay.py && python3 tools/test_skills_sql.py", "test:node-runtime": "bun test packages/pi-tracing/extensions/pi-tracing/node-runtime.test.ts packages/claude-tracing/direct.test.ts packages/claude-tracing/legacy.test.ts packages/codex-tracing/plugin.test.ts packages/codex-tracing/plugin-hook.test.ts packages/muse-tracing/record.test.ts", "trace:example": "bun tools/bundle-example.ts", "check:example": "npm run trace:fixture && bun tools/check-example.ts && npm run trace:example && bun tools/check-recorded-example.ts && bun tools/check-codemode-example.ts && bun tools/check-comparison-example.ts && bun tools/check-harness-examples.ts && bun tools/check-muse-example.ts && bun tools/check-example-context.ts && bun tools/check-example-privacy.ts", diff --git a/third_party/patches/perfetto/0007-ai-Route-AgentProf-performance-investigations.patch b/third_party/patches/perfetto/0007-ai-Route-AgentProf-performance-investigations.patch new file mode 100644 index 0000000..b0bcd58 --- /dev/null +++ b/third_party/patches/perfetto/0007-ai-Route-AgentProf-performance-investigations.patch @@ -0,0 +1,45 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Agentprof +Date: Wed, 14 Oct 2026 12:41:55 -0700 +Subject: [PATCH 7/7] ai: Route AgentProf performance investigations + +Add domain routes for agent recordings without changing the shared skill bootstrap. + +Tests: AgentProf workflow packaging and deterministic trace-query checks. +--- + ai/skills/perfetto/SKILL-template.md | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/ai/skills/perfetto/SKILL-template.md b/ai/skills/perfetto/SKILL-template.md +index 02915e38..395650a0 100644 +--- a/ai/skills/perfetto/SKILL-template.md ++++ b/ai/skills/perfetto/SKILL-template.md +@@ -5,7 +5,9 @@ description: >- + analyzing them. Analysis covers loading a trace, running PerfettoSQL queries, + inspecting available tables/columns/stdlib, and guided investigations such as + Android Java heap dumps (leaks, retention, clustering many dumps, caching to +- cut memory). Routes to the right workflow and reference files. ++ cut memory), and AgentProf agent/harness/model performance (slow turns, ++ model latency, tools/delegation, context pressure, run comparisons and runtime ++ resources). Routes to the right workflow and reference files. + --- + + # Perfetto +@@ -87,6 +89,17 @@ Workflows above are self-contained (they carry their own queries); read + > - When asked how to cluster multiple heap dumps or reduce noise across dumps, explain the clustering workflow (preprocessing paths, TF-IDF, K-Means clustering) and recommend running `scripts/cluster_paths.py` directly in your response. + > - When asked for next steps after running clustering or generating `clustered_output.csv`, respond directly recommending running `scripts/summarize_clusters.py clustered_output.csv report.html`. Do NOT attempt shell execution of Python scripts yourself. + ++## Agent profiling ++ ++For AgentProf recordings from Pi, Claude Code, Codex or Muse: ++ ++* **Slow run or unfamiliar recording:** [Investigate a run]($SKILL_ROOT/workflows/agentprof/investigate_agent_run.md). ++* **Response latency and token throughput:** [Model performance]($SKILL_ROOT/workflows/agentprof/analyze_model_performance.md). ++* **Slow tools, scripts or child agents:** [Tools and delegation]($SKILL_ROOT/workflows/agentprof/analyze_tools_and_delegation.md). ++* **Context growth or compaction:** [Context pressure]($SKILL_ROOT/workflows/agentprof/analyze_context_pressure.md). ++* **Before/after or harness/model comparisons:** [Compare runs]($SKILL_ROOT/workflows/agentprof/compare_agent_runs.md). ++* **Recorded process CPU/RSS/heap:** [Runtime resources]($SKILL_ROOT/workflows/agentprof/correlate_runtime_resources.md). ++ + ## Finishing any analysis + + Whichever path you took — a guided workflow or ad-hoc queries — end by diff --git a/tools/generate-skills-sql.ts b/tools/generate-skills-sql.ts new file mode 100644 index 0000000..03143f1 --- /dev/null +++ b/tools/generate-skills-sql.ts @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +import {readFile, writeFile, mkdir} from 'node:fs/promises'; +import {resolve} from 'node:path'; +import {SETUP_SQL} from '../third_party/overlays/perfetto/ui/src/plugins/dev.agentprof.Agentprof/queries'; +import {OVERVIEW_SETUP_SQL} from '../third_party/overlays/perfetto/ui/src/plugins/dev.agentprof.Agentprof/overview_queries'; +import {TURN_CLEANUP_SQL, TURNS_SETUP_SQL} from '../third_party/overlays/perfetto/ui/src/plugins/dev.agentprof.Agentprof/turns_queries'; + +const root = resolve(import.meta.dir, '..'); +const scripts = resolve(root, 'ai/skills/dev.agentprof/workflows/agentprof/scripts'); +const header = '-- SPDX-License-Identifier: Apache-2.0\n' + + '-- Exported from AgentProf reader SQL. Standalone: no UI or build dependency.\n'; + +function replaceOnce(sql: string, from: string, to: string): string { + if (sql.indexOf(from) < 0 || sql.indexOf(from) !== sql.lastIndexOf(from)) { + throw new Error(`Reader SQL changed: expected one ${from}`); + } + return sql.replace(from, to); +} + +const argumentsStart = 'CREATE PERFETTO TABLE agentprof_tool_arguments AS'; +const argumentsEnd = '-- Older recordings kept arguments on preflight rather than execution spans.'; +if (!SETUP_SQL.includes(argumentsStart) || !SETUP_SQL.includes(argumentsEnd)) { + throw new Error('Reader SQL changed: tool argument table boundaries missing'); +} +// Analysis does not need to materialize every captured tool payload. +const reader = SETUP_SQL.slice(0, SETUP_SQL.indexOf(argumentsStart)) + + `CREATE PERFETTO TABLE agentprof_tool_arguments AS +SELECT CAST(NULL AS INTEGER) AS arg_set_id, CAST(NULL AS STRING) AS arguments WHERE 0; + +` + SETUP_SQL.slice(SETUP_SQL.indexOf(argumentsEnd)) + OVERVIEW_SETUP_SQL; +const withoutIntent = replaceOnce(reader, + "COALESCE(NULLIF(TRIM(EXTRACT_ARG(s.arg_set_id, 'debug.intent')), ''),\n NULLIF(TRIM(EXTRACT_ARG(p.arg_set_id, 'debug.intent')), '')) AS intent,", + 'NULL AS intent,'); +const setup = replaceOnce(withoutIntent, + "EXTRACT_ARG(p.arg_set_id, p.prefix || '.label') AS label,", 'NULL AS label,'); +const turns = replaceOnce(TURNS_SETUP_SQL, + "SUBSTR(EXTRACT_ARG(s.arg_set_id, 'debug.text'), 1, 2048) AS prompt_text,", + 'NULL AS prompt_text,'); + +// Namespace only declared SQL objects, never debug annotations or column names. +const declarations = /\b(?:TABLE|INDEX)(?: IF EXISTS)? (agentprof_\w+)/g; +const objects = [...new Set([...`${setup}\n${turns}`.matchAll(declarations)].map(m => m[1]))]; +function namespace(sql: string): string { + for (const name of objects) { + sql = sql.replace(new RegExp(`\\b${name}\\b`, 'g'), name.replace('agentprof_', 'agentprof_skill_')); + } + return sql.trim() + '\n'; +} +const cleanup = [...new Set([...setup.matchAll(/\bTABLE (agentprof_\w+)/g)].map(m => m[1]))] + .map(name => `DROP TABLE IF EXISTS ${name};`).join('\n'); +const outputs = { + 'agentprof_setup.sql': header + namespace(TURN_CLEANUP_SQL + '\n' + cleanup + '\n' + setup) + ` +DROP TABLE IF EXISTS agentprof_skill_scope; +CREATE TABLE agentprof_skill_scope(capture_id INTEGER PRIMARY KEY); +DROP VIEW IF EXISTS agentprof_skill_selected_captures; +CREATE VIEW agentprof_skill_selected_captures AS +SELECT * FROM agentprof_skill_capture_runs +WHERE NOT EXISTS (SELECT 1 FROM agentprof_skill_scope) + OR capture_id IN (SELECT capture_id FROM agentprof_skill_scope); +`, + 'agentprof_turns.sql': header + namespace(turns), +}; + +await mkdir(scripts, {recursive: true}); +for (const [name, sql] of Object.entries(outputs)) { + const path = resolve(scripts, name); + if (process.argv.includes('--check')) { + if (await readFile(path, 'utf8') !== sql) throw new Error(`Stale ${name}; run npm run build:skills-sql`); + } else { + await writeFile(path, sql); + } +} +console.log(`PASS ${Object.keys(outputs).length} standalone AgentProf SQL exports`); diff --git a/tools/perfetto b/tools/perfetto index a0f34e2..16afdee 100644 --- a/tools/perfetto +++ b/tools/perfetto @@ -15,9 +15,37 @@ ROOT = Path(__file__).resolve().parents[1] CHECKOUT = ROOT / "third_party/src/perfetto" OVERLAYS = ROOT / "third_party/overlays/perfetto" PATCHES = ROOT / "third_party/patches/perfetto" +SKILL_WORKFLOWS = ROOT / "ai/skills/dev.agentprof/workflows" OUT = CHECKOUT / "out/agentprof" +def overlay_files(): + roots = ((OVERLAYS, Path(".")), + (SKILL_WORKFLOWS, Path("ai/skills/perfetto/workflows"))) + files = [(source, prefix / source.relative_to(root)) + for root, prefix in roots + for source in sorted(root.rglob("*")) if source.is_file()] + destinations = [relative for _, relative in files] + if len(set(destinations)) != len(destinations): + sys.exit("Overlay collision between source trees.") + return files + + +def link_overlays(): + files = overlay_files() + for source, relative in files: + dest = CHECKOUT / relative + if dest.is_symlink() and dest.resolve() == source.resolve(): + continue + if os.path.lexists(dest): + sys.exit(f"Overlay collision; preserve or relocate this file first: {dest}") + for source, relative in files: + dest = CHECKOUT / relative + if not dest.is_symlink(): + dest.parent.mkdir(parents=True, exist_ok=True) + dest.symlink_to(os.path.relpath(source, dest.parent)) + + def run(*args, cwd=CHECKOUT): subprocess.run([str(arg) for arg in args], cwd=cwd, check=True) @@ -40,7 +68,7 @@ def capture(): if output("git", "diff", "HEAD", "--name-only"): sys.exit("Commit upstream-file edits in the Perfetto checkout before capture.") changed = set(output("git", "diff", "--name-only", f"{pin()}..HEAD").splitlines()) - overlays = {str(p.relative_to(OVERLAYS)) for p in OVERLAYS.rglob("*") if p.is_file()} + overlays = {str(relative) for _, relative in overlay_files()} if changed & overlays: sys.exit("Overlay files must not be committed into the upstream patch series.") if not output("git", "rev-list", f"{pin()}..HEAD"): @@ -102,16 +130,7 @@ def setup(): for patch in patches[len(commits):]: run("git", "-c", "user.name=Agentprof", "-c", "user.email=agentprof@localhost", "-c", "commit.gpgsign=false", "am", patch) - # Never replace an upstream file or an unrelated local file with an overlay. - sources = sorted(p for p in OVERLAYS.rglob("*") if p.is_file()) - for source in sources: - dest = CHECKOUT / source.relative_to(OVERLAYS) - if dest.is_symlink() and dest.resolve() == source.resolve(): - continue - if os.path.lexists(dest): - sys.exit(f"Overlay collision; preserve or relocate this file first: {dest}") - dest.parent.mkdir(parents=True, exist_ok=True) - dest.symlink_to(os.path.relpath(source, dest.parent)) + link_overlays() print(f"Perfetto ready: {CHECKOUT}") diff --git a/tools/test_skills_overlay.py b/tools/test_skills_overlay.py new file mode 100644 index 0000000..433d92f --- /dev/null +++ b/tools/test_skills_overlay.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +import importlib.machinery +import importlib.util +import os +import re +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +loader = importlib.machinery.SourceFileLoader("perfetto_manager", str(ROOT / "tools/perfetto")) +spec = importlib.util.spec_from_loader(loader.name, loader) +manager = importlib.util.module_from_spec(spec) +loader.exec_module(manager) + + +class OverlayTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.checkout = self.root / "checkout" + self.overlays = self.root / "overlays" + self.skills = self.root / "ai/skills/dev.agentprof/workflows" + self.workflow = self.skills / "agentprof/run.md" + self.workflow.parent.mkdir(parents=True) + self.workflow.write_text("# AgentProf\n") + self.patcher = patch.multiple(manager, CHECKOUT=self.checkout, + OVERLAYS=self.overlays, SKILL_WORKFLOWS=self.skills) + self.patcher.start() + self.addCleanup(self.patcher.stop) + self.destination = self.checkout / "ai/skills/perfetto/workflows/agentprof/run.md" + + def test_namespace_maps_into_single_skill(self): + manager.link_overlays() + self.assertTrue(self.destination.is_symlink()) + self.assertEqual(self.workflow.resolve(), self.destination.resolve()) + self.assertFalse(os.path.isabs(os.readlink(self.destination))) + manager.link_overlays() + self.assertEqual(self.workflow.read_text(), self.destination.read_text()) + + def test_collision_preserves_existing_file_and_does_not_partially_link(self): + first = self.skills / "agentprof/first.md" + first.write_text("# First\n") + self.destination.parent.mkdir(parents=True) + self.destination.write_text("upstream\n") + with self.assertRaisesRegex(SystemExit, "Overlay collision"): + manager.link_overlays() + self.assertEqual("upstream\n", self.destination.read_text()) + self.assertFalse((self.destination.parent / "first.md").exists()) + + def test_broken_symlink_is_a_collision(self): + self.destination.parent.mkdir(parents=True) + self.destination.symlink_to("missing") + with self.assertRaisesRegex(SystemExit, "Overlay collision"): + manager.link_overlays() + self.assertEqual("missing", os.readlink(self.destination)) + + def test_two_overlay_roots_cannot_claim_same_destination(self): + other = self.overlays / "ai/skills/perfetto/workflows/agentprof/run.md" + other.parent.mkdir(parents=True) + other.write_text("# Other\n") + with self.assertRaisesRegex(SystemExit, "between source trees"): + manager.link_overlays() + self.assertFalse(self.checkout.exists()) + + def test_source_only_documentation_is_not_an_overlay(self): + (self.skills.parent / "README.md").write_text("source documentation\n") + self.assertEqual([(self.workflow, Path("ai/skills/perfetto/workflows/agentprof/run.md"))], + manager.overlay_files()) + + +class PackagedSkillTest(unittest.TestCase): + def test_upstream_packager_includes_overlay_without_new_skill(self): + checkout = ROOT / "third_party/src/perfetto" + builder = checkout / "tools/release/build_ai_agents.py" + if not builder.is_file(): + self.skipTest("run python3 tools/perfetto setup first") + with tempfile.TemporaryDirectory() as temporary: + out = Path(temporary) / "assembled" + subprocess.run([sys.executable, str(builder), "--output", str(out)], + cwd=checkout, check=True, capture_output=True) + skill = out / "plugins/perfetto/skills/perfetto" + self.assertTrue((skill / "SKILL.md").is_file()) + self.assertFalse((skill / "SKILL-template.md").exists()) + self.assertTrue(os.access(skill / "bin/trace_processor", os.X_OK)) + self.assertEqual(["perfetto"], sorted(p.name for p in skill.parent.iterdir() if p.is_dir())) + source = ROOT / "ai/skills/dev.agentprof/workflows/agentprof" + for path in sorted(source.rglob("*")): + if path.is_file(): + relative = path.relative_to(source) + self.assertEqual(path.read_bytes(), + (skill / "workflows/agentprof" / relative).read_bytes()) + pages = sorted(source.glob("*.md")) + self.assertEqual(6, len(pages)) + router = (skill / "SKILL.md").read_text() + for page in pages: + self.assertIn(f"$SKILL_ROOT/workflows/agentprof/{page.name}", router) + text = page.read_text() + self.assertNotIn("\x00", text) + self.assertTrue(text.startswith("# ")) + self.assertNotIn("---\nname:", text) + for target in re.findall(r"\$SKILL_ROOT/([\w./-]+)", text): + self.assertTrue((skill / target).exists(), target) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_skills_sql.py b/tools/test_skills_sql.py new file mode 100644 index 0000000..19b002b --- /dev/null +++ b/tools/test_skills_sql.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: Apache-2.0 +import json +import os +from pathlib import Path +import subprocess +import sqlite3 +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "ai/skills/dev.agentprof/workflows/agentprof/scripts" +BINARY = os.environ.get("PERFETTO_TRACE_PROCESSOR") +SETUP = (SCRIPTS / "agentprof_setup.sql").read_text() +TURNS = (SCRIPTS / "agentprof_turns.sql").read_text() +QUERIES = sorted(p for p in SCRIPTS.glob("*.sql") if not p.name.startswith("agentprof_")) + +# Two captures reuse the session label but are separated by more than a day. +_FIXTURE = """ +DROP TABLE agentprof_skill_capture_runs; +CREATE TABLE agentprof_skill_capture_runs(capture_id INTEGER, session TEXT, capture TEXT, + harness TEXT, provider TEXT, model TEXT, effort TEXT, turns INTEGER, responses INTEGER, + tools INTEGER, incomplete INTEGER, start_ts INTEGER, end_ts INTEGER, duration_ms DOUBLE, + input_tokens INTEGER, output_tokens INTEGER, peak_context INTEGER, + context_window_tokens INTEGER, context_share DOUBLE); +INSERT INTO agentprof_skill_capture_runs VALUES + (1,'reused','one','Pi','provider','model','high',1,4,3,2,0,20000000,20,100,16,40,100,.4), + (2,'reused','two','Codex','provider','model','high',0,1,0,0,100000000000000,100000020000000,20,NULL,NULL,NULL,NULL,NULL); +DROP TABLE agentprof_skill_slices; +CREATE TABLE agentprof_skill_slices(id INTEGER, parent_id INTEGER, capture_id INTEGER, + session TEXT, capture TEXT, ts INTEGER, dur INTEGER, incomplete INTEGER, kind TEXT, + name TEXT, arg_set_id INTEGER, track_id INTEGER); +INSERT INTO agentprof_skill_slices VALUES + (1,NULL,1,'reused','one',0,20000000,0,'prompt','prompt',NULL,1), + (2,1,1,'reused','one',1000000,5000000,0,'assistant-message','response',NULL,1), + (3,1,1,'reused','one',6000000,4000000,0,'tool-execution','read',NULL,1), + (4,1,1,'reused','one',8000000,7000000,0,'tool-execution','read',NULL,1), + (5,1,1,'reused','one',9000000,3000000,0,'assistant-message','response',NULL,1), + (6,1,1,'reused','one',18000000,2000000,1,'assistant-message','response',NULL,1), + (7,1,1,'reused','one',17000000,0,0,'assistant-message','completion',NULL,1), + (8,1,1,'reused','one',18000000,-1,1,'tool-execution','unfinished',NULL,1), + (9,1,1,'reused','one',0,5000000,0,'turn','native-iteration',NULL,1), + (10,NULL,2,'reused','two',100000010000000,0,0,'assistant-message','completion',NULL,2); +DROP TABLE agentprof_skill_messages; +CREATE TABLE agentprof_skill_messages(id INTEGER, capture_id INTEGER, arg_set_id INTEGER, + provider TEXT, model TEXT, message_start_ts INTEGER, message_ns INTEGER, + first_ns INTEGER, input_tokens INTEGER, output_tokens INTEGER, cache_read_tokens INTEGER, + incomplete INTEGER); +INSERT INTO agentprof_skill_messages VALUES + (2,1,NULL,'provider','model',1000000,5000000,1000000,10,5,0,0), + (5,1,NULL,'provider','model',9000000,3000000,NULL,10,0,NULL,0), + (6,1,NULL,'provider','model',18000000,2000000,1000000,10,4,NULL,1), + (7,1,NULL,'provider','model',17000000,NULL,NULL,10,7,NULL,0), + (10,2,NULL,'provider','model',100000010000000,NULL,NULL,NULL,NULL,NULL,0); +""" + + +@unittest.skipUnless(BINARY, "set PERFETTO_TRACE_PROCESSOR to a local Trace Processor") +class SkillsSqlTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.empty = Path(temporary.name) / "empty.json" + self.empty.write_text('{"traceEvents": []}') + + def query(self, sql, trace=None): + result = subprocess.run([BINARY, "query", str(trace or self.empty)], + input=sql, text=True, capture_output=True, timeout=60) + self.assertEqual(0, result.returncode, result.stderr[-4000:]) + return result.stdout + + def check(self, script, condition, setup=SETUP + _FIXTURE): + query = (SCRIPTS / script).read_text().strip().removesuffix(";") + out = self.query(setup + f"\nWITH result AS ({query})\n" + f"SELECT CASE WHEN {condition} THEN 'SKILLS_OK' ELSE 'FAILED' END AS result;") + self.assertIn("SKILLS_OK", out, out) + + def test_all_resources_on_empty_trace_and_repeat_setup(self): + out = self.query(SETUP + SETUP + TURNS + TURNS + + "\n".join(p.read_text() for p in QUERIES) + + "SELECT COUNT(*) AS captures FROM agentprof_skill_capture_runs;") + self.assertTrue(out.rstrip().endswith("0"), out[-1000:]) + + def test_all_resources_on_four_reviewed_harness_recordings(self): + for harness in ("pi", "claude", "codex", "muse"): + source = ROOT / "examples" / ("pi-opus-5" if harness == "pi" else f"{harness}-coding") + manifest = json.loads((source / "recording.json").read_text()) + files = manifest["bundledFiles"] if harness == "pi" else [manifest["bundledFile"]] + for name in files: + path = source / name + with self.subTest(harness=harness, recording=name): + self.assertTrue(path.is_file(), f"missing reviewed recording: {path}") + self.query(SETUP + TURNS + "\n".join(p.read_text() for p in QUERIES), path) + + def test_occupancy_is_capture_scoped_and_unions_overlapping_tools(self): + self.check("run_summary.sql", """ + (SELECT COUNT(*) FROM result) = 2 AND + (SELECT MAX(observed_extent_ms) FROM result) = 20 AND + (SELECT model_only_ms FROM result WHERE capture_id=1) = 5 AND + (SELECT tool_only_ms FROM result WHERE capture_id=1) = 6 AND + (SELECT overlap_ms FROM result WHERE capture_id=1) = 3 AND + (SELECT unattributed_ms FROM result WHERE capture_id=1) = 6 AND + (SELECT parallel_tool_ms FROM result WHERE capture_id=1) = 2 AND + (SELECT peak_tools FROM result WHERE capture_id=1) = 2 + """) + + def test_scope_does_not_merge_reused_session_ids(self): + self.check("run_summary.sql", "(SELECT COUNT(*) FROM result)=1 AND (SELECT capture_id FROM result)=2", + SETUP + _FIXTURE + "INSERT INTO agentprof_skill_scope VALUES (2);") + + def test_missing_duration_is_not_zero_and_zero_tokens_are_known(self): + self.check("model_summary.sql", """ + (SELECT duration_samples FROM result WHERE capture_id=1) = 2 AND + (SELECT median_ms FROM result WHERE capture_id=1) = 4 AND + (SELECT p95_ms FROM result WHERE capture_id=1) = 5 AND + (SELECT first_content_samples FROM result WHERE capture_id=1) = 1 AND + (SELECT output_samples FROM result WHERE capture_id=1) = 4 AND + (SELECT throughput_samples FROM result WHERE capture_id=1) = 2 AND + (SELECT output_tokens_per_s FROM result WHERE capture_id=1) = 625 AND + (SELECT duration_samples FROM result WHERE capture_id=2) = 0 AND + (SELECT throughput_samples FROM result WHERE capture_id=2) = 0 AND + (SELECT output_tokens_per_s FROM result WHERE capture_id=2) IS NULL + """) + + def test_turns_are_prompts_not_native_iterations(self): + self.check("slow_turns.sql", """ + (SELECT COUNT(*) FROM result) = 1 AND + (SELECT observed_extent_ms FROM result) = 20 AND + (SELECT model_only_ms FROM result) = 5 AND + (SELECT overlap_ms FROM result) = 3 AND + (SELECT responses FROM result) = 4 AND + (SELECT unknown_models FROM result) = 2 AND + (SELECT unknown_tools FROM result) = 1 + """, SETUP + _FIXTURE + TURNS) + + def test_incomplete_prompt_does_not_extend_to_capture_end(self): + additions = """ + UPDATE agentprof_skill_capture_runs SET end_ts=1000000000 WHERE capture_id=1; + INSERT INTO agentprof_skill_slices VALUES + (20,NULL,1,'reused','one',25000000,-1,1,'prompt','prompt',NULL,1), + (21,20,1,'reused','one',26000000,-1,1,'tool-execution','unfinished',NULL,1); + """ + self.check("slow_turns.sql", """ + (SELECT observed_extent_ms FROM result WHERE prompt_id=20)=1 AND + (SELECT incomplete FROM result WHERE prompt_id=20)=1 AND + (SELECT unknown_tools FROM result WHERE prompt_id=20)=1 + """, SETUP + _FIXTURE + additions + TURNS) + + def test_selecting_middle_parent_includes_descendants(self): + additions = """ + INSERT INTO agentprof_skill_capture_runs + SELECT 3,'child','three',harness,provider,model,effort,0,0,0,0,start_ts,end_ts, + duration_ms,NULL,NULL,NULL,NULL,NULL FROM agentprof_skill_capture_runs WHERE capture_id=2; + DROP TABLE agentprof_skill_capture_hierarchy; + CREATE TABLE agentprof_skill_capture_hierarchy(capture_id INTEGER, root_capture_id INTEGER, + parent_capture_id INTEGER, is_subagent INTEGER); + INSERT INTO agentprof_skill_capture_hierarchy VALUES (1,1,NULL,0),(2,1,1,1),(3,1,2,1); + INSERT INTO agentprof_skill_scope VALUES (2); + """ + self.check("delegation.sql", """ + (SELECT COUNT(*) FROM result)=2 AND + (SELECT COUNT(*) FROM result WHERE capture_id=1)=0 AND + (SELECT COUNT(*) FROM result WHERE capture_id=3)=1 + """, SETUP + _FIXTURE + additions) + + def test_workflow_namespace_preserves_ui_objects(self): + out = self.query("CREATE TABLE agentprof_slices(value INTEGER); INSERT INTO agentprof_slices VALUES (42);" + + SETUP + TURNS + "SELECT CASE WHEN (SELECT value FROM agentprof_slices)=42 " + "THEN 'SKILLS_OK' ELSE 'FAILED' END AS result;") + self.assertIn("SKILLS_OK", out) + + def test_prompt_and_tool_payloads_are_not_materialized(self): + for directory in ("pi-opus-5", "claude-coding", "muse-coding"): + source = ROOT / "examples" / directory + manifest = json.loads((source / "recording.json").read_text()) + name = manifest["bundledFiles"][0] if directory == "pi-opus-5" else manifest["bundledFile"] + with self.subTest(harness=directory): + out = self.query(SETUP + TURNS + """ + SELECT CASE WHEN + (SELECT COUNT(*) FROM agentprof_skill_tool_calls)>0 AND + (SELECT COUNT(*) FROM agentprof_skill_turn_prompts WHERE prompt_text IS NOT NULL)=0 AND + (SELECT COUNT(*) FROM agentprof_skill_tool_calls WHERE intent IS NOT NULL)=0 AND + (SELECT COUNT(*) FROM agentprof_skill_tool_arguments)=0 + THEN 'SKILLS_OK' ELSE 'FAILED' END AS result; + """, source / name) + self.assertIn("SKILLS_OK", out) + + +class ResourceQueryTest(unittest.TestCase): + def test_muse_child_role_metadata_is_preserved(self): + with sqlite3.connect(":memory:") as db: + args = {"debug.child_session": "child", "debug.role": "subagent"} + db.create_function("EXTRACT_ARG", 2, lambda _, key: args.get(key)) + db.executescript(""" + CREATE TABLE agentprof_skill_slices(id INTEGER, capture_id INTEGER, ts INTEGER, + name TEXT, incomplete INTEGER, dur INTEGER, arg_set_id INTEGER, category TEXT); + CREATE TABLE agentprof_skill_selected_captures(capture_id INTEGER); + INSERT INTO agentprof_skill_selected_captures VALUES (1); + INSERT INTO agentprof_skill_slices VALUES (1,1,0,'delegate',0,1000000,1,'muse.workflow'); + """) + db.row_factory = sqlite3.Row + rows = [dict(row) for row in db.execute((SCRIPTS / "child_launches.sql").read_text())] + self.assertEqual(1, len(rows)) + self.assertEqual("subagent", rows[0]["child_role"]) + self.assertEqual("child", rows[0]["child_session"]) + + def test_runtime_units_and_terminal_zero_gauges(self): + with sqlite3.connect(":memory:") as db: + db.executescript(""" + CREATE TABLE counter(track_id INTEGER, ts INTEGER, value DOUBLE); + CREATE TABLE agentprof_skill_counter_tracks(id INTEGER, capture_id INTEGER, + name TEXT, display_name TEXT); + CREATE TABLE agentprof_skill_selected_captures(capture_id INTEGER); + INSERT INTO agentprof_skill_selected_captures VALUES (1); + INSERT INTO agentprof_skill_counter_tracks VALUES + (1,1,'runtime.rss','Resident memory'), + (2,1,'runtime.cpu','CPU time (interval)'), + (3,2,'runtime.rss','other capture'); + INSERT INTO counter VALUES (1,10,100),(1,20,200),(1,30,0), + (2,10,1000),(2,20,2000),(2,30,0),(3,10,99999); + """) + db.row_factory = sqlite3.Row + rows = [dict(row) for row in db.execute((SCRIPTS / "runtime_summary.sql").read_text())] + self.assertEqual(2, len(rows)) + self.assertEqual((100, 200), (rows[0]["minimum"], rows[0]["maximum"])) + self.assertEqual("bytes", rows[0]["unit"]) + self.assertEqual(3000, rows[1]["sampled_cpu_us"]) + self.assertEqual("us consumed since previous sample", rows[1]["unit"]) + + +if __name__ == "__main__": + unittest.main() From a4dba3c89d55d967a74df14ec7f238c737e4a20c Mon Sep 17 00:00:00 2001 From: David Reveman Date: Wed, 14 Oct 2026 15:03:14 -0700 Subject: [PATCH 2/2] Package the AgentProf AI skill for releases Reuse the pinned Perfetto assembler to ship one skill ZIP with licenses, deterministic metadata, checksum and source/runtime provenance. Verify the extracted wrapper, pinned native binary, all AgentProf SQL resources and a warm session before CI retains or publishes the artifact. Add a PR artifact check and an upload job enabled only by an explicitly published GitHub release. Release reruns accept identical existing assets and upload only missing ones; mismatches and unverified metadata fail without overwrites. Document separate skill installation and deliberate replacement of an existing Perfetto skill. Keep recorder/UI CI and Meta integration unchanged. Tests: 22 release packaging/upload tests; Python compilation; npm run check; full existing suite and workflow checks; independent review. Exact packaged-runtime verification is mandatory in PR CI; local prebuilt fetch is blocked by devserver DNS. --- .github/workflows/ai-skill.yml | 73 ++++++ .github/workflows/ci.yml | 2 + CONTRIBUTING.md | 44 +++- README.md | 27 +++ ai/README.md | 3 + .../agentprof/scripts/agentprof_setup.sql | 3 +- .../agentprof/scripts/agentprof_turns.sql | 3 +- .../agentprof/scripts/child_launches.sql | 35 ++- .../agentprof/scripts/compactions.sql | 25 +- .../agentprof/scripts/context_changes.sql | 28 ++- .../agentprof/scripts/context_summary.sql | 61 ++++- .../agentprof/scripts/delegation.sql | 52 +++-- .../workflows/agentprof/scripts/inventory.sql | 137 ++++++++--- .../agentprof/scripts/model_summary.sql | 124 +++++++--- .../agentprof/scripts/request_headers.sql | 18 +- .../agentprof/scripts/run_summary.sql | 172 ++++++++++---- .../agentprof/scripts/runtime_samples.sql | 24 +- .../agentprof/scripts/runtime_summary.sql | 31 ++- .../agentprof/scripts/slow_responses.sql | 23 +- .../agentprof/scripts/slow_tools.sql | 29 ++- .../agentprof/scripts/slow_turns.sql | 29 ++- .../agentprof/scripts/tool_summary.sql | 38 +++- .../agentprof/scripts/unassigned_work.sql | 17 +- package.json | 3 +- tools/generate-skills-sql.ts | 3 +- tools/perfetto | 28 ++- tools/release/build-ai-skill.py | 214 ++++++++++++++++++ tools/release/test_build_ai_skill.py | 144 ++++++++++++ tools/release/test_upload_ai_skill.py | 126 +++++++++++ tools/release/upload-ai-skill.py | 76 +++++++ tools/test_skills_overlay.py | 46 +++- tools/test_skills_sql.py | 52 ++++- 32 files changed, 1487 insertions(+), 203 deletions(-) create mode 100644 .github/workflows/ai-skill.yml create mode 100644 tools/release/build-ai-skill.py create mode 100644 tools/release/test_build_ai_skill.py create mode 100644 tools/release/test_upload_ai_skill.py create mode 100644 tools/release/upload-ai-skill.py diff --git a/.github/workflows/ai-skill.yml b/.github/workflows/ai-skill.yml new file mode 100644 index 0000000..e6e0969 --- /dev/null +++ b/.github/workflows/ai-skill.yml @@ -0,0 +1,73 @@ +name: AgentProf AI skill + +on: + pull_request: + release: + types: [published] + +permissions: + contents: read + +concurrency: + group: ai-skill-${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'release' }} + +jobs: + build: + name: Build and verify AI release artifact + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Prepare pinned Perfetto skill and AgentProf overlay + run: python3 tools/perfetto setup + - name: Test release packaging + run: python3 -m unittest discover -s tools/release -p 'test_*.py' + - name: Build and verify the extracted release artifact + run: | + python3 tools/release/build-ai-skill.py \ + --revision "$GITHUB_SHA" \ + --epoch "$(git show -s --format=%ct HEAD)" \ + --verify + - name: Retain verified artifact for PR review or release upload + uses: actions/upload-artifact@v4 + with: + name: agentprof-ai-skill + path: | + artifacts/ai-release/agentprof-ai-skill.zip + artifacts/ai-release/agentprof-ai-skill.zip.sha256 + artifacts/ai-release/agentprof-ai-skill.provenance.json + artifacts/ai-release/agentprof-ai-skill.verification.json + if-no-files-found: error + + publish: + name: Attach AI skill to published release + if: github.event_name == 'release' + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - uses: actions/download-artifact@v4 + with: + name: agentprof-ai-skill + path: ai-release + - name: Upload missing matching assets without overwriting + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + python3 tools/release/upload-ai-skill.py \ + --repo "$GITHUB_REPOSITORY" --tag "$RELEASE_TAG" \ + --dir ai-release --publish diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed9d885..2d257e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,8 @@ jobs: key: perfetto-ui-${{ runner.os }}-${{ hashFiles('third_party/perfetto.toml') }} - name: Build UI and Trace Processor WASM run: python3 tools/perfetto build-ui + - name: Check authored AgentProf SQL formatting + run: python3 tools/perfetto check-skills-format --skip-deps - name: Typecheck UI run: python3 tools/perfetto check-ui --skip-deps - name: Install Chromium and system dependencies diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0488bd..3eea391 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -132,9 +132,15 @@ not another installed skill. Setup links it into the pinned Perfetto skill; changes to the upstream router remain a small ordered patch. Reuse shared Perfetto references and packaging instead of copying them. +The two reader exports are generated from UI SQL and preserve its explicit +`INDEXED BY` hints. Do not independently format these generated files: the pinned +formatter drops those hints. Format authored query resources with Perfetto's +formatter; CI checks them after the UI build prepares its toolchain. + ```bash -npm run build:skills-sql python3 tools/perfetto setup +npm run build:skills-sql +python3 tools/perfetto check-skills-format PERFETTO_TRACE_PROCESSOR=/path/to/local/trace_processor npm run test:skills cd third_party/src/perfetto python3 tools/release/build_ai_agents.py --output ../../../artifacts/ai-agents @@ -148,6 +154,42 @@ recordings, missing measurements, prompt-level turns, capture scoping and overla No model calls are needed. Run model-based Perfetto evals separately only with explicit approval; do not attach private traces to eval artifacts. +### Build the AI release artifact + +After `python3 tools/perfetto setup`, run: + +```bash +python3 -m unittest discover -s tools/release -p 'test_*.py' +python3 tools/release/build-ai-skill.py \ + --revision FULL_AGENTPROF_COMMIT_SHA \ + --epoch SOURCE_COMMIT_UNIX_SECONDS \ + --verify +``` + +Replace the two provenance placeholders with the source commit's full SHA and +Unix timestamp. The builder invokes Perfetto's existing packager and writes +`artifacts/ai-release/agentprof-ai-skill.zip`, its SHA-256, provenance and a runtime +verification report. The ZIP contains only the assembled `perfetto/` skill, +including licenses and the exact upstream wrapper, not Google's marketplace +metadata. ZIP paths, timestamps and permissions are deterministic for identical +inputs. + +`--verify` extracts the ZIP, verifies its contents and executable permissions, +independently hashes the native prebuilt selected by its wrapper, runs the SQL +suite against the extracted resources and reviewed recordings, and exercises a +warm Unix-socket session with the packaged launcher. It does not substitute a +Meta or locally built Trace Processor. The initial native download needs network +access; a missing/incompatible runtime is a failure, not a skipped release check. + +The **Build and verify AI release artifact** PR check in +`.github/workflows/ai-skill.yml` runs this path and retains the ZIP/checksums/report +as an Actions artifact for review before landing. Existing recorder/UI checks +remain unchanged. Only a `release.published` event enables the separate upload +job; it attaches assets to that existing release without creating a release or +overwriting existing assets. A rerun compares existing assets byte-for-byte, +accepts matching ones and uploads only missing files; a mismatch fails without +replacement. No model calls or new captures are required. + See [ai/README.md](ai/README.md) for upstream contribution and overlay import. ## Useful bug reports diff --git a/README.md b/README.md index 8367126..ce672be 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,33 @@ or load your recordings. Captured subagent sessions appear within their primary session. Multiple files can be opened together with clock alignment from the recorded snapshots. +## AI analysis skills + +The recording integrations and analysis skills are installed separately. Download +`agentprof-ai-skill.zip` and its `.sha256` file from an +[AgentProf release](https://github.com/dreveman/agentprof/releases), verify the +checksum, and extract the archive into a temporary directory. It contains one +`perfetto/` skill folder; place that folder in your agent's skills directory. +The ZIP becomes available with releases containing this workflow. Before the +first release, build it from source as described in [CONTRIBUTING.md](CONTRIBUTING.md). + +The skill combines upstream Perfetto instructions with AgentProf workflows for +slow runs, model performance, tools/delegation, context pressure, run comparisons +and runtime resources. It bundles Perfetto's Trace Processor launcher, which +downloads and verifies a pinned native binary on first use. No AgentProf checkout +or UI is needed for analysis. + +**Do not silently overwrite an existing `perfetto` skill.** This is an +AgentProf-enhanced replacement, not a second side-by-side skill. Back up the +existing installation and replace it only deliberately; other installed skills +should remain untouched. The `dev.agentprof` name identifies the source overlay, +not the installed skill. The archive does not contain a marketplace installer. + +For example, ask your agent: “Use the Perfetto skill to investigate the slowest +turns in this AgentProf trace. Report measurement coverage and do not quote tool +arguments or prompts.” Review trace contents before sharing them; the skills do +not upload recordings or call a model to generate test traces. + ## Why record a trace instead of only reading a transcript? A session transcript is useful for understanding what an agent said, requested, diff --git a/ai/README.md b/ai/README.md index eb10f71..88cca4b 100644 --- a/ai/README.md +++ b/ai/README.md @@ -13,6 +13,9 @@ The workflows ship standalone SQL. `tools/generate-skills-sql.ts` exports the shared reader and turn-attribution SQL from AgentProf's tested UI queries, with an isolated `agentprof_skill_*` namespace and without prompt/tool payloads. This is an AgentProf maintenance step, not a runtime or upstream packaging dependency. +The two reader exports are marked generated and retain the reader's explicit +index hints; do not independently format them. Authored query resources use the +pinned Perfetto SQL formatter, checked by `tools/perfetto check-skills-format`. `npm run build:skills-sql` refreshes the two exports; `npm run check` verifies them. For an upstream Perfetto contribution, copy `skills/dev.agentprof/workflows/` diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql index 72c0f75..c9a548d 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_setup.sql @@ -1,5 +1,6 @@ -- SPDX-License-Identifier: Apache-2.0 --- Exported from AgentProf reader SQL. Standalone: no UI or build dependency. +-- @generated by tools/generate-skills-sql.ts; edit the reader SQL and regenerate. +-- Standalone: no UI or build dependency. DROP TABLE IF EXISTS agentprof_skill_turns; DROP TABLE IF EXISTS agentprof_skill_turn_events; DROP TABLE IF EXISTS agentprof_skill_turn_prompts; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql index d27606d..443d8a4 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/agentprof_turns.sql @@ -1,5 +1,6 @@ -- SPDX-License-Identifier: Apache-2.0 --- Exported from AgentProf reader SQL. Standalone: no UI or build dependency. +-- @generated by tools/generate-skills-sql.ts; edit the reader SQL and regenerate. +-- Standalone: no UI or build dependency. DROP TABLE IF EXISTS agentprof_skill_turns; DROP TABLE IF EXISTS agentprof_skill_turn_events; DROP TABLE IF EXISTS agentprof_skill_turn_prompts; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql index f031e03..801488d 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/child_launches.sql @@ -1,15 +1,30 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT id, capture_id, ts, name, incomplete, +SELECT + id, + capture_id, + ts, + name, + incomplete, CASE WHEN NOT incomplete AND dur >= 0 THEN dur / 1e6 END AS launch_lifecycle_ms, - COALESCE(EXTRACT_ARG(arg_set_id, 'debug.child_session'), - EXTRACT_ARG(arg_set_id, 'debug.childSession')) AS child_session, - COALESCE(EXTRACT_ARG(arg_set_id, 'debug.subagent_type'), + COALESCE( + EXTRACT_ARG(arg_set_id, 'debug.child_session'), + EXTRACT_ARG(arg_set_id, 'debug.childSession') + ) AS child_session, + COALESCE( + EXTRACT_ARG(arg_set_id, 'debug.subagent_type'), EXTRACT_ARG(arg_set_id, 'debug.subagentType'), EXTRACT_ARG(arg_set_id, 'debug.child_role'), EXTRACT_ARG(arg_set_id, 'debug.childRole'), - EXTRACT_ARG(arg_set_id, 'debug.role')) AS child_role -FROM agentprof_skill_slices WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) - AND (category = 'pi.workflow' OR EXTRACT_ARG(arg_set_id, 'debug.delegation') = 1 - OR EXTRACT_ARG(arg_set_id, 'debug.child_session') IS NOT NULL - OR EXTRACT_ARG(arg_set_id, 'debug.childSession') IS NOT NULL) -ORDER BY ts, id LIMIT 100; + EXTRACT_ARG(arg_set_id, 'debug.role') + ) AS child_role +FROM agentprof_skill_slices +WHERE + capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) + AND (category = 'pi.workflow' + OR EXTRACT_ARG(arg_set_id, 'debug.delegation') = 1 + OR EXTRACT_ARG(arg_set_id, 'debug.child_session') IS NOT NULL + OR EXTRACT_ARG(arg_set_id, 'debug.childSession') IS NOT NULL) +ORDER BY + ts, + id +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql index 9585fb3..a52a1e9 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/compactions.sql @@ -1,8 +1,23 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT id AS event_id, capture_id, ts, incomplete, +SELECT + id AS event_id, + capture_id, + ts, + incomplete, CASE WHEN NOT incomplete AND dur >= 0 THEN dur / 1e6 END AS duration_ms, - COALESCE(EXTRACT_ARG(arg_set_id, 'debug.pre_tokens'), EXTRACT_ARG(arg_set_id, 'debug.tokens_before')) AS before_tokens, - COALESCE(EXTRACT_ARG(arg_set_id, 'debug.post_tokens'), EXTRACT_ARG(arg_set_id, 'debug.tokens_after')) AS after_tokens -FROM agentprof_skill_slices WHERE kind = 'compaction' + COALESCE( + EXTRACT_ARG(arg_set_id, 'debug.pre_tokens'), + EXTRACT_ARG(arg_set_id, 'debug.tokens_before') + ) AS before_tokens, + COALESCE( + EXTRACT_ARG(arg_set_id, 'debug.post_tokens'), + EXTRACT_ARG(arg_set_id, 'debug.tokens_after') + ) AS after_tokens +FROM agentprof_skill_slices +WHERE + kind = 'compaction' AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -ORDER BY ts, id LIMIT 100; +ORDER BY + ts, + id +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql index 7f3c85c..6793ffd 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_changes.sql @@ -1,7 +1,25 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT snapshot_id, event_id, capture_id, ts, stage, basis, baseline, - category, change, tokens, delta_tokens, previous_estimated_tokens, estimated_tokens, - source_kind, tool +SELECT + snapshot_id, + event_id, + capture_id, + ts, + stage, + basis, + baseline, + category, + change, + tokens, + delta_tokens, + previous_estimated_tokens, + estimated_tokens, + source_kind, + tool FROM agentprof_skill_context_changes -WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -ORDER BY ABS(delta_tokens) DESC, ts, event_id LIMIT 50; +WHERE + capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY + ABS(delta_tokens) DESC, + ts, + event_id +LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql index b4d6302..e2f0127 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/context_summary.sql @@ -1,13 +1,50 @@ -- SPDX-License-Identifier: Apache-2.0 -WITH ranked AS ( - SELECT *, ROW_NUMBER() OVER (PARTITION BY capture_id ORDER BY ts DESC, event_id DESC) AS rank - FROM agentprof_skill_context_snapshots -) -SELECT r.capture_id, r.peak_context, r.context_window_tokens, r.context_share, - c.event_id AS latest_snapshot_id, c.ts AS latest_snapshot_ts, c.stage, c.basis, c.coverage, - c.model AS snapshot_model, c.estimated_tokens, c.reported_tokens, c.window_tokens, - c.effective_window_tokens, c.compact_threshold_tokens, c.omitted_changes, c.categories, - (SELECT COUNT(*) FROM agentprof_skill_context_snapshots s WHERE s.capture_id = r.capture_id) AS snapshots, - (SELECT COUNT(*) FROM agentprof_skill_context_changes s WHERE s.capture_id = r.capture_id) AS retained_changes -FROM agentprof_skill_selected_captures r LEFT JOIN ranked c ON c.capture_id = r.capture_id AND c.rank = 1 -ORDER BY r.start_ts, r.capture_id LIMIT 100; +WITH + ranked AS ( + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY + capture_id + ORDER BY ts DESC, event_id DESC + ) AS rank + FROM agentprof_skill_context_snapshots + ) +SELECT + r.capture_id, + r.peak_context, + r.context_window_tokens, + r.context_share, + c.event_id AS latest_snapshot_id, + c.ts AS latest_snapshot_ts, + c.stage, + c.basis, + c.coverage, + c.model AS snapshot_model, + c.estimated_tokens, + c.reported_tokens, + c.window_tokens, + c.effective_window_tokens, + c.compact_threshold_tokens, + c.omitted_changes, + c.categories, + ( + SELECT COUNT(*) + FROM agentprof_skill_context_snapshots AS s + WHERE + s.capture_id = r.capture_id + ) AS snapshots, + ( + SELECT COUNT(*) + FROM agentprof_skill_context_changes AS s + WHERE + s.capture_id = r.capture_id + ) AS retained_changes +FROM agentprof_skill_selected_captures AS r +LEFT JOIN ranked AS c + ON c.capture_id = r.capture_id + AND c.rank = 1 +ORDER BY + r.start_ts, + r.capture_id +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql index beca189..537cff1 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/delegation.sql @@ -1,16 +1,38 @@ -- SPDX-License-Identifier: Apache-2.0 -WITH RECURSIVE selected(capture_id) AS ( - SELECT capture_id FROM agentprof_skill_selected_captures - UNION - SELECT h.capture_id FROM agentprof_skill_capture_hierarchy h - JOIN selected s ON h.parent_capture_id = s.capture_id -) -SELECT c.capture_id, h.root_capture_id, h.parent_capture_id, h.is_subagent, - c.harness, c.start_ts, c.end_ts, c.duration_ms AS retained_extent_ms, - p.start_ts AS parent_start_ts, p.end_ts AS parent_end_ts, - CASE WHEN h.parent_capture_id IS NOT NULL THEN - MAX(0, MIN(c.end_ts, p.end_ts) - MAX(c.start_ts, p.start_ts)) / 1e6 END AS parent_child_extent_overlap_ms -FROM agentprof_skill_capture_runs c JOIN agentprof_skill_capture_hierarchy h USING(capture_id) -LEFT JOIN agentprof_skill_capture_runs p ON p.capture_id = h.parent_capture_id -WHERE c.capture_id IN (SELECT capture_id FROM selected) -ORDER BY c.start_ts, c.capture_id LIMIT 100; +WITH RECURSIVE + selected(capture_id) AS ( + SELECT capture_id FROM agentprof_skill_selected_captures + UNION + SELECT h.capture_id + FROM agentprof_skill_capture_hierarchy AS h + JOIN selected AS s + ON h.parent_capture_id = s.capture_id + ) +SELECT + c.capture_id, + h.root_capture_id, + h.parent_capture_id, + h.is_subagent, + c.harness, + c.start_ts, + c.end_ts, + c.duration_ms AS retained_extent_ms, + p.start_ts AS parent_start_ts, + p.end_ts AS parent_end_ts, + CASE + WHEN h.parent_capture_id IS NOT NULL THEN MAX( + 0, + MIN(c.end_ts, p.end_ts) - MAX(c.start_ts, p.start_ts) + ) + / 1e6 + END AS parent_child_extent_overlap_ms +FROM agentprof_skill_capture_runs AS c +JOIN agentprof_skill_capture_hierarchy AS h USING (capture_id) +LEFT JOIN agentprof_skill_capture_runs AS p + ON p.capture_id = h.parent_capture_id +WHERE + c.capture_id IN (SELECT capture_id FROM selected) +ORDER BY + c.start_ts, + c.capture_id +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql index 35f4b89..ae8fa05 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/inventory.sql @@ -1,30 +1,109 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT r.capture_id, r.session, r.capture, r.harness, r.provider, r.model, r.effort, - r.context_window_tokens, r.duration_ms AS retained_extent_ms, - r.turns AS prompts, r.responses, r.tools, r.incomplete, - (SELECT COUNT(*) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id - AND NOT incomplete AND message_ns >= 0) AS measured_responses, - (SELECT COUNT(*) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id - AND NOT incomplete AND first_ns >= 0) AS first_content_samples, - (SELECT COUNT(output_tokens) FROM agentprof_skill_messages m - WHERE m.capture_id = r.capture_id) AS output_usage_samples, - (SELECT COUNT(*) FROM agentprof_skill_context_snapshots c - WHERE c.capture_id = r.capture_id) AS context_snapshots, - (SELECT COUNT(DISTINCT t.name) FROM agentprof_skill_counter_tracks t - WHERE t.capture_id = r.capture_id AND t.name IN - ('runtime.rss', 'runtime.heap', 'runtime.cpu')) AS runtime_metrics, - (SELECT MAX(c.value) FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id - WHERE t.capture_id = r.capture_id AND t.name = 'tracing.droppedEvents') AS dropped_events, - (SELECT MAX(c.value) FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id - WHERE t.capture_id = r.capture_id AND t.name = 'tracing.laneOverflows') AS lane_overflows, - (SELECT MAX(c.value) FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id - WHERE t.capture_id = r.capture_id AND t.name = 'tracing.queueDepth') AS queue_depth_peak, - (SELECT SUM(EXTRACT_ARG(s.arg_set_id, 'debug.unavailable_child_sessions')) - FROM agentprof_skill_slices s WHERE s.capture_id = r.capture_id AND s.kind = 'capture') AS unavailable_children, - (SELECT COUNT(DISTINCT COALESCE(t.machine_id, 0)) FROM agentprof_skill_slices s - JOIN track t ON t.id = s.track_id WHERE s.capture_id = r.capture_id) AS machines, - (SELECT COALESCE(SUM(value), 0) FROM stats WHERE name IN - ('clock_sync_failure_no_path', 'clock_sync_failure_undeferrable_packet_loss', - 'clock_sync_unrelatable_clock_domains', 'trace_sorter_negative_timestamp_dropped', - 'invalid_clock_snapshots')) AS trace_clock_errors -FROM agentprof_skill_selected_captures r ORDER BY r.start_ts, r.capture_id LIMIT 100; +SELECT + r.capture_id, + r.session, + r.capture, + r.harness, + r.provider, + r.model, + r.effort, + r.context_window_tokens, + r.duration_ms AS retained_extent_ms, + r.turns AS prompts, + r.responses, + r.tools, + r.incomplete, + ( + SELECT COUNT(*) + FROM agentprof_skill_messages AS m + WHERE + m.capture_id = r.capture_id + AND NOT incomplete + AND message_ns >= 0 + ) AS measured_responses, + ( + SELECT COUNT(*) + FROM agentprof_skill_messages AS m + WHERE + m.capture_id = r.capture_id + AND NOT incomplete + AND first_ns >= 0 + ) AS first_content_samples, + ( + SELECT COUNT(output_tokens) + FROM agentprof_skill_messages AS m + WHERE + m.capture_id = r.capture_id + ) AS output_usage_samples, + ( + SELECT COUNT(*) + FROM agentprof_skill_context_snapshots AS c + WHERE + c.capture_id = r.capture_id + ) AS context_snapshots, + ( + SELECT COUNT(DISTINCT t.name) + FROM agentprof_skill_counter_tracks AS t + WHERE + t.capture_id = r.capture_id + AND t.name IN ('runtime.rss', 'runtime.heap', 'runtime.cpu') + ) AS runtime_metrics, + ( + SELECT MAX(c.value) + FROM counter AS c + JOIN agentprof_skill_counter_tracks AS t + ON t.id = c.track_id + WHERE + t.capture_id = r.capture_id + AND t.name = 'tracing.droppedEvents' + ) AS dropped_events, + ( + SELECT MAX(c.value) + FROM counter AS c + JOIN agentprof_skill_counter_tracks AS t + ON t.id = c.track_id + WHERE + t.capture_id = r.capture_id + AND t.name = 'tracing.laneOverflows' + ) AS lane_overflows, + ( + SELECT MAX(c.value) + FROM counter AS c + JOIN agentprof_skill_counter_tracks AS t + ON t.id = c.track_id + WHERE + t.capture_id = r.capture_id + AND t.name = 'tracing.queueDepth' + ) AS queue_depth_peak, + ( + SELECT SUM(EXTRACT_ARG(s.arg_set_id, 'debug.unavailable_child_sessions')) + FROM agentprof_skill_slices AS s + WHERE + s.capture_id = r.capture_id + AND s.kind = 'capture' + ) AS unavailable_children, + ( + SELECT COUNT(DISTINCT COALESCE(t.machine_id, 0)) + FROM agentprof_skill_slices AS s + JOIN track AS t + ON t.id = s.track_id + WHERE + s.capture_id = r.capture_id + ) AS machines, + ( + SELECT COALESCE(SUM(value), 0) + FROM stats + WHERE + name IN ( + 'clock_sync_failure_no_path', + 'clock_sync_failure_undeferrable_packet_loss', + 'clock_sync_unrelatable_clock_domains', + 'trace_sorter_negative_timestamp_dropped', + 'invalid_clock_snapshots' + ) + ) AS trace_clock_errors +FROM agentprof_skill_selected_captures AS r +ORDER BY + r.start_ts, + r.capture_id +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql index 99ef3a1..8c0030d 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/model_summary.sql @@ -1,31 +1,103 @@ -- SPDX-License-Identifier: Apache-2.0 -WITH responses AS ( - SELECT *, COALESCE(EXTRACT_ARG(arg_set_id, 'debug.timing'), 'recorded-span') AS timing - FROM agentprof_skill_messages - WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -), measured AS ( - SELECT *, ROW_NUMBER() OVER (PARTITION BY capture_id, provider, model, timing ORDER BY message_ns, id) AS rank, - COUNT(*) OVER (PARTITION BY capture_id, provider, model, timing) AS samples - FROM responses WHERE NOT incomplete AND message_ns >= 0 -), durations AS ( - SELECT capture_id, provider, model, timing, - AVG(CASE WHEN rank IN ((samples + 1) / 2, (samples + 2) / 2) THEN message_ns END) / 1e6 AS median_ms, - MAX(CASE WHEN rank = (95 * samples + 99) / 100 THEN message_ns END) / 1e6 AS p95_ms - FROM measured GROUP BY capture_id, provider, model, timing -) -SELECT r.capture_id, r.provider, r.model, r.timing, COUNT(*) AS responses, +WITH + responses AS ( + SELECT + *, + COALESCE(EXTRACT_ARG(arg_set_id, 'debug.timing'), 'recorded-span') AS timing + FROM agentprof_skill_messages + WHERE + capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) + ), + measured AS ( + SELECT + *, + ROW_NUMBER() OVER ( + PARTITION BY + capture_id, + provider, + model, + timing + ORDER BY message_ns, id + ) AS rank, + COUNT(*) OVER (PARTITION BY capture_id, provider, model, timing) AS samples + FROM responses + WHERE + NOT incomplete + AND message_ns >= 0 + ), + durations AS ( + SELECT + capture_id, + provider, + model, + timing, + AVG( + CASE + WHEN rank IN ((samples + 1) / 2, (samples + 2) / 2) THEN message_ns + END + ) + / 1e6 AS median_ms, + MAX(CASE WHEN rank = (95 * samples + 99) / 100 THEN message_ns END) / 1e6 AS p95_ms + FROM measured + GROUP BY + capture_id, + provider, + model, + timing + ) +SELECT + r.capture_id, + r.provider, + r.model, + r.timing, + COUNT(*) AS responses, SUM(r.incomplete) AS incomplete, COUNT(CASE WHEN NOT r.incomplete AND r.message_ns >= 0 THEN 1 END) AS duration_samples, - AVG(CASE WHEN NOT r.incomplete AND r.message_ns >= 0 THEN r.message_ns END) / 1e6 AS mean_ms, - MAX(d.median_ms) AS median_ms, MAX(d.p95_ms) AS p95_ms, + AVG(CASE WHEN NOT r.incomplete AND r.message_ns >= 0 THEN r.message_ns END) + / 1e6 AS mean_ms, + MAX(d.median_ms) AS median_ms, + MAX(d.p95_ms) AS p95_ms, AVG(CASE WHEN NOT r.incomplete AND r.first_ns >= 0 THEN r.first_ns END) / 1e6 AS first_content_mean_ms, COUNT(CASE WHEN NOT r.incomplete AND r.first_ns >= 0 THEN 1 END) AS first_content_samples, - SUM(r.input_tokens) AS reported_input_tokens, COUNT(r.input_tokens) AS input_samples, - SUM(r.output_tokens) AS reported_output_tokens, COUNT(r.output_tokens) AS output_samples, - SUM(r.cache_read_tokens) AS reported_cache_read_tokens, COUNT(r.cache_read_tokens) AS cache_read_samples, - SUM(CASE WHEN NOT r.incomplete AND r.message_ns > 0 AND r.output_tokens IS NOT NULL THEN r.output_tokens END) * 1e9 / - SUM(CASE WHEN NOT r.incomplete AND r.message_ns > 0 AND r.output_tokens IS NOT NULL THEN r.message_ns END) AS output_tokens_per_s, - COUNT(CASE WHEN NOT r.incomplete AND r.message_ns > 0 AND r.output_tokens IS NOT NULL THEN 1 END) AS throughput_samples -FROM responses r LEFT JOIN durations d ON d.capture_id = r.capture_id AND d.provider IS r.provider - AND d.model IS r.model AND d.timing = r.timing -GROUP BY r.capture_id, r.provider, r.model, r.timing ORDER BY responses DESC LIMIT 100; + SUM(r.input_tokens) AS reported_input_tokens, + COUNT(r.input_tokens) AS input_samples, + SUM(r.output_tokens) AS reported_output_tokens, + COUNT(r.output_tokens) AS output_samples, + SUM(r.cache_read_tokens) AS reported_cache_read_tokens, + COUNT(r.cache_read_tokens) AS cache_read_samples, + SUM( + CASE + WHEN NOT r.incomplete + AND r.message_ns > 0 + AND r.output_tokens IS NOT NULL THEN r.output_tokens + END + ) + * 1e9 + / SUM( + CASE + WHEN NOT r.incomplete + AND r.message_ns > 0 + AND r.output_tokens IS NOT NULL THEN r.message_ns + END + ) AS output_tokens_per_s, + COUNT( + CASE + WHEN NOT r.incomplete + AND r.message_ns > 0 + AND r.output_tokens IS NOT NULL THEN 1 + END + ) AS throughput_samples +FROM responses AS r +LEFT JOIN durations AS d + ON d.capture_id = r.capture_id + AND d.provider IS r.provider + AND d.model IS r.model + AND d.timing = r.timing +GROUP BY + r.capture_id, + r.provider, + r.model, + r.timing +ORDER BY + responses DESC +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql index 946796f..232267c 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/request_headers.sql @@ -1,8 +1,18 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT id, capture_id, ts, name, +SELECT + id, + capture_id, + ts, + name, CASE WHEN NOT incomplete AND dur >= 0 THEN dur / 1e6 END AS request_span_ms, EXTRACT_ARG(arg_set_id, 'debug.phase') AS phase, - incomplete, EXTRACT_ARG(arg_set_id, 'debug.timing') AS timing -FROM agentprof_skill_slices WHERE kind = 'provider-request' + incomplete, + EXTRACT_ARG(arg_set_id, 'debug.timing') AS timing +FROM agentprof_skill_slices +WHERE + kind = 'provider-request' AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -ORDER BY dur DESC, id LIMIT 50; +ORDER BY + dur DESC, + id +LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql index ed27d17..00218ad 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/run_summary.sql @@ -1,51 +1,135 @@ -- SPDX-License-Identifier: Apache-2.0 -WITH intervals AS ( - SELECT capture_id, message_start_ts AS ts, message_start_ts + message_ns AS end_ts, - 1 AS models, 0 AS tools, 0 AS scripts FROM agentprof_skill_messages - WHERE NOT incomplete AND message_ns > 0 - UNION ALL - SELECT capture_id, ts, ts + dur, 0, kind = 'tool-execution', kind IN ('script', 'model-call') - FROM agentprof_skill_slices WHERE NOT incomplete AND dur > 0 +WITH + intervals AS ( + SELECT + capture_id, + message_start_ts AS ts, + message_start_ts + message_ns AS end_ts, + 1 AS models, + 0 AS tools, + 0 AS scripts + FROM agentprof_skill_messages + WHERE + NOT incomplete + AND message_ns > 0 + UNION ALL + SELECT + capture_id, + ts, + ts + dur, + 0, + kind = 'tool-execution', + kind IN ('script', 'model-call') + FROM agentprof_skill_slices + WHERE + NOT incomplete + AND dur > 0 AND kind IN ('tool-execution', 'script', 'model-call') -), bounds AS ( - SELECT capture_id, MIN(ts) AS start_ts, MAX(end_ts) AS end_ts FROM ( - SELECT capture_id, start_ts AS ts, end_ts FROM agentprof_skill_selected_captures - UNION ALL SELECT capture_id, ts, end_ts FROM intervals - ) GROUP BY capture_id -), boundaries AS ( - SELECT capture_id, ts, models, tools, scripts FROM intervals - UNION ALL SELECT capture_id, end_ts, -models, -tools, -scripts FROM intervals -), deltas AS ( - SELECT capture_id, ts, SUM(models) AS models, SUM(tools) AS tools, SUM(scripts) AS scripts - FROM boundaries GROUP BY capture_id, ts -), activity AS ( - SELECT capture_id, ts, LEAD(ts) OVER (PARTITION BY capture_id ORDER BY ts) - ts AS dur, - SUM(models) OVER (PARTITION BY capture_id ORDER BY ts) AS models, - SUM(tools) OVER (PARTITION BY capture_id ORDER BY ts) AS tools, - SUM(scripts) OVER (PARTITION BY capture_id ORDER BY ts) AS scripts FROM deltas -), totals AS ( - SELECT capture_id, - SUM(CASE WHEN models > 0 AND tools = 0 THEN dur ELSE 0 END) AS model_only_ns, - SUM(CASE WHEN tools > 0 AND models = 0 THEN dur ELSE 0 END) AS tool_only_ns, - SUM(CASE WHEN models > 0 AND tools > 0 THEN dur ELSE 0 END) AS overlap_ns, - SUM(CASE WHEN scripts > 0 AND models = 0 AND tools = 0 THEN dur ELSE 0 END) AS script_only_ns, - SUM(CASE WHEN tools > 1 THEN dur ELSE 0 END) AS parallel_tool_ns, - MAX(tools) AS peak_tools FROM activity WHERE dur > 0 GROUP BY capture_id -) -SELECT r.capture_id, r.session, r.harness, r.provider, r.model, r.effort, - r.turns AS prompts, r.responses, r.tools, r.incomplete, + ), + bounds AS ( + SELECT capture_id, MIN(ts) AS start_ts, MAX(end_ts) AS end_ts + FROM ( + SELECT capture_id, start_ts AS ts, end_ts + FROM agentprof_skill_selected_captures + UNION ALL + SELECT capture_id, ts, end_ts FROM intervals + ) + GROUP BY + capture_id + ), + boundaries AS ( + SELECT capture_id, ts, models, tools, scripts FROM intervals + UNION ALL + SELECT capture_id, end_ts, -models, -tools, -scripts FROM intervals + ), + deltas AS ( + SELECT + capture_id, + ts, + SUM(models) AS models, + SUM(tools) AS tools, + SUM(scripts) AS scripts + FROM boundaries + GROUP BY + capture_id, + ts + ), + activity AS ( + SELECT + capture_id, + ts, + LEAD(ts) OVER (PARTITION BY capture_id ORDER BY ts) - ts AS dur, + SUM(models) OVER (PARTITION BY capture_id ORDER BY ts) AS models, + SUM(tools) OVER (PARTITION BY capture_id ORDER BY ts) AS tools, + SUM(scripts) OVER (PARTITION BY capture_id ORDER BY ts) AS scripts + FROM deltas + ), + totals AS ( + SELECT + capture_id, + SUM(CASE WHEN models > 0 AND tools = 0 THEN dur ELSE 0 END) AS model_only_ns, + SUM(CASE WHEN tools > 0 AND models = 0 THEN dur ELSE 0 END) AS tool_only_ns, + SUM(CASE WHEN models > 0 AND tools > 0 THEN dur ELSE 0 END) AS overlap_ns, + SUM( + CASE WHEN scripts > 0 AND models = 0 AND tools = 0 THEN dur ELSE 0 END + ) AS script_only_ns, + SUM(CASE WHEN tools > 1 THEN dur ELSE 0 END) AS parallel_tool_ns, + MAX(tools) AS peak_tools + FROM activity + WHERE + dur > 0 + GROUP BY + capture_id + ) +SELECT + r.capture_id, + r.session, + r.harness, + r.provider, + r.model, + r.effort, + r.turns AS prompts, + r.responses, + r.tools, + r.incomplete, (b.end_ts - b.start_ts) / 1e6 AS observed_extent_ms, COALESCE(t.model_only_ns, 0) / 1e6 AS model_only_ms, COALESCE(t.tool_only_ns, 0) / 1e6 AS tool_only_ms, COALESCE(t.overlap_ns, 0) / 1e6 AS overlap_ms, COALESCE(t.script_only_ns, 0) / 1e6 AS script_only_ms, - MAX(0, b.end_ts - b.start_ts - COALESCE(t.model_only_ns, 0) - COALESCE(t.tool_only_ns, 0) - - COALESCE(t.overlap_ns, 0) - COALESCE(t.script_only_ns, 0)) / 1e6 AS unattributed_ms, - COALESCE(t.parallel_tool_ns, 0) / 1e6 AS parallel_tool_ms, COALESCE(t.peak_tools, 0) AS peak_tools, - r.input_tokens AS reported_input_counter_peak, r.output_tokens AS reported_output_counter_peak, - r.peak_context, r.context_window_tokens, r.context_share, - (SELECT COUNT(*) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id - AND NOT incomplete AND message_ns >= 0) AS measured_responses, - (SELECT COUNT(output_tokens) FROM agentprof_skill_messages m WHERE m.capture_id = r.capture_id) AS output_usage_samples -FROM agentprof_skill_selected_captures r JOIN bounds b USING(capture_id) -LEFT JOIN totals t USING(capture_id) ORDER BY r.start_ts, r.capture_id LIMIT 100; + MAX( + 0, + b.end_ts - b.start_ts - COALESCE(t.model_only_ns, 0) + - COALESCE(t.tool_only_ns, 0) + - COALESCE(t.overlap_ns, 0) + - COALESCE(t.script_only_ns, 0) + ) + / 1e6 AS unattributed_ms, + COALESCE(t.parallel_tool_ns, 0) / 1e6 AS parallel_tool_ms, + COALESCE(t.peak_tools, 0) AS peak_tools, + r.input_tokens AS reported_input_counter_peak, + r.output_tokens AS reported_output_counter_peak, + r.peak_context, + r.context_window_tokens, + r.context_share, + ( + SELECT COUNT(*) + FROM agentprof_skill_messages AS m + WHERE + m.capture_id = r.capture_id + AND NOT incomplete + AND message_ns >= 0 + ) AS measured_responses, + ( + SELECT COUNT(output_tokens) + FROM agentprof_skill_messages AS m + WHERE + m.capture_id = r.capture_id + ) AS output_usage_samples +FROM agentprof_skill_selected_captures AS r +JOIN bounds AS b USING (capture_id) +LEFT JOIN totals AS t USING (capture_id) +ORDER BY + r.start_ts, + r.capture_id +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql index 2a56943..7b74c2d 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_samples.sql @@ -1,7 +1,21 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT t.capture_id, t.id AS track_id, c.ts, t.display_name AS metric, c.value, - CASE WHEN t.name = 'runtime.cpu' THEN 'us consumed since previous sample' ELSE 'bytes' END AS unit -FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id -WHERE t.name IN ('runtime.rss', 'runtime.heap', 'runtime.cpu') +SELECT + t.capture_id, + t.id AS track_id, + c.ts, + t.display_name AS metric, + c.value, + CASE + WHEN t.name = 'runtime.cpu' THEN 'us consumed since previous sample' + ELSE 'bytes' + END AS unit +FROM counter AS c +JOIN agentprof_skill_counter_tracks AS t + ON t.id = c.track_id +WHERE + t.name IN ('runtime.rss', 'runtime.heap', 'runtime.cpu') AND t.capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -ORDER BY c.ts DESC, t.id LIMIT 200; +ORDER BY + c.ts DESC, + t.id +LIMIT 200; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql index 0585beb..6327ce5 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/runtime_summary.sql @@ -1,11 +1,30 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT t.capture_id, t.id AS track_id, t.display_name AS metric, - CASE WHEN t.name = 'runtime.cpu' THEN 'us consumed since previous sample' ELSE 'bytes' END AS unit, - COUNT(*) AS samples, MIN(c.ts) AS first_sample_ts, MAX(c.ts) AS last_sample_ts, +SELECT + t.capture_id, + t.id AS track_id, + t.display_name AS metric, + CASE + WHEN t.name = 'runtime.cpu' THEN 'us consumed since previous sample' + ELSE 'bytes' + END AS unit, + COUNT(*) AS samples, + MIN(c.ts) AS first_sample_ts, + MAX(c.ts) AS last_sample_ts, MIN(CASE WHEN t.name = 'runtime.cpu' OR c.value > 0 THEN c.value END) AS minimum, MAX(CASE WHEN t.name = 'runtime.cpu' OR c.value > 0 THEN c.value END) AS maximum, CASE WHEN t.name = 'runtime.cpu' THEN SUM(c.value) END AS sampled_cpu_us -FROM counter c JOIN agentprof_skill_counter_tracks t ON t.id = c.track_id -WHERE t.name IN ('runtime.rss', 'runtime.heap', 'runtime.cpu') +FROM counter AS c +JOIN agentprof_skill_counter_tracks AS t + ON t.id = c.track_id +WHERE + t.name IN ('runtime.rss', 'runtime.heap', 'runtime.cpu') AND t.capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -GROUP BY t.capture_id, t.id, t.display_name, t.name ORDER BY t.capture_id, t.id LIMIT 100; +GROUP BY + t.capture_id, + t.id, + t.display_name, + t.name +ORDER BY + t.capture_id, + t.id +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql index 6365caf..24fdeec 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_responses.sql @@ -1,9 +1,22 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT id, capture_id, provider, model, +SELECT + id, + capture_id, + provider, + model, EXTRACT_ARG(arg_set_id, 'debug.timing') AS timing, - message_start_ts AS ts, CASE WHEN NOT incomplete AND message_ns >= 0 THEN message_ns / 1e6 END AS duration_ms, + message_start_ts AS ts, + CASE WHEN NOT incomplete AND message_ns >= 0 THEN message_ns / 1e6 END AS duration_ms, CASE WHEN NOT incomplete AND first_ns >= 0 THEN first_ns / 1e6 END AS first_content_ms, - input_tokens, output_tokens, cache_read_tokens, incomplete + input_tokens, + output_tokens, + cache_read_tokens, + incomplete FROM agentprof_skill_messages -WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -ORDER BY incomplete, message_ns DESC, id LIMIT 50; +WHERE + capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY + incomplete, + message_ns DESC, + id +LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql index 32b80c1..7f66d3b 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_tools.sql @@ -1,9 +1,26 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT s.id, s.capture_id, s.ts, s.kind, s.name, +SELECT + s.id, + s.capture_id, + s.ts, + s.kind, + s.name, CASE WHEN NOT s.incomplete AND s.dur >= 0 THEN s.dur / 1e6 END AS duration_ms, - s.incomplete, s.is_error, EXTRACT_ARG(s.arg_set_id, 'debug.timing') AS timing, + s.incomplete, + s.is_error, + EXTRACT_ARG(s.arg_set_id, 'debug.timing') AS timing, s.parent_call_id, - (SELECT COUNT(DISTINCT c.id) FROM agentprof_skill_script_children c WHERE c.script_id = s.id) AS script_descendants -FROM agentprof_skill_tool_calls s -WHERE s.capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -ORDER BY s.incomplete DESC, s.dur DESC, s.id LIMIT 50; + ( + SELECT COUNT(DISTINCT c.id) + FROM agentprof_skill_script_children AS c + WHERE + c.script_id = s.id + ) AS script_descendants +FROM agentprof_skill_tool_calls AS s +WHERE + s.capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY + s.incomplete DESC, + s.dur DESC, + s.id +LIMIT 50; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql index 0301cde..79f524d 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/slow_turns.sql @@ -1,10 +1,25 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT prompt_id, capture_id, turn_number, incomplete, +SELECT + prompt_id, + capture_id, + turn_number, + incomplete, (display_end_ts - start_ts) / 1e6 AS observed_extent_ms, - models_only_ns / 1e6 AS model_only_ms, tools_only_ns / 1e6 AS tool_only_ms, - overlap_ns / 1e6 AS overlap_ms, unattributed_ns / 1e6 AS unattributed_ms, - model_calls AS responses, unmatched_requests, tools, unknown_models, unknown_tools, - input_tokens, output_tokens + models_only_ns / 1e6 AS model_only_ms, + tools_only_ns / 1e6 AS tool_only_ms, + overlap_ns / 1e6 AS overlap_ms, + unattributed_ns / 1e6 AS unattributed_ms, + model_calls AS responses, + unmatched_requests, + tools, + unknown_models, + unknown_tools, + input_tokens, + output_tokens FROM agentprof_skill_turns -WHERE capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -ORDER BY display_end_ts - start_ts DESC, prompt_id LIMIT 20; +WHERE + capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) +ORDER BY + display_end_ts - start_ts DESC, + prompt_id +LIMIT 20; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql index f3be151..9a19961 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/tool_summary.sql @@ -1,14 +1,36 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT capture_id, kind, name AS operation, +SELECT + capture_id, + kind, + name AS operation, COALESCE(EXTRACT_ARG(arg_set_id, 'debug.timing'), 'recorded-span') AS timing, - COUNT(*) AS calls, SUM(incomplete) AS incomplete, + COUNT(*) AS calls, + SUM(incomplete) AS incomplete, SUM(NOT incomplete AND dur >= 0) AS duration_samples, SUM(CASE WHEN NOT incomplete AND dur >= 0 THEN dur END) / 1e6 AS summed_work_ms, MAX(CASE WHEN NOT incomplete AND dur >= 0 THEN dur END) / 1e6 AS longest_ms, - COUNT(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.is_error'), - EXTRACT_ARG(arg_set_id, 'debug.agentprof_tool_is_error'))) AS outcome_samples, - SUM(COALESCE(EXTRACT_ARG(arg_set_id, 'debug.is_error'), - EXTRACT_ARG(arg_set_id, 'debug.agentprof_tool_is_error')) = 1) AS errors -FROM agentprof_skill_slices WHERE kind IN ('tool-execution', 'script', 'model-call') + COUNT( + COALESCE( + EXTRACT_ARG(arg_set_id, 'debug.is_error'), + EXTRACT_ARG(arg_set_id, 'debug.agentprof_tool_is_error') + ) + ) AS outcome_samples, + SUM( + COALESCE( + EXTRACT_ARG(arg_set_id, 'debug.is_error'), + EXTRACT_ARG(arg_set_id, 'debug.agentprof_tool_is_error') + ) + = 1 + ) AS errors +FROM agentprof_skill_slices +WHERE + kind IN ('tool-execution', 'script', 'model-call') AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -GROUP BY capture_id, kind, name, timing ORDER BY longest_ms DESC LIMIT 100; +GROUP BY + capture_id, + kind, + name, + timing +ORDER BY + longest_ms DESC +LIMIT 100; diff --git a/ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql b/ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql index 7c648c1..5bf8d5a 100644 --- a/ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql +++ b/ai/skills/dev.agentprof/workflows/agentprof/scripts/unassigned_work.sql @@ -1,6 +1,17 @@ -- SPDX-License-Identifier: Apache-2.0 -SELECT capture_id, kind, COUNT(*) AS unassigned_events, +SELECT + capture_id, + kind, + COUNT(*) AS unassigned_events, SUM(incomplete OR measured_ns IS NULL) AS unknown_or_incomplete -FROM agentprof_skill_turn_events WHERE prompt_id IS NULL +FROM agentprof_skill_turn_events +WHERE + prompt_id IS NULL AND capture_id IN (SELECT capture_id FROM agentprof_skill_selected_captures) -GROUP BY capture_id, kind ORDER BY capture_id, kind LIMIT 100; +GROUP BY + capture_id, + kind +ORDER BY + capture_id, + kind +LIMIT 100; diff --git a/package.json b/package.json index 97cd32e..dbe9d7a 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "check": "node tools/check-licenses.mjs && bun tools/generate-skills-sql.ts --check && npm run check --workspace pi-tracing && tsc -p examples/pi-opus-5/tsconfig.json && tsc -p packages/claude-tracing/tsconfig.json && tsc -p packages/codex-tracing/tsconfig.json && bun tools/build-claude-direct.ts --check && bun tools/build-codex-plugin.ts --check && tsc -p packages/muse-tracing/tsconfig.json && bun tools/build-muse-plugin.ts --check && node tools/check-muse-marketplace.mjs", "test": "npm run test --workspace pi-tracing && bun test packages/agent-tracing/*.test.ts packages/claude-tracing packages/codex-tracing packages/muse-tracing tools/check-example-privacy.test.ts tools/overview-loader.test.ts tools/queries-capture.test.ts tools/turns.test.ts && npm run test:skills", "build:skills-sql": "bun tools/generate-skills-sql.ts", - "test:skills": "python3 tools/test_skills_overlay.py && python3 tools/test_skills_sql.py", + "test:skills": "python3 tools/test_skills_overlay.py && python3 tools/test_skills_sql.py && python3 -m unittest discover -s tools/release -p 'test_*.py'", + "build:ai-skill": "python3 tools/release/build-ai-skill.py", "test:node-runtime": "bun test packages/pi-tracing/extensions/pi-tracing/node-runtime.test.ts packages/claude-tracing/direct.test.ts packages/claude-tracing/legacy.test.ts packages/codex-tracing/plugin.test.ts packages/codex-tracing/plugin-hook.test.ts packages/muse-tracing/record.test.ts", "trace:example": "bun tools/bundle-example.ts", "check:example": "npm run trace:fixture && bun tools/check-example.ts && npm run trace:example && bun tools/check-recorded-example.ts && bun tools/check-codemode-example.ts && bun tools/check-comparison-example.ts && bun tools/check-harness-examples.ts && bun tools/check-muse-example.ts && bun tools/check-example-context.ts && bun tools/check-example-privacy.ts", diff --git a/tools/generate-skills-sql.ts b/tools/generate-skills-sql.ts index 03143f1..af3afb9 100644 --- a/tools/generate-skills-sql.ts +++ b/tools/generate-skills-sql.ts @@ -8,7 +8,8 @@ import {TURN_CLEANUP_SQL, TURNS_SETUP_SQL} from '../third_party/overlays/perfett const root = resolve(import.meta.dir, '..'); const scripts = resolve(root, 'ai/skills/dev.agentprof/workflows/agentprof/scripts'); const header = '-- SPDX-License-Identifier: Apache-2.0\n' + - '-- Exported from AgentProf reader SQL. Standalone: no UI or build dependency.\n'; + '-- @generated by tools/generate-skills-sql.ts; edit the reader SQL and regenerate.\n' + + '-- Standalone: no UI or build dependency.\n'; function replaceOnce(sql: string, from: string, to: string): string { if (sql.indexOf(from) < 0 || sql.indexOf(from) !== sql.lastIndexOf(from)) { diff --git a/tools/perfetto b/tools/perfetto index 16afdee..7fab033 100644 --- a/tools/perfetto +++ b/tools/perfetto @@ -136,7 +136,7 @@ def setup(): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("command", choices=["setup", "capture", "build-ui", "dev-server", "check-ui", "smoke-ui"]) + parser.add_argument("command", choices=["setup", "capture", "build-ui", "dev-server", "check-ui", "check-skills-format", "smoke-ui"]) parser.add_argument("--skip-deps", action="store_true") parser.add_argument("--existing-server", action="store_true", help="smoke-ui: test a server already running on port 10000") @@ -159,7 +159,31 @@ def main(): return if not args.skip_deps: run(sys.executable, "tools/install-build-deps", "--ui") - if args.command == "check-ui": + if args.command == "check-skills-format": + scripts = SKILL_WORKFLOWS / "agentprof/scripts" + generated = {"agentprof_setup.sql", "agentprof_turns.sql"} + header = ("-- SPDX-License-Identifier: Apache-2.0\n" + "-- @generated by tools/generate-skills-sql.ts; edit the reader SQL and regenerate.\n") + sources = [] + found_generated = set() + for source in sorted(scripts.glob("*.sql")): + text = source.read_text() + if source.name in generated: + if not text.startswith(header): + sys.exit(f"Invalid generated SQL header: {source}") + found_generated.add(source.name) + else: + if "@generated" in text: + sys.exit(f"Authored SQL cannot be marked generated: {source}") + sources.append(source) + if found_generated != generated or not sources: + sys.exit("Missing generated or authored AgentProf SQL resources.") + if not (OUT / "build.ninja").is_file(): + run(sys.executable, "tools/gn", "gen", + "--args=is_debug=false skip_buildtools_check=true", OUT) + os.environ["OUT"] = str(OUT) + run(sys.executable, "python/tools/code_format_sql.py", "--check-only", *sources) + elif args.command == "check-ui": run("ui/node_modules/.bin/tsc", "--project", "ui/tsconfig.json", "--noEmit") elif args.command == "build-ui": run("ui/build", "--out", OUT, *extra) diff --git a/tools/release/build-ai-skill.py b/tools/release/build-ai-skill.py new file mode 100644 index 0000000..c12d0d1 --- /dev/null +++ b/tools/release/build-ai-skill.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Package Perfetto's assembled skill with the AgentProf workflow overlay.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import platform +import re +import runpy +import stat +import subprocess +import sys +import tempfile +import time +import tomllib +import zipfile + +ROOT = Path(__file__).resolve().parents[2] +CHECKOUT = ROOT / "third_party/src/perfetto" +OVERLAY = ROOT / "ai/skills/dev.agentprof/workflows/agentprof" +ARCHIVE = "agentprof-ai-skill.zip" +PROVENANCE = "AGENTPROF_RELEASE.json" + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def wrapper_manifest(wrapper): + return runpy.run_path(str(wrapper))["TRACE_PROCESSOR_SHELL_MANIFEST"] + + +def provenance(revision, epoch, wrapper, perfetto_version): + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("revision must be a full lowercase 40-character commit SHA") + if epoch < 0: + raise ValueError("epoch must be nonnegative") + manifest = wrapper_manifest(wrapper) + expected_version = json.loads((CHECKOUT / "ai/extensions/claude-code/plugin.json").read_text())["version"] + if perfetto_version != expected_version or any(f"/{perfetto_version}/" not in item["url"] for item in manifest): + raise ValueError("Perfetto manifests and bundled runtime version disagree") + return { + "agentprof_revision": revision, + "agentprof_version": json.loads((ROOT / "package.json").read_text())["version"], + "perfetto_pin": tomllib.loads((ROOT / "third_party/perfetto.toml").read_text())["perfetto"]["pin"], + "perfetto_version": perfetto_version, + "build_epoch": epoch, + "trace_processor_wrapper_sha256": digest(wrapper.read_bytes()), + "trace_processor_prebuilts": manifest, + } + + +def write_archive(skill, archive, metadata): + timestamp = time.gmtime(max(315532800, metadata["build_epoch"]))[:6] + if timestamp[0] > 2107: + raise ValueError("epoch exceeds ZIP's timestamp range") + (skill / "LICENSE").write_bytes((ROOT / "LICENSE").read_bytes()) + (skill / PROVENANCE).write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as output: + for path in sorted(skill.rglob("*")): + if path.is_symlink(): + raise ValueError(f"unexpected symlink in assembled skill: {path.name}") + if not path.is_file(): + continue + mode = 0o755 if path.stat().st_mode & 0o111 else 0o644 + info = zipfile.ZipInfo("perfetto/" + path.relative_to(skill).as_posix(), timestamp) + info.create_system = 3 + info.external_attr = (stat.S_IFREG | mode) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + output.writestr(info, path.read_bytes(), compresslevel=9) + checksum = digest(archive.read_bytes()) + archive.with_suffix(".zip.sha256").write_text(f"{checksum} {archive.name}\n") + return checksum + + +def extract_verified(archive, destination): + expected = archive.with_suffix(".zip.sha256").read_text().strip() + if expected != f"{digest(archive.read_bytes())} {archive.name}": + raise ValueError("archive checksum mismatch") + with zipfile.ZipFile(archive) as bundle: + seen = set() + for info in bundle.infolist(): + path = PurePosixPath(info.filename) + mode = info.external_attr >> 16 + if (path.is_absolute() or ".." in path.parts or "\\" in info.filename + or not path.parts or path.parts[0] != "perfetto" + or path.as_posix() != info.filename or info.filename in seen + or not stat.S_ISREG(mode) or stat.S_IMODE(mode) not in (0o644, 0o755)): + raise ValueError(f"unsafe or duplicate ZIP entry: {info.filename}") + seen.add(info.filename) + target = destination.joinpath(*path.parts) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(bundle.read(info)) + target.chmod(mode & 0o777) + skill = destination / "perfetto" + metadata = json.loads((skill / PROVENANCE).read_text()) + if (skill / "SKILL-template.md").exists() or not (skill / "SKILL.md").is_file(): + raise ValueError("expected one assembled perfetto skill, not a source template") + router = (skill / "SKILL.md").read_text() + if not router.startswith("---\nname: perfetto\n"): + raise ValueError("invalid skill identity") + if (skill / "LICENSE").read_bytes() != (ROOT / "LICENSE").read_bytes(): + raise ValueError("license mismatch") + wrapper = skill / "bin/trace_processor" + if (wrapper.read_bytes() != (CHECKOUT / "tools/trace_processor").read_bytes() + or not wrapper.stat().st_mode & stat.S_IXUSR): + raise ValueError("bundled wrapper must be upstream-identical and executable") + expected_metadata = provenance(metadata["agentprof_revision"], metadata["build_epoch"], + wrapper, metadata["perfetto_version"]) + if metadata != expected_metadata: + raise ValueError("release provenance mismatch") + for source in sorted(OVERLAY.rglob("*")): + if not source.is_file(): + continue + relative = source.relative_to(OVERLAY) + packaged = skill / "workflows/agentprof" / relative + if packaged.read_bytes() != source.read_bytes(): + raise ValueError(f"workflow export mismatch: {relative}") + if source.suffix == ".md": + if f"$SKILL_ROOT/workflows/agentprof/{relative.as_posix()}" not in router: + raise ValueError(f"missing workflow route: {relative}") + for target in re.findall(r"\$SKILL_ROOT/([\w./-]+)", source.read_text()): + if not (skill / target).is_file(): + raise ValueError(f"missing workflow resource: {target}") + return skill, metadata + + +def verify_runtime(skill, metadata): + wrapper = skill / "bin/trace_processor" + module = runpy.run_path(str(wrapper)) + manifest = module["TRACE_PROCESSOR_SHELL_MANIFEST"] + entry = next(item for item in manifest if item.get("platform") == sys.platform.lower() + and platform.machine().lower() in item.get("machine", [])) + # Independently hash the cache entry before executing it; the upstream + # launcher trusts SHA-named files already present in its cache. + native = Path(module["get_perfetto_prebuilt"](manifest)) + if digest(native.read_bytes()) != entry["sha256"]: + raise ValueError("cached Trace Processor does not match the bundled manifest") + version = subprocess.check_output([str(wrapper), "--version"], text=True, timeout=60).strip() + if re.findall(r"\bv\d+(?:\.\d+)+\b", version) != [metadata["perfetto_version"]]: + raise ValueError(f"native Trace Processor version disagrees with {metadata['perfetto_version']}: {version}") + subprocess.run([str(wrapper), "query", "--help"], check=True, capture_output=True, timeout=60) + env = {**os.environ, "PERFETTO_TRACE_PROCESSOR": str(wrapper), "AGENTPROF_SKILL_ROOT": str(skill)} + subprocess.run([sys.executable, str(ROOT / "tools/test_skills_sql.py")], env=env, check=True, timeout=300) + with tempfile.TemporaryDirectory(prefix="ap-ai-") as temporary: + root = Path(temporary) + trace = root / "empty.json" + trace.write_text('{"traceEvents": []}') + socket = root / "trace.sock" + log = root / "server.log" + with log.open("w") as output: + server = subprocess.Popen([str(wrapper), "server", "unix", "--path", str(socket), + "--idle-timeout", "60s", str(trace)], + stdout=output, stderr=output) + try: + deadline = time.monotonic() + 30 + while not socket.exists(): + if server.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError(f"packaged warm session failed: {log.read_text()}") + time.sleep(.05) + for script in ("agentprof_setup.sql", "agentprof_turns.sql", "inventory.sql"): + subprocess.run([str(wrapper), "query", "--remote", str(socket), "--query-file", + str(skill / "workflows/agentprof/scripts" / script)], + check=True, capture_output=True, timeout=60) + finally: + if server.poll() is None: + server.terminate() + try: + server.wait(timeout=10) + except subprocess.TimeoutExpired: + server.kill() + server.wait(timeout=10) + return {"native_version": version, "native_sha256": entry["sha256"], + "platform": entry["arch"], "packaged_sql_tests": True, "warm_session": True} + + +def build(out, revision, epoch, verify=False): + out.mkdir(parents=True, exist_ok=True) + (out / "agentprof-ai-skill.verification.json").unlink(missing_ok=True) + with tempfile.TemporaryDirectory(prefix="agentprof-ai-build-") as temporary: + assembled = Path(temporary) / "assembled" + subprocess.run([sys.executable, str(CHECKOUT / "tools/release/build_ai_agents.py"), + "--output", str(assembled)], check=True, cwd=CHECKOUT) + skill = assembled / "plugins/perfetto/skills/perfetto" + metadata = provenance(revision, epoch, skill / "bin/trace_processor", + json.loads((assembled / "BRANCH_METADATA.json").read_text())["tag"]) + archive = out / ARCHIVE + checksum = write_archive(skill, archive, metadata) + (out / "agentprof-ai-skill.provenance.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + with tempfile.TemporaryDirectory(prefix="agentprof-ai-check-") as temporary: + skill, metadata = extract_verified(archive, Path(temporary)) + if verify: + runtime = verify_runtime(skill, metadata) + report = {"archive_sha256": checksum, "runtime": runtime} + (out / "agentprof-ai-skill.verification.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(f"Built {archive} ({checksum})") + return archive + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=ROOT / "artifacts/ai-release") + parser.add_argument("--revision", required=True, help="full AgentProf source commit SHA") + parser.add_argument("--epoch", required=True, type=int, help="source commit Unix timestamp (SOURCE_DATE_EPOCH)") + parser.add_argument("--verify", action="store_true", help="verify the exact packaged runtime (may download its pinned prebuilt)") + args = parser.parse_args() + build(args.out, args.revision, args.epoch, args.verify) + + +if __name__ == "__main__": + main() diff --git a/tools/release/test_build_ai_skill.py b/tools/release/test_build_ai_skill.py new file mode 100644 index 0000000..7dffc1d --- /dev/null +++ b/tools/release/test_build_ai_skill.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: Apache-2.0 +import importlib.util +import json +from pathlib import Path +import stat +import sys +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +SPEC = importlib.util.spec_from_file_location("build_ai_skill", Path(__file__).with_name("build-ai-skill.py")) +builder = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(builder) +REVISION = "0123456789abcdef" * 2 + "01234567" +EPOCH = 1700000000 + + +@unittest.skipUnless((builder.CHECKOUT / "tools/release/build_ai_agents.py").is_file(), + "run python3 tools/perfetto setup first") +class ReleaseSkillTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.archive = builder.build(self.root / "out", REVISION, EPOCH) + + def extract(self): + return builder.extract_verified(self.archive, self.root / "extracted") + + def test_zip_is_one_skill_not_a_marketplace(self): + with zipfile.ZipFile(self.archive) as bundle: + names = bundle.namelist() + self.assertTrue(all(name.startswith("perfetto/") for name in names)) + self.assertNotIn("perfetto/SKILL-template.md", names) + self.assertIn("perfetto/SKILL.md", names) + self.assertIn("perfetto/LICENSE", names) + self.assertIn("perfetto/AGENTPROF_RELEASE.json", names) + mode = bundle.getinfo("perfetto/bin/trace_processor").external_attr >> 16 + self.assertTrue(stat.S_ISREG(mode)) + self.assertEqual(0o755, stat.S_IMODE(mode)) + skill, metadata = self.extract() + self.assertEqual(REVISION, metadata["agentprof_revision"]) + self.assertEqual(EPOCH, metadata["build_epoch"]) + for path in builder.OVERLAY.rglob("*"): + if path.is_file(): + self.assertEqual(path.read_bytes(), (skill / "workflows/agentprof" / path.relative_to(builder.OVERLAY)).read_bytes()) + + def test_rebuild_is_byte_identical(self): + second = builder.build(self.root / "second", REVISION, EPOCH) + self.assertEqual(self.archive.read_bytes(), second.read_bytes()) + self.assertEqual(self.archive.with_suffix(".zip.sha256").read_bytes(), + second.with_suffix(".zip.sha256").read_bytes()) + + def test_checksum_mismatch_is_rejected(self): + self.archive.with_suffix(".zip.sha256").write_text("invalid\n") + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + self.extract() + + def test_wrapper_or_workflow_changes_are_rejected(self): + skill, _ = self.extract() + with patch.object(builder, "OVERLAY", self.root / "changed-overlay"): + overlay = builder.OVERLAY + overlay.mkdir() + (overlay / "investigate_agent_run.md").write_text("changed source\n") + with self.assertRaisesRegex(ValueError, "export mismatch"): + builder.extract_verified(self.archive, self.root / "different") + self.assertTrue((skill / "bin/trace_processor").is_file()) + + def test_archive_traversal_is_rejected_before_writing(self): + self.archive.unlink() + with zipfile.ZipFile(self.archive, "w") as bundle: + info = zipfile.ZipInfo("perfetto/../../escaped") + info.external_attr = (stat.S_IFREG | 0o644) << 16 + bundle.writestr(info, b"do not extract") + self.archive.with_suffix(".zip.sha256").write_text(f"{builder.digest(self.archive.read_bytes())} {self.archive.name}\n") + with self.assertRaisesRegex(ValueError, "unsafe"): + self.extract() + self.assertFalse((self.root / "escaped").exists()) + + def test_runtime_verification_uses_extracted_skill(self): + calls = [] + def verify(skill, metadata): + self.assertIn("agentprof-ai-check-", str(skill)) + self.assertEqual(REVISION, metadata["agentprof_revision"]) + self.assertTrue((skill / "workflows/agentprof/scripts/agentprof_setup.sql").is_file()) + calls.append(skill) + return {"test": "verified"} + with patch.object(builder, "verify_runtime", side_effect=verify): + archive = builder.build(self.root / "verified", REVISION, EPOCH, verify=True) + self.assertEqual(1, len(calls)) + report = json.loads((archive.parent / "agentprof-ai-skill.verification.json").read_text()) + self.assertEqual(builder.digest(archive.read_bytes()), report["archive_sha256"]) + + def test_poisoned_native_cache_is_rejected_before_execution(self): + skill, metadata = self.extract() + native = self.root / "native" + native.write_bytes(b"not the pinned binary") + manifest = [{"platform": sys.platform, "machine": [builder.platform.machine().lower()], + "sha256": "0" * 64, "arch": "fake"}] + module = {"TRACE_PROCESSOR_SHELL_MANIFEST": manifest, + "get_perfetto_prebuilt": lambda _: str(native)} + with patch.object(builder.runpy, "run_path", return_value=module), \ + patch.object(builder.subprocess, "check_output") as execute: + with self.assertRaisesRegex(ValueError, "cached Trace Processor"): + builder.verify_runtime(skill, metadata) + execute.assert_not_called() + + def test_wrong_native_version_is_rejected_before_sql_execution(self): + skill, metadata = self.extract() + native = self.root / "native" + native.write_bytes(b"runtime fixture") + manifest = [{"platform": sys.platform, "machine": [builder.platform.machine().lower()], + "sha256": builder.digest(native.read_bytes()), "arch": "fake"}] + module = {"TRACE_PROCESSOR_SHELL_MANIFEST": manifest, + "get_perfetto_prebuilt": lambda _: str(native)} + with patch.object(builder.runpy, "run_path", return_value=module), \ + patch.object(builder.subprocess, "check_output", return_value="Perfetto v57.1 (test)"), \ + patch.object(builder.subprocess, "run") as execute: + with self.assertRaisesRegex(ValueError, "native Trace Processor version disagrees"): + builder.verify_runtime(skill, metadata) + execute.assert_not_called() + + def test_unverified_rebuild_does_not_keep_stale_runtime_report(self): + report = self.archive.parent / "agentprof-ai-skill.verification.json" + report.write_text("stale verification\n") + builder.build(self.archive.parent, REVISION, EPOCH) + self.assertFalse(report.exists()) + + def test_manifest_version_mismatch_is_rejected(self): + wrapper = builder.CHECKOUT / "tools/trace_processor" + with self.assertRaisesRegex(ValueError, "version disagree"): + builder.provenance(REVISION, EPOCH, wrapper, "not-the-shipped-runtime") + + def test_revision_and_epoch_are_validated(self): + wrapper = builder.CHECKOUT / "tools/trace_processor" + with self.assertRaisesRegex(ValueError, "full lowercase"): + builder.provenance("HEAD", EPOCH, wrapper, "test") + with self.assertRaisesRegex(ValueError, "nonnegative"): + builder.provenance(REVISION, -1, wrapper, "test") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/test_upload_ai_skill.py b/tools/release/test_upload_ai_skill.py new file mode 100644 index 0000000..e21874f --- /dev/null +++ b/tools/release/test_upload_ai_skill.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +SPEC = importlib.util.spec_from_file_location("upload_ai_skill", Path(__file__).with_name("upload-ai-skill.py")) +uploader = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(uploader) + + +class UploadSkillTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + provenance = json.dumps({"perfetto_version": "v57.2", + "trace_processor_prebuilts": [{"arch": "linux-amd64", "sha256": "a" * 64}]}) + with zipfile.ZipFile(self.root / uploader.ASSETS[0], "w") as archive: + archive.writestr("perfetto/AGENTPROF_RELEASE.json", provenance) + checksum = uploader.hashlib.sha256((self.root / uploader.ASSETS[0]).read_bytes()).hexdigest() + (self.root / uploader.ASSETS[1]).write_text(f"{checksum} {uploader.ASSETS[0]}\n") + (self.root / uploader.ASSETS[2]).write_text(provenance) + (self.root / uploader.ASSETS[3]).write_text(json.dumps({"archive_sha256": checksum, + "runtime": {"warm_session": True, "packaged_sql_tests": True, + "native_version": "Perfetto v57.2 (test)", "native_sha256": "a" * 64, + "platform": "linux-amd64"}})) + self.existing = set() + self.mismatch = False + self.draft = False + self.calls = [] + + def gh(self, *args): + self.calls.append(args) + if args[:2] == ("release", "view"): + return json.dumps({"isDraft": self.draft, "tagName": "v1.0", + "assets": [{"name": name} for name in self.existing]}) + if args[:2] == ("release", "download"): + destination = Path(args[args.index("--dir") + 1]) + for name in self.existing: + (destination / name).write_bytes(b"different" if self.mismatch + else (self.root / name).read_bytes()) + return "" + + def upload(self, publish=False): + with patch.object(uploader, "gh", side_effect=self.gh): + return uploader.upload("owner/repo", "v1.0", self.root, publish) + + def test_default_is_dry_run(self): + self.assertEqual(list(uploader.ASSETS), self.upload()) + self.assertFalse(any(call[:2] == ("release", "upload") for call in self.calls)) + + def test_first_publication_uploads_all_assets(self): + self.assertEqual(list(uploader.ASSETS), self.upload(publish=True)) + call = next(call for call in self.calls if call[:2] == ("release", "upload")) + self.assertNotIn("--clobber", call) + + def test_matching_complete_rerun_does_not_upload(self): + self.existing = set(uploader.ASSETS) + self.assertEqual([], self.upload(publish=True)) + self.assertFalse(any(call[:2] == ("release", "upload") for call in self.calls)) + + def test_partial_upload_recovers_only_missing_assets(self): + self.existing = {uploader.ASSETS[0], uploader.ASSETS[1]} + self.assertEqual(list(uploader.ASSETS[2:]), self.upload(publish=True)) + call = next(call for call in self.calls if call[:2] == ("release", "upload")) + self.assertNotIn(self.root / uploader.ASSETS[0], call) + self.assertIn(self.root / uploader.ASSETS[2], call) + + def test_mismatch_refuses_every_upload(self): + self.existing = {uploader.ASSETS[0]} + self.mismatch = True + with self.assertRaisesRegex(ValueError, "refusing overwrite"): + self.upload(publish=True) + self.assertFalse(any(call[:2] == ("release", "upload") for call in self.calls)) + + def test_draft_release_is_not_published(self): + self.draft = True + with self.assertRaisesRegex(ValueError, "published release"): + self.upload(publish=True) + self.assertFalse(any(call[:2] == ("release", "upload") for call in self.calls)) + + def test_changed_provenance_prevents_any_remote_call(self): + (self.root / uploader.ASSETS[2]).write_text("stale provenance") + with self.assertRaisesRegex(ValueError, "checksum-bound archive"): + self.upload(publish=True) + self.assertEqual([], self.calls) + + def test_truthy_nonboolean_verification_is_rejected(self): + path = self.root / uploader.ASSETS[3] + report = json.loads(path.read_text()) + report["runtime"]["warm_session"] = "false" + report["runtime"]["packaged_sql_tests"] = "false" + path.write_text(json.dumps(report)) + with self.assertRaisesRegex(ValueError, "matching runtime verification"): + self.upload(publish=True) + self.assertEqual([], self.calls) + + def test_wrong_native_identity_is_rejected(self): + path = self.root / uploader.ASSETS[3] + report = json.loads(path.read_text()) + report["runtime"]["native_version"] = "Perfetto v57.1 (test)" + path.write_text(json.dumps(report)) + with self.assertRaisesRegex(ValueError, "matching runtime verification"): + self.upload(publish=True) + self.assertEqual([], self.calls) + + def test_missing_provenance_prevents_any_remote_call(self): + (self.root / uploader.ASSETS[2]).unlink() + with self.assertRaisesRegex(ValueError, "all four"): + self.upload(publish=True) + self.assertEqual([], self.calls) + + def test_missing_runtime_verification_is_not_published(self): + (self.root / uploader.ASSETS[3]).write_text(json.dumps({"archive_sha256": "wrong", + "runtime": {"warm_session": True, "packaged_sql_tests": True}})) + with self.assertRaisesRegex(ValueError, "matching runtime verification"): + self.upload(publish=True) + self.assertEqual([], self.calls) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/upload-ai-skill.py b/tools/release/upload-ai-skill.py new file mode 100644 index 0000000..1f85997 --- /dev/null +++ b/tools/release/upload-ai-skill.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Upload verified AI skill assets to an existing published GitHub release.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess +import tempfile +import zipfile + +ASSETS = ("agentprof-ai-skill.zip", "agentprof-ai-skill.zip.sha256", + "agentprof-ai-skill.provenance.json", "agentprof-ai-skill.verification.json") + + +def gh(*args): + return subprocess.check_output(["gh", *map(str, args)], text=True) + + +def upload(repo, tag, directory, publish=False): + files = {name: directory / name for name in ASSETS} + if any(not path.is_file() for path in files.values()): + raise ValueError("all four verified release assets are required") + checksum = hashlib.sha256(files[ASSETS[0]].read_bytes()).hexdigest() + if files[ASSETS[1]].read_text().strip() != f"{checksum} {ASSETS[0]}": + raise ValueError("local archive checksum mismatch") + provenance_bytes = files[ASSETS[2]].read_bytes() + with zipfile.ZipFile(files[ASSETS[0]]) as archive: + if provenance_bytes != archive.read("perfetto/AGENTPROF_RELEASE.json"): + raise ValueError("provenance differs from the checksum-bound archive") + provenance = json.loads(provenance_bytes) + report = json.loads(files[ASSETS[3]].read_text()) + runtime = report["runtime"] + matching_prebuilt = any(item["arch"] == runtime.get("platform") + and item["sha256"] == runtime.get("native_sha256") + for item in provenance["trace_processor_prebuilts"]) + if (report["archive_sha256"] != checksum or runtime.get("warm_session") is not True + or runtime.get("packaged_sql_tests") is not True or not matching_prebuilt + or re.findall(r"\bv\d+(?:\.\d+)+\b", runtime.get("native_version", "")) + != [provenance["perfetto_version"]]): + raise ValueError("artifact has no matching runtime verification") + release = json.loads(gh("release", "view", tag, "--repo", repo, + "--json", "isDraft,tagName,assets")) + if release["isDraft"] or release["tagName"] != tag: + raise ValueError("target must be the existing published release") + existing = {asset["name"] for asset in release["assets"]} & files.keys() + if existing: + with tempfile.TemporaryDirectory(prefix="agentprof-ai-existing-") as temporary: + arguments = ["release", "download", tag, "--repo", repo, "--dir", temporary] + for name in sorted(existing): + arguments.extend(("--pattern", name)) + gh(*arguments) + for name in existing: + if files[name].read_bytes() != (Path(temporary) / name).read_bytes(): + raise ValueError(f"existing release asset differs: {name}; refusing overwrite") + missing = [files[name] for name in ASSETS if name not in existing] + if publish and missing: + gh("release", "upload", tag, *missing, "--repo", repo) + print(f"{len(existing)} matching assets; {len(missing)} {'uploaded' if publish else 'would upload'}") + return [path.name for path in missing] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--dir", type=Path, required=True) + parser.add_argument("--publish", action="store_true", help="upload missing assets after checking existing ones") + args = parser.parse_args() + upload(args.repo, args.tag, args.dir, args.publish) + + +if __name__ == "__main__": + main() diff --git a/tools/test_skills_overlay.py b/tools/test_skills_overlay.py index 433d92f..70b77bd 100644 --- a/tools/test_skills_overlay.py +++ b/tools/test_skills_overlay.py @@ -29,7 +29,8 @@ def setUp(self): self.workflow.parent.mkdir(parents=True) self.workflow.write_text("# AgentProf\n") self.patcher = patch.multiple(manager, CHECKOUT=self.checkout, - OVERLAYS=self.overlays, SKILL_WORKFLOWS=self.skills) + OVERLAYS=self.overlays, SKILL_WORKFLOWS=self.skills, + OUT=self.checkout / "out/agentprof") self.patcher.start() self.addCleanup(self.patcher.stop) self.destination = self.checkout / "ai/skills/perfetto/workflows/agentprof/run.md" @@ -67,6 +68,49 @@ def test_two_overlay_roots_cannot_claim_same_destination(self): manager.link_overlays() self.assertFalse(self.checkout.exists()) + def prepare_sql_sources(self): + scripts = self.skills / "agentprof/scripts" + scripts.mkdir() + header = ("-- SPDX-License-Identifier: Apache-2.0\n" + "-- @generated by tools/generate-skills-sql.ts; edit the reader SQL and regenerate.\n") + for name in ("agentprof_setup.sql", "agentprof_turns.sql"): + (scripts / name).write_text(header + "SELECT 1;\n") + (scripts / "inventory.sql").write_text("SELECT 1;\n") + return scripts + + def run_format_check(self): + with patch.object(manager, "setup"), patch.object(manager, "run") as run, \ + patch.object(sys, "argv", ["perfetto", "check-skills-format", "--skip-deps"]), \ + patch.dict(os.environ): + manager.main() + return run.call_args_list + + def test_formatter_initializes_fresh_output(self): + scripts = self.prepare_sql_sources() + calls = self.run_format_check() + self.assertEqual(2, len(calls)) + self.assertEqual("tools/gn", calls[0].args[1]) + self.assertEqual("--check-only", calls[1].args[2]) + self.assertEqual((scripts / "inventory.sql",), calls[1].args[3:]) + + def test_formatter_preserves_existing_output_configuration(self): + self.prepare_sql_sources() + (manager.OUT / "build.ninja").parent.mkdir(parents=True) + (manager.OUT / "build.ninja").write_text("existing configuration\n") + self.assertEqual(1, len(self.run_format_check())) + + def test_authored_sql_cannot_bypass_formatting(self): + scripts = self.prepare_sql_sources() + (scripts / "inventory.sql").write_text("-- copied from an @generated query\nSELECT 1;\n") + with self.assertRaisesRegex(SystemExit, "Authored SQL"): + self.run_format_check() + + def test_generated_sql_requires_its_exact_ownership_header(self): + scripts = self.prepare_sql_sources() + (scripts / "agentprof_turns.sql").write_text("-- @generated\nSELECT 1;\n") + with self.assertRaisesRegex(SystemExit, "Invalid generated SQL header"): + self.run_format_check() + def test_source_only_documentation_is_not_an_overlay(self): (self.skills.parent / "README.md").write_text("source documentation\n") self.assertEqual([(self.workflow, Path("ai/skills/perfetto/workflows/agentprof/run.md"))], diff --git a/tools/test_skills_sql.py b/tools/test_skills_sql.py index 19b002b..961abf2 100644 --- a/tools/test_skills_sql.py +++ b/tools/test_skills_sql.py @@ -2,13 +2,17 @@ import json import os from pathlib import Path +import re import subprocess import sqlite3 import tempfile import unittest +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] -SCRIPTS = ROOT / "ai/skills/dev.agentprof/workflows/agentprof/scripts" +SCRIPTS = (Path(os.environ["AGENTPROF_SKILL_ROOT"]) / "workflows/agentprof/scripts" + if "AGENTPROF_SKILL_ROOT" in os.environ + else ROOT / "ai/skills/dev.agentprof/workflows/agentprof/scripts") BINARY = os.environ.get("PERFETTO_TRACE_PROCESSOR") SETUP = (SCRIPTS / "agentprof_setup.sql").read_text() TURNS = (SCRIPTS / "agentprof_turns.sql").read_text() @@ -68,6 +72,9 @@ def query(self, sql, trace=None): self.assertEqual(0, result.returncode, result.stderr[-4000:]) return result.stdout + def query_resource(self, resource, trace=None): + return self.query(SETUP + TURNS + resource.read_text(), trace) + def check(self, script, condition, setup=SETUP + _FIXTURE): query = (SCRIPTS / script).read_text().strip().removesuffix(";") out = self.query(setup + f"\nWITH result AS ({query})\n" @@ -76,9 +83,11 @@ def check(self, script, condition, setup=SETUP + _FIXTURE): def test_all_resources_on_empty_trace_and_repeat_setup(self): out = self.query(SETUP + SETUP + TURNS + TURNS + - "\n".join(p.read_text() for p in QUERIES) + "SELECT COUNT(*) AS captures FROM agentprof_skill_capture_runs;") self.assertTrue(out.rstrip().endswith("0"), out[-1000:]) + for resource in QUERIES: + with self.subTest(resource=resource.name): + self.query_resource(resource) def test_all_resources_on_four_reviewed_harness_recordings(self): for harness in ("pi", "claude", "codex", "muse"): @@ -87,9 +96,10 @@ def test_all_resources_on_four_reviewed_harness_recordings(self): files = manifest["bundledFiles"] if harness == "pi" else [manifest["bundledFile"]] for name in files: path = source / name - with self.subTest(harness=harness, recording=name): - self.assertTrue(path.is_file(), f"missing reviewed recording: {path}") - self.query(SETUP + TURNS + "\n".join(p.read_text() for p in QUERIES), path) + self.assertTrue(path.is_file(), f"missing reviewed recording: {path}") + for resource in QUERIES: + with self.subTest(harness=harness, recording=name, resource=resource.name): + self.query_resource(resource, path) def test_occupancy_is_capture_scoped_and_unions_overlapping_tools(self): self.check("run_summary.sql", """ @@ -185,6 +195,38 @@ def test_prompt_and_tool_payloads_are_not_materialized(self): self.assertIn("SKILLS_OK", out) +class ResourceInvocationTest(unittest.TestCase): + def test_generated_turn_setup_preserves_reader_index_hints(self): + reader = (ROOT / "third_party/overlays/perfetto/ui/src/plugins/dev.agentprof.Agentprof/turns_queries.ts").read_text() + expected = [name.replace("agentprof_", "agentprof_skill_", 1) + for name in re.findall(r"\bINDEXED BY (\w+)", reader)] + self.assertTrue(expected) + self.assertEqual(expected, re.findall(r"\bINDEXED BY (\w+)", TURNS)) + self.assertIn("@generated by tools/generate-skills-sql.ts", TURNS) + self.assertIn("@generated by tools/generate-skills-sql.ts", SETUP) + + def test_recordings_execute_each_resource_in_its_own_cli_call(self): + case = SkillsSqlTest("test_all_resources_on_four_reviewed_harness_recordings") + with patch.object(case, "query", return_value="") as query: + case.test_all_resources_on_four_reviewed_harness_recordings() + self.assertEqual(7 * len(QUERIES), query.call_count) + for calls in (query.call_args_list[i:i + len(QUERIES)] + for i in range(0, query.call_count, len(QUERIES))): + self.assertEqual([SETUP + TURNS + resource.read_text() for resource in QUERIES], + [call.args[0] for call in calls]) + + def test_empty_results_do_not_hide_batched_output(self): + case = SkillsSqlTest("test_all_resources_on_empty_trace_and_repeat_setup") + with patch.object(case, "query", return_value="0") as query: + case.test_all_resources_on_empty_trace_and_repeat_setup() + self.assertEqual(1 + len(QUERIES), query.call_count) + self.assertEqual(SETUP + SETUP + TURNS + TURNS + + "SELECT COUNT(*) AS captures FROM agentprof_skill_capture_runs;", + query.call_args_list[0].args[0]) + self.assertEqual([SETUP + TURNS + resource.read_text() for resource in QUERIES], + [call.args[0] for call in query.call_args_list[1:]]) + + class ResourceQueryTest(unittest.TestCase): def test_muse_child_role_metadata_is_preserved(self): with sqlite3.connect(":memory:") as db: