From d053fb6a446bed50196cc46a0a025060ec63cfe3 Mon Sep 17 00:00:00 2001 From: DrDrewCain Date: Sat, 12 Sep 2026 23:47:07 -0500 Subject: [PATCH] Show per-task token usage and reporting coverage --- README.md | 13 +++++++ scripts/fixtures/agent-input-server.py | 4 +- scripts/test-agent-inputs-native.cjs | 10 ++++- src/agents/AgentsPage.tsx | 6 +-- src/agents/RunPanel.tsx | 13 ++++--- src/agents/UsageDetails.tsx | 17 ++++++++ src/agents/agents.css | 7 ++++ src/agents/runs.ts | 25 ++++++------ src/agents/usage.ts | 54 ++++++++++++++++++++++++++ src/capabilities.ts | 6 +-- tests/agent-usage.test.ts | 45 +++++++++++++++++++++ 11 files changed, 174 insertions(+), 26 deletions(-) create mode 100644 src/agents/UsageDetails.tsx create mode 100644 src/agents/usage.ts create mode 100644 tests/agent-usage.test.ts diff --git a/README.md b/README.md index 550b38c..a34bad0 100644 --- a/README.md +++ b/README.md @@ -594,3 +594,16 @@ the source discards the pending result. Reloading does not replay inference. The browser rechecks source evidence before and after the response; a removed or changed source refuses the interpretation. Cancellation stops display and browser transport, but cannot guarantee the provider stops work already received. + +When the connected server advertises `agents.usage`, completed workflow results +show **Reported token usage** for each model task or handoff hop. Prompt, +completion, and total counts each include their reporting coverage; missing +reports display **Unknown**, and historical tasks without telemetry say that it +was not recorded. Saved results keep their original counts across restart and +repeated reads. Human replies have no model usage, and a final handoff is shown +once with its hop. + +This view requires the native workflow usage contract and explicitly requests +`include_usage=true`. Older servers retain the existing result view. The counts +are provider reports for completed tasks, not billing totals or quality scores; +failed and interrupted attempts can consume tokens outside these receipts. diff --git a/scripts/fixtures/agent-input-server.py b/scripts/fixtures/agent-input-server.py index 991ed86..1c41307 100644 --- a/scripts/fixtures/agent-input-server.py +++ b/scripts/fixtures/agent-input-server.py @@ -9,6 +9,7 @@ from scone_memory import HashEmbedder, InMemoryDocumentStore, InMemoryVectorIndex, MemoryEngine from scone_memory.agents.catalog import AgentCatalog, AgentDefinition, AgentModel from scone_memory.agents.evidence_loop import ToolStep +from scone_memory.agents.usage import ModelTokenUsage from scone_memory.agents.plan_store import AgentPlanStore from scone_memory.agents.run_service import AgentRunService from scone_memory.api.app import create_app @@ -24,7 +25,8 @@ def __init__(self, name): async def complete(self, messages, tools): with (state / 'calls.jsonl').open('a') as output: output.write(json.dumps({'model': self.name, 'messages': messages}) + '\n') - return ToolStep(content=self.name + ' completed the task.') + usage = ModelTokenUsage(prompt_tokens=120, completion_tokens=18, total_tokens=138) if self.name == 'careful' else ModelTokenUsage(prompt_tokens=25) + return ToolStep(content=self.name + ' completed the task.', usage=usage) catalog = AgentCatalog(models=[AgentModel(name, name.title() + ' local', '1', lambda name=name: Model(name)) for name in ('fast', 'careful')], agents=[AgentDefinition( agent_id='worker', instructions='Use the provided direction.', models=('fast', 'careful'), diff --git a/scripts/test-agent-inputs-native.cjs b/scripts/test-agent-inputs-native.cjs index b51f916..9525a92 100644 --- a/scripts/test-agent-inputs-native.cjs +++ b/scripts/test-agent-inputs-native.cjs @@ -24,7 +24,7 @@ for(const mobile of [false,true])test(`human replies survive process restart wit t.after(async()=>{await browser?.close();await stop();fs.rmSync(state,{recursive:true,force:true});}); await start();browser=await engines[process.env.SCONE_BROWSER_ENGINE||'chromium'].launch({headless:true,executablePath:process.env.SCONE_BROWSER_PATH}); const page=await browser.newPage({viewport:mobile?{width:390,height:844}:{width:1380,height:1000}});page.setDefaultTimeout(7000); - const errors=[];page.on('pageerror',error=>errors.push(error.message)); + const errors=[],resultRequests=[];page.on('pageerror',error=>errors.push(error.message));page.on('request',request=>{if(new URL(request.url()).pathname.endsWith('/result'))resultRequests.push(request.url());}); await page.goto(base+'/agents'); await page.getByLabel('Workflow identifier',{exact:true}).fill('interactive'); await page.getByLabel('Task type',{exact:true}).selectOption('input'); @@ -58,6 +58,14 @@ for(const mobile of [false,true])test(`human replies survive process restart wit const completed=calls();assert.deepEqual(completed.map(call=>call.model),['fast','careful']);assert.match(JSON.stringify(completed[1].messages),/Take the northern route/); await view.getByRole('button',{name:'Check status',exact:true}).click();await page.getByText('careful completed the task.',{exact:true}).waitFor();assert.equal(calls().length,2); assert.equal(await page.getByRole('heading',{name:'task-1 · Human input',exact:true}).count(),1); + const careful=page.getByRole('heading',{name:'task-2 · worker · careful',exact:true}).locator('..'); + const fast=page.getByRole('heading',{name:'task-3 · worker · fast',exact:true}).locator('..'); + assert.equal(await careful.getByRole('cell',{name:'138',exact:true}).count(),1); + assert.equal(await fast.getByRole('cell',{name:'Unknown',exact:true}).count(),2); + assert.equal(await page.getByRole('table',{name:'Reported token usage'}).count(),2); + assert(resultRequests.length>0&&resultRequests.every(url=>new URL(url).searchParams.get('include_usage')==='true')); + assert.equal(calls().length,2); + if(process.env.SCONE_USAGE_SCREENSHOTS)await page.screenshot({path:path.join(process.env.SCONE_USAGE_SCREENSHOTS,`usage-${mobile?'mobile':'desktop'}.png`),fullPage:true}); await page.getByRole('button',{name:/interactive.*Revision 1/}).first().click(); await page.getByRole('button',{name:'Prepare a new run',exact:true}).count().then(async count=>{if(count)await page.getByRole('button',{name:'Prepare a new run',exact:true}).click();}); await page.getByLabel('Run identifier',{exact:true}).fill('cancel-waiting');await page.getByLabel('Question',{exact:true}).fill('Another trip.'); diff --git a/src/agents/AgentsPage.tsx b/src/agents/AgentsPage.tsx index a79e1ad..710d896 100644 --- a/src/agents/AgentsPage.tsx +++ b/src/agents/AgentsPage.tsx @@ -10,7 +10,7 @@ import {parseRunPolicy} from './runs'; const secureRequest={cache:'no-store',redirect:'error',credentials:'omit',referrerPolicy:'no-referrer'} as const; -type Ready={inputsAvailable:boolean;handoffsAvailable:boolean;runsAvailable:boolean;maxParallel:number;space:string;catalog:AgentChoice[];items:SavedPlan[];next_after:string|null}; +type Ready={usageAvailable:boolean;inputsAvailable:boolean;handoffsAvailable:boolean;runsAvailable:boolean;maxParallel:number;space:string;catalog:AgentChoice[];items:SavedPlan[];next_after:string|null}; function Editor({api,space,catalog,initial,onSave,onDirty,inputsAvailable}:{api:ApiClient;space:string;catalog:AgentChoice[];initial:SavedPlan|null;onSave:(plan:SavedPlan)=>void;onDirty:()=>void;inputsAvailable:boolean}){ const first=catalog[0]; const newTask=(id:string):AgentTask=>({task_id:id,agent_id:first?.agent_id??'',model_id:first?.default_model??'',prompt:'',depends_on:[]}); @@ -74,7 +74,7 @@ export function AgentsPage({api,enabled}:{api:ApiClient;enabled:boolean}){ if(!caps.features['agents.catalog']||!caps.features['agents.plans'])throw Error('Agent workflow configuration is not enabled on this server.'); const [catalog,page]=await Promise.all([api.request('/v1/agents/catalog',options).then(parseCatalog),api.request('/v1/agent-plans?limit=20',options).then(value=>parsePlanPage(value,status.space as string))]); const maxParallel=caps.features['agents.parallel']?parseRunPolicy(await api.request('/v1/agents/run-policy',options),status.space):1; - if(!controller.signal.aborted)setLoaded({api,data:{inputsAvailable:caps.features['agents.inputs'],handoffsAvailable:caps.features['agents.handoffs'],maxParallel,runsAvailable:caps.features['agents.runs'],space:status.space,catalog,...page}}); + if(!controller.signal.aborted)setLoaded({api,data:{usageAvailable:caps.features['agents.usage'],inputsAvailable:caps.features['agents.inputs'],handoffsAvailable:caps.features['agents.handoffs'],maxParallel,runsAvailable:caps.features['agents.runs'],space:status.space,catalog,...page}}); })().catch(error=>{if(!controller.signal.aborted)setError(error instanceof Error?error.message:'Agent configuration could not be loaded.');}).finally(()=>{if(!controller.signal.aborted)setLoading(false);}); return()=>controller.abort(); },[api,enabled,refresh]); @@ -95,7 +95,7 @@ export function AgentsPage({api,enabled}:{api:ApiClient;enabled:boolean}){ {(()=>{ const props={api,inputsAvailable:data.inputsAvailable,space:data.space,catalog:data.catalog,initial:selection.plan,onDirty:()=>setDirty(true),onSave:(saved:SavedPlan)=>{setDirty(false);setSelection(current=>({...current,plan:saved}));setLoaded(current=>current?.api===api?{api,data:{...current.data,items:[saved,...current.data.items.filter(item=>item.plan.workflow_id!==saved.plan.workflow_id)]}}:current);}}; return (selection.plan?isHandoffPlan(selection.plan.plan):selection.handoff)?data.handoffsAvailable?:

This server does not support handoff configuration.

:selection.plan&&isInteractivePlan(selection.plan.plan)&&!data.inputsAvailable?

This server does not support human input workflows.

:; - })()}{data.runsAvailable&&} {error&&

{error}

} + })()}{data.runsAvailable&&} {error&&

{error}

} } ; } diff --git a/src/agents/RunPanel.tsx b/src/agents/RunPanel.tsx index c89787e..e66d340 100644 --- a/src/agents/RunPanel.tsx +++ b/src/agents/RunPanel.tsx @@ -1,3 +1,4 @@ +import {UsageDetails} from './UsageDetails'; import {useEffect,useRef,useState} from 'react'; import {ApiError,type ApiClient} from '../api'; import {isHandoffPlan,isInputTask,isInteractivePlan,type SavedPlan} from './plans'; @@ -14,7 +15,7 @@ function message(error:unknown):string{ } return error instanceof Error?error.message:'The run response is unavailable.'; } -function RunView({api,space,id,expected,onChanged,inputsAvailable}:{api:ApiClient;space:string;id:string;expected?:RunSubmission;onChanged:()=>void;inputsAvailable:boolean}){ +function RunView({api,space,id,expected,onChanged,inputsAvailable,usageAvailable}:{api:ApiClient;space:string;id:string;expected?:RunSubmission;onChanged:()=>void;inputsAvailable:boolean;usageAvailable:boolean}){ const [version,setVersion]=useState(0),[request,setRequest]=useState(null),[status,setStatus]=useState(null),[result,setResult]=useState(null),[issue,setIssue]=useState(''),[busy,setBusy]=useState(false),[cancelling,setCancelling]=useState(false); const [inputs,setInputs]=useState([]); const active=useRef(null); @@ -34,7 +35,7 @@ function RunView({api,space,id,expected,onChanged,inputsAvailable}:{api:ApiClien if(isInteractivePlan(original.plan)&&inputsAvailable){const prompts=parseInputPage(await api.request(runAddress(id)+'/inputs',options()),original);if(controller.signal.aborted)return;verifiedInputs=prompts;setInputs(prompts);} terminal=!progress.active_local&&(progress.status!=='running'||progress.outcome_unknown); if(!progress.active_local&&['completed','verification_unavailable'].includes(progress.status)){ - const verified=parseRunResult(await api.request(runAddress(id)+'/result',options()),original); + const verified=parseRunResult(await api.request(runAddress(id)+'/result'+(usageAvailable?'?include_usage=true':''),options()),original,usageAvailable); if(isInteractivePlan(original.plan)){if(!inputsAvailable)throw Error('This server does not support human input workflows.');matchInputResults(verified,verifiedInputs);} if(!controller.signal.aborted)setResult(verified); }else if((progress.active_local||progress.status==='running')&&!progress.outcome_unknown){ @@ -47,7 +48,7 @@ function RunView({api,space,id,expected,onChanged,inputsAvailable}:{api:ApiClien await poll(); })().catch(error=>{if(!controller.signal.aborted){setIssue(message(error));setBusy(false);}}); return()=>{controller.abort();if(timer)clearTimeout(timer);}; - },[api,space,id,version,expected,inputsAvailable]); + },[api,space,id,version,expected,inputsAvailable,usageAvailable]); const canCancel=!!status&&(status.active_local||(!status.outcome_unknown&&['awaiting_input','registered','created'].includes(status.status))); const cancel=async()=>{ if(cancelling||!canCancel)return; @@ -67,7 +68,7 @@ function RunView({api,space,id,expected,onChanged,inputsAvailable}:{api:ApiClien {request&&
Original request · {request.plan.workflow_id} · Revision {request.revision} · {isHandoffPlan(request.plan)?`Up to ${request.plan.max_handoffs} handoffs`:`Up to ${request.max_parallel} simultaneous tasks`}

{request.question}

{isHandoffPlan(request.plan)?<>

Starting agent: {request.plan.root_agent}

    {request.plan.agents.map(agent=>
  • {agent.agent_id} · {agent.model_id} · {agent.can_handoff_to.length?`May hand off to ${agent.can_handoff_to.join(', ')}`:'Must finish without handing off'}
    • )}
:
    {request.plan.tasks.map(task=>
  • {task.task_id}: {isInputTask(task)?'Human input':`${task.agent_id} · ${task.model_id}`}{task.depends_on.length?` · Receives ${task.depends_on.join(', ')}`:''}
    • )}
}
} {request&&status&&inputsAvailable&&inputs.length>0&&{setVersion(n=>n+1);onChanged();}}/>} {result?.outcome==='handoff_limit'&&

The handoff limit was reached. These are partial results; no final answer was produced.

} - {result&&
{result.tasks.map(output=>output.kind==='human_input'?

{output.task_id} · Human input

Reply used by this workflow

{output.text}
:

{output.task_id} · {output.agent_id} · {output.model_id}{result.finalTask===output.task_id?' · Final answer':''}

{output.handoff_to!==undefined&&

{output.handoff_to===null?'Agent finished':`Handed off to ${output.handoff_to}`}

}

{output.source_status==='retained'?`${output.evidence_ids.length} retained evidence references`:'No retained evidence · Treat this as ungrounded model output'} · {output.model_calls} model calls · {output.tool_calls} tool calls

{output.text}
)}
} + {result&&
{result.tasks.map(output=>output.kind==='human_input'?

{output.task_id} · Human input

Reply used by this workflow

{output.text}
:

{output.task_id} · {output.agent_id} · {output.model_id}{result.finalTask===output.task_id?' · Final answer':''}

{output.handoff_to!==undefined&&

{output.handoff_to===null?'Agent finished':`Handed off to ${output.handoff_to}`}

}

{output.source_status==='retained'?`${output.evidence_ids.length} retained evidence references`:'No retained evidence · Treat this as ungrounded model output'} · {output.model_calls} model calls · {output.tool_calls} tool calls

{result.reusedTasks?.includes(output.task_id)&&

Saved task result reused

}{output.usage!==undefined&&}
{output.text}
)}
} {issue&&

{issue}

} ; } @@ -85,7 +86,7 @@ function RunHistory({api,space,version,onSelect}:{api:ApiClient;space:string;ver useEffect(()=>{const controller=new AbortController();active.current=controller;setItems([]);setAfter(null);void load(controller,null);return()=>controller.abort();},[api,space,version]); return

Run history in {space}

    {items.map(run=>
    • )}
{!busy&&!items.length&&!issue&&

No runs saved yet.

}{busy&&

Loading runs…

}{after&&}{issue&&

{issue}

}
; } -export function RunPanel({api,space,plan,dirty,maxParallel,handoffsAvailable,inputsAvailable}:{api:ApiClient;space:string;plan:SavedPlan|null;dirty:boolean;maxParallel:number;handoffsAvailable:boolean;inputsAvailable:boolean}){ +export function RunPanel({api,space,plan,dirty,maxParallel,handoffsAvailable,inputsAvailable,usageAvailable}:{api:ApiClient;space:string;plan:SavedPlan|null;dirty:boolean;maxParallel:number;handoffsAvailable:boolean;inputsAvailable:boolean;usageAvailable:boolean}){ const [runId,setRunId]=useState(()=>crypto.randomUUID()),[parallel,setParallel]=useState(1),[question,setQuestion]=useState(''),[busy,setBusy]=useState(false),[attempted,setAttempted]=useState(false),[issue,setIssue]=useState(''),[submitted,setSubmitted]=useState(null),[selected,setSelected]=useState<{id:string;version:number;expected?:RunSubmission}|null>(null),[history,setHistory]=useState(0),[lookup,setLookup]=useState(''); const active=useRef(null); const supported=!plan||(isInteractivePlan(plan.plan)?inputsAvailable:!isHandoffPlan(plan.plan)||handoffsAvailable); @@ -108,7 +109,7 @@ export function RunPanel({api,space,plan,dirty,maxParallel,handoffsAvailable,inp {attempted&&
A new run calls the selected models again.
} {issue&&

{issue}

}
{event.preventDefault();try{inspect(lookup);setIssue('');}catch(error){setIssue(message(error));}}}>
- {selected&&setHistory(n=>n+1)}/>} + {selected&&setHistory(n=>n+1)}/>} ; } diff --git a/src/agents/UsageDetails.tsx b/src/agents/UsageDetails.tsx new file mode 100644 index 0000000..00c97c2 --- /dev/null +++ b/src/agents/UsageDetails.tsx @@ -0,0 +1,17 @@ +import {tokenUsageRows,type ToolTokenUsage} from './usage'; + +export function UsageDetails({usage}:{usage:ToolTokenUsage|null}) { + if (usage === null) return

Token usage was not recorded for this saved task.

; + return
+ + + + {tokenUsageRows(usage).map(row => + + + + )} +
Reported token usage
CategoryTokensReporting calls
{row.label}{row.tokens === null ? 'Unknown' : row.tokens.toLocaleString()}{row.reportedCalls} of {row.modelCalls}
+

Provider-reported counts for this task. Missing reports stay unknown. These counts are not a billing total.

+
; +} diff --git a/src/agents/agents.css b/src/agents/agents.css index 4e134f6..1794756 100644 --- a/src/agents/agents.css +++ b/src/agents/agents.css @@ -1,2 +1,9 @@ .agents-page{padding:32px;max-width:1500px;margin:auto}.agents-page>header{max-width:760px;margin-bottom:24px}.agents-layout{display:grid;grid-template-columns:250px minmax(0,1fr);gap:28px}.agents-layout aside ul{list-style:none;padding:0;display:grid;gap:8px}.agents-layout aside button{width:100%;text-align:left;margin-bottom:8px}.agents-layout small{display:block;margin-top:4px;color:var(--muted,#777)}.agent-editor fieldset{border:0;padding:0;min-width:0}.agent-editor label{display:grid;gap:6px;font-size:14px}.agent-editor input:not([type=checkbox]),.agent-editor select,.agent-editor textarea{width:100%;box-sizing:border-box;padding:10px;border:1px solid var(--border,#ddd);border-radius:8px;font:inherit;background:var(--surface,#fff);color:inherit}.agent-task{border:1px solid var(--border,#ddd);border-radius:12px;padding:20px;margin:18px 0}.agent-task-heading,.agent-actions{display:flex;align-items:center;justify-content:space-between;gap:12px;flex-wrap:wrap}.agent-fields{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px;margin-bottom:16px}.agent-editor .agent-dependencies{margin-top:16px}.agent-dependencies label{display:inline-flex;align-items:center;margin:8px 18px 0 0}.agent-notice{padding:12px 16px;background:var(--surface-muted,#f4f4f4);border-radius:8px;line-height:1.5}.agent-actions{justify-content:flex-start}.agent-actions span{color:var(--muted,#777)}@media(max-width:850px){.agents-layout{grid-template-columns:1fr}.agents-layout aside ul{grid-template-columns:repeat(2,minmax(0,1fr))}.agent-fields{grid-template-columns:1fr}.agents-page{padding:20px}} .agent-runs{border-top:1px solid var(--border,#ddd);margin-top:32px;padding-top:20px}.agent-runs fieldset{border:0;padding:0;min-width:0;display:grid;gap:12px}.agent-runs label{display:grid;gap:6px}.agent-runs input,.agent-runs textarea,.agent-runs select{box-sizing:border-box;width:100%;max-width:800px;padding:10px;font:inherit;color:inherit;border:1px solid var(--border,#ddd);border-radius:8px;background:var(--surface,#fff)}.agent-runs fieldset button{justify-self:start}.agent-runs .agent-actions{margin:16px 0}.agent-run-lookup{display:flex;align-items:end;gap:12px;margin:24px 0;flex-wrap:wrap}.agent-run-lookup label{flex:1;min-width:180px;max-width:600px}.agent-run-view{border:1px solid var(--border,#ddd);border-radius:12px;padding:20px;margin:24px 0;overflow-wrap:anywhere}.agent-run-view article{border-top:1px solid var(--border,#ddd);padding:12px 0}.agent-output{white-space:pre-wrap;overflow-wrap:anywhere}.agent-run-list{list-style:none;padding:0;display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,260px),1fr));gap:12px}.agent-run-list button{width:100%;text-align:left;overflow-wrap:anywhere}.agent-run-list small{display:block;margin-top:6px}.agent-runs h3{overflow-wrap:anywhere} + +.agent-usage {margin:1rem 0;max-width:36rem} +.agent-usage table {width:100%;border-collapse:collapse;font-size:.875rem} +.agent-usage caption {text-align:left;font-weight:600;margin-bottom:.5rem} +.agent-usage th,.agent-usage td {padding:.4rem .5rem;text-align:left;border-bottom:1px solid var(--line, #d8d4ce)} +.agent-usage td {font-variant-numeric:tabular-nums} +.agent-usage-note {font-size:.8rem;line-height:1.5;opacity:.8} diff --git a/src/agents/runs.ts b/src/agents/runs.ts index a50a513..23d7584 100644 --- a/src/agents/runs.ts +++ b/src/agents/runs.ts @@ -1,9 +1,10 @@ +import {usageFields,type ToolTokenUsage} from './usage.ts'; import {bindingIds,identifier,isHandoffPlan,isInputTask,parseSavedPlan,record,type WorkflowPlan,type HandoffPlan} from './plans.ts'; export interface RunStatus {space:string;run_id:string;created_at:string;workflow_id:string;plan_revision:number;status:string;active_local:boolean;completed_steps:string[];inflight:string|null;outcome_unknown:boolean;error_class:string|null;max_parallel:number;inflight_steps:string[];waiting_steps:string[]} export interface RunRequest {space:string;run_id:string;question:string;created_at:string;plan:WorkflowPlan;revision:number;bindings:Record;max_parallel:number} -export interface TaskOutput {kind?:'model';task_id:string;agent_id:string;model_id:string;text:string;source_status:'retained'|'none';evidence_ids:string[];model_calls:number;tool_calls:number;handoff_to?:string|null} +export interface TaskOutput {kind?:'model';task_id:string;agent_id:string;model_id:string;text:string;source_status:'retained'|'none';evidence_ids:string[];model_calls:number;tool_calls:number;handoff_to?:string|null;usage?:ToolTokenUsage|null} export interface HumanOutput {kind:'human_input';task_id:string;text:string;activation_id:string} -export interface RunResult {tasks:(TaskOutput|HumanOutput)[];outcome?:'completed'|'handoff_limit';finalTask?:string|null} +export interface RunResult {reusedTasks?:string[];tasks:(TaskOutput|HumanOutput)[];outcome?:'completed'|'handoff_limit';finalTask?:string|null} function fail():never{throw Error('The run response could not be verified.');} function text(value:unknown,max:number):string{if(typeof value!=='string'||!value.trim()||value.length>max)fail();return value;} function integer(value:unknown,min:number,max=Number.MAX_SAFE_INTEGER):number{if(typeof value!=='number'||!Number.isSafeInteger(value)||valuemax)fail();return value;} @@ -53,15 +54,15 @@ export function matchRun(status:RunStatus,request:RunRequest):void{ if(isHandoffPlan(request.plan)&&JSON.stringify(status.completed_steps)!==JSON.stringify(ids.slice(0,status.completed_steps.length)))fail(); if(isHandoffPlan(request.plan)&&status.inflight!==null&&status.inflight!==ids[status.completed_steps.length])fail(); } -function parseOutput(value:unknown,expected:{task_id:string;agent_id:string;model_id:string;binding:string;depends_on:string[]}):{output:TaskOutput;packets:string[]}{ +function parseOutput(value:unknown,expected:{task_id:string;agent_id:string;model_id:string;binding:string;depends_on:string[]},includeUsage:boolean):{output:TaskOutput;packets:string[]}{ const output=record(value); if(output.task_id!==expected.task_id||output.agent_id!==expected.agent_id||output.model_id!==expected.model_id||output.binding!==expected.binding||JSON.stringify(names(output.depends_on))!==JSON.stringify(expected.depends_on))fail(); const evidence_ids=list(output.evidence_ids,2048).map(id=>text(id,4096)); const packets=list(output.evidence_packets,32).map(packet=>text(packet,1_048_576)); if(output.source_status!==(evidence_ids.length?'retained':'none'))fail(); - return {output:{task_id:expected.task_id,agent_id:expected.agent_id,model_id:expected.model_id,text:text(output.text,64000),source_status:output.source_status as 'retained'|'none',evidence_ids,model_calls:integer(output.model_calls,1,17),tool_calls:integer(output.tool_calls,0,16)},packets}; + return {output:{task_id:expected.task_id,agent_id:expected.agent_id,model_id:expected.model_id,text:text(output.text,64000),source_status:output.source_status as 'retained'|'none',evidence_ids,model_calls:integer(output.model_calls,1,17),tool_calls:integer(output.tool_calls,0,16),...usageFields(output,integer(output.model_calls,1,17),includeUsage)},packets}; } -function parseHandoffResult(row:Record,request:RunRequest,plan:HandoffPlan):RunResult{ +function parseHandoffResult(row:Record,request:RunRequest,plan:HandoffPlan,includeUsage:boolean):RunResult{ const hops=list(row.hops,plan.max_handoffs+1),ids=hopIds(plan).slice(0,hops.length),tasks:TaskOutput[]=[]; if(!hops.length||row.space!==request.space||row.run_id!==request.run_id||!['completed','handoff_limit'].includes(String(row.status)))fail(); if(names(row.reused_hops).some(id=>!ids.includes(id)))fail(); @@ -70,7 +71,7 @@ function parseHandoffResult(row:Record,request:RunRequest,plan:H const hop=record(value),agent=plan.agents.find(agent=>agent.agent_id===current); if(!agent)fail(); const expected={task_id:ids[index],agent_id:agent.agent_id,model_id:agent.model_id,binding:request.bindings[agent.agent_id],depends_on:index?[ids[index-1]]:[]}; - const parsed=parseOutput(hop.output,expected);packetCount+=parsed.packets.length; + const parsed=parseOutput(hop.output,expected,includeUsage);packetCount+=parsed.packets.length; const next=hop.handoff_to===null?null:identifier(hop.handoff_to); if(next!==null&&!agent.can_handoff_to.includes(next))fail(); if(index,request:RunRequest,plan:H if(index===hops.length-1){ if(next===null){ if(row.status!=='completed')fail(); - const final=parseOutput(row.final,expected); + const final=parseOutput(row.final,expected,includeUsage); if(JSON.stringify(final)!==JSON.stringify(parsed))fail(); }else if(row.status!=='handoff_limit'||hops.length!==plan.max_handoffs+1||row.final!==null)fail(); } } if(packetCount>128)fail(); - return {tasks,outcome:current===null?'completed':'handoff_limit',finalTask:current===null?ids[ids.length-1]:null}; + return {tasks,reusedTasks:names(row.reused_hops),outcome:current===null?'completed':'handoff_limit',finalTask:current===null?ids[ids.length-1]:null}; } -export function parseRunResult(value:unknown,request:RunRequest):RunResult{ - if(isHandoffPlan(request.plan))return parseHandoffResult(record(value),request,request.plan); +export function parseRunResult(value:unknown,request:RunRequest,includeUsage=false):RunResult{ + if(isHandoffPlan(request.plan))return parseHandoffResult(record(value),request,request.plan,includeUsage); const row=record(value),results=record(row.results),known=new Set(request.plan.tasks.map(task=>task.task_id)); if(row.space!==request.space||row.run_id!==request.run_id||row.status!=='completed'||Object.keys(results).length!==known.size)fail(); if(names(row.reused_steps).some(id=>!known.has(id)))fail(); @@ -103,10 +104,10 @@ export function parseRunResult(value:unknown,request:RunRequest):RunResult{ const evidence_ids=list(output.evidence_ids,2048).map(id=>text(id,4096)); const evidence_packets=list(output.evidence_packets,32).map(packet=>text(packet,1_048_576));packets+=evidence_packets.length; if(output.source_status!==(evidence_ids.length?'retained':'none'))fail(); - return {task_id:task.task_id,agent_id:task.agent_id,model_id:task.model_id,text:text(output.text,64000),source_status:output.source_status as 'retained'|'none',evidence_ids,model_calls:integer(output.model_calls,1,17),tool_calls:integer(output.tool_calls,0,16)}; + return {task_id:task.task_id,agent_id:task.agent_id,model_id:task.model_id,text:text(output.text,64000),source_status:output.source_status as 'retained'|'none',evidence_ids,model_calls:integer(output.model_calls,1,17),tool_calls:integer(output.tool_calls,0,16),...usageFields(output,integer(output.model_calls,1,17),includeUsage)}; }); if(packets>128)fail(); - return {tasks}; + return {tasks,reusedTasks:names(row.reused_steps)}; } export type RunSubmission=Omit; diff --git a/src/agents/usage.ts b/src/agents/usage.ts new file mode 100644 index 0000000..5cae80a --- /dev/null +++ b/src/agents/usage.ts @@ -0,0 +1,54 @@ +import {record} from './plans.ts'; + +export interface ModelTokenUsage { + readonly prompt_tokens: number | null; + readonly completion_tokens: number | null; + readonly total_tokens: number | null; +} +export interface ToolTokenUsage {readonly calls: readonly ModelTokenUsage[]} +export interface TokenUsageRow {label:string; tokens:number|null; reportedCalls:number; modelCalls:number} + +function invalid(): never {throw Error('The model token usage could not be verified.');} +function count(value: unknown): number | null { + if (value === null) return null; + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0 || value > 1e9) invalid(); + return value; +} +function exact(value: Record, keys: string[]): void { + if (Object.keys(value).length !== keys.length || keys.some(key => !Object.hasOwn(value, key))) invalid(); +} +export function parseTokenUsage(value: unknown, modelCalls: number): ToolTokenUsage | null { + if (!Number.isSafeInteger(modelCalls) || modelCalls < 1 || modelCalls > 17) invalid(); + if (value === null) return null; + const row = record(value); + exact(row, ['calls']); + if (!Array.isArray(row.calls) || row.calls.length !== modelCalls) invalid(); + const calls = row.calls.map((raw: unknown): ModelTokenUsage => { + const call = record(raw); + exact(call, ['prompt_tokens', 'completion_tokens', 'total_tokens']); + const prompt = count(call.prompt_tokens), completion = count(call.completion_tokens), total = count(call.total_tokens); + const known = [prompt, completion].filter((value): value is number => value !== null); + const sum = known.reduce((sum, value) => sum + value, 0); + if (total !== null && (total < sum || (known.length === 2 && total !== sum))) invalid(); + return Object.freeze({prompt_tokens:prompt, completion_tokens:completion, total_tokens:total}); + }); + return Object.freeze({calls:Object.freeze(calls)}); +} +export function usageFields(row: Record, modelCalls: number, includeUsage: boolean): {usage?: ToolTokenUsage | null} { + if (typeof includeUsage !== 'boolean') invalid(); + if (!includeUsage) { + if (Object.hasOwn(row, 'usage')) invalid(); + return {}; + } + if (!Object.hasOwn(row, 'usage')) invalid(); + return {usage:parseTokenUsage(row.usage, modelCalls)}; +} +export function tokenUsageRows(usage: ToolTokenUsage | null): TokenUsageRow[] { + if (usage === null) return []; + const fields = [['Prompt', 'prompt_tokens'], ['Completion', 'completion_tokens'], ['Total', 'total_tokens']] as const; + return fields.map(([label, field]) => { + const counts = usage.calls.map(call => call[field]).filter((value): value is number => value !== null); + return {label, tokens:usage.calls.length > 0 && counts.length === usage.calls.length ? counts.reduce((sum, value) => sum + value, 0) : null, + reportedCalls:counts.length, modelCalls:usage.calls.length}; + }); +} diff --git a/src/capabilities.ts b/src/capabilities.ts index 798b324..73116ea 100644 --- a/src/capabilities.ts +++ b/src/capabilities.ts @@ -1,7 +1,7 @@ export const FEATURE_KEYS = ['recall', 'facts.read', 'facts.review', 'facts.close', 'facts.exclude', 'facts.include', 'events.read', 'metrics.read', 'scopes.read', 'status.read'] as const; export type Feature = typeof FEATURE_KEYS[number]; -type OptionalFeature = 'documents.video.understand' | 'documents.sync' | 'agents.inputs' | 'recall.parts' | 'agents.handoffs' | 'agents.parallel' | 'agents.runs' | 'agents.catalog' | 'agents.plans' | 'episodes.forget' | 'documents.ocr.tables' | 'documents.jobs' | 'documents.files' | 'graph.knowledge_walk' | 'documents.provenance' | 'recall.graph_boost' | 'graph.knowledge_seeds' | 'graph.knowledge_paging' | 'graph.timeline' | 'graph.sources' | 'graph.export' | 'graph.path' | 'graph.knowledge' | 'entities.read' | 'graph.report' | 'images.understand' | 'models.manage' | 'episodes.attachments' | 'episodes.list' | 'episodes.read' | 'facts.links' | 'integrity.read' | 'profile.read' | 'processing.distill' | 'processing.derive' | 'jobs.read' | 'recall.conditions'; +type OptionalFeature = 'agents.usage' | 'documents.video.understand' | 'documents.sync' | 'agents.inputs' | 'recall.parts' | 'agents.handoffs' | 'agents.parallel' | 'agents.runs' | 'agents.catalog' | 'agents.plans' | 'episodes.forget' | 'documents.ocr.tables' | 'documents.jobs' | 'documents.files' | 'graph.knowledge_walk' | 'documents.provenance' | 'recall.graph_boost' | 'graph.knowledge_seeds' | 'graph.knowledge_paging' | 'graph.timeline' | 'graph.sources' | 'graph.export' | 'graph.path' | 'graph.knowledge' | 'entities.read' | 'graph.report' | 'images.understand' | 'models.manage' | 'episodes.attachments' | 'episodes.list' | 'episodes.read' | 'facts.links' | 'integrity.read' | 'profile.read' | 'processing.distill' | 'processing.derive' | 'jobs.read' | 'recall.conditions'; export interface Capabilities { schema_version: 1; implementation: string; @@ -14,7 +14,7 @@ export function parseCapabilities(value: unknown): Capabilities { if (data.schema_version !== 1 || typeof data.implementation !== 'string' || !data.implementation.trim() || !data.features || typeof data.features !== 'object' || Array.isArray(data.features)) throw Error('Unsupported capability response'); const features = data.features as Record; - for(const key of ['documents.video.understand','documents.sync','documents.ocr.tables','agents.inputs','recall.parts','agents.handoffs','agents.parallel','agents.runs','agents.catalog','agents.plans','episodes.forget','documents.jobs','documents.files','documents.provenance','processing.distill','processing.derive','graph.knowledge','entities.read','graph.report','graph.path','graph.export','graph.sources','graph.timeline','graph.knowledge_paging','graph.knowledge_seeds','graph.knowledge_walk','recall.graph_boost']){ + for(const key of ['agents.usage','documents.video.understand','documents.sync','documents.ocr.tables','agents.inputs','recall.parts','agents.handoffs','agents.parallel','agents.runs','agents.catalog','agents.plans','episodes.forget','documents.jobs','documents.files','documents.provenance','processing.distill','processing.derive','graph.knowledge','entities.read','graph.report','graph.path','graph.export','graph.sources','graph.timeline','graph.knowledge_paging','graph.knowledge_seeds','graph.knowledge_walk','recall.graph_boost']){ if(features[key]!==undefined&&typeof features[key]!=='boolean')throw Error('Invalid processing capability flag'); } if (FEATURE_KEYS.some(key => typeof features[key] !== 'boolean')) throw Error('Incomplete or invalid capability flags'); @@ -28,5 +28,5 @@ export function parseCapabilities(value: unknown): Capabilities { if(features['recall.conditions']!==undefined&&typeof features['recall.conditions']!=='boolean')throw Error('Invalid recall conditions capability'); if(features['images.understand']!==undefined&&typeof features['images.understand']!=='boolean')throw Error('Invalid image understanding capability'); if(features['models.manage']!==undefined&&typeof features['models.manage']!=='boolean')throw Error('Invalid model management capability'); - return {schema_version:1, implementation:data.implementation, features:{'documents.video.understand':features['documents.video.understand']===true,'documents.sync':features['documents.sync']===true,'documents.ocr.tables':features['documents.ocr.tables']===true,'agents.inputs':features['agents.inputs']===true,'recall.parts':features['recall.parts']===true,'agents.handoffs':features['agents.handoffs']===true,'agents.parallel':features['agents.parallel']===true,'agents.runs':features['agents.runs']===true,'agents.catalog':features['agents.catalog']===true,'agents.plans':features['agents.plans']===true,'episodes.forget':features['episodes.forget']===true,'documents.jobs':features['documents.jobs']===true,'documents.files':features['documents.files']===true,'graph.knowledge_walk':features['graph.knowledge_walk']===true,'documents.provenance':features['documents.provenance']===true,'recall.graph_boost':features['recall.graph_boost']===true,'graph.knowledge_seeds':features['graph.knowledge_seeds']===true,'graph.knowledge_paging':features['graph.knowledge_paging']===true,'graph.timeline':features['graph.timeline']===true,'graph.sources':features['graph.sources']===true,'graph.export':features['graph.export']===true,'graph.path':features['graph.path']===true,'graph.report':features['graph.report']===true,'graph.knowledge':features['graph.knowledge']===true,'entities.read':features['entities.read']===true,...Object.fromEntries(FEATURE_KEYS.map(key => [key, features[key]])), 'episodes.attachments':features['episodes.attachments'] === true, 'episodes.list':features['episodes.list'] === true, 'episodes.read':features['episodes.read'] === true, 'facts.links':features['facts.links'] === true, 'integrity.read':features['integrity.read'] === true, 'profile.read':features['profile.read'] === true, 'processing.distill':features['processing.distill']===true, 'processing.derive':features['processing.derive']===true, 'jobs.read':features['jobs.read']===true, 'recall.conditions':features['recall.conditions']===true, 'models.manage':features['models.manage']===true, 'images.understand':features['images.understand']===true} as Capabilities['features']}; + return {schema_version:1, implementation:data.implementation, features:{'agents.usage':features['agents.usage']===true,'documents.video.understand':features['documents.video.understand']===true,'documents.sync':features['documents.sync']===true,'documents.ocr.tables':features['documents.ocr.tables']===true,'agents.inputs':features['agents.inputs']===true,'recall.parts':features['recall.parts']===true,'agents.handoffs':features['agents.handoffs']===true,'agents.parallel':features['agents.parallel']===true,'agents.runs':features['agents.runs']===true,'agents.catalog':features['agents.catalog']===true,'agents.plans':features['agents.plans']===true,'episodes.forget':features['episodes.forget']===true,'documents.jobs':features['documents.jobs']===true,'documents.files':features['documents.files']===true,'graph.knowledge_walk':features['graph.knowledge_walk']===true,'documents.provenance':features['documents.provenance']===true,'recall.graph_boost':features['recall.graph_boost']===true,'graph.knowledge_seeds':features['graph.knowledge_seeds']===true,'graph.knowledge_paging':features['graph.knowledge_paging']===true,'graph.timeline':features['graph.timeline']===true,'graph.sources':features['graph.sources']===true,'graph.export':features['graph.export']===true,'graph.path':features['graph.path']===true,'graph.report':features['graph.report']===true,'graph.knowledge':features['graph.knowledge']===true,'entities.read':features['entities.read']===true,...Object.fromEntries(FEATURE_KEYS.map(key => [key, features[key]])), 'episodes.attachments':features['episodes.attachments'] === true, 'episodes.list':features['episodes.list'] === true, 'episodes.read':features['episodes.read'] === true, 'facts.links':features['facts.links'] === true, 'integrity.read':features['integrity.read'] === true, 'profile.read':features['profile.read'] === true, 'processing.distill':features['processing.distill']===true, 'processing.derive':features['processing.derive']===true, 'jobs.read':features['jobs.read']===true, 'recall.conditions':features['recall.conditions']===true, 'models.manage':features['models.manage']===true, 'images.understand':features['images.understand']===true} as Capabilities['features']}; } diff --git a/tests/agent-usage.test.ts b/tests/agent-usage.test.ts new file mode 100644 index 0000000..52bfebb --- /dev/null +++ b/tests/agent-usage.test.ts @@ -0,0 +1,45 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {parseTokenUsage,tokenUsageRows} from '../src/agents/usage.ts'; +import {parseRunRequest,parseRunResult} from '../src/agents/runs.ts'; +const count={prompt_tokens:10,completion_tokens:2,total_tokens:12}; +const task={task_id:'find',agent_id:'research',model_id:'careful',prompt:'Find evidence',depends_on:[]}; +const snapshot={space:'alpha',run_id:'run-1',created_at:'2026-09-11T00:00:00Z',cancel_requested_at:null,question:'What happened?',scope:{},exclude_session_id:null,plan:{space:'alpha',revision:1,updated_at:'2026-09-11T00:00:00Z',plan:{workflow_id:'report',tasks:[task]},bindings:{find:'a'.repeat(64)}}}; +const request=parseRunRequest(snapshot,'alpha','run-1'); +const receipt={...task,binding:'a'.repeat(64),text:'Answer',source_status:'none',evidence_ids:[],evidence_packets:[],model_calls:1,tool_calls:0}; +function result(output:unknown){return {space:'alpha',run_id:'run-1',status:'completed',results:{find:output},reused_steps:['find']};} + +test('negotiated model usage is typed and reused task remains identifiable',()=>{ + const parsed=parseRunResult(result({...receipt,usage:{calls:[count]}}),request,true); + assert.deepEqual(parsed.tasks[0].usage,{calls:[count]}); + assert.deepEqual(parsed.reusedTasks,['find']); + assert.equal(parseRunResult(result({...receipt,usage:null}),request,true).tasks[0].usage,null); + assert.throws(()=>parseRunResult(result(receipt),request,true)); + assert.throws(()=>parseRunResult(result({...receipt,usage:{calls:[count]}}),request)); +}); + +test('unknown coverage never looks like zero or a complete subtotal',()=>{ + const parsed=parseTokenUsage({calls:[count,{prompt_tokens:20,completion_tokens:null,total_tokens:null}]},2); + assert.deepEqual(tokenUsageRows(parsed),[ + {label:'Prompt',tokens:30,reportedCalls:2,modelCalls:2}, + {label:'Completion',tokens:null,reportedCalls:1,modelCalls:2}, + {label:'Total',tokens:null,reportedCalls:1,modelCalls:2}, + ]); + assert.equal(tokenUsageRows(parseTokenUsage({calls:[{prompt_tokens:0,completion_tokens:0,total_tokens:0}]},1))[2].tokens,0); +}); + +for(const malformed of [undefined,{},[],{calls:[]},{calls:[count,count]},{calls:[{...count,total_tokens:11}]}, + {calls:[{...count,prompt_tokens:true}]},{calls:[{...count,prompt_tokens:'10'}]}, + {calls:[{...count,prompt_tokens:1.2}]},{calls:[{...count,prompt_tokens:-1}]}, + {calls:[{...count,prompt_tokens:1_000_000_001}]},{calls:[{...count,prompt_tokens:NaN}]}, + {calls:[{...count,prompt_tokens:Infinity}]},{calls:[{prompt_tokens:10}]}, + {calls:[{...count,private:'SECRET'}]},{calls:[count],total_tokens:12}]){ + test('malformed usage refuses '+JSON.stringify(malformed),()=>assert.throws(()=>parseTokenUsage(malformed,1))); +} + +test('native maximum totals remain safe integers and detached',()=>{ + const raw={calls:Array.from({length:17},()=>({prompt_tokens:1e9,completion_tokens:0,total_tokens:1e9}))}; + const parsed=parseTokenUsage(raw,17);raw.calls[0].total_tokens=0; + assert.equal(tokenUsageRows(parsed)[2].tokens,17e9); + assert.throws(()=>parseTokenUsage({...raw,calls:[...raw.calls,count]},18)); +});