From 17d28e23045c4e1d94b9db9b5f4da52de9006a4f Mon Sep 17 00:00:00 2001 From: DrDrewCain Date: Sun, 13 Sep 2026 00:32:29 -0500 Subject: [PATCH] Edit and preserve task output requirements in the console --- README.md | 22 ++++++ scripts/fixtures/agent-output-server.py | 57 +++++++++++++++ scripts/test-agent-output-native.cjs | 81 ++++++++++++++++++++++ src/agents/AgentsPage.tsx | 20 +++--- src/agents/OutputRequirementsEditor.tsx | 40 +++++++++++ src/agents/agents.css | 5 ++ src/agents/output-requirements.ts | 92 +++++++++++++++++++++++++ src/agents/plans.ts | 7 +- src/capabilities.ts | 6 +- tests/agent-output-requirements.test.ts | 44 ++++++++++++ 10 files changed, 360 insertions(+), 14 deletions(-) create mode 100644 scripts/fixtures/agent-output-server.py create mode 100644 scripts/test-agent-output-native.cjs create mode 100644 src/agents/OutputRequirementsEditor.tsx create mode 100644 src/agents/output-requirements.ts create mode 100644 tests/agent-output-requirements.test.ts diff --git a/README.md b/README.md index a34bad0..443460a 100644 --- a/README.md +++ b/README.md @@ -607,3 +607,25 @@ This view requires the native workflow usage contract and explicitly requests `include_usage=true`. Older servers retain the existing result view. The counts are provider reports for completed tasks, not billing totals or quality scores; failed and interrupted attempts can consume tokens outside these receipts. + +### Task output requirements + +When the host advertises `agents.output_requirements`, each model task can choose +text or JSON-object output, answer instructions, and byte/line limits. Hosts with +`agents.output_schema` additionally accept an optional JSON schema. Schema drafts +must be bounded JSON objects without duplicate keys; the host validates schema +semantics and rejects invalid model answers. The console keeps authored local +`$defs`/`$ref` intact through saving, loading and immutable run requests. Human +input tasks retain their existing plain-text reply controls. + +Changing requirements creates a new plan revision and requires a new run. Shape +validation does not establish factual accuracy. The console refuses nonfinite or +unsafe integer schema values, and schema drafts that lose decimal precision or underflow, rather than silently rounding them; its browser +number representation is narrower than Python's integers. Servers without these +capabilities retain the existing workflow editor. + +`scripts/test-agent-output-native.cjs` exercises the packaged console against +actual local native HTTP and encrypted journals at desktop and mobile widths, +including malformed drafts, selected models, invalid answer withholding and +process restart without additional inference. It uses the same explicit local +Python/browser environment variables as the other native browser fixtures. diff --git a/scripts/fixtures/agent-output-server.py b/scripts/fixtures/agent-output-server.py new file mode 100644 index 0000000..18eacef --- /dev/null +++ b/scripts/fixtures/agent-output-server.py @@ -0,0 +1,57 @@ +"""Real local agent HTTP fixture with persistent journals and scripted models.""" +import asyncio +import json +from pathlib import Path +import sys + +import uvicorn +from fastapi.responses import HTMLResponse +from scone_memory import HashEmbedder, InMemoryDocumentStore, InMemoryVectorIndex, MemoryEngine +from scone_memory.agents.catalog import AgentCatalog, AgentDefinition, AgentModel +from scone_memory.agents.evidence_loop import ToolStep +from scone_memory.agents.usage import ModelTokenUsage +from scone_memory.agents.plan_store import AgentPlanStore +from scone_memory.agents.run_service import AgentRunService +from scone_memory.api.app import create_app +from scone_memory.retrieval.recall_scope import RecallScope + + +async def run(): + state, html, port = Path(sys.argv[1]), Path(sys.argv[2]), int(sys.argv[3]) + memory = await MemoryEngine(InMemoryDocumentStore(), InMemoryVectorIndex(), HashEmbedder()).open() + class Model: + def __init__(self, name): + self.name = name + async def complete(self, messages, tools): + with (state / 'calls.jsonl').open('a') as output: + output.write(json.dumps({'model': self.name, 'messages': messages}) + '\n') + usage = ModelTokenUsage(prompt_tokens=120, completion_tokens=18, total_tokens=138) if self.name == 'careful' else ModelTokenUsage(prompt_tokens=25) + return ToolStep(content='{"name":"Juniper"}' if self.name == 'careful' else 'Invalid plain-text result', usage=usage) + catalog = AgentCatalog(models=[AgentModel(name, name.title() + ' local', '1', lambda name=name: Model(name)) + for name in ('fast', 'careful')], agents=[AgentDefinition( + agent_id='worker', instructions='Use the provided direction.', models=('fast', 'careful'), + default_model='fast', initial_search=False)]) + plans = AgentPlanStore(state / 'plans.sqlite', key=b'k' * 32) + service = AgentRunService(state / 'runs', key=b'k' * 32, catalog=catalog, plans=plans, memory=memory, + scope_for=lambda space: RecallScope.validated()) + app = create_app(memory, {'agent-fixture': 'alpha'}, agent_catalog=catalog, + agent_plan_store=plans, agent_run_service=service) + @app.get('/agents') + async def console(): + return HTMLResponse(html.read_text().replace('__SCONE_TOKEN__', 'agent-fixture')) + server = uvicorn.Server(uvicorn.Config(app, host='127.0.0.1', port=port, log_level='warning')) + async def commands(): + await asyncio.to_thread(sys.stdin.readline) + server.should_exit = True + commands_task = asyncio.create_task(commands()) + try: + await server.serve() + finally: + commands_task.cancel() + await asyncio.gather(commands_task, return_exceptions=True) + await service.aclose() + plans.close() + await memory.close() + + +asyncio.run(run()) diff --git a/scripts/test-agent-output-native.cjs b/scripts/test-agent-output-native.cjs new file mode 100644 index 0000000..3631a36 --- /dev/null +++ b/scripts/test-agent-output-native.cjs @@ -0,0 +1,81 @@ +const {test}=require('node:test'); +const assert=require('node:assert/strict'); +const {spawn}=require('node:child_process'); +const {once}=require('node:events'); +const fs=require('node:fs'),os=require('node:os'),path=require('node:path'),net=require('node:net'); +const engines=require(process.env.SCONE_PLAYWRIGHT_MODULE||'playwright'); +const {testPython}=require('./fixture-host.cjs'); +async function availablePort(){const server=net.createServer();await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const port=server.address().port;await new Promise(resolve=>server.close(resolve));return port;} +for(const mobile of [false,true])test(`task schemas survive editing, execution and restart, mobile=${mobile}`,{timeout:60000},async t=>{ + const state=fs.mkdtempSync(path.join(os.tmpdir(),'scone-task-contract-browser-')),port=await availablePort(),base=`http://127.0.0.1:${port}`; + let server,closed,logs='',browser; + const stop=async()=>{if(!server)return;if(server.exitCode===null&&server.signalCode===null)server.stdin.end('stop\n');const timer=setTimeout(()=>server.kill('SIGKILL'),5000);try{await closed;}finally{clearTimeout(timer);}assert.notEqual(server.signalCode,'SIGKILL',logs);assert.equal(server.exitCode,0,logs);server=null;}; + const start=async()=>{ + server=spawn(testPython(),['-u',path.join(__dirname,'fixtures/agent-output-server.py'),state,path.join(__dirname,'../dist/console.html'),String(port)],{stdio:['pipe','pipe','pipe']}); + closed=once(server,'close');server.stderr.on('data',part=>{logs=(logs+part).slice(-8000);});server.stdout.resume(); + for(let i=0;i<150;i++){if(server.exitCode!==null)throw Error(logs);try{if((await fetch(base+'/healthz',{signal:AbortSignal.timeout(300)})).ok)return;}catch{}await new Promise(resolve=>setTimeout(resolve,50));}throw Error('Fixture did not become ready: '+logs); + }; + const calls=()=>fs.existsSync(path.join(state,'calls.jsonl'))?fs.readFileSync(path.join(state,'calls.jsonl'),'utf8').trim().split('\n').filter(Boolean).map(JSON.parse):[]; + t.after(async()=>{await browser?.close();await stop();fs.rmSync(state,{recursive:true,force:true});}); + await start();browser=await engines[process.env.SCONE_BROWSER_ENGINE||'chromium'].launch({headless:true,executablePath:process.env.SCONE_BROWSER_PATH}); + const page=await browser.newPage({viewport:mobile?{width:390,height:844}:{width:1380,height:1100}});page.setDefaultTimeout(7000); + const errors=[],writes=[];page.on('pageerror',error=>errors.push(error.message));page.on('request',request=>{if(request.method()==='PUT')writes.push(request.postDataJSON());}); + await page.goto(base+'/agents'); + await page.getByLabel('Workflow identifier',{exact:true}).fill('names'); + await page.getByLabel('Model',{exact:true}).selectOption('careful'); + await page.getByLabel('Task instructions',{exact:true}).fill('Extract the supported name.'); + assert.equal(await page.getByLabel('Answer format',{exact:true}).count(),1,await page.locator('main').innerText()); + await page.getByLabel('Answer format',{exact:true}).selectOption('json_object'); + await page.getByLabel('Maximum answer bytes',{exact:true}).fill('256'); + await page.getByLabel('Maximum answer lines (optional)',{exact:true}).fill('1'); + await page.getByLabel('Answer instructions (optional)',{exact:true}).fill('Use the name field.'); + const field=page.getByLabel('JSON schema (optional)',{exact:true}); + await field.fill('{"properties":'); + await page.getByRole('button',{name:'Save workflow',exact:true}).click(); + assert.equal(writes.length,0);assert.equal(calls().length,0); + await page.getByRole('button',{name:'Add task',exact:true}).click(); + const second=page.getByRole('region',{name:'Task 2',exact:true}); + await second.getByLabel('Answer format',{exact:true}).selectOption('json_object'); + await second.getByLabel('JSON schema (optional)',{exact:true}).fill('{"renamed":'); + await second.getByLabel('Task identifier',{exact:true}).fill('answer'); + assert.equal(await second.getByLabel('JSON schema (optional)',{exact:true}).inputValue(),'{"renamed":'); + await page.getByRole('button',{name:'Remove task 1',exact:true}).click(); + assert.equal(await field.inputValue(),'{"renamed":','removing another node retains this task’s invalid draft'); + await page.getByLabel('Model',{exact:true}).selectOption('careful'); + await page.getByLabel('Task instructions',{exact:true}).fill('Extract the supported name.'); + await page.getByLabel('Maximum answer bytes',{exact:true}).fill('256'); + await page.getByLabel('Maximum answer lines (optional)',{exact:true}).fill('1'); + await page.getByLabel('Answer instructions (optional)',{exact:true}).fill('Use the name field.'); + const schema={$defs:{name:{type:'string',minLength:1}},properties:{name:{$ref:'#/$defs/name'}},required:['name'],additionalProperties:false}; + await field.fill(JSON.stringify(schema,null,2)); + await page.getByRole('button',{name:'Save workflow',exact:true}).click(); + await page.getByText('Saved revision 1.',{exact:true}).waitFor(); + assert.deepEqual(writes[0].plan.tasks[0].answer_requirements.output_schema,schema); + await field.evaluate(element=>{element.scrollTop=0;element.blur();});await page.evaluate(()=>scrollTo(0,0)); + if(process.env.SCONE_CONTRACT_SCREENSHOTS)await page.screenshot({path:path.join(process.env.SCONE_CONTRACT_SCREENSHOTS,`contracts-${mobile?'mobile':'desktop'}.png`),fullPage:true}); + await page.getByLabel('Run identifier',{exact:true}).fill('valid');await page.getByLabel('Question',{exact:true}).fill('What is the name?'); + await page.getByRole('button',{name:'Start run',exact:true}).click(); + const view=page.getByRole('region',{name:'Selected run',exact:true}); + await view.getByRole('status').filter({hasText:'completed · 1 tasks completed'}).waitFor(); + await page.getByText('{"name":"Juniper"}',{exact:true}).waitFor(); + assert.equal(calls().length,1);assert.match(JSON.stringify(calls()[0]),/Use the name field/); + await stop();await start();await page.reload(); + await page.getByRole('button',{name:/names.*Revision 1/}).first().click(); + assert.deepEqual(JSON.parse(await field.inputValue()),schema); + assert.equal(await page.getByLabel('Maximum answer bytes',{exact:true}).inputValue(),'256'); + await page.getByRole('button',{name:/valid · names/}).click(); + await page.getByText('{"name":"Juniper"}',{exact:true}).waitFor();assert.equal(calls().length,1); + await page.getByLabel('Model',{exact:true}).selectOption('fast'); + await page.getByRole('button',{name:'Save workflow',exact:true}).click();await page.getByText('Saved revision 2.',{exact:true}).waitFor(); + const prepare=page.getByRole('button',{name:'Prepare a new run',exact:true});if(await prepare.count())await prepare.click(); + await page.getByLabel('Run identifier',{exact:true}).fill('invalid');await page.getByLabel('Question',{exact:true}).fill('What is the name?'); + await page.getByRole('button',{name:'Start run',exact:true}).click(); + await view.getByRole('status').filter({hasText:'failed'}).waitFor(); + assert.equal(calls().length,2);assert.equal(await page.getByText('Invalid plain-text result',{exact:true}).count(),0); + await field.fill('{'); + await page.getByText('Save your workflow changes before starting a run.',{exact:true}).waitFor(); + let discardQuestion='';page.once('dialog',async dialog=>{discardQuestion=dialog.message();await dialog.dismiss();}); + await page.getByRole('button',{name:'Reload saved workflows',exact:true}).click(); + assert.match(discardQuestion,/Discard your unsaved/);assert.equal(await field.inputValue(),'{'); + assert(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth));assert.deepEqual(errors,[]); +}); diff --git a/src/agents/AgentsPage.tsx b/src/agents/AgentsPage.tsx index 710d896..6bd419f 100644 --- a/src/agents/AgentsPage.tsx +++ b/src/agents/AgentsPage.tsx @@ -4,17 +4,20 @@ import {parseCapabilities} from '../capabilities'; import {WorkspaceState} from '../components/WorkspaceState'; import {isHandoffPlan,isInputTask,isInteractivePlan,parseCatalog,parsePlanPage,parseSavedEdit,planAddress,record,replaceTask,validatePlan,type AgentChoice,type TaskNode,type AgentTask,type SavedPlan} from './plans'; import './agents.css'; +import {OutputRequirementsEditor} from './OutputRequirementsEditor'; +import {requireOutputCapabilities} from './output-requirements'; import {RunPanel} from './RunPanel'; import {HandoffEditor} from './HandoffEditor'; import {parseRunPolicy} from './runs'; const secureRequest={cache:'no-store',redirect:'error',credentials:'omit',referrerPolicy:'no-referrer'} as const; -type Ready={usageAvailable:boolean;inputsAvailable:boolean;handoffsAvailable:boolean;runsAvailable:boolean;maxParallel:number;space:string;catalog:AgentChoice[];items:SavedPlan[];next_after:string|null}; -function Editor({api,space,catalog,initial,onSave,onDirty,inputsAvailable}:{api:ApiClient;space:string;catalog:AgentChoice[];initial:SavedPlan|null;onSave:(plan:SavedPlan)=>void;onDirty:()=>void;inputsAvailable:boolean}){ +type Ready={requirementsAvailable:boolean;schemaAvailable:boolean;usageAvailable:boolean;inputsAvailable:boolean;handoffsAvailable:boolean;runsAvailable:boolean;maxParallel:number;space:string;catalog:AgentChoice[];items:SavedPlan[];next_after:string|null}; +function Editor({api,space,catalog,initial,onSave,onDirty,inputsAvailable,requirementsAvailable,schemaAvailable}:{api:ApiClient;space:string;catalog:AgentChoice[];initial:SavedPlan|null;onSave:(plan:SavedPlan)=>void;onDirty:()=>void;inputsAvailable:boolean;requirementsAvailable:boolean;schemaAvailable:boolean}){ const first=catalog[0]; const newTask=(id:string):AgentTask=>({task_id:id,agent_id:first?.agent_id??'',model_id:first?.default_model??'',prompt:'',depends_on:[]}); const [plan,setPlan]=useState<{workflow_id:string;tasks:TaskNode[]}>((initial&&!isHandoffPlan(initial.plan)?initial.plan:null)??{workflow_id:'',tasks:[newTask('task-1')]}),[revision,setRevision]=useState(initial?.revision??0),[issue,setIssue]=useState(''),[busy,setBusy]=useState(false),[notice,setNotice]=useState(''),[reviewRequired,setReviewRequired]=useState(initial?.configuration_current===false); + const [taskKeys,setTaskKeys]=useState(()=>plan.tasks.map(()=>crypto.randomUUID())); const active=useRef(null); useEffect(()=>()=>active.current?.abort(),[]); const update=(index:number,change:TaskNode)=>{ @@ -26,7 +29,7 @@ function Editor({api,space,catalog,initial,onSave,onDirty,inputsAvailable}:{api: setIssue('');setNotice(''); const controller=new AbortController();active.current=controller; try{ - const {workflow_id,tasks}=plan;const valid=validatePlan({...(tasks.some(isInputTask)?{kind:'interactive'}:{}),workflow_id,tasks},catalog);setBusy(true); + const {workflow_id,tasks}=plan;const valid=validatePlan({...(tasks.some(isInputTask)?{kind:'interactive'}:{}),workflow_id,tasks},catalog);requireOutputCapabilities(valid,requirementsAvailable,schemaAvailable);setBusy(true); const result=await api.request(planAddress(valid.workflow_id),{...secureRequest,method:'PUT',body:JSON.stringify({expected_revision:revision,plan:valid}),cache:'no-store',signal:AbortSignal.any([controller.signal,AbortSignal.timeout(15000)])}); const saved=parseSavedEdit(result,space,valid,revision); if(isHandoffPlan(saved.plan))throw Error('The server returned a different workflow type.'); @@ -41,8 +44,8 @@ function Editor({api,space,catalog,initial,onSave,onDirty,inputsAvailable}:{api: {plan.tasks.map((task,index)=>{ const agent=isInputTask(task)?undefined:catalog.find(choice=>choice.agent_id===task.agent_id); - return
-

Task {index+1}

+ return
+

Task {index+1}

{inputsAvailable&&}
{!isInputTask(task)&&<>}{isInputTask(task)&&}