Skip to content

fix: Separate active inheritance from upload witness #1090

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Pigbibi merged 1 commit into from
Nov 8, 2026
Merged

fix: Separate active inheritance from upload witness #1090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 28 additions & 6 deletions .github/workflows/deploy-strategy-switch-console.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1654,7 +1654,8 @@ jobs:
const statusCandidate = process.env.RELEASE_SOURCE_SHA === '68e3d4b69e706e2100f860c473ec657e9baf05ba';
const cpuCandidate = process.env.RELEASE_SOURCE_SHA === '974354b98ebe0cd55931baa03034d887d4fac560';
const integrated = controlRuntimeCandidate || controlAdminCandidate || historyCandidate || declarationCandidate || returnsCandidate || settingsCandidate || dailyCandidate || statusCandidate || cpuCandidate || process.env.RELEASE_SOURCE_SHA === '0dd2c0b3e73e890bc2c39ab0bae0eeb48c58470a';
const reviewedMain = process.env.RELEASE_SOURCE_SHA === '084ba4281cfff3304cf226ce511a2fc0a6b67b08'
const currentReviewedMain = process.env.RELEASE_SOURCE_SHA === '084ba4281cfff3304cf226ce511a2fc0a6b67b08';
const reviewedMain = currentReviewedMain
|| process.env.RELEASE_SOURCE_SHA === 'c1b737d6e721eb9fe71a12c613fbb49d3fa51563';
const betaRelease = ibkrDaily || integrated || reviewedMain;
const flowMode = process.env.EXTERNAL_FLOW_ENABLE === 'true';
Expand Down Expand Up @@ -1873,13 +1874,22 @@ jobs:
form.set('worker.js', new File([bundle], 'worker.js', { type: 'application/javascript+module' }));
const betaBody = betaRelease ? JSON.stringify({ ...metadata, modules: [{name:'worker.js', content_type:'application/javascript+module', content_base64:bundle.toString('base64')}] }) : null;
let previousLatest = prior;
const separateWitness = currentReviewedMain && resume === 'off' && !flowMode;
if (pr764 || betaRelease) {
// Pin the observed latest version as a concurrency boundary. The
// legacy path inherits latest; IBKR inherits the active version.
// The legacy path inherits latest. IBKR and other beta releases
// inherit the active version and require that version to be the
// newest upload. Current reviewed main keeps the active version
// as the only configuration donor and records the newest upload
// separately as a concurrency witness.
stage = 'inherit_baseline';
const items = (await request('/versions?page=1&per_page=10'))?.items;
previousLatest = resume !== 'off' ? '2b097d2e-ffb5-4f08-af3b-9cc1d4b9e9ee' : ibkrDaily ? '26996cc3-732a-4774-af02-b46842913d36' : prior;
if (!Array.isArray(items) || !items.length || items.length > 10 || items[0]?.id !== previousLatest) fail('inherit_baseline_mismatch');
if (separateWitness) {
if (!Array.isArray(items) || !items.length || items.length > 10 || !isUuid(items[0]?.id)) fail('inherit_baseline_mismatch');
previousLatest = items[0].id;
} else {
previousLatest = resume !== 'off' ? '2b097d2e-ffb5-4f08-af3b-9cc1d4b9e9ee' : ibkrDaily ? '26996cc3-732a-4774-af02-b46842913d36' : prior;
if (!Array.isArray(items) || !items.length || items.length > 10 || items[0]?.id !== previousLatest) fail('inherit_baseline_mismatch');
}
}
stage = 'pre_upload_active'; await active(priorDeployment, prior);
if (resume === 'off') {
Expand All @@ -1888,6 +1898,9 @@ jobs:
} else {
uploaded = '2b097d2e-ffb5-4f08-af3b-9cc1d4b9e9ee';
}
// A stable unactivated upload may be the witness. Reject it before
// any candidate read so it cannot be selected for activation.
if (separateWitness && previousLatest !== prior && uploaded === previousLatest) fail('uploaded_matches_witness');
if (resume !== 'off' || integrated || reviewedMain) {
stage = 'candidate_module';
const candidate = await request(`/versions/${uploaded}?include=modules`, 'GET', undefined, true);
Expand All @@ -1902,7 +1915,7 @@ jobs:
if (typeof module.content_base64 !== 'string') fail('candidate_module_content_missing');
if (module.content_base64 !== bundle.toString('base64')) fail('candidate_module_mismatch');
}
if (uploaded === prior) fail();
if (uploaded === prior || (separateWitness && uploaded === previousLatest)) fail();
stage = 'uploaded_configuration'; const next = resources(await request(`/versions/${uploaded}`), uploaded);
const comparable = value => ({ bindings: [...value.bindings].sort((a, b) => a.name.localeCompare(b.name)),
script_runtime: value.script_runtime, handlers: value.script.handlers,
Expand Down Expand Up @@ -1934,6 +1947,15 @@ jobs:
// precondition here. No automatic retry or force fallback on an unknown result.
stage = 'pre_deploy_active'; await active(priorDeployment, prior);
stage = 'pre_deploy_latest'; await latest(uploaded);
if (separateWitness) {
// latest(uploaded) reads one version id. Re-read the bounded list
// so the witness observed before upload is still the immediate
// predecessor immediately before deployment.
stage = 'pre_deploy_lineage';
const items = (await request('/versions?page=1&per_page=10'))?.items;
if (!Array.isArray(items) || items.length < 2 || items.length > 10
|| items[0]?.id !== uploaded || items[1]?.id !== previousLatest) fail('inherit_lineage_mismatch');
}
if (resume !== 'off') {
const items = (await request('/versions?page=1&per_page=10'))?.items;
if (!Array.isArray(items) || items.length < 2 || items.length > 10 || items[0]?.id !== uploaded
Expand Down
17 changes: 14 additions & 3 deletions docs/console-execution-evidence.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,20 @@ checking out the source. The build must reproduce that map and leave no tracked
or untracked changes; the legacy frontend overlay is not applied. This pin does
not enable strategy-profile synchronization or Meridian activation.

Publication uses version-pinned inheritance from the active Worker version for
every binding, including secrets. Latest-version lineage checks reject concurrent
uploads; module and configuration readbacks must match before activation.
Publication for the current pin inherits every binding, including secrets,
from the sole version receiving 100% traffic. That active version is the
configuration donor. The newest uploaded version is a separate concurrency
witness and does not have to be the active version. Missing, invalid or
oversized witness evidence stops before upload. The witness is never the
inheritance donor and is never selected for activation. After upload, and again
immediately before deployment, the bounded newest-first list must be the new
version followed by that same witness. A competing upload, or any change to the
sole active deployment, stops activation. Module equality, migration-tag and
configuration readbacks stay in place. The historical reviewed main still
requires its newest upload to be the active version. This entry has no
compare-and-swap. A race after the final read remains possible, and an unknown
write result is not retried. An older unactivated upload is not activated to
make the witness check pass.

This mode requires `release_frontend_assets=true`, `sync_secrets=false`,
`sync_strategy_profiles=false`, `skip_binance_receiver_checks=false`,
Expand Down
42 changes: 35 additions & 7 deletions tests/console_deploy_safety_validation.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -755,13 +755,18 @@ async function releaseCase({pr764=false,ibkrDaily=false,activeOverride=null,inhe
releaseCases++;return {calls,stdout,stderr};
}
// Exact integrated candidate: explicit inheritance, verified module, one non-forced activation.
async function integratedReleaseCase({ reviewedMain=false, reviewedMainSource="c1b737d6e721eb9fe71a12c613fbb49d3fa51563", lineage=null, settings=false, daily=false, statusCandidate=false, cpu=false, active=null, latest=null, moduleContent=syntheticBundle.toString("base64"), uploadedTag=undefined, postTag=runtime.migration_tag, flow=false, oldFlag=null, drift=false } = {}, expected=true) {
async function integratedReleaseCase({ reviewedMain=false, reviewedMainSource="c1b737d6e721eb9fe71a12c613fbb49d3fa51563", lineage=null, settings=false, daily=false, statusCandidate=false, cpu=false, active=null, latest=null, moduleContent=syntheticBundle.toString("base64"), uploadedTag=undefined, postTag=runtime.migration_tag, flow=false, oldFlag=null, drift=false, baselineItems=undefined, preUploadVersion=null, preDeployWitness=null, uploadId=null, ambiguousActive=false, expectFailure=null } = {}, expected=true) {
const calls=[], stdout=[], stderr=[], prior=settings?"7ffe9227-686a-48ad-baa9-72665969d918":daily?"14f0c9c2-09f9-40e9-bfbe-17165f499495":statusCandidate?"175ba768-484a-4fe6-9cea-e5a101bc1618":cpu?"90a0c8e9-b5ad-447a-998f-5d585ec28063":"600ec84d-a804-4ede-8d1d-0a34819ce46a";
active ??= prior; latest ??= active;
const separateWitness = reviewedMain && reviewedMainSource === inboxSource && !flow;
const responses=responseSequence(false,active);
responses.splice(3,0,wrap({items:[{id:latest}]}));
if (ambiguousActive) responses[0].result.deployments[0].versions.push({version_id:"00000000-0000-4000-8000-000000000024",percentage:0});
responses.splice(3,0,wrap({items:baselineItems===undefined?[{id:latest}]:baselineItems}));
responses.splice(6,0,wrap({id:newVersion,modules:[{name:"worker.js",content_type:"application/javascript+module",content_base64:moduleContent}]}));
responses.splice(8,0,wrap({items:[{id:newVersion},{id:lineage ?? latest}]}));
responses.splice(8,0,wrap({items:[{id:uploadId ?? newVersion},{id:lineage ?? latest}]}));
if (uploadId) responses[5].result.id = uploadId;
if (preUploadVersion) responses[4].result.deployments[0].versions[0].version_id = preUploadVersion;
if (separateWitness) responses.splice(11,0,wrap({items:[{id:uploadId ?? newVersion},{id:preDeployWitness ?? lineage ?? latest}]}));
if (flow) {
if (oldFlag) responses[1].result.resources.bindings.push(oldFlag);
for (const i of [7, 10]) responses[i].result.resources.bindings.push({name:"CONSOLE_EXTERNAL_FLOW_ENABLED",type:"plain_text",text:"true"});
Expand All @@ -786,8 +791,18 @@ async function integratedReleaseCase({ reviewedMain=false, reviewedMainSource="c
if(expected){assert.equal(posts.length,2);assert.ok(posts[0].url.endsWith("/versions?deploy=false"));
assert.ok(!calls.some(c=>c.url.includes("bindings_inherit=strict")), "never use latest-version secret inheritance");
const upload=JSON.parse(posts[0].options.body);assert.deepEqual(upload.bindings,[...remoteBindings.map(b=>({type:"inherit",name:b.name,version_id:prior})),...(flow?[{name:"CONSOLE_EXTERNAL_FLOW_ENABLED",type:"plain_text",text:"true"}]:[])]);
assert.ok(posts[1].url.endsWith("/deployments?force=false"));if (flow) assert.equal(JSON.parse(stdout.at(-1)).external_flow_enabled,true); else assert.equal(JSON.parse(stdout.at(-1)).version_id,newVersion);
} else {if (postTag !== null) assert.equal(posts.filter(x=>x.url.includes("/deployments?")).length,0); else { assert.equal(posts.filter(x=>x.url.includes("/deployments?")).length,1); assert.equal(JSON.parse(stderr.at(-1)).outcome,"UNKNOWN_REQUIRES_READBACK"); } assert.equal(JSON.parse(stderr.at(-1)).retry,false);}
assert.ok(posts[1].url.endsWith("/deployments?force=false"));
const activated=JSON.parse(posts[1].options.body).versions[0].version_id;
assert.equal(activated,newVersion);
if (separateWitness) {
assert.ok(upload.bindings.every(binding=>binding.version_id===prior));
if (latest!==prior) assert.ok(upload.bindings.every(binding=>binding.version_id!==latest));
assert.notEqual(activated,latest);
assert.equal(JSON.parse(stdout.at(-1)).prior_version,prior);
}
if (flow) assert.equal(JSON.parse(stdout.at(-1)).external_flow_enabled,true); else assert.equal(JSON.parse(stdout.at(-1)).version_id,newVersion);
} else {if (postTag !== null) assert.equal(posts.filter(x=>x.url.includes("/deployments?")).length,0); else { assert.equal(posts.filter(x=>x.url.includes("/deployments?")).length,1); assert.equal(JSON.parse(stderr.at(-1)).outcome,"UNKNOWN_REQUIRES_READBACK"); } assert.equal(JSON.parse(stderr.at(-1)).retry,false);
if (expectFailure) { const error=JSON.parse(stderr.at(-1)); for (const [key,value] of Object.entries(expectFailure)) assert.equal(error[key],value,key); }}
assert.ok(!JSON.stringify({stdout,stderr}).includes(privateRelease));
}
// Execute the actual publisher for the exact merged source, with fake provider responses.
Expand All @@ -797,9 +812,22 @@ await integratedReleaseCase({reviewedMain:true,lineage:priorVersion},false);
await integratedReleaseCase({reviewedMain:true,moduleContent:Buffer.from("wrong").toString("base64")},false);
await integratedReleaseCase({reviewedMain:true,uploadedTag:"wrong-tag"},false);
await integratedReleaseCase({reviewedMain:true,postTag:null},false);
const orphanWitness="00000000-0000-4000-8000-000000000021";
const driftedWitness="00000000-0000-4000-8000-000000000022";
const driftedActive="00000000-0000-4000-8000-000000000023";
const stopped={retry:false,deploy_attempted:false};
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource});
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:priorVersion},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,lineage:priorVersion},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness});
const baselineStop={...stopped,stage:"inherit_baseline",checkpoint:"inherit_baseline_mismatch",upload_attempted:false,outcome:"NOT_ATTEMPTED"};
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,baselineItems:[],expectFailure:baselineStop},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,baselineItems:null,expectFailure:baselineStop},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,baselineItems:[{id:"not-a-uuid"}],expectFailure:baselineStop},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,baselineItems:Array.from({length:11},(_,i)=>({id:`00000000-0000-4000-8000-${String(i).padStart(12,"0")}`})),expectFailure:baselineStop},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,ambiguousActive:true,expectFailure:{...stopped,stage:"pre_release_active",upload_attempted:false,outcome:"NOT_ATTEMPTED"}},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,preUploadVersion:driftedActive,expectFailure:{...stopped,stage:"pre_upload_active",upload_attempted:false,outcome:"NOT_ATTEMPTED"}},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,lineage:driftedWitness,expectFailure:{...stopped,stage:"inherit_lineage",checkpoint:"inherit_lineage_mismatch",upload_attempted:true,outcome:"UPLOAD_REQUIRES_READBACK_NOT_ACTIVATED"}},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,preDeployWitness:driftedWitness,expectFailure:{...stopped,stage:"pre_deploy_lineage",checkpoint:"inherit_lineage_mismatch",upload_attempted:true,outcome:"UPLOAD_REQUIRES_READBACK_NOT_ACTIVATED"}},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,latest:orphanWitness,uploadId:orphanWitness,expectFailure:{...stopped,checkpoint:"uploaded_matches_witness",upload_attempted:true,outcome:"UPLOAD_REQUIRES_READBACK_NOT_ACTIVATED"}},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,moduleContent:Buffer.from("wrong").toString("base64")},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,uploadedTag:"wrong-tag"},false);
await integratedReleaseCase({reviewedMain:true,reviewedMainSource:inboxSource,postTag:null},false);
Expand Down
Loading