From 9a16e837933cd61a3deaf05314d8f7f36a70a2e5 Mon Sep 17 00:00:00 2001 From: Bob Killen Date: Sun, 11 Oct 2026 09:45:51 -0500 Subject: [PATCH] fix(validate-members): hold sources.landscape.collectedAt to the snapshot validate-members.mjs cross-checked the landscape provenance block in members.json against enduser-landscape.json on revision and sourceUrl only. collectedAt, the third field the generator writes from the same snapshot in the same statement, was held to nothing, and it is the one of the three the site renders: DirectoryFreshness formats it into the "last synced" note. A members.json whose collectedAt was stale or unparseable relative to the snapshot passed validation while the page printed a date the data did not support, or silently dropped the sentence. Fold collectedAt into the existing comparison. It sits on the snapshot root, not on snapshot.source, which is where generate-members.mjs copies it from. tests/validate-members-membership.test.mjs gains the case (a different valid timestamp, an unparseable one, null and absent all fail; the matching value passes) and its fixture now carries collectedAt like the generator's output. tests/e2e-data-fixture-validity.test.mjs records that the data-variant members.json overlay, which sets collectedAt to "not a timestamp" on purpose, now gains exactly this error, replacing the row's note that it should. Closes #1270 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Bob Killen --- scripts/validate-members.mjs | 6 ++++- tests/e2e-data-fixture-validity.test.mjs | 11 +++----- tests/validate-members-membership.test.mjs | 31 ++++++++++++++++++++-- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/scripts/validate-members.mjs b/scripts/validate-members.mjs index 4600d92b..3077b18d 100644 --- a/scripts/validate-members.mjs +++ b/scripts/validate-members.mjs @@ -417,9 +417,13 @@ if (landscape) { message: `membership source ID set mismatch (missing: ${missing.join(', ') || 'none'}; unexpected: ${unexpected.join(', ') || 'none'})`, }); } + // collectedAt is the one field of this block the site renders + // (DirectoryFreshness), and the generator copies it from the snapshot root + // rather than from snapshot.source like the other two. if ( data.sources?.landscape?.revision !== landscape.source?.revision || - data.sources?.landscape?.sourceUrl !== landscape.source?.sourceUrl + data.sources?.landscape?.sourceUrl !== landscape.source?.sourceUrl || + data.sources?.landscape?.collectedAt !== landscape.collectedAt ) { errors.push({ path: 'members.json.sources.landscape', diff --git a/tests/e2e-data-fixture-validity.test.mjs b/tests/e2e-data-fixture-validity.test.mjs index f2eadbf1..e348f182 100644 --- a/tests/e2e-data-fixture-validity.test.mjs +++ b/tests/e2e-data-fixture-validity.test.mjs @@ -181,13 +181,10 @@ const EXPECTED = new Map([ [ 'data-variant/members.json', { - // Recorded as an empty delta because that is what the gate does today, - // not because the shape is benign: DirectoryFreshness renders - // sources.landscape.collectedAt and validate-members.mjs never reads it, - // unlike every sibling validator's document-level timestamp. If that gap - // is closed this entry gains the new error, which is the signal. - why: 'makes sources.landscape.collectedAt unparseable to degrade the DirectoryFreshness note; validate-members.mjs does not check that field', - adds: [], + why: 'makes sources.landscape.collectedAt unparseable to degrade the DirectoryFreshness note; validate-members.mjs holds that field to the snapshot like revision and sourceUrl', + adds: [ + 'members.json.sources.landscape: landscape provenance must match enduser-landscape.json', + ], }, ], [ diff --git a/tests/validate-members-membership.test.mjs b/tests/validate-members-membership.test.mjs index 1ba1eaff..ca43a222 100644 --- a/tests/validate-members-membership.test.mjs +++ b/tests/validate-members-membership.test.mjs @@ -37,10 +37,12 @@ function member(id, membershipStatus, membershipSources = []) { }; } +const COLLECTED_AT = '2026-10-01T00:00:00.000Z'; + function snapshot(sourceIds) { return { generated: true, - collectedAt: '2026-10-01T00:00:00.000Z', + collectedAt: COLLECTED_AT, source: { repository: 'https://github.com/cncf/landscape', revision: REVISION, @@ -54,7 +56,7 @@ function snapshot(sourceIds) { }; } -function run(members, landscape) { +function run(members, landscape, provenance = {}) { return runScriptWithFixtures(SCRIPT, { 'data/members.json': JSON.stringify({ description: 'directory', @@ -64,6 +66,8 @@ function run(members, landscape) { repository: 'https://github.com/cncf/landscape', revision: REVISION, sourceUrl: SOURCE_URL, + collectedAt: COLLECTED_AT, + ...provenance, }, }, schema: {}, @@ -172,6 +176,29 @@ test('rejects duplicate, missing, unexpected, and stale landscape provenance', ( assert.match(result.stderr, /landscape provenance must match/); }); +// The generator writes revision, sourceUrl and collectedAt into +// sources.landscape from one snapshot in one statement, and collectedAt is +// the only one of the three DirectoryFreshness renders. A members.json that +// kept the snapshot's revision but a different sync time (a partial +// regeneration, a merge that took one side) states a date the data does not +// support, so it is held to the snapshot like the other two. +test('rejects a collectedAt that differs from the snapshot it claims', () => { + const memberSource = source('member-source', 'member'); + const members = [member('one', 'member', [memberSource])]; + const landscape = snapshot(['member-source']); + for (const collectedAt of [ + '1999-01-01T00:00:00.000Z', + 'not a timestamp', + null, + undefined, + ]) { + const result = run(members, landscape, { collectedAt }); + assert.equal(result.status, 1, `collectedAt ${String(collectedAt)}`); + assert.match(result.stderr, /landscape provenance must match/); + } + assert.equal(run(members, landscape).status, 0); +}); + test('requires membership fields for modern landscape-backed output', () => { const legacyShape = member('legacy-shape', 'unknown'); delete legacyShape.membershipStatus;