Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# The comparables gate (docs/engineering/skills/repository-guidance.md,
# "Research entries need a comparables record"): a change to what it checks,
# to the lists of entries and pages it exempts, or to the route map other
# repositories read, asks for Max's review.
/website/src/data/comparables.ts @MaxGhenis
/website/src/data/comparablesGrandfathered.json @MaxGhenis
/website/src/data/analysisRoutes.ts @MaxGhenis
/website/src/data/analysisRoutes.json @MaxGhenis
/website/scripts/build-analysis-routes.ts @MaxGhenis
/website/src/__tests__/content/comparables*.test.ts @MaxGhenis
/website/src/__tests__/content/posts-and-apps-data.test.ts @MaxGhenis
/website/src/__tests__/content/routing.ts @MaxGhenis
/website/src/__tests__/content/fixtures/comparables_fresh_cases.json @MaxGhenis
/.github/workflows/comparables-gate.yml @MaxGhenis
/.github/CODEOWNERS @MaxGhenis
122 changes: 122 additions & 0 deletions .github/workflows/comparables-gate.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
name: Comparables gate

# Every research entry not on the list of entries live before the gate
# (website/src/data/comparablesGrandfathered.json) must name a complete record
# of its comparables search in PolicyEngine/policyengine-scorecard. The Tests
# job checks each block's shape without the network; this job downloads the
# scorecard's published index from its main branch and checks that every
# block resolves to a search the index lists, and that the grandfather list
# has not grown after the cutover. The scorecard is public, so the download
# needs no token. A failed download fails the job: the gate fails closed.
#
# It runs on every pull request (no path filter, so it can be a required
# check), on every push to main, and weekly, so a record removed from the
# scorecard shows up without waiting for the next change. A failed weekly
# run opens an issue.

on:
pull_request:
push:
branches: [main]
schedule:
- cron: "17 9 * * 1"
workflow_dispatch:

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

env:
# Pinned here, not read from the change under test, so a change cannot
# point the gate at another index. A test checks comparables.ts agrees.
SCORECARD_INDEX_URL: https://raw.githubusercontent.com/PolicyEngine/policyengine-scorecard/main/data/comparables/index.json

jobs:
resolve:
name: Comparables resolve against the scorecard
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Set up Bun
uses: oven-sh/setup-bun@v2

# Same Node as the Tests job (pr.yaml): vitest runs on Node.
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: "26"

- name: Install dependencies
run: bun install --frozen-lockfile

- name: Download the scorecard's comparables index
run: |
curl --fail --silent --show-error --location --retry 5 --retry-all-errors \
--output "$RUNNER_TEMP/comparables-index.json" "$SCORECARD_INDEX_URL"
node -e "const i=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));console.log(Object.keys(i.analyses).length+' records in the scorecard index ('+i.schema+')')" "$RUNNER_TEMP/comparables-index.json"
echo "COMPARABLES_INDEX_PATH=$RUNNER_TEMP/comparables-index.json" >> "$GITHUB_ENV"

# The registries, grandfather list and featured banner on the branch this
# change merges into: the base of a pull request, or main before a push.
# None on a scheduled or manual run. A base the checkout cannot read
# fails the job, rather than passing as if there were none.
- name: The registries before this change
env:
BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
run: |
dir="$RUNNER_TEMP/comparables-base"
mkdir -p "$dir"
if [ -n "$BASE_SHA" ] && [ "$BASE_SHA" != "0000000000000000000000000000000000000000" ]; then
if ! git cat-file -e "$BASE_SHA^{commit}"; then
echo "::error::cannot read the base commit $BASE_SHA"
exit 1
fi
for path in \
app/src/data/apps/apps.json \
app/src/data/posts/posts.json \
website/src/data/comparablesGrandfathered.json \
website/src/components/home/FeaturedResearchBanner.tsx; do
# a file the base does not have yet is absent, not an error
if git cat-file -e "$BASE_SHA:$path" 2>/dev/null; then
git show "$BASE_SHA:$path" > "$dir/$(basename "$path")"
fi
done
fi
ls -la "$dir"
echo "COMPARABLES_BASE_DIR=$dir" >> "$GITHUB_ENV"

- name: Every research entry passes the gate
working-directory: ./website
env:
COMPARABLES_GATE: "1"
run: bun run test src/__tests__/content/comparables.live.test.ts

report:
name: Report a failed weekly run
needs: resolve
if: failure() && github.event_name == 'schedule'
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Open or update the gate's issue
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
title="Comparables gate failing on main"
number=$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number')
body="The weekly comparables gate failed: $RUN_URL. A research entry's comparables block no longer resolves against the scorecard index, or the index changed shape. See website/src/data/comparables.ts."
if [ -n "$number" ]; then
gh issue comment "$number" --body "$body"
else
gh issue create --title "$title" --body "$body"
fi
7 changes: 7 additions & 0 deletions app/src/data/posts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ This folder contains the blog posts for PolicyEngine.
- `authors.json` - Author information
- `images/` - Legacy image folder (do not use for new posts)

## Comparables

A new post needs a `comparables` block in its `posts.json` entry, a record of its
comparables search (or, for an event or announcement, a `no_estimate` declaration): see
"Research entries need a comparables record" in
`docs/engineering/skills/repository-guidance.md`.

## Adding Images to Blog Posts

**Important**: Images should be stored in `app/public/assets/posts/{post-name}/` and referenced with absolute paths.
Expand Down
13 changes: 13 additions & 0 deletions docs/engineering/skills/repository-guidance.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,19 @@ The `app/` directory holds the calculator code. `calculator-app/` (Next.js) comp
- `app/public/assets/` holds static assets that the website build copies in. `calculator-app/public` is a symlink to `app/public`, so everything in `app/public` is served on app.policyengine.org.
- Website pages and components go in `website/src/`. New website rewrites go in `website/next.config.ts`, not `website/vercel.json`.

## Research entries need a comparables record

A research entry is an `apps.json` app with `displayWithResearch` or the `policy` tag, or any `posts.json` post. Every research entry needs a `comparables` block naming the record of its comparables search in PolicyEngine/policyengine-scorecard (`docs/COMPARABLES.md` there; the `policyengine-comparables-review` skill runs the search), unless it is on `website/src/data/comparablesGrandfathered.json`, the list of entries live before the gate. The date an entry carries does not exempt it.

- Merge the scorecard PR first, then paste the block `python -m scorecard_db.comparables record analysis-<country>-<slug>` prints (its latest search). The search is dated on or before the entry's date, no more than `MAX_SEARCH_AGE_DAYS` before it, and is not a backfill.
- On the day a change lists or re-lists an entry (a new entry, or a new date or block), the `Comparables gate` workflow also requires that the cited search is no more than `MAX_SEARCH_AGE_DAYS` old that day, whatever date the entry carries; that it compared the results files the analysis reads now; and that no gap it found is still pending.
- A post that is plainly not an analysis (tagged `org`, `event`, `giving`, `conferences`, `year-in-review` or `april-fools`, and none of the analysis tags in `comparables.ts`) may declare `{"status": "no_estimate", "reason": "..."}` instead. An app never may.
- Taking an entry off the site takes it off the grandfather list too, so it comes back only with a record. Entries may join the list only before `COMPARABLES_CUTOVER_DATE`; after it the list only shrinks.
- Every page the site proxies to another deployment (a rewrite in `next.config.ts`, most from `website/src/data/appZoneRoutes.ts`, in any form next.config accepts) is a research app's page, a page live before the gate, or on the grandfather file's `infrastructure` list of pages that publish no analysis. Add a dashboard through `appZoneRoutes.ts` with its `apps.json` entry.
- A card a change adds to the featured research banner must link a research entry whose comparables search is fresh that day under the scorecard's promotion window, grandfathered entries included.
- `website/src/data/analysisRoutes.json` maps every path and deployment host that leads to a research app, for PolicyEngine/newsletters' check; after a routing or registry change run `bun run scripts/build-analysis-routes.ts` in `website/` (a test fails while it is stale).
- The Tests job checks each block's shape and these lists. The `Comparables gate` workflow runs on every pull request, checks that each block resolves against the scorecard's index on main, the listing and banner rules above, and that the list has not grown after the cutover, and opens an issue when its weekly run fails.

## Visual standards

Detailed visual standards are documented in `docs/engineering/skills/`. Read the relevant document before changing interface code:
Expand Down
40 changes: 40 additions & 0 deletions website/scripts/build-analysis-routes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/**
* Writes src/data/analysisRoutes.json, the map of every path and deployment
* host that leads to a research app (src/data/analysisRoutes.ts), which
* PolicyEngine/newsletters reads. posts-and-apps-data.test.ts fails while the
* committed file differs from what this builds.
*
* bun run scripts/build-analysis-routes.ts
*/
import { writeFileSync } from "node:fs";
import appsData from "../src/data/apps.json";
import postsData from "../src/data/posts/posts.json";
import {
buildAnalysisRoutes,
unlistedPagePaths,
} from "../src/data/analysisRoutes";
import { entryKey, researchEntries } from "../src/data/comparables";
import { loadRouting } from "../src/__tests__/content/routing";

const { rewrites, redirects } = await loadRouting();
const entries = researchEntries(
appsData as Parameters<typeof researchEntries>[0],
postsData as Parameters<typeof researchEntries>[1],
);
const appKeys = new Set(entries.filter((e) => e.kind === "app").map(entryKey));
const routes = buildAnalysisRoutes(
rewrites,
redirects,
appsData as never,
appKeys,
);
writeFileSync(
new URL("../src/data/analysisRoutes.json", import.meta.url),
`${JSON.stringify(routes, null, 2)}\n`,
);
const unlisted = unlistedPagePaths(rewrites, appKeys);
console.log(`${routes.paths.length} paths, ${routes.hosts.length} hosts`);
console.log(
"proxied pages that are no research app's:",
JSON.stringify(unlisted),
);
105 changes: 105 additions & 0 deletions website/src/__tests__/content/comparables.live.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// @vitest-environment node
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import appsData from "@/data/apps.json";
import {
bannerHrefs,
changedEntries,
checkIndex,
type ComparablesIndex,
GRANDFATHERED,
type Grandfathered,
grandfatherAdditions,
promotionProblems,
publicationProblems,
requiresComparables,
researchEntries,
resolveComparables,
validateComparables,
} from "@/data/comparables";
import postsData from "@/data/posts/posts.json";

// Every research entry against the scorecard's published index. The
// Comparables gate workflow downloads the index from
// PolicyEngine/policyengine-scorecard's main branch, sets
// COMPARABLES_INDEX_PATH, and sets COMPARABLES_GATE=1 so that a missing
// index fails rather than skips. COMPARABLES_BASE_DIR holds the registries,
// grandfather list and featured banner on the branch the change merges into
// (absent when there is none, as on a scheduled run). Without
// COMPARABLES_GATE (the Tests job, local runs) this file skips, and
// posts-and-apps-data.test.ts still checks every block's shape.
const gate = process.env.COMPARABLES_GATE === "1";
const indexPath = process.env.COMPARABLES_INDEX_PATH;
const baseDir = process.env.COMPARABLES_BASE_DIR;
const BANNER = new URL(
"../../components/home/FeaturedResearchBanner.tsx",
import.meta.url,
);

if (gate && !indexPath) {
throw new Error(
"COMPARABLES_GATE is set but COMPARABLES_INDEX_PATH is not: the gate has no index to check against",
);
}

function fromBase<T>(name: string): T | null {
if (!baseDir) return null;
const path = join(baseDir, name);
if (!existsSync(path) || !readFileSync(path, "utf8").trim()) return null;
const text = readFileSync(path, "utf8");
return (name.endsWith(".json") ? JSON.parse(text) : text) as T;
}

describe.skipIf(!gate)("comparables against the scorecard index", () => {
// read inside the tests: a skipped describe still runs its body
const loadIndex = () =>
JSON.parse(readFileSync(indexPath as string, "utf8")) as ComparablesIndex;
const entries = researchEntries(
appsData as Parameters<typeof researchEntries>[0],
postsData as Parameters<typeof researchEntries>[1],
);
const today = new Date().toISOString().slice(0, 10);

test("the index is the scorecard's, and applies the same entry rules", () => {
expect(checkIndex(loadIndex())).toEqual([]);
});

test("every research entry has a well-formed block where it needs one, and every block resolves to a complete scorecard record", () => {
const index = loadIndex();
const errors = entries.flatMap((entry) => [
...validateComparables(entry),
...resolveComparables(entry, index),
]);
expect(errors).toEqual([]);
});

test("an entry this change lists cites a search fresh today, against the current results, with no gap pending", () => {
const apps = fromBase<Parameters<typeof researchEntries>[0]>("apps.json");
const posts = fromBase<Parameters<typeof researchEntries>[1]>("posts.json");
if (!apps || !posts) return; // no base: a scheduled run, or the first list
const index = loadIndex();
const changed = changedEntries(
researchEntries(apps, posts),
entries,
).filter((entry) => requiresComparables(entry));
expect(
changed.flatMap((e) => publicationProblems(e, index, today)),
).toEqual([]);
});

test("a card this change features links an analysis whose comparables are fresh today", () => {
const base = fromBase<string>("FeaturedResearchBanner.tsx");
if (base === null) return;
const before = new Set(bannerHrefs(base));
const added = bannerHrefs(readFileSync(BANNER, "utf8")).filter(
(href) => !before.has(href),
);
expect(promotionProblems(added, entries, loadIndex(), today)).toEqual([]);
});

test("the grandfather list grows only before the cutover", () => {
const base = fromBase<Grandfathered>("comparablesGrandfathered.json");
expect(grandfatherAdditions(base, GRANDFATHERED, today)).toEqual([]);
});
});
Loading
Loading