From 74505038cc06057c8d35cbdeaf2ac8a8a6aa7587 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 11:52:41 -0400 Subject: [PATCH 1/5] Refuse to send an edition whose analyses lack a fresh comparables search Max, 2026-10-21: "even if we've already published something, we should do a fresh comp review before big pushes like newsletters". Every policyengine.org analysis an edition dated on or after 2026-10-26 links must have a comparables search in PolicyEngine/policyengine-scorecard within its promotion window and no gap still pending. A new CI job and upload-newsletter both enforce it; /create-newsletter and /upload-draft say how to satisfy it. Co-Authored-By: Claude Opus 5.5 --- .claude/commands/create-newsletter.md | 15 +- .claude/commands/upload-draft.md | 6 +- .github/workflows/ci.yml | 24 +++ CLAUDE.md | 8 + src/newsletter_uploader/cli.py | 14 ++ src/newsletter_uploader/comparables.py | 258 ++++++++++++++++++++++ tests/test_comparables.py | 283 +++++++++++++++++++++++++ 7 files changed, 606 insertions(+), 2 deletions(-) create mode 100644 src/newsletter_uploader/comparables.py create mode 100644 tests/test_comparables.py diff --git a/.claude/commands/create-newsletter.md b/.claude/commands/create-newsletter.md index 264e338..25fcd61 100644 --- a/.claude/commands/create-newsletter.md +++ b/.claude/commands/create-newsletter.md @@ -31,7 +31,20 @@ Create a new newsletter HTML file from PolicyEngine research posts and optional - Add research sections - Add footer with Mailchimp merge tags -4. **Show result**: +4. **Check comparables are fresh** (required before sending, Max 2026-10-21): + - Run `python -m newsletter_uploader.comparables editions/`. Every + policyengine.org analysis the edition links needs a comparables search in + PolicyEngine/policyengine-scorecard dated within the scorecard's promotion + window (7 days by default) before the edition's date, with no gap still pending. + - For each analysis it lists, run the comparables review with search kind + `prepromotion` (`/comparables-review` and the `policyengine-comparables-review` + skill in the PolicyEngine plugin), merge the scorecard PR, and check again. + CI and `upload-newsletter` both refuse an edition that fails this. + - If the fresh search finds a comparable whose figure differs from ours by more than + the threshold, the edition says so beside our figure, or leaves the analysis out + until the gap is explained. + +5. **Show result**: - Path to generated file - Preview of content - Next steps: upload with `upload-newsletter` command diff --git a/.claude/commands/upload-draft.md b/.claude/commands/upload-draft.md index dcb4636..8182260 100644 --- a/.claude/commands/upload-draft.md +++ b/.claude/commands/upload-draft.md @@ -17,7 +17,11 @@ This is a wrapper around the `upload-newsletter` command that: 1. Validates the HTML file exists 2. Prompts for required fields if missing -3. Runs the upload command +3. Runs the upload command, which first checks that every policyengine.org analysis + the edition links has a fresh comparables search in the scorecard (editions dated on + or after the cutover; `src/newsletter_uploader/comparables.py`) and refuses to upload + otherwise. Fix it by running the comparables review (search kind `prepromotion`), + never by renaming or re-dating the file. 4. Shows the Mailchimp edit URL ## Arguments diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68738e7..0d0da0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,30 @@ jobs: chmod +x scripts/lint-newsletters.sh ./scripts/lint-newsletters.sh + comparables: + # Max, 2026-10-21: "even if we've already published something, we should + # do a fresh comp review before big pushes like newsletters". Every + # policyengine.org analysis an edition dated on or after the cutover links + # needs a comparables search in PolicyEngine/policyengine-scorecard within + # the scorecard's promotion window (src/newsletter_uploader/comparables.py). + # It downloads the scorecard index and app-v2's registries only when such + # an edition exists, and fails if it cannot read them. + name: Fresh comparables for promoted analyses + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install + run: pip install -e . + + - name: Every analysis an edition links has a fresh comparables search + run: python -m newsletter_uploader.comparables editions + test: runs-on: ubuntu-latest strategy: diff --git a/CLAUDE.md b/CLAUDE.md index d8d8331..38a45b3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,6 +46,14 @@ All newsletter HTML files follow this pattern: - Responsive table layout in footer (Mailchimp CAN-SPAM compliance) - Mailchimp merge tags in footer - External assets use full URLs (no relative paths) +4. **Fresh comparables before sending (Max, 2026-10-21).** Every policyengine.org + analysis the edition links needs a comparables search recorded in + PolicyEngine/policyengine-scorecard within the scorecard's promotion window before + the edition's date (`python -m newsletter_uploader.comparables editions/`). + For any it lists, run the comparables review with search kind `prepromotion` (the + PolicyEngine plugin's `policyengine-comparables-review` skill) first. CI and + `upload-newsletter` refuse an edition dated on or after the cutover that fails it. + The same applies to the social posts kept beside an edition. ## Email Compatibility diff --git a/src/newsletter_uploader/cli.py b/src/newsletter_uploader/cli.py index 05b3649..a3f32ad 100644 --- a/src/newsletter_uploader/cli.py +++ b/src/newsletter_uploader/cli.py @@ -5,6 +5,7 @@ import click +from . import comparables from .audience import AudienceType from .mailchimp_client import MailchimpClient from .uploader import NewsletterUploader @@ -79,6 +80,19 @@ def main(html_file, audience, subject, preview, title, api_key, list_id, campaig ) sys.exit(1) + # Fresh comparables before a push (Max, 2026-10-21): every analysis an + # edition dated on or after the cutover links needs a recent comparables + # search in the scorecard (newsletter_uploader/comparables.py). + due, problems = comparables.run_check([html_file]) + if problems: + click.echo("❌ Comparables are not fresh for this edition:", err=True) + for problem in problems: + click.echo(f" {problem}", err=True) + click.echo(comparables.HOW_TO_FIX, err=True) + sys.exit(1) + if due: + click.echo("✓ Comparables are fresh for every analysis this edition links") + # Create client and uploader if campaign_id: click.echo("📧 Updating Mailchimp draft campaign") diff --git a/src/newsletter_uploader/comparables.py b/src/newsletter_uploader/comparables.py new file mode 100644 index 0000000..c7dc9e9 --- /dev/null +++ b/src/newsletter_uploader/comparables.py @@ -0,0 +1,258 @@ +"""Fresh comparables before a newsletter goes out. + +Max, 2026-10-21: "even if we've already published something, we should do a +fresh comp review before big pushes like newsletters". Every policyengine.org +analysis an edition links must have a comparables search recorded in +PolicyEngine/policyengine-scorecard (docs/COMPARABLES.md there) dated within +the scorecard's promotion window before the edition's date, and no gap found +by then still unexplained. The 21 October 2026 UK edition re-promoted the +childcare dashboard at £0.59bn; a fresh search would have put CenTax's £0.80bn +static cost for the 30 hours alone beside it. + +What counts as an analysis is decided by policyengine-app-v2's registries: an +app listed with research, or a post, that publishes an estimate (tagged +``policy``, or carrying a reviewed comparables block) and has not declared +``no_estimate``. Links to anything else (the home page, calculators, the +research index) are not checked. + +Editions are ``editions/-.html`` (and the ``.md`` social +posts beside them). Only editions dated on or after ``PROMOTION_CUTOVER_DATE`` +are checked, so the archive never changes its verdict. The registries and the +scorecard's index are downloaded from their repositories' main branches only +when such an edition exists; a failed download fails the check. + + python -m newsletter_uploader.comparables editions +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import re +import sys +import time +import urllib.request +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Tuple + +PROMOTION_CUTOVER_DATE = "2026-10-26" + +APPS_URL = ( + "https://raw.githubusercontent.com/PolicyEngine/policyengine-app-v2/main/" + "app/src/data/apps/apps.json" +) +POSTS_URL = ( + "https://raw.githubusercontent.com/PolicyEngine/policyengine-app-v2/main/" + "app/src/data/posts/posts.json" +) +INDEX_URL = ( + "https://raw.githubusercontent.com/PolicyEngine/policyengine-scorecard/main/" + "data/comparables/index.json" +) +INDEX_SCHEMA = "comparables/index/1" + +COUNTRIES = ("uk", "us", "ca", "ng", "il") +_LINK = re.compile( + r"https?://(?:www\.)?policyengine\.org/(uk|us|ca|ng|il)/(research/)?" + r"([A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*)(?=[/?#\"'\s<>)\].,;:!]|$)" +) +_EDITION = re.compile(r"^(\d{4}-\d{2}-\d{2})-") + +Key = Tuple[str, str, str] # (country, "app" | "post", slug) + + +def edition_date(path: Path) -> Optional[str]: + match = _EDITION.match(path.name) + if not match: + return None + try: + dt.date.fromisoformat(match.group(1)) + except ValueError: + return None + return match.group(1) + + +def post_slug(filename: str) -> str: + """The website's post slug (policyengine-app-v2 postTransformers.ts).""" + stem = filename[: filename.index(".")] if "." in filename else filename + return stem.lower().replace("_", "-") + + +def analysis_links(text: str) -> List[Key]: + """Every policyengine.org country page an edition links, once each.""" + seen: List[Key] = [] + for country, research, slug in _LINK.findall(text): + key = ( + country.lower(), + "post" if research else "app", + slug.lower().replace("_", "-"), + ) + if key not in seen: + seen.append(key) + return seen + + +def research_entries(apps: Sequence[dict], posts: Sequence[dict]) -> Dict[Key, dict]: + entries: Dict[Key, dict] = {} + for app in apps: + if app.get("displayWithResearch") and app.get("slug") and app.get("countryId"): + entries[(app["countryId"], "app", app["slug"])] = app + for post in posts: + for tag in post.get("tags", []): + if tag in COUNTRIES: + entries[(tag, "post", post_slug(post["filename"]))] = post + return entries + + +def publishes_estimate(entry: dict) -> bool: + record = entry.get("comparables") or {} + if record.get("status") == "no_estimate": + return False + return "policy" in entry.get("tags", []) or record.get("status") == "reviewed" + + +def fresh(index_entry: dict, on: str, window_days: int) -> List[str]: + """Why an analysis may not be promoted on ``on``: the same rule as the + scorecard's ``comparables.fresh``.""" + start = (dt.date.fromisoformat(on) - dt.timedelta(days=window_days)).isoformat() + reasons = [] + searches = index_entry.get("searches", []) + if not any(start <= s["date"] <= on for s in searches): + earlier = [s["date"] for s in searches if s["date"] <= on] + last = max(earlier) if earlier else "none" + reasons.append( + f"no comparables search between {start} and {on} (last: {last}); " + "run a prepromotion search before sending" + ) + pending = [ + c["claim"] + for c in index_entry.get("comparisons", []) + if c.get("status") == "pending" and c.get("found_on", "9999") <= on + ] + if pending: + reasons.append( + f"gaps still pending by {on}: {', '.join(pending)}; " + "explain them before sending" + ) + return reasons + + +def check_edition( + path: Path, + text: str, + entries: Dict[Key, dict], + index: dict, +) -> List[str]: + """Every reason this edition may not go out.""" + on = edition_date(path) + if on is None: + return [f"{path.name}: no YYYY-MM-DD date at the start of the file name"] + window = index["promotion_window_days"] + errors = [] + for country, kind, slug in analysis_links(text): + entry = entries.get((country, kind, slug)) + if entry is None or not publishes_estimate(entry): + continue + source = f"analysis-{country}-{slug}" + page = ( + f"policyengine.org/{country}/{'research/' if kind == 'post' else ''}{slug}" + ) + recorded = index["analyses"].get(source) + if recorded is None or recorded.get("kind") != kind: + errors.append( + f"{path.name}: {page} has no comparables record in the scorecard; " + f"run the comparables review ({source}) before sending" + ) + continue + for reason in fresh(recorded, on, window): + errors.append(f"{path.name}: {page}: {reason}") + return errors + + +def editions_to_check(paths: Sequence[Path], cutover: str) -> List[Path]: + out = [] + for path in sorted(paths): + date = edition_date(path) + if date is not None and date >= cutover: + out.append(path) + return out + + +def _load(where: str) -> object: + if not where.startswith("https://"): + return json.loads(Path(where).read_text()) + last_error: Optional[Exception] = None + for attempt in range(4): + try: + with urllib.request.urlopen(where, timeout=30) as response: + return json.loads(response.read().decode("utf-8")) + except Exception as error: # noqa: BLE001 - every failure retries, then fails + last_error = error + time.sleep(2**attempt) + raise SystemExit(f"could not read {where}: {last_error}") + + +def run_check( + files: Sequence[Path], + apps: str = APPS_URL, + posts: str = POSTS_URL, + index_source: str = INDEX_URL, + cutover: Optional[str] = None, +) -> Tuple[List[Path], List[str]]: + """Check every edition dated on or after the cutover (the module's + ``PROMOTION_CUTOVER_DATE`` unless given). Returns the editions checked and + every error; downloads nothing when none is due.""" + due = editions_to_check(files, cutover or PROMOTION_CUTOVER_DATE) + if not due: + return [], [] + index = _load(index_source) + if not isinstance(index, dict) or index.get("schema") != INDEX_SCHEMA: + return due, [f"the scorecard index at {index_source} is not {INDEX_SCHEMA}"] + entries = research_entries(_load(apps), _load(posts)) + errors: List[str] = [] + for path in due: + errors.extend(check_edition(path, path.read_text(), entries, index)) + return due, errors + + +HOW_TO_FIX = ( + "Run the comparables review for each analysis above (the " + "policyengine-comparables-review skill, search kind prepromotion), merge its " + "scorecard PR, then check again." +) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(prog="python -m newsletter_uploader.comparables") + parser.add_argument("paths", nargs="*", default=["editions"]) + parser.add_argument("--apps", default=APPS_URL) + parser.add_argument("--posts", default=POSTS_URL) + parser.add_argument("--index", default=INDEX_URL) + parser.add_argument("--cutover", default=None) + args = parser.parse_args(argv) + + files: List[Path] = [] + for raw in args.paths: + path = Path(raw) + if path.is_dir(): + files.extend(p for p in path.iterdir() if p.suffix in (".html", ".md")) + else: + files.append(path) + due, errors = run_check(files, args.apps, args.posts, args.index, args.cutover) + if not due: + cutover = args.cutover or PROMOTION_CUTOVER_DATE + print(f"No edition dated on or after {cutover}; nothing to check.") + return 0 + for path in due: + failed = any(e.startswith(f"{path.name}:") for e in errors) + print(f"{'FAIL' if failed else 'ok '} {path.name}") + for error in errors: + print(f" {error}") + if errors: + print("\n" + HOW_TO_FIX) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_comparables.py b/tests/test_comparables.py new file mode 100644 index 0000000..35006e6 --- /dev/null +++ b/tests/test_comparables.py @@ -0,0 +1,283 @@ +"""Fresh comparables before a newsletter goes out (comparables.py). + +Invariants: +- an edition dated before the cutover is never checked, and nothing is + downloaded for it; +- a linked analysis that publishes an estimate passes exactly when the + scorecard records a search in the window before the edition's date and no + gap found by then is pending (checked over every offset and window in a grid, + the finite-domain form of a property test); +- links to anything that is not such an analysis are never checked; +- the upload CLI refuses a stale edition before it touches Mailchimp. +""" + +import datetime as dt +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from newsletter_uploader import comparables as cmp +from newsletter_uploader.cli import main as cli_main + +EDITIONS = Path(__file__).resolve().parent.parent / "editions" +WINDOW = 7 +PUSH = "2026-11-02" + +APPS = [ + { + "slug": "childcare-100k-threshold", + "countryId": "uk", + "displayWithResearch": True, + "tags": ["uk", "featured", "policy", "interactives"], + }, + { + "slug": "marriage", + "countryId": "uk", + "displayWithResearch": True, + "tags": ["uk", "policy"], + "comparables": {"status": "no_estimate", "reason": "a household calculator"}, + }, + {"slug": "tool", "countryId": "us", "tags": ["us", "policy"]}, +] +POSTS = [ + {"filename": "UK_Two_Child_Limit.md", "tags": ["uk", "policy"]}, + {"filename": "giving-tuesday.md", "tags": ["us", "giving"]}, +] + + +def shift(date, days): + return (dt.date.fromisoformat(date) + dt.timedelta(days=days)).isoformat() + + +def index_with(search_dates, pending_found=(), kind="app"): + return { + "schema": cmp.INDEX_SCHEMA, + "promotion_window_days": WINDOW, + "analyses": { + "analysis-uk-childcare-100k-threshold": { + "kind": kind, + "searches": [{"date": d, "kind": "reharvest"} for d in search_dates], + "comparisons": [ + {"claim": f"c{i}", "status": "pending", "found_on": found} + for i, found in enumerate(pending_found) + ], + } + }, + } + + +def edition(tmp_path, date, body): + path = tmp_path / f"{date}-uk.html" + path.write_text(f"{body}") + return path + + +CHILDCARE_LINK = ( + 'dashboard' +) + + +class TestLinks: + def test_finds_apps_and_posts_once(self): + text = ( + CHILDCARE_LINK + + 'x' + + '' + + "post https://policyengine.org/us/research/giving-tuesday." + ) + assert cmp.analysis_links(text) == [ + ("uk", "app", "childcare-100k-threshold"), + ("uk", "post", "uk-two-child-limit"), + ("us", "post", "giving-tuesday"), + ] + + def test_ignores_other_hosts_and_bare_country_pages(self): + text = ( + 'calc' + 'home' + 'other' + ) + assert cmp.analysis_links(text) == [] + + def test_post_slug_matches_the_website(self): + assert cmp.post_slug("UK_Two_Child_Limit.md") == "uk-two-child-limit" + + +class TestRegistry: + def test_research_entries_are_listed_apps_and_posts_by_country(self): + entries = cmp.research_entries(APPS, POSTS) + assert ("uk", "app", "childcare-100k-threshold") in entries + assert ("us", "app", "tool") not in entries # not listed with research + assert ("uk", "post", "uk-two-child-limit") in entries + + def test_what_publishes_an_estimate(self): + entries = cmp.research_entries(APPS, POSTS) + assert cmp.publishes_estimate( + entries[("uk", "app", "childcare-100k-threshold")] + ) + assert not cmp.publishes_estimate(entries[("uk", "app", "marriage")]) + assert not cmp.publishes_estimate(entries[("us", "post", "giving-tuesday")]) + + +class TestFresh: + @pytest.mark.parametrize("window", [1, 3, 7, 14]) + def test_fresh_exactly_when_a_search_falls_in_the_window(self, window): + for offset in range(-20, 6): + searches = [shift(PUSH, offset)] + in_window = -window <= offset <= 0 + assert ( + cmp.fresh( + index_with(searches)["analyses"][ + "analysis-uk-childcare-100k-threshold" + ], + PUSH, + window, + ) + == [] + ) == in_window, (window, offset) + + def test_a_wider_window_is_never_stricter(self): + for offset in range(-20, 6): + entry = index_with([shift(PUSH, offset)])["analyses"][ + "analysis-uk-childcare-100k-threshold" + ] + for narrow in range(1, 15): + for wide in range(narrow, 15): + if cmp.fresh(entry, PUSH, narrow) == []: + assert cmp.fresh(entry, PUSH, wide) == [] + + def test_a_pending_gap_blocks_only_once_found(self): + for offset in range(-10, 6): + entry = index_with([PUSH], pending_found=[shift(PUSH, offset)])["analyses"][ + "analysis-uk-childcare-100k-threshold" + ] + assert bool(cmp.fresh(entry, PUSH, WINDOW)) == (offset <= 0) + + +class TestEdition: + def check(self, tmp_path, body, index): + path = edition(tmp_path, PUSH, body) + return cmp.check_edition( + path, path.read_text(), cmp.research_entries(APPS, POSTS), index + ) + + def test_fresh_record_passes(self, tmp_path): + assert self.check(tmp_path, CHILDCARE_LINK, index_with([shift(PUSH, -2)])) == [] + + def test_stale_record_fails(self, tmp_path): + errors = self.check(tmp_path, CHILDCARE_LINK, index_with([shift(PUSH, -12)])) + assert len(errors) == 1 + assert "run a prepromotion search" in errors[0] + + def test_missing_record_fails(self, tmp_path): + index = index_with([PUSH]) + index["analyses"] = {} + errors = self.check(tmp_path, CHILDCARE_LINK, index) + assert "has no comparables record" in errors[0] + + def test_a_record_of_another_kind_does_not_count(self, tmp_path): + errors = self.check(tmp_path, CHILDCARE_LINK, index_with([PUSH], kind="post")) + assert "has no comparables record" in errors[0] + + def test_links_without_an_estimate_are_not_checked(self, tmp_path): + body = ( + 'calc' + 'post' + 'tool' + ) + index = index_with([]) + index["analyses"] = {} + assert self.check(tmp_path, body, index) == [] + + def test_a_policy_post_needs_a_record(self, tmp_path): + body = 'p' + errors = self.check(tmp_path, body, index_with([PUSH])) + assert "analysis-uk-uk-two-child-limit" in errors[0] + + def test_an_undated_file_is_reported(self, tmp_path): + path = tmp_path / "draft.html" + path.write_text(CHILDCARE_LINK) + errors = cmp.check_edition(path, path.read_text(), {}, index_with([PUSH])) + assert "no YYYY-MM-DD date" in errors[0] + + +class TestRun: + def write_sources(self, tmp_path, index): + apps, posts, idx = (tmp_path / n for n in ("apps.json", "posts.json", "i.json")) + apps.write_text(json.dumps(APPS)) + posts.write_text(json.dumps(POSTS)) + idx.write_text(json.dumps(index)) + return str(apps), str(posts), str(idx) + + def test_nothing_before_the_cutover_is_checked_or_downloaded(self, tmp_path): + old = edition(tmp_path, shift(cmp.PROMOTION_CUTOVER_DATE, -1), CHILDCARE_LINK) + due, errors = cmp.run_check( + [old], "/nonexistent/apps", "/nonexistent/posts", "/nonexistent/index" + ) + assert (due, errors) == ([], []) + + def test_the_archive_is_never_checked(self): + archive = [p for p in EDITIONS.iterdir() if p.suffix in (".html", ".md")] + due = cmp.editions_to_check(archive, cmp.PROMOTION_CUTOVER_DATE) + assert all(cmp.edition_date(p) >= cmp.PROMOTION_CUTOVER_DATE for p in due), due + + def test_main_passes_and_fails(self, tmp_path, capsys): + path = edition(tmp_path, PUSH, CHILDCARE_LINK) + ok = self.write_sources(tmp_path, index_with([PUSH])) + argv = [str(path), "--apps", ok[0], "--posts", ok[1], "--index", ok[2]] + assert cmp.main(argv + ["--cutover", "2026-10-26"]) == 0 + stale = self.write_sources(tmp_path, index_with([shift(PUSH, -30)])) + argv = [str(path), "--apps", stale[0], "--posts", stale[1], "--index", stale[2]] + assert cmp.main(argv + ["--cutover", "2026-10-26"]) == 1 + assert "prepromotion" in capsys.readouterr().out + + def test_a_wrong_index_schema_fails(self, tmp_path): + path = edition(tmp_path, PUSH, CHILDCARE_LINK) + index = index_with([PUSH]) + index["schema"] = "something/else" + sources = self.write_sources(tmp_path, index) + due, errors = cmp.run_check([path], *sources, cutover="2026-10-26") + assert due and "is not comparables/index/1" in errors[0] + + def test_an_unreadable_source_fails_closed(self, tmp_path, monkeypatch): + monkeypatch.setattr(cmp.time, "sleep", lambda _s: None) + path = edition(tmp_path, PUSH, CHILDCARE_LINK) + with pytest.raises(SystemExit, match="could not read"): + cmp.run_check( + [path], + index_source="https://127.0.0.1:9/index.json", + cutover="2026-10-26", + ) + + +class TestUploadRefusesStaleEditions: + def test_upload_stops_before_mailchimp(self, tmp_path, mocker): + path = edition(tmp_path, PUSH, CHILDCARE_LINK) + mocker.patch.object(cmp, "PROMOTION_CUTOVER_DATE", "2026-10-26") + sources = { + cmp.APPS_URL: APPS, + cmp.POSTS_URL: POSTS, + cmp.INDEX_URL: index_with([shift(PUSH, -30)]), + } + mocker.patch.object(cmp, "_load", side_effect=lambda where: sources[where]) + client = mocker.patch("newsletter_uploader.cli.MailchimpClient") + result = CliRunner().invoke( + cli_main, + [ + str(path), + "--audience", + "uk", + "--subject", + "s", + "--preview", + "p", + "--api-key", + "key-us5", + ], + ) + assert result.exit_code == 1 + assert "Comparables are not fresh" in result.output + client.assert_not_called() From 79183cc6c4784486492770c44dab576bbcb6b0c4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 12:06:46 -0400 Subject: [PATCH 2/5] Close the review's holes in the newsletter comparables check Links are recognised in every common form (no scheme, any case, escaped, blog/ and dated slugs, the site's redirects, the apps' own hosts), and tracking links and shorteners are refused. Posts are found by slug under any country. Every analysis is checked unless it may declare no estimate. The index's shape is validated, and fresh now requires the search to have compared the current results, tested against the scorecard's own cases. Uploads refuse undated files and judge on the later of the edition's date and today; CI checks only the editions a change adds or modifies. The draft update recipe goes through upload-newsletter. Co-Authored-By: Claude Opus 5.5 --- .claude/commands/create-newsletter.md | 16 +- .claude/commands/upload-draft.md | 10 +- .github/workflows/ci.yml | 32 +- .gitignore | 1 + CLAUDE.md | 44 +- pyproject.toml | 1 + src/newsletter_uploader/cli.py | 11 +- src/newsletter_uploader/comparables.py | 635 ++++- tests/conftest.py | 38 + tests/fixtures/comparables_fresh_cases.json | 2354 +++++++++++++++++++ tests/test_cli.py | 2 +- tests/test_comparables.py | 844 +++++-- 12 files changed, 3599 insertions(+), 389 deletions(-) create mode 100644 tests/fixtures/comparables_fresh_cases.json diff --git a/.claude/commands/create-newsletter.md b/.claude/commands/create-newsletter.md index 25fcd61..771185e 100644 --- a/.claude/commands/create-newsletter.md +++ b/.claude/commands/create-newsletter.md @@ -32,17 +32,19 @@ Create a new newsletter HTML file from PolicyEngine research posts and optional - Add footer with Mailchimp merge tags 4. **Check comparables are fresh** (required before sending, Max 2026-10-21): - - Run `python -m newsletter_uploader.comparables editions/`. Every - policyengine.org analysis the edition links needs a comparables search in - PolicyEngine/policyengine-scorecard dated within the scorecard's promotion - window (7 days by default) before the edition's date, with no gap still pending. + - Name the file for its send date, then run + `python -m newsletter_uploader.comparables --require-dates editions/`. + Every policyengine.org analysis the edition links needs a comparables search in + PolicyEngine/policyengine-scorecard on the send date or in the scorecard's + promotion window before it (7 days by default, counting both ends), against the + results the analysis reads now, with no gap still pending. - For each analysis it lists, run the comparables review with search kind `prepromotion` (`/comparables-review` and the `policyengine-comparables-review` skill in the PolicyEngine plugin), merge the scorecard PR, and check again. CI and `upload-newsletter` both refuse an edition that fails this. - - If the fresh search finds a comparable whose figure differs from ours by more than - the threshold, the edition says so beside our figure, or leaves the analysis out - until the gap is explained. + - The check confirms the search is fresh and every gap over the threshold has a + written cause; it does not put the comparable in the edition. Whether an edition + shows a comparable beside our figure is not yet decided. 5. **Show result**: - Path to generated file diff --git a/.claude/commands/upload-draft.md b/.claude/commands/upload-draft.md index 8182260..b5d6333 100644 --- a/.claude/commands/upload-draft.md +++ b/.claude/commands/upload-draft.md @@ -18,10 +18,12 @@ This is a wrapper around the `upload-newsletter` command that: 1. Validates the HTML file exists 2. Prompts for required fields if missing 3. Runs the upload command, which first checks that every policyengine.org analysis - the edition links has a fresh comparables search in the scorecard (editions dated on - or after the cutover; `src/newsletter_uploader/comparables.py`) and refuses to upload - otherwise. Fix it by running the comparables review (search kind `prepromotion`), - never by renaming or re-dating the file. + the edition links has a fresh comparables search in the scorecard, judged on the + later of the edition's date and today (`src/newsletter_uploader/comparables.py`), and + refuses to upload otherwise, or when the file name does not start with its date. Fix + it by running the comparables review (search kind `prepromotion`), never by renaming + or re-dating the file. To change a draft already uploaded, run the same command with + `--campaign-id`; it runs the same check. 4. Shows the Mailchimp edit URL ## Arguments diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d0da0f..ddbe2a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,15 +20,20 @@ jobs: comparables: # Max, 2026-10-21: "even if we've already published something, we should # do a fresh comp review before big pushes like newsletters". Every - # policyengine.org analysis an edition dated on or after the cutover links - # needs a comparables search in PolicyEngine/policyengine-scorecard within - # the scorecard's promotion window (src/newsletter_uploader/comparables.py). - # It downloads the scorecard index and app-v2's registries only when such - # an edition exists, and fails if it cannot read them. + # policyengine.org analysis an edition links needs a comparables search in + # PolicyEngine/policyengine-scorecard within the scorecard's promotion + # window before the edition's date (src/newsletter_uploader/comparables.py). + # It checks the editions this change adds or modifies (on a push, the + # pushed range), so an edition already sent is not re-judged against + # later data; each must be named for its date. It downloads the scorecard + # index and app-v2's registries only when such an edition is due, and + # fails if it cannot read them. name: Fresh comparables for promoted analyses runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v6 @@ -38,8 +43,21 @@ jobs: - name: Install run: pip install -e . - - name: Every analysis an edition links has a fresh comparables search - run: python -m newsletter_uploader.comparables editions + - name: Every analysis a changed edition links has a fresh comparables search + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then + BASE_SHA=$(git rev-list --max-parents=0 HEAD | tail -1) + fi + git diff --name-only --diff-filter=AMR "$BASE_SHA" HEAD -- 'editions/**' 'social-posts*.md' \ + | grep -iE '\.(html?|md)$' > "$RUNNER_TEMP/changed.txt" || true + echo "Changed editions:"; cat "$RUNNER_TEMP/changed.txt" + if [ ! -s "$RUNNER_TEMP/changed.txt" ]; then + echo "No edition changed; nothing to check." + exit 0 + fi + xargs -d '\n' python -m newsletter_uploader.comparables --require-dates < "$RUNNER_TEMP/changed.txt" test: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 2a76fa1..8ce9e11 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ __pycache__/ .coverage htmlcov/ .pytest_cache/ +.hypothesis/ diff --git a/CLAUDE.md b/CLAUDE.md index 38a45b3..198562a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,12 +48,17 @@ All newsletter HTML files follow this pattern: - External assets use full URLs (no relative paths) 4. **Fresh comparables before sending (Max, 2026-10-21).** Every policyengine.org analysis the edition links needs a comparables search recorded in - PolicyEngine/policyengine-scorecard within the scorecard's promotion window before - the edition's date (`python -m newsletter_uploader.comparables editions/`). + PolicyEngine/policyengine-scorecard on the send date or in the scorecard's promotion + window before it (7 days by default, counting both ends), against the results the + analysis reads now, with no gap still pending + (`python -m newsletter_uploader.comparables --require-dates editions/`). For any it lists, run the comparables review with search kind `prepromotion` (the - PolicyEngine plugin's `policyengine-comparables-review` skill) first. CI and - `upload-newsletter` refuse an edition dated on or after the cutover that fails it. - The same applies to the social posts kept beside an edition. + PolicyEngine plugin's `policyengine-comparables-review` skill) first. Name the file + for its send date: CI checks the editions a change adds or modifies on their dates, + and `upload-newsletter` refuses an undated file and judges the edition on the later + of its date and the day you upload. The same applies to the social posts kept beside + an edition. The check confirms the search is fresh; it does not put the comparable + in the edition. ## Email Compatibility @@ -90,7 +95,7 @@ https://raw.githubusercontent.com/PolicyEngine/newsletters/branch-name/assets/im ## File Naming Conventions -- Newsletter editions: `YYYY-MM-DD.html` (e.g., `2024-11-04.html`) +- Newsletter editions: `YYYY-MM-DD-{audience}.html` or `YYYY-MM-DD.html` (e.g., `2024-11-04.html`); the date at the start is the send date, and the comparables check reads it - Images: Descriptive names in `assets/images/` ## Configuration @@ -141,28 +146,25 @@ The command creates a **draft campaign** (not sent) that you can review, test, a ### Updating Existing Drafts -**IMPORTANT:** When making changes to a newsletter that has already been uploaded to Mailchimp, **update the existing campaign** instead of creating a new one. Use the Mailchimp API to update the campaign content: +**IMPORTANT:** When making changes to a newsletter that has already been uploaded to Mailchimp, **update the existing campaign** instead of creating a new one, with the same command and the campaign's ID. It runs the comparables check (step 4 above) before it touches Mailchimp, so an analysis added to a draft is checked too: +```bash +upload-newsletter editions/2025-10-01-uk.html \ + --campaign-id \ + --subject "UK Happy Hour Tomorrow + New Research" \ + --preview "Join us for drinks and discussion" +``` + +Do not put HTML to the campaign's content endpoint directly: that skips the check. + +To delete an accidentally created duplicate campaign: ```python import requests API_KEY = "your-api-key" SERVER = "us5" # from API key suffix -CAMPAIGN_ID = "existing-campaign-id" - -url = f"https://{SERVER}.api.mailchimp.com/3.0/campaigns/{CAMPAIGN_ID}/content" -auth = ("anystring", API_KEY) - -with open("editions/your-newsletter.html", "r") as f: - html_content = f.read() - -response = requests.put(url, auth=auth, json={"html": html_content}) -``` - -To delete an accidentally created duplicate campaign: -```python url = f"https://{SERVER}.api.mailchimp.com/3.0/campaigns/{CAMPAIGN_ID}" -requests.delete(url, auth=auth) +requests.delete(url, auth=("anystring", API_KEY)) ``` ### Development diff --git a/pyproject.toml b/pyproject.toml index 2b6ba6e..552a33a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dev = [ "pytest-cov>=4.1.0", "pytest-mock>=3.11.0", "responses>=0.23.0", + "hypothesis>=6.0", "black>=23.0.0", "ruff>=0.1.0", ] diff --git a/src/newsletter_uploader/cli.py b/src/newsletter_uploader/cli.py index a3f32ad..059bcc7 100644 --- a/src/newsletter_uploader/cli.py +++ b/src/newsletter_uploader/cli.py @@ -81,9 +81,12 @@ def main(html_file, audience, subject, preview, title, api_key, list_id, campaig sys.exit(1) # Fresh comparables before a push (Max, 2026-10-21): every analysis an - # edition dated on or after the cutover links needs a recent comparables - # search in the scorecard (newsletter_uploader/comparables.py). - due, problems = comparables.run_check([html_file]) + # edition links needs a recent comparables search in the scorecard, + # judged on the later of the edition's date and today + # (newsletter_uploader/comparables.py). An undated file is refused. + due, problems = comparables.run_check( + [html_file], today=comparables.today_utc(), require_dates=True + ) if problems: click.echo("❌ Comparables are not fresh for this edition:", err=True) for problem in problems: @@ -111,7 +114,7 @@ def main(html_file, audience, subject, preview, title, api_key, list_id, campaig # Upload newsletter click.echo("Reading HTML file...") - html_content = html_file.read_text() + html_content = html_file.read_text(encoding="utf-8") click.echo(f"✓ Read {len(html_content)} characters") if campaign_id: diff --git a/src/newsletter_uploader/comparables.py b/src/newsletter_uploader/comparables.py index c7dc9e9..624dc42 100644 --- a/src/newsletter_uploader/comparables.py +++ b/src/newsletter_uploader/comparables.py @@ -3,39 +3,62 @@ Max, 2026-10-21: "even if we've already published something, we should do a fresh comp review before big pushes like newsletters". Every policyengine.org analysis an edition links must have a comparables search recorded in -PolicyEngine/policyengine-scorecard (docs/COMPARABLES.md there) dated within -the scorecard's promotion window before the edition's date, and no gap found -by then still unexplained. The 21 October 2026 UK edition re-promoted the -childcare dashboard at £0.59bn; a fresh search would have put CenTax's £0.80bn -static cost for the 30 hours alone beside it. - -What counts as an analysis is decided by policyengine-app-v2's registries: an -app listed with research, or a post, that publishes an estimate (tagged -``policy``, or carrying a reviewed comparables block) and has not declared -``no_estimate``. Links to anything else (the home page, calculators, the -research index) are not checked. - -Editions are ``editions/-.html`` (and the ``.md`` social -posts beside them). Only editions dated on or after ``PROMOTION_CUTOVER_DATE`` -are checked, so the archive never changes its verdict. The registries and the -scorecard's index are downloaded from their repositories' main branches only -when such an edition exists; a failed download fails the check. +PolicyEngine/policyengine-scorecard (docs/COMPARABLES.md there): + +- dated on the push date or in the scorecard's promotion window before it + (the window counts both ends: 7 days means the push date and the 7 days + before it); +- against the results files the analysis reads now (the search's + ``results_pins`` equal the record's ``current_pins``); +- with no gap found by the push date still pending. + +The 20 October 2026 UK edition re-promoted the childcare dashboard at +£0.59bn; a fresh search would have found CenTax's £0.80bn static cost for the +30 hours alone. The check does not make an edition print the comparable. + +What counts as an analysis is decided by policyengine-app-v2's registries and +the rules the scorecard's index publishes (``entry_rules``): an app listed +with research or tagged ``policy``, or any post, unless it is a post that is +plainly not an analysis (an event, an announcement) and declares +``no_estimate``. A link is recognised in any common form: with or without a +scheme or ``www.``, in any case, HTML-escaped or percent-encoded, through +``/research/`` or ``/blog/``, with a dated post slug, through one of the +site's redirects to an analysis (``ALIASES``), or to the app's own host. +Tracking links and link shorteners are refused, because where they lead +cannot be checked. + +An edition is a file whose name starts with its date, ``YYYY-MM-DD`` +(``editions/2026-11-02-uk.html``, ``editions/2026-11-02.html``, and the +``.md`` social posts beside them). ``upload-newsletter`` refuses an undated +file and judges freshness on the later of the edition's date and the day it +runs; CI checks the editions a change adds or modifies, on their dates. Only +pushes on or after ``PROMOTION_CUTOVER_DATE`` are checked. The registries and +the scorecard's index are downloaded from their repositories' main branches +only when an edition is due; a failed download, or an index of the wrong +shape, fails the check. python -m newsletter_uploader.comparables editions + python -m newsletter_uploader.comparables --require-dates editions/ + +``fresh`` is the scorecard's ``comparables.fresh``; both are tested against +``tests/fixtures/comparables_fresh_cases.json``, a copy of the scorecard's. """ from __future__ import annotations import argparse import datetime as dt +import html import json import re import sys import time -import urllib.request +import urllib.parse from pathlib import Path from typing import Dict, List, Optional, Sequence, Tuple +import requests + PROMOTION_CUTOVER_DATE = "2026-10-26" APPS_URL = ( @@ -51,84 +74,290 @@ "data/comparables/index.json" ) INDEX_SCHEMA = "comparables/index/1" - -COUNTRIES = ("uk", "us", "ca", "ng", "il") -_LINK = re.compile( - r"https?://(?:www\.)?policyengine\.org/(uk|us|ca|ng|il)/(research/)?" - r"([A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)*)(?=[/?#\"'\s<>)\].,;:!]|$)" +# The scorecard's policy keeps its window in this range too. +MAX_WINDOW_DAYS = 90 + +EDITION_SUFFIXES = (".html", ".htm", ".md") +_EDITION = re.compile(r"^(\d{4}-\d{2}-\d{2})(?=[-_.]|$)") +_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_DATED_SLUG = re.compile(r"^\d{4}-\d{2}-\d{2}-(.+)$") +_SITE = re.compile( + r"(?)\]]*)?", + re.IGNORECASE, +) +_SEGMENT = re.compile(r"^[a-z0-9_-]+$") +# Links whose destination cannot be read from the link itself: Mailchimp's +# click tracking (its profile and unsubscribe links are fine) and shorteners. +TRACKING = (("list-manage.com", "/track/"),) +UNCHECKABLE_HOSTS = ( + "bit.ly", + "t.co", + "tinyurl.com", + "ow.ly", + "buff.ly", + "lnkd.in", + "shorturl.at", + "rebrand.ly", + "cutt.ly", ) -_EDITION = re.compile(r"^(\d{4}-\d{2}-\d{2})-") -Key = Tuple[str, str, str] # (country, "app" | "post", slug) +# Redirects on policyengine.org that land on an analysis (policyengine-app-v2 +# website/next.config.ts, redirects()). Each target must be a research entry +# in the registry the check downloads, or the check fails: the table is then +# stale. +ALIASES: Dict[Tuple[str, str], Tuple[str, str]] = { + ("us", "obbba-household-explorer"): ("us", "obbba-households"), + ("us", "obbba-household-by-household"): ("us", "obbba-households"), + ("us", "obbba-scatter"): ("us", "obbba-households"), + ("us", "obba-household-explorer"): ("us", "obbba-households"), + ("us", "ob3-households"): ("us", "obbba-households"), + ("us", "research/obbba-household-explorer"): ("us", "obbba-households"), + ("us", "research/obbba-household-by-household"): ("us", "obbba-households"), + ("us", "aca-reforms-calculator"): ("us", "aca-calc"), + ("us", "cliff-watch"): ("us", "cliffwatch"), +} +# A linked analysis: ("app", country, slug) or ("post", "", slug). +Key = Tuple[str, str, str] -def edition_date(path: Path) -> Optional[str]: - match = _EDITION.match(path.name) - if not match: - return None + +def today_utc() -> str: + return dt.datetime.now(dt.timezone.utc).date().isoformat() + + +def is_date(value: object) -> bool: + if not isinstance(value, str) or not _DATE.match(value): + return False try: - dt.date.fromisoformat(match.group(1)) + dt.date.fromisoformat(value) except ValueError: + return False + return True + + +def edition_date(path: Path) -> Optional[str]: + match = _EDITION.match(path.name) + if not match or not is_date(match.group(1)): return None return match.group(1) def post_slug(filename: str) -> str: """The website's post slug (policyengine-app-v2 postTransformers.ts).""" - stem = filename[: filename.index(".")] if "." in filename else filename - return stem.lower().replace("_", "-") - - -def analysis_links(text: str) -> List[Key]: - """Every policyengine.org country page an edition links, once each.""" - seen: List[Key] = [] - for country, research, slug in _LINK.findall(text): - key = ( - country.lower(), - "post" if research else "app", - slug.lower().replace("_", "-"), - ) - if key not in seen: - seen.append(key) + return filename[: filename.find(".")].lower().replace("_", "-") + + +def _decode(text: str) -> str: + # HTML entities, then percent-encoding, twice over for a link that was + # escaped both ways + for _ in range(2): + text = urllib.parse.unquote(html.unescape(text)) + return text + + +def _segments(path: str) -> List[str]: + return [s for s in path.split("/") if s] + + +def _clean_slug(segment: str) -> Optional[str]: + match = re.match(r"^[a-z0-9_-]+", segment.lower()) + if not match: + return None + slug = match.group(0).rstrip("-_").replace("_", "-") + return slug or None + + +def site_paths(text: str) -> List[List[str]]: + """The path segments of every policyengine.org link, lowercased, once each.""" + seen: List[List[str]] = [] + for match in _SITE.finditer(_decode(text)): + segments = [s.lower() for s in _segments(match.group(1) or "")] + if segments not in seen: + seen.append(segments) return seen -def research_entries(apps: Sequence[dict], posts: Sequence[dict]) -> Dict[Key, dict]: - entries: Dict[Key, dict] = {} - for app in apps: - if app.get("displayWithResearch") and app.get("slug") and app.get("countryId"): - entries[(app["countryId"], "app", app["slug"])] = app - for post in posts: - for tag in post.get("tags", []): - if tag in COUNTRIES: - entries[(tag, "post", post_slug(post["filename"]))] = post - return entries +def other_links(text: str) -> List[Tuple[str, str]]: + """(host, path) of every link to another host, lowercased, once each.""" + seen: List[Tuple[str, str]] = [] + for match in _URL.finditer(_decode(text)): + host = match.group(1).lower().rstrip(".") + if host == "policyengine.org" or host.endswith(".policyengine.org"): + continue + link = (host, (match.group(2) or "/").split("?")[0].split("#")[0].lower()) + if link not in seen: + seen.append(link) + return seen -def publishes_estimate(entry: dict) -> bool: - record = entry.get("comparables") or {} - if record.get("status") == "no_estimate": - return False - return "policy" in entry.get("tags", []) or record.get("status") == "reviewed" +def _host(url: str) -> Tuple[str, str]: + parsed = urllib.parse.urlsplit(url) + return (parsed.hostname or "").lower(), (parsed.path or "/").rstrip( + "/" + ).lower() or "/" + + +class Registry: + """The research entries in policyengine-app-v2's registries, under the + rules the scorecard's index publishes.""" + + def __init__(self, apps: Sequence[dict], posts: Sequence[dict], rules: dict): + self.rules = rules + countries = rules["country_tags"] + self.countries = set(countries) + self.entries: Dict[Key, dict] = {} + self.hosts: Dict[str, List[Key]] = {} + for app in apps: + tags = app.get("tags") or [] + if not (app.get("slug") and app.get("countryId")): + continue + if not (app.get("displayWithResearch") or "policy" in tags): + continue + key: Key = ("app", app["countryId"], app["slug"]) + self.entries[key] = { + "kind": "app", + "countries": [app["countryId"]], + "slug": app["slug"], + "tags": tags, + "comparables": app.get("comparables"), + } + source = app.get("source") or "" + host, _ = _host(source) + if host.endswith(".vercel.app") or host.endswith(".github.io"): + self.hosts.setdefault(host, []).append(key) + for post in posts: + tags = post.get("tags") or [] + slug = post_slug(post["filename"]) + self.entries[("post", "", slug)] = { + "kind": "post", + "countries": [t for t in tags if t in self.countries] + or [rules["default_country"]], + "slug": slug, + "tags": tags, + "comparables": post.get("comparables"), + } + + def stale_aliases(self) -> List[str]: + return [ + f"{c}/{path} -> {tc}/{ts}" + for (c, path), (tc, ts) in ALIASES.items() + if ("app", tc, ts) not in self.entries + ] + + def from_path(self, segments: List[str]) -> Tuple[Optional[Key], Optional[str]]: + """The entry a policyengine.org path names, or why none is known.""" + if len(segments) < 2 or segments[0] not in self.countries: + return None, None + country = segments[0] + if segments[1] in ("research", "blog"): + slug = _clean_slug(segments[2]) if len(segments) > 2 else None + if slug is None: + return None, None + alias = ALIASES.get((country, f"research/{slug}")) + if alias: + return ("app",) + alias, None + if ("post", "", slug) in self.entries: + return ("post", "", slug), None + dated = _DATED_SLUG.match(slug) + if dated and ("post", "", dated.group(1)) in self.entries: + return ("post", "", dated.group(1)), None + return None, f"policyengine.org/{country}/research/{slug} is not a post" + slug = _clean_slug(segments[1]) + alias = ALIASES.get((country, slug or "")) + if alias: + return ("app",) + alias, None + if slug and ("app", country, slug) in self.entries: + return ("app", country, slug), None + return None, None + + def from_host(self, host: str) -> List[Key]: + """The research apps served from a host: the host in an app's source, + or one of its Vercel preview deployments (-git--...).""" + if host in self.hosts: + return self.hosts[host] + for known, keys in self.hosts.items(): + project = known.split(".")[0] + if host.endswith(".vercel.app") and host.startswith(f"{project}-git-"): + return keys + return [] + + def may_declare_no_estimate(self, entry: dict) -> bool: + tags = set(entry["tags"]) + return ( + entry["kind"] == "post" + and bool(tags & set(self.rules["no_estimate_tags"])) + and not tags & set(self.rules["analysis_tags"]) + ) + + def publishes_estimate(self, entry: dict) -> bool: + record = entry.get("comparables") or {} + return not ( + isinstance(record, dict) + and record.get("status") == "no_estimate" + and self.may_declare_no_estimate(entry) + ) + + +def linked_analyses(text: str, registry: Registry) -> Tuple[List[Key], List[str]]: + """Every research entry an edition links, and every link the check refuses.""" + keys: List[Key] = [] + refused: List[str] = [] + for segments in site_paths(text): + key, problem = registry.from_path(segments) + if problem: + refused.append(problem) + if key and key not in keys: + keys.append(key) + for host, path in other_links(text): + tracked = any( + (host == h or host.endswith(f".{h}")) and path.startswith(prefix) + for h, prefix in TRACKING + ) + if tracked or any( + host == h or host.endswith(f".{h}") for h in UNCHECKABLE_HOSTS + ): + refused.append( + f"{host}{path} hides where it leads (a tracking link or a shortener); " + "link the policyengine.org page itself" + ) + continue + for key in registry.from_host(host): + if key not in keys: + keys.append(key) + return keys, refused def fresh(index_entry: dict, on: str, window_days: int) -> List[str]: - """Why an analysis may not be promoted on ``on``: the same rule as the - scorecard's ``comparables.fresh``.""" + """Why an analysis may not be promoted on ``on``: the scorecard's + ``comparables.fresh``. A pending gap with no ``found_on`` counts as + found.""" start = (dt.date.fromisoformat(on) - dt.timedelta(days=window_days)).isoformat() reasons = [] - searches = index_entry.get("searches", []) - if not any(start <= s["date"] <= on for s in searches): + pins = sorted(index_entry.get("current_pins") or []) + searches = index_entry["searches"] + recent = [s for s in searches if start <= s["date"] <= on] + if not recent: earlier = [s["date"] for s in searches if s["date"] <= on] last = max(earlier) if earlier else "none" reasons.append( f"no comparables search between {start} and {on} (last: {last}); " "run a prepromotion search before sending" ) + elif not any(sorted(s.get("results_pins") or []) == pins for s in recent): + reasons.append( + f"no search between {start} and {on} compared the results the page reads " + "now; run a prepromotion search against them" + ) pending = [ c["claim"] for c in index_entry.get("comparisons", []) - if c.get("status") == "pending" and c.get("found_on", "9999") <= on + if c.get("status") == "pending" and (c.get("found_on") or "") <= on ] if pending: reasons.append( @@ -138,55 +367,181 @@ def fresh(index_entry: dict, on: str, window_days: int) -> List[str]: return reasons +def check_index(index: object) -> List[str]: + """Every reason the downloaded index cannot be trusted.""" + if not isinstance(index, dict): + return ["the scorecard index is not a JSON object"] + errors = [] + if index.get("schema") != INDEX_SCHEMA: + errors.append( + f"the scorecard index schema is {index.get('schema')!r}, not {INDEX_SCHEMA}" + ) + window = index.get("promotion_window_days") + if ( + not isinstance(window, int) + or isinstance(window, bool) + or not 1 <= window <= MAX_WINDOW_DAYS + ): + errors.append( + f"the scorecard index's promotion_window_days is {window!r}, " + f"not a whole number of days from 1 to {MAX_WINDOW_DAYS}" + ) + rules = index.get("entry_rules") + if ( + not isinstance(rules, dict) + or not all( + isinstance(rules.get(k), list) + for k in ("analysis_tags", "no_estimate_tags", "country_tags") + ) + or not isinstance(rules.get("default_country"), str) + ): + errors.append("the scorecard index publishes no entry_rules") + analyses = index.get("analyses") + if not isinstance(analyses, dict): + return errors + ["the scorecard index has no analyses object"] + for source, entry in analyses.items(): + where = f"the scorecard index's {source}" + if not isinstance(entry, dict): + errors.append(f"{where} is not an object") + continue + searches = entry.get("searches") + if not isinstance(searches, list) or not searches: + errors.append(f"{where} lists no searches") + else: + for s in searches: + if not isinstance(s, dict) or not is_date(s.get("date")): + errors.append(f"{where} has a search with no real date") + elif not isinstance(s.get("results_pins"), list): + errors.append(f"{where} has a search with no results_pins") + if not isinstance(entry.get("current_pins"), list): + errors.append(f"{where} has no current_pins") + comparisons = entry.get("comparisons", []) + if not isinstance(comparisons, list) or not all( + isinstance(c, dict) and isinstance(c.get("status"), str) + for c in comparisons + ): + errors.append(f"{where} has a comparison with no status") + return errors + + +def record_for( + key: Key, entry: dict, index: dict +) -> Tuple[Optional[str], Optional[dict]]: + """The scorecard record an entry names, or the one its name implies.""" + analyses = index["analyses"] + block = entry.get("comparables") + if isinstance(block, dict) and isinstance(block.get("scorecardSource"), str): + candidates = [block["scorecardSource"]] + else: + candidates = [f"analysis-{c}-{entry['slug']}" for c in entry["countries"]] + for source in candidates: + recorded = analyses.get(source) + if ( + recorded + and recorded.get("kind") == entry["kind"] + and recorded.get("slug") == entry["slug"] + ): + return source, recorded + return candidates[0], None + + +def _page(key: Key) -> str: + kind, country, slug = key + return ( + f"policyengine.org/{country}/{slug}" + if kind == "app" + else f"policyengine.org/research/{slug}" + ) + + def check_edition( - path: Path, - text: str, - entries: Dict[Key, dict], - index: dict, + name: str, text: str, on: str, registry: Registry, index: dict ) -> List[str]: - """Every reason this edition may not go out.""" - on = edition_date(path) - if on is None: - return [f"{path.name}: no YYYY-MM-DD date at the start of the file name"] + """Every reason this edition may not go out on ``on``.""" window = index["promotion_window_days"] - errors = [] - for country, kind, slug in analysis_links(text): - entry = entries.get((country, kind, slug)) - if entry is None or not publishes_estimate(entry): + keys, refused = linked_analyses(text, registry) + errors = [f"{name}: {problem}" for problem in refused] + for key in keys: + entry = registry.entries[key] + if not registry.publishes_estimate(entry): continue - source = f"analysis-{country}-{slug}" - page = ( - f"policyengine.org/{country}/{'research/' if kind == 'post' else ''}{slug}" - ) - recorded = index["analyses"].get(source) - if recorded is None or recorded.get("kind") != kind: + source, recorded = record_for(key, entry, index) + if recorded is None: errors.append( - f"{path.name}: {page} has no comparables record in the scorecard; " + f"{name}: {_page(key)} has no comparables record in the scorecard; " f"run the comparables review ({source}) before sending" ) continue for reason in fresh(recorded, on, window): - errors.append(f"{path.name}: {page}: {reason}") + errors.append(f"{name}: {_page(key)}: {reason}") + # A page the registry no longer lists (taken down, or not yet listed) + # that still has a scorecard record is held to its record. + for segments in site_paths(text): + if len(segments) < 2 or segments[0] not in registry.countries: + continue + slug = _clean_slug(segments[1]) + source = f"analysis-{segments[0]}-{slug}" + recorded = index["analyses"].get(source) + if ( + slug is None + or ("app", segments[0], slug) in keys + or recorded is None + or recorded.get("kind") != "app" + ): + continue + page = f"policyengine.org/{segments[0]}/{slug}" + for reason in fresh(recorded, on, window): + errors.append(f"{name}: {page}: {reason}") return errors -def editions_to_check(paths: Sequence[Path], cutover: str) -> List[Path]: - out = [] - for path in sorted(paths): +def edition_files(paths: Sequence[Path]) -> List[Path]: + """The files named, and every edition-like file under a directory named.""" + files: List[Path] = [] + for path in paths: + if path.is_dir(): + files.extend( + p + for p in sorted(path.rglob("*")) + if p.is_file() and p.suffix.lower() in EDITION_SUFFIXES + ) + else: + files.append(path) + return files + + +def due_editions( + files: Sequence[Path], cutover: str, today: Optional[str], require_dates: bool +) -> Tuple[List[Tuple[Path, str]], List[str]]: + """Each edition to check with the date to judge it on, and every file + that cannot be dated. The date is the edition's, or today when that is + later (an upload after the edition's date).""" + due: List[Tuple[Path, str]] = [] + errors: List[str] = [] + for path in files: date = edition_date(path) - if date is not None and date >= cutover: - out.append(path) - return out + if date is None: + if require_dates: + errors.append( + f"{path.name}: no YYYY-MM-DD date at the start of the file name, " + "so its comparables cannot be checked; name it -" + ) + continue + on = max(date, today) if today else date + if on >= cutover: + due.append((path, on)) + return due, errors def _load(where: str) -> object: if not where.startswith("https://"): - return json.loads(Path(where).read_text()) + return json.loads(Path(where).read_text(encoding="utf-8")) last_error: Optional[Exception] = None for attempt in range(4): try: - with urllib.request.urlopen(where, timeout=30) as response: - return json.loads(response.read().decode("utf-8")) + response = requests.get(where, timeout=30) + response.raise_for_status() + return response.json() except Exception as error: # noqa: BLE001 - every failure retries, then fails last_error = error time.sleep(2**attempt) @@ -195,58 +550,86 @@ def _load(where: str) -> object: def run_check( files: Sequence[Path], - apps: str = APPS_URL, - posts: str = POSTS_URL, - index_source: str = INDEX_URL, + apps: Optional[str] = None, + posts: Optional[str] = None, + index_source: Optional[str] = None, cutover: Optional[str] = None, + today: Optional[str] = None, + require_dates: bool = False, ) -> Tuple[List[Path], List[str]]: - """Check every edition dated on or after the cutover (the module's - ``PROMOTION_CUTOVER_DATE`` unless given). Returns the editions checked and - every error; downloads nothing when none is due.""" - due = editions_to_check(files, cutover or PROMOTION_CUTOVER_DATE) + """Check every edition due on or after the cutover. The sources and the + cutover default to the module's ``APPS_URL``, ``POSTS_URL``, + ``INDEX_URL`` and ``PROMOTION_CUTOVER_DATE``, read when called. Returns + the editions checked and every error; downloads nothing when none is + due.""" + apps = apps or APPS_URL + posts = posts or POSTS_URL + index_source = index_source or INDEX_URL + due, errors = due_editions( + files, cutover or PROMOTION_CUTOVER_DATE, today, require_dates + ) if not due: - return [], [] + return [], errors index = _load(index_source) - if not isinstance(index, dict) or index.get("schema") != INDEX_SCHEMA: - return due, [f"the scorecard index at {index_source} is not {INDEX_SCHEMA}"] - entries = research_entries(_load(apps), _load(posts)) - errors: List[str] = [] - for path in due: - errors.extend(check_edition(path, path.read_text(), entries, index)) - return due, errors + problems = check_index(index) + if problems: + return [path for path, _ in due], errors + problems + registry = Registry(_load(apps), _load(posts), index["entry_rules"]) + stale = registry.stale_aliases() + if stale: + return [path for path, _ in due], errors + [ + "the redirect table in comparables.py names an analysis the " + f"registry no longer lists: {a}" + for a in stale + ] + for path, on in due: + text = path.read_text(encoding="utf-8") + errors.extend(check_edition(path.name, text, on, registry, index)) + return [path for path, _ in due], errors HOW_TO_FIX = ( "Run the comparables review for each analysis above (the " "policyengine-comparables-review skill, search kind prepromotion), merge its " - "scorecard PR, then check again." + "scorecard PR, then check again; raw.githubusercontent.com caches the index " + "for up to 5 minutes after a merge." ) def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(prog="python -m newsletter_uploader.comparables") parser.add_argument("paths", nargs="*", default=["editions"]) - parser.add_argument("--apps", default=APPS_URL) - parser.add_argument("--posts", default=POSTS_URL) - parser.add_argument("--index", default=INDEX_URL) + parser.add_argument("--apps", default=None) + parser.add_argument("--posts", default=None) + parser.add_argument("--index", default=None) parser.add_argument("--cutover", default=None) + parser.add_argument( + "--require-dates", + action="store_true", + help="fail a file whose name does not start with its date", + ) args = parser.parse_args(argv) - files: List[Path] = [] - for raw in args.paths: - path = Path(raw) - if path.is_dir(): - files.extend(p for p in path.iterdir() if p.suffix in (".html", ".md")) - else: - files.append(path) - due, errors = run_check(files, args.apps, args.posts, args.index, args.cutover) - if not due: - cutover = args.cutover or PROMOTION_CUTOVER_DATE + files = edition_files([Path(raw) for raw in args.paths]) + due, errors = run_check( + files, + args.apps, + args.posts, + args.index, + args.cutover, + require_dates=args.require_dates, + ) + cutover = args.cutover or PROMOTION_CUTOVER_DATE + if not due and not errors: print(f"No edition dated on or after {cutover}; nothing to check.") return 0 - for path in due: - failed = any(e.startswith(f"{path.name}:") for e in errors) - print(f"{'FAIL' if failed else 'ok '} {path.name}") + file_errors = { + path.name for path in due if any(e.startswith(f"{path.name}:") for e in errors) + } + general = [e for e in errors if not any(e.startswith(f"{p.name}:") for p in files)] + if not general: + for path in due: + print(f"{'FAIL' if path.name in file_errors else 'ok '} {path.name}") for error in errors: print(f" {error}") if errors: diff --git a/tests/conftest.py b/tests/conftest.py index d6635d7..d58c364 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,45 @@ """Pytest configuration and shared fixtures.""" +import json + import pytest +from newsletter_uploader import comparables + + +@pytest.fixture(autouse=True) +def offline_comparables(tmp_path_factory, monkeypatch): + """The comparables check reads local, empty registries and a valid empty + index, so no test reaches the network; tests that need data pass their + own sources.""" + root = tmp_path_factory.mktemp("comparables-sources") + (root / "apps.json").write_text("[]") + (root / "posts.json").write_text("[]") + (root / "index.json").write_text( + json.dumps( + { + "schema": comparables.INDEX_SCHEMA, + "promotion_window_days": 7, + "entry_rules": { + "analysis_tags": ["policy"], + "no_estimate_tags": ["event"], + "country_tags": ["uk", "us", "ca", "ng", "il"], + "default_country": "us", + "search_kinds": [ + "prepublish", + "backfill", + "reharvest", + "prepromotion", + ], + }, + "analyses": {}, + } + ) + ) + monkeypatch.setattr(comparables, "APPS_URL", str(root / "apps.json")) + monkeypatch.setattr(comparables, "POSTS_URL", str(root / "posts.json")) + monkeypatch.setattr(comparables, "INDEX_URL", str(root / "index.json")) + @pytest.fixture def api_key(): diff --git a/tests/fixtures/comparables_fresh_cases.json b/tests/fixtures/comparables_fresh_cases.json new file mode 100644 index 0000000..75ae48b --- /dev/null +++ b/tests/fixtures/comparables_fresh_cases.json @@ -0,0 +1,2354 @@ +[ + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 3 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-24", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-25", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-26", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-10-30", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": true, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-02", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-10-31", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-02", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + }, + { + "entry": { + "comparisons": [ + { + "claim": "c0", + "found_on": "2026-11-04", + "status": "pending" + } + ], + "current_pins": [ + "PolicyEngine/test@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "searches": [ + { + "date": "2026-11-03", + "kind": "reharvest", + "results_pins": [ + "x" + ] + } + ] + }, + "fresh": false, + "on": "2026-11-02", + "window": 7 + } +] diff --git a/tests/test_cli.py b/tests/test_cli.py index 2d36adc..80b8f25 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,7 @@ def runner(): @pytest.fixture def html_file(tmp_path): """Create a test HTML file.""" - html_path = tmp_path / "test.html" + html_path = tmp_path / "2026-01-01-test.html" html_path.write_text("Test") return html_path diff --git a/tests/test_comparables.py b/tests/test_comparables.py index 35006e6..e2b7729 100644 --- a/tests/test_comparables.py +++ b/tests/test_comparables.py @@ -1,29 +1,45 @@ """Fresh comparables before a newsletter goes out (comparables.py). Invariants: -- an edition dated before the cutover is never checked, and nothing is - downloaded for it; -- a linked analysis that publishes an estimate passes exactly when the - scorecard records a search in the window before the edition's date and no - gap found by then is pending (checked over every offset and window in a grid, - the finite-domain form of a property test); -- links to anything that is not such an analysis are never checked; -- the upload CLI refuses a stale edition before it touches Mailchimp. +- every way of writing a link to a research entry finds it (Hypothesis): with + or without a scheme or www., in any case, escaped or encoded, with any + trailing punctuation, through /research/ or /blog/ or a dated slug, through + a redirect in ALIASES, or to the app's own host; +- a post is found by its slug under any country; +- every research entry is checked unless it is a post that is plainly not an + analysis and declares no_estimate; +- a linked analysis passes exactly when the scorecard's fresh says so, on the + cases the scorecard checks its own fresh against; +- an index of the wrong shape fails with a message, never a traceback; +- an undated file fails wherever dates are required; an upload is judged on + the later of its date and today, before Mailchimp is touched; +- tracking links and shorteners are refused. """ +import copy import datetime as dt import json from pathlib import Path import pytest from click.testing import CliRunner +from hypothesis import given, settings +from hypothesis import strategies as st from newsletter_uploader import comparables as cmp from newsletter_uploader.cli import main as cli_main -EDITIONS = Path(__file__).resolve().parent.parent / "editions" +FIXTURES = Path(__file__).resolve().parent / "fixtures" WINDOW = 7 PUSH = "2026-11-02" +PIN = "PolicyEngine/test@" + "b" * 40 + ":" + "a" * 64 +RULES = { + "analysis_tags": ["policy", "autumn-budget", "tax", "technical"], + "no_estimate_tags": ["org", "event", "giving"], + "country_tags": ["uk", "us", "ca", "ng", "il"], + "default_country": "us", + "search_kinds": ["prepublish", "backfill", "reharvest", "prepromotion"], +} APPS = [ { @@ -31,6 +47,28 @@ "countryId": "uk", "displayWithResearch": True, "tags": ["uk", "featured", "policy", "interactives"], + "source": "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold", + }, + { + "slug": "scottish-budget-2026-27", + "countryId": "uk", + "displayWithResearch": True, + "tags": ["uk", "policy"], + "source": "https://post-scottish-budget-dashboard.vercel.app/", + }, + { + "slug": "obbba-households", + "countryId": "us", + "displayWithResearch": True, + "tags": ["us", "policy"], + "source": "https://obbba-household-by-household.vercel.app/us/obbba-households", + }, + {"slug": "aca-calc", "countryId": "us", "tags": ["us", "policy"]}, + { + "slug": "cliffwatch", + "countryId": "us", + "displayWithResearch": True, + "tags": ["us"], }, { "slug": "marriage", @@ -39,11 +77,34 @@ "tags": ["uk", "policy"], "comparables": {"status": "no_estimate", "reason": "a household calculator"}, }, - {"slug": "tool", "countryId": "us", "tags": ["us", "policy"]}, + {"slug": "year-in-review", "countryId": "us", "tags": ["us", "featured"]}, ] POSTS = [ {"filename": "UK_Two_Child_Limit.md", "tags": ["uk", "policy"]}, - {"filename": "giving-tuesday.md", "tags": ["us", "giving"]}, + {"filename": "fuel-duty-freeze.md", "tags": ["uk", "autumn-budget", "featured"]}, + {"filename": "childcare-expansion-2023.md", "tags": ["uk", "policy"]}, + { + "filename": "giving-tuesday.md", + "tags": ["us", "giving"], + "comparables": { + "status": "no_estimate", + "reason": "an appeal, not an analysis", + }, + }, + { + "filename": "giving-and-policy.md", + "tags": ["us", "giving", "policy"], + "comparables": {"status": "no_estimate", "reason": "policy wins over this"}, + }, + {"filename": "api-v2.md", "tags": ["global", "technical"]}, + { + "filename": "joint-post.md", + "tags": ["uk", "us", "policy"], + "comparables": { + "status": "reviewed", + "scorecardSource": "analysis-uk-joint-post", + }, + }, ] @@ -51,233 +112,578 @@ def shift(date, days): return (dt.date.fromisoformat(date) + dt.timedelta(days=days)).isoformat() -def index_with(search_dates, pending_found=(), kind="app"): +def registry(): + return cmp.Registry(APPS, POSTS, RULES) + + +def record(kind, slug, search_dates, pending_found=(), pins_match=True): + return { + "kind": kind, + "slug": slug, + "current_pins": [PIN], + "searches": [ + { + "date": d, + "kind": "reharvest", + "results_pins": [PIN] if pins_match else [], + } + for d in search_dates + ], + "comparisons": [ + {"claim": f"c{i}", "status": "pending", "found_on": found} + for i, found in enumerate(pending_found) + ], + } + + +def index(analyses): return { "schema": cmp.INDEX_SCHEMA, "promotion_window_days": WINDOW, - "analyses": { - "analysis-uk-childcare-100k-threshold": { - "kind": kind, - "searches": [{"date": d, "kind": "reharvest"} for d in search_dates], - "comparisons": [ - {"claim": f"c{i}", "status": "pending", "found_on": found} - for i, found in enumerate(pending_found) - ], - } - }, + "entry_rules": RULES, + "analyses": copy.deepcopy(analyses), } -def edition(tmp_path, date, body): - path = tmp_path / f"{date}-uk.html" - path.write_text(f"{body}") - return path +CHILDCARE = ("app", "uk", "childcare-100k-threshold") +FRESH_CHILDCARE = { + "analysis-uk-childcare-100k-threshold": record( + "app", "childcare-100k-threshold", [PUSH] + ) +} +STALE_CHILDCARE = { + "analysis-uk-childcare-100k-threshold": record( + "app", "childcare-100k-threshold", ["2026-01-01"] + ) +} -CHILDCARE_LINK = ( - 'dashboard' -) +def keys(text): + found, _ = cmp.linked_analyses(text, registry()) + return found -class TestLinks: - def test_finds_apps_and_posts_once(self): - text = ( - CHILDCARE_LINK - + 'x' - + '' - + "post https://policyengine.org/us/research/giving-tuesday." - ) - assert cmp.analysis_links(text) == [ - ("uk", "app", "childcare-100k-threshold"), - ("uk", "post", "uk-two-child-limit"), - ("us", "post", "giving-tuesday"), - ] +# --- recognising a link --- + +SCHEMES = ["https://", "http://", "//", "", "HTTPS://"] +HOSTS = [ + "policyengine.org", + "www.policyengine.org", + "PolicyEngine.org", + "policyengine.org:443", +] +TRAILS = [ + "", + "/", + "?utm_source=x", + "#top", + ".", + ",", + ")", + "**", + "_", + "—", + "’", + " ", + "&utm=1", + "|", + "*", + "/deep/path", +] + - def test_ignores_other_hosts_and_bare_country_pages(self): - text = ( - 'calc' - 'home' - 'other' +def written(path, scheme, host, trail, encode): + url = f"{scheme}{host}{path}{trail}" + if encode == "html": + url = url.replace("/", "/").replace("-", "-") + elif encode == "percent": + url = url.replace("-", "%2D") + return f"

Read this now

" + + +@settings(max_examples=300, deadline=None) +@given( + target=st.sampled_from( + [ + ("/uk/childcare-100k-threshold", CHILDCARE), + ("/UK/Childcare-100k-Threshold", CHILDCARE), + ("//uk/childcare-100k-threshold", CHILDCARE), + ("/uk/research/uk-two-child-limit", ("post", "", "uk-two-child-limit")), + ("/uk/research/UK_Two_Child_Limit", ("post", "", "uk-two-child-limit")), + ("/uk/blog/uk-two-child-limit", ("post", "", "uk-two-child-limit")), + ( + "/uk/research/2025-01-01-uk-two-child-limit", + ("post", "", "uk-two-child-limit"), + ), + ( + "/us/research/childcare-expansion-2023", + ("post", "", "childcare-expansion-2023"), + ), + ("/us/ob3-households", ("app", "us", "obbba-households")), + ("/us/obbba-household-explorer/x", ("app", "us", "obbba-households")), + ( + "/us/research/obbba-household-explorer", + ("app", "us", "obbba-households"), + ), + ("/us/aca-reforms-calculator", ("app", "us", "aca-calc")), + ("/us/cliff-watch", ("app", "us", "cliffwatch")), + ] + ), + scheme=st.sampled_from(SCHEMES), + host=st.sampled_from(HOSTS), + trail=st.sampled_from(TRAILS), + encode=st.sampled_from(["none", "html", "percent"]), +) +def test_every_way_of_writing_a_link_finds_its_entry( + target, scheme, host, trail, encode +): + path, key = target + assert key in keys(written(path, scheme, host, trail, encode)) + + +def test_the_scottish_budget_social_post_link_is_found(): + text = "Explore it at policyengine.org/uk/scottish-budget-2026-27 today." + assert keys(text) == [("app", "uk", "scottish-budget-2026-27")] + + +def test_an_apps_own_host_and_its_previews_are_found(): + assert keys( + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold" + ) == [CHILDCARE] + assert keys("uk-childcare-100k-threshold.vercel.app") == [CHILDCARE] + assert keys( + "https://uk-childcare-100k-threshold-git-dev-policy-engine.vercel.app/" + ) == [CHILDCARE] + assert keys("https://unrelated.vercel.app/") == [] + + +def test_other_pages_are_not_analyses(): + text = ( + "policyengine.org policyengine.org/uk policyengine.org/uk/research " + "policyengine.org/us/donate hello@policyengine.org " + "legacy.policyengine.org/uk/x " + "policyengine.org/us/year-in-review" + ) + assert keys(text) == [] + + +def test_tracking_links_and_shorteners_are_refused(): + _, refused = cmp.linked_analyses( + "https://policyengine.us5.list-manage.com/track/click?u=1 bit.ly/abc https://t.co/x", + registry(), + ) + assert len(refused) == 3 + _, refused = cmp.linked_analyses( + "https://policyengine.us5.list-manage.com/unsubscribe?u=1 " + "https://policyengine.us5.list-manage.com/profile?u=1", + registry(), + ) + assert refused == [] + + +def test_a_research_link_to_no_post_is_refused(): + _, refused = cmp.linked_analyses( + "policyengine.org/uk/research/no-such-post", registry() + ) + assert refused == ["policyengine.org/uk/research/no-such-post is not a post"] + + +def test_post_slug_is_the_websites(): + assert cmp.post_slug("UK_Two_Child_Limit.md") == "uk-two-child-limit" + assert cmp.post_slug("a.b.md") == "a" + + +# --- which entries are checked --- + + +def test_every_entry_is_checked_unless_it_may_declare_no_estimate(): + reg = registry() + checked = {k for k, e in reg.entries.items() if reg.publishes_estimate(e)} + assert ("post", "", "fuel-duty-freeze") in checked # an untagged costing + assert ("post", "", "giving-and-policy") in checked # policy wins + assert ("app", "uk", "marriage") in checked # an app never declares it + assert ("post", "", "api-v2") in checked + assert ("post", "", "giving-tuesday") not in checked + # an app neither listed with research nor tagged policy is not an entry + assert ("app", "us", "year-in-review") not in reg.entries + + +def test_a_post_with_no_country_tag_is_filed_under_the_default(): + assert registry().entries[("post", "", "api-v2")]["countries"] == ["us"] + + +def test_a_post_is_checked_against_the_record_its_block_names(): + reg = registry() + key = ("post", "", "joint-post") + entry = reg.entries[key] + analyses = {"analysis-uk-joint-post": record("post", "joint-post", [PUSH])} + assert cmp.record_for(key, entry, index(analyses))[0] == "analysis-uk-joint-post" + errors = cmp.check_edition( + "e.html", "policyengine.org/us/research/joint-post", PUSH, reg, index(analyses) + ) + assert errors == [] + + +def test_a_post_linked_under_another_country_is_still_checked(): + errors = cmp.check_edition( + "e.html", + "https://policyengine.org/us/research/childcare-expansion-2023", + PUSH, + registry(), + index({}), + ) + assert errors and "analysis-uk-childcare-expansion-2023" in errors[0] + + +def test_a_record_of_another_kind_or_slug_does_not_count(): + analyses = { + "analysis-uk-childcare-100k-threshold": record( + "post", "childcare-100k-threshold", [PUSH] ) - assert cmp.analysis_links(text) == [] + } + errors = cmp.check_edition( + "e.html", + "policyengine.org/uk/childcare-100k-threshold", + PUSH, + registry(), + index(analyses), + ) + assert errors and "no comparables record" in errors[0] - def test_post_slug_matches_the_website(self): - assert cmp.post_slug("UK_Two_Child_Limit.md") == "uk-two-child-limit" +def test_a_page_off_the_registry_is_held_to_its_scorecard_record(): + reg = cmp.Registry( + [a for a in APPS if a["slug"] != "childcare-100k-threshold"], POSTS, RULES + ) + link = "policyengine.org/uk/childcare-100k-threshold" + assert cmp.check_edition("e.html", link, PUSH, reg, index(STALE_CHILDCARE)) + assert cmp.check_edition("e.html", link, PUSH, reg, index(FRESH_CHILDCARE)) == [] + assert cmp.check_edition("e.html", link, PUSH, reg, index({})) == [] -class TestRegistry: - def test_research_entries_are_listed_apps_and_posts_by_country(self): - entries = cmp.research_entries(APPS, POSTS) - assert ("uk", "app", "childcare-100k-threshold") in entries - assert ("us", "app", "tool") not in entries # not listed with research - assert ("uk", "post", "uk-two-child-limit") in entries - def test_what_publishes_an_estimate(self): - entries = cmp.research_entries(APPS, POSTS) - assert cmp.publishes_estimate( - entries[("uk", "app", "childcare-100k-threshold")] - ) - assert not cmp.publishes_estimate(entries[("uk", "app", "marriage")]) - assert not cmp.publishes_estimate(entries[("us", "post", "giving-tuesday")]) - - -class TestFresh: - @pytest.mark.parametrize("window", [1, 3, 7, 14]) - def test_fresh_exactly_when_a_search_falls_in_the_window(self, window): - for offset in range(-20, 6): - searches = [shift(PUSH, offset)] - in_window = -window <= offset <= 0 - assert ( - cmp.fresh( - index_with(searches)["analyses"][ - "analysis-uk-childcare-100k-threshold" - ], - PUSH, - window, - ) - == [] - ) == in_window, (window, offset) - - def test_a_wider_window_is_never_stricter(self): - for offset in range(-20, 6): - entry = index_with([shift(PUSH, offset)])["analyses"][ - "analysis-uk-childcare-100k-threshold" - ] - for narrow in range(1, 15): - for wide in range(narrow, 15): - if cmp.fresh(entry, PUSH, narrow) == []: - assert cmp.fresh(entry, PUSH, wide) == [] - - def test_a_pending_gap_blocks_only_once_found(self): - for offset in range(-10, 6): - entry = index_with([PUSH], pending_found=[shift(PUSH, offset)])["analyses"][ - "analysis-uk-childcare-100k-threshold" - ] - assert bool(cmp.fresh(entry, PUSH, WINDOW)) == (offset <= 0) - - -class TestEdition: - def check(self, tmp_path, body, index): - path = edition(tmp_path, PUSH, body) - return cmp.check_edition( - path, path.read_text(), cmp.research_entries(APPS, POSTS), index - ) +def test_a_stale_alias_table_fails_the_check(): + reg = cmp.Registry([a for a in APPS if a["slug"] != "aca-calc"], POSTS, RULES) + assert reg.stale_aliases() == ["us/aca-reforms-calculator -> us/aca-calc"] + + +# --- freshness: the scorecard's rule --- - def test_fresh_record_passes(self, tmp_path): - assert self.check(tmp_path, CHILDCARE_LINK, index_with([shift(PUSH, -2)])) == [] - - def test_stale_record_fails(self, tmp_path): - errors = self.check(tmp_path, CHILDCARE_LINK, index_with([shift(PUSH, -12)])) - assert len(errors) == 1 - assert "run a prepromotion search" in errors[0] - - def test_missing_record_fails(self, tmp_path): - index = index_with([PUSH]) - index["analyses"] = {} - errors = self.check(tmp_path, CHILDCARE_LINK, index) - assert "has no comparables record" in errors[0] - - def test_a_record_of_another_kind_does_not_count(self, tmp_path): - errors = self.check(tmp_path, CHILDCARE_LINK, index_with([PUSH], kind="post")) - assert "has no comparables record" in errors[0] - - def test_links_without_an_estimate_are_not_checked(self, tmp_path): - body = ( - 'calc' - 'post' - 'tool' - ) - index = index_with([]) - index["analyses"] = {} - assert self.check(tmp_path, body, index) == [] - - def test_a_policy_post_needs_a_record(self, tmp_path): - body = 'p' - errors = self.check(tmp_path, body, index_with([PUSH])) - assert "analysis-uk-uk-two-child-limit" in errors[0] - - def test_an_undated_file_is_reported(self, tmp_path): - path = tmp_path / "draft.html" - path.write_text(CHILDCARE_LINK) - errors = cmp.check_edition(path, path.read_text(), {}, index_with([PUSH])) - assert "no YYYY-MM-DD date" in errors[0] - - -class TestRun: - def write_sources(self, tmp_path, index): - apps, posts, idx = (tmp_path / n for n in ("apps.json", "posts.json", "i.json")) - apps.write_text(json.dumps(APPS)) - posts.write_text(json.dumps(POSTS)) - idx.write_text(json.dumps(index)) - return str(apps), str(posts), str(idx) - - def test_nothing_before_the_cutover_is_checked_or_downloaded(self, tmp_path): - old = edition(tmp_path, shift(cmp.PROMOTION_CUTOVER_DATE, -1), CHILDCARE_LINK) - due, errors = cmp.run_check( - [old], "/nonexistent/apps", "/nonexistent/posts", "/nonexistent/index" - ) - assert (due, errors) == ([], []) - - def test_the_archive_is_never_checked(self): - archive = [p for p in EDITIONS.iterdir() if p.suffix in (".html", ".md")] - due = cmp.editions_to_check(archive, cmp.PROMOTION_CUTOVER_DATE) - assert all(cmp.edition_date(p) >= cmp.PROMOTION_CUTOVER_DATE for p in due), due - - def test_main_passes_and_fails(self, tmp_path, capsys): - path = edition(tmp_path, PUSH, CHILDCARE_LINK) - ok = self.write_sources(tmp_path, index_with([PUSH])) - argv = [str(path), "--apps", ok[0], "--posts", ok[1], "--index", ok[2]] - assert cmp.main(argv + ["--cutover", "2026-10-26"]) == 0 - stale = self.write_sources(tmp_path, index_with([shift(PUSH, -30)])) - argv = [str(path), "--apps", stale[0], "--posts", stale[1], "--index", stale[2]] - assert cmp.main(argv + ["--cutover", "2026-10-26"]) == 1 - assert "prepromotion" in capsys.readouterr().out - - def test_a_wrong_index_schema_fails(self, tmp_path): - path = edition(tmp_path, PUSH, CHILDCARE_LINK) - index = index_with([PUSH]) - index["schema"] = "something/else" - sources = self.write_sources(tmp_path, index) - due, errors = cmp.run_check([path], *sources, cutover="2026-10-26") - assert due and "is not comparables/index/1" in errors[0] - - def test_an_unreadable_source_fails_closed(self, tmp_path, monkeypatch): - monkeypatch.setattr(cmp.time, "sleep", lambda _s: None) - path = edition(tmp_path, PUSH, CHILDCARE_LINK) - with pytest.raises(SystemExit, match="could not read"): - cmp.run_check( - [path], - index_source="https://127.0.0.1:9/index.json", - cutover="2026-10-26", - ) +def test_fresh_agrees_with_the_scorecards_cases(): + cases = json.loads((FIXTURES / "comparables_fresh_cases.json").read_text()) + assert len(cases) > 50 + for case in cases: + assert (cmp.fresh(case["entry"], case["on"], case["window"]) == []) == case[ + "fresh" + ], case -class TestUploadRefusesStaleEditions: - def test_upload_stops_before_mailchimp(self, tmp_path, mocker): - path = edition(tmp_path, PUSH, CHILDCARE_LINK) - mocker.patch.object(cmp, "PROMOTION_CUTOVER_DATE", "2026-10-26") - sources = { - cmp.APPS_URL: APPS, - cmp.POSTS_URL: POSTS, - cmp.INDEX_URL: index_with([shift(PUSH, -30)]), - } - mocker.patch.object(cmp, "_load", side_effect=lambda where: sources[where]) - client = mocker.patch("newsletter_uploader.cli.MailchimpClient") - result = CliRunner().invoke( - cli_main, - [ - str(path), - "--audience", - "uk", - "--subject", - "s", - "--preview", - "p", - "--api-key", - "key-us5", - ], + +def test_a_pending_gap_with_no_found_on_counts_as_found(): + entry = record("app", "x", [PUSH]) + entry["comparisons"] = [{"claim": "c", "status": "pending"}] + assert cmp.fresh(entry, PUSH, WINDOW) + + +@settings(max_examples=200, deadline=None) +@given( + offsets=st.lists(st.integers(-60, 60), min_size=1, max_size=5), + window=st.integers(1, 30), + pending=st.one_of(st.none(), st.integers(-30, 30)), +) +def test_an_edition_passes_exactly_when_its_analysis_is_fresh(offsets, window, pending): + dates = sorted(shift(PUSH, o) for o in offsets) + rec = record( + "app", + "childcare-100k-threshold", + dates, + pending_found=() if pending is None else [shift(PUSH, pending)], + ) + idx = dict( + index({"analysis-uk-childcare-100k-threshold": rec}), + promotion_window_days=window, + ) + errors = cmp.check_edition( + "e.html", "policyengine.org/uk/childcare-100k-threshold", PUSH, registry(), idx + ) + in_window = any(-window <= o <= 0 for o in offsets) + blocked = pending is not None and pending <= 0 + assert (errors == []) == (in_window and not blocked) + + +def test_a_search_against_old_results_is_not_fresh(): + rec = record("app", "childcare-100k-threshold", [PUSH], pins_match=False) + errors = cmp.check_edition( + "e.html", + "policyengine.org/uk/childcare-100k-threshold", + PUSH, + registry(), + index({"analysis-uk-childcare-100k-threshold": rec}), + ) + assert errors and "compared the results the page reads now" in errors[0] + + +# --- the index's shape --- + + +def good_index(): + return index(FRESH_CHILDCARE) + + +@pytest.mark.parametrize( + "breaks", + [ + lambda i: i.update(schema="comparables/index/0"), + lambda i: i.pop("promotion_window_days"), + lambda i: i.update(promotion_window_days="7"), + lambda i: i.update(promotion_window_days=True), + lambda i: i.update(promotion_window_days=7.9), + lambda i: i.update(promotion_window_days=3650), + lambda i: i.update(promotion_window_days=0), + lambda i: i.pop("analyses"), + lambda i: i.update(analyses=[]), + lambda i: i.pop("entry_rules"), + lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].update( + searches=None + ), + lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].update( + searches=[] + ), + lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"]["searches"][ + 0 + ].update(date="soon"), + lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"]["searches"][ + 0 + ].pop("results_pins"), + lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].pop( + "current_pins" + ), + lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].update( + comparisons=[{"claim": "c"}] + ), + ], +) +def test_an_index_of_the_wrong_shape_fails_with_a_message(breaks, tmp_path): + idx = good_index() + breaks(idx) + assert cmp.check_index(idx) + path = tmp_path / "index.json" + path.write_text(json.dumps(idx)) + edition = tmp_path / f"{PUSH}-uk.html" + edition.write_text("policyengine.org/uk/childcare-100k-threshold", encoding="utf-8") + due, errors = cmp.run_check([edition], index_source=str(path), cutover="2026-10-26") + assert due == [edition] and errors + + +def test_a_good_index_passes(): + assert cmp.check_index(good_index()) == [] + + +# --- editions and their dates --- + + +@pytest.mark.parametrize( + "name, date", + [ + ("2026-11-02-uk.html", "2026-11-02"), + ("2026-11-02.html", "2026-11-02"), + ("2026-11-02_uk.html", "2026-11-02"), + ("2026-11-02-uk-social.md", "2026-11-02"), + ("uk-2026-11-02.html", None), + ("draft.html", None), + ("2026-11-31-uk.html", None), + ("2026-11-0212.html", None), + ], +) +def test_edition_dates(name, date): + assert cmp.edition_date(Path(name)) == date + + +def write_sources(tmp_path, analyses, apps=APPS, posts=POSTS): + for name, data in ( + ("apps.json", apps), + ("posts.json", posts), + ("index.json", index(analyses)), + ): + (tmp_path / name).write_text(json.dumps(data)) + return dict( + apps=str(tmp_path / "apps.json"), + posts=str(tmp_path / "posts.json"), + index_source=str(tmp_path / "index.json"), + ) + + +def make_edition(tmp_path, name, body="policyengine.org/uk/childcare-100k-threshold"): + path = tmp_path / name + path.write_text(f"{body}", encoding="utf-8") + return path + + +def test_an_undated_file_fails_where_dates_are_required(tmp_path): + sources = write_sources(tmp_path, FRESH_CHILDCARE) + for name in ("draft.html", "uk-2026-11-02.html", "2026-11-31-uk.html"): + path = make_edition(tmp_path, name) + due, errors = cmp.run_check([path], **sources, require_dates=True) + assert due == [] and errors and "no YYYY-MM-DD date" in errors[0] + assert cmp.run_check([path], **sources) == ([], []) + + +def test_nothing_before_the_cutover_is_checked_or_downloaded(tmp_path): + path = make_edition(tmp_path, "2026-10-25-uk.html") + missing = str(tmp_path / "missing.json") + assert cmp.run_check([path], missing, missing, missing, cutover="2026-10-26") == ( + [], + [], + ) + + +def test_an_upload_after_the_editions_date_is_judged_today(tmp_path): + sources = write_sources( + tmp_path, + { + "analysis-uk-childcare-100k-threshold": record( + "app", "childcare-100k-threshold", ["2026-11-20"] + ) + }, + ) + path = make_edition(tmp_path, "2026-10-25-uk.html") + # judged on its own date before the cutover: not due + assert cmp.run_check([path], **sources, cutover="2026-10-26")[0] == [] + # uploaded on 2026-11-21: due, and a search on the 20th counts + assert cmp.run_check( + [path], **sources, cutover="2026-10-26", today="2026-11-21" + ) == ([path], []) + # uploaded on 2026-12-21: the search is stale + due, errors = cmp.run_check( + [path], **sources, cutover="2026-10-26", today="2026-12-21" + ) + assert due == [path] and errors + + +def test_a_directory_is_scanned_for_every_edition_like_file(tmp_path): + (tmp_path / "sub").mkdir() + for name in ( + "2026-11-02-uk.HTML", + "sub/2026-11-03-uk.htm", + "2026-11-04-social.md", + "notes.txt", + ): + (tmp_path / name).write_text("x", encoding="utf-8") + names = sorted(p.name for p in cmp.edition_files([tmp_path])) + assert names == ["2026-11-02-uk.HTML", "2026-11-03-uk.htm", "2026-11-04-social.md"] + + +def test_main_passes_and_fails(tmp_path, capsys): + sources = write_sources(tmp_path, FRESH_CHILDCARE) + path = make_edition(tmp_path, f"{PUSH}-uk.html") + args = [ + str(path), + "--apps", + sources["apps"], + "--posts", + sources["posts"], + "--index", + sources["index_source"], + ] + assert cmp.main(args) == 0 + assert f"ok {path.name}" in capsys.readouterr().out + write_sources(tmp_path, STALE_CHILDCARE) + assert cmp.main(args) == 1 + out = capsys.readouterr().out + assert f"FAIL {path.name}" in out and "run a prepromotion search" in out + + +def test_main_prints_no_verdicts_when_the_index_is_bad(tmp_path, capsys): + sources = write_sources(tmp_path, FRESH_CHILDCARE) + bad = dict(index(FRESH_CHILDCARE), schema="nope") + Path(sources["index_source"]).write_text(json.dumps(bad)) + path = make_edition(tmp_path, f"{PUSH}-uk.html") + args = [ + str(path), + "--apps", + sources["apps"], + "--posts", + sources["posts"], + "--index", + sources["index_source"], + ] + assert cmp.main(args) == 1 + out = capsys.readouterr().out + assert "ok " not in out and "schema" in out + + +def test_an_unreadable_source_fails_closed(tmp_path, monkeypatch): + monkeypatch.setattr(cmp.time, "sleep", lambda _s: None) + + def refuse(*_a, **_k): + raise OSError("network down") + + monkeypatch.setattr(cmp.requests, "get", refuse) + path = make_edition(tmp_path, f"{PUSH}-uk.html") + with pytest.raises(SystemExit, match="could not read"): + cmp.run_check( + [path], + index_source="https://example.invalid/index.json", + cutover="2026-10-26", ) + + +# --- the upload command --- + + +def upload(path, mocker, extra=()): + client = mocker.patch("newsletter_uploader.cli.MailchimpClient") + result = CliRunner().invoke( + cli_main, + [ + str(path), + "--audience", + "uk", + "--subject", + "s", + "--preview", + "p", + "--api-key", + "k-us5", + *extra, + ], + ) + return result, client + + +@pytest.mark.parametrize("campaign", [(), ("--campaign-id", "abc")]) +def test_upload_refuses_a_stale_edition_before_mailchimp( + tmp_path, mocker, monkeypatch, campaign +): + sources = write_sources(tmp_path, STALE_CHILDCARE) + monkeypatch.setattr(cmp, "APPS_URL", sources["apps"]) + monkeypatch.setattr(cmp, "POSTS_URL", sources["posts"]) + monkeypatch.setattr(cmp, "INDEX_URL", sources["index_source"]) + path = make_edition(tmp_path, "2026-11-02-uk.html") + result, client = upload(path, mocker, campaign) + assert result.exit_code == 1 + assert "not fresh" in result.output + client.assert_not_called() + + +@pytest.mark.parametrize( + "name", ["draft.html", "2026-11-02_uk.html", "uk-2026-11-02.html"] +) +def test_upload_refuses_an_undated_file_before_mailchimp(tmp_path, mocker, name): + path = make_edition(tmp_path, name) + result, client = upload(path, mocker) + if cmp.edition_date(path) is None: assert result.exit_code == 1 - assert "Comparables are not fresh" in result.output + assert "no YYYY-MM-DD date" in result.output client.assert_not_called() + + +def test_upload_judges_an_old_edition_on_today(tmp_path, mocker, monkeypatch): + sources = write_sources(tmp_path, STALE_CHILDCARE) + monkeypatch.setattr(cmp, "APPS_URL", sources["apps"]) + monkeypatch.setattr(cmp, "POSTS_URL", sources["posts"]) + monkeypatch.setattr(cmp, "INDEX_URL", sources["index_source"]) + monkeypatch.setattr(cmp, "today_utc", lambda: "2026-11-30") + path = make_edition(tmp_path, "2026-10-01-uk.html") + result, client = upload(path, mocker) + assert result.exit_code == 1 + client.assert_not_called() From 810bc5d005e34d25cde7dae1aac0a9a8716fc254 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 13:58:48 -0400 Subject: [PATCH 3/5] Read where links lead from the site's own route map, and fail closed in CI Links are matched against policyengine-app-v2's analysisRoutes.json (every path, redirect and deployment host that leads to a research app) and the scorecard's records (each analysis's page and deployments), so tracker paths, country-wide zone routes, non-Vercel hosts, GitHub Pages projects and a delisted dashboard's own URL are all checked. URLs are normalised (dot segments, a trailing dot on the host). The index and route map are validated strictly, and every source is read at the commit its main branch points to. CI fails on an unreadable base, compares a new branch with the empty tree, and refuses social posts outside editions/. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 23 +- CLAUDE.md | 12 +- README.md | 2 +- src/newsletter_uploader/comparables.py | 525 +++++++++++++++---------- tests/conftest.py | 10 + tests/test_comparables.py | 495 ++++++++++++++++------- 6 files changed, 709 insertions(+), 358 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddbe2a9..7ec22fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,12 +46,29 @@ jobs: - name: Every analysis a changed edition links has a fresh comparables search env: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + # reads each source at the commit its main branch points to + GITHUB_TOKEN: ${{ github.token }} run: | + set -euo pipefail + # The commit to compare with: the pull request's base, or main before + # the push. A new branch's first push has no such commit, so its + # files are compared with the empty tree. A base the checkout + # cannot read fails the job rather than passing with nothing checked. if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then - BASE_SHA=$(git rev-list --max-parents=0 HEAD | tail -1) + BASE_SHA=$(git hash-object -t tree /dev/null) + elif ! git cat-file -e "$BASE_SHA^{commit}"; then + echo "::error::cannot read the base commit $BASE_SHA" + exit 1 fi - git diff --name-only --diff-filter=AMR "$BASE_SHA" HEAD -- 'editions/**' 'social-posts*.md' \ - | grep -iE '\.(html?|md)$' > "$RUNNER_TEMP/changed.txt" || true + git diff --name-only --diff-filter=AMR "$BASE_SHA" HEAD > "$RUNNER_TEMP/all-changed.txt" + # Social posts live in editions/, named --social.md; + # one anywhere else could not be checked. + if grep -iE '(^|/)[^/]*social[^/]*\.md$' "$RUNNER_TEMP/all-changed.txt" | grep -v '^editions/' > "$RUNNER_TEMP/misplaced.txt"; then + echo "::error::social posts go in editions/ as --social.md, so their comparables can be checked:" + cat "$RUNNER_TEMP/misplaced.txt" + exit 1 + fi + grep -iE '^editions/.*\.(html?|md)$' "$RUNNER_TEMP/all-changed.txt" > "$RUNNER_TEMP/changed.txt" || true echo "Changed editions:"; cat "$RUNNER_TEMP/changed.txt" if [ ! -s "$RUNNER_TEMP/changed.txt" ]; then echo "No edition changed; nothing to check." diff --git a/CLAUDE.md b/CLAUDE.md index 198562a..d999d68 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Repository Overview -This is a simple newsletter repository containing HTML email templates for PolicyEngine's Mailchimp campaigns. No build tools, testing frameworks, or development servers are needed - this is pure HTML/CSS. +This repository holds the HTML email templates for PolicyEngine's Mailchimp campaigns, and a Python package (`newsletter_uploader`) that uploads them and checks, before a send, that every analysis an edition links has a fresh comparables search. The editions themselves are plain HTML/CSS with no build step. ## Repository Structure @@ -57,7 +57,8 @@ All newsletter HTML files follow this pattern: for its send date: CI checks the editions a change adds or modifies on their dates, and `upload-newsletter` refuses an undated file and judges the edition on the later of its date and the day you upload. The same applies to the social posts kept beside - an edition. The check confirms the search is fresh; it does not put the comparable + an edition, which live in `editions/` as `--social.md` (CI fails a social + post anywhere else). The check confirms the search is fresh; it does not put the comparable in the edition. ## Email Compatibility @@ -106,7 +107,7 @@ https://raw.githubusercontent.com/PolicyEngine/newsletters/branch-name/assets/im ## Uploading to Mailchimp -This repo includes a Python package (`newsletter_uploader`) with full test coverage. +This repo includes a Python package (`newsletter_uploader`), tested with pytest. ### Installation @@ -163,6 +164,7 @@ import requests API_KEY = "your-api-key" SERVER = "us5" # from API key suffix +CAMPAIGN_ID = "duplicate-campaign-id" url = f"https://{SERVER}.api.mailchimp.com/3.0/campaigns/{CAMPAIGN_ID}" requests.delete(url, auth=("anystring", API_KEY)) ``` @@ -170,7 +172,7 @@ requests.delete(url, auth=("anystring", API_KEY)) ### Development ```bash -# Run tests (93% coverage) +# Run tests (with coverage: pytest --cov=newsletter_uploader) pytest -v # Format and lint @@ -180,7 +182,7 @@ ruff check src/ tests/ The package is structured with: - `src/newsletter_uploader/` - Core package modules -- `tests/` - Comprehensive test suite +- `tests/` - pytest suite - CI runs on GitHub Actions for Python 3.8-3.11 ## Claude Code Automations diff --git a/README.md b/README.md index 52f9d90..ed34346 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,6 @@ ruff check src/ tests/ - `audience.py` - Audience targeting logic - `uploader.py` - Newsletter uploader - `cli.py` - Command-line interface -- `tests/` - Test suite (93% coverage) +- `tests/` - pytest suite - `editions/` - Newsletter HTML files - `.env` - Contains `MAILCHIMP_API_KEY` (gitignored) diff --git a/src/newsletter_uploader/comparables.py b/src/newsletter_uploader/comparables.py index 624dc42..aa2e584 100644 --- a/src/newsletter_uploader/comparables.py +++ b/src/newsletter_uploader/comparables.py @@ -20,22 +20,28 @@ the rules the scorecard's index publishes (``entry_rules``): an app listed with research or tagged ``policy``, or any post, unless it is a post that is plainly not an analysis (an event, an announcement) and declares -``no_estimate``. A link is recognised in any common form: with or without a -scheme or ``www.``, in any case, HTML-escaped or percent-encoded, through -``/research/`` or ``/blog/``, with a dated post slug, through one of the -site's redirects to an analysis (``ALIASES``), or to the app's own host. -Tracking links and link shorteners are refused, because where they lead -cannot be checked. +``no_estimate``. Where a link leads is read from policyengine-app-v2's route +map (``website/src/data/analysisRoutes.json``: every path, redirect and +deployment host that leads to a research app, built from the site's own +routing) and from the scorecard's records (each analysis's page and the +``deployments`` it is also served from, which keep a dashboard taken off the +site from being promoted through its own URL). A link is recognised in any +common form: with or without a scheme or ``www.``, in any case, HTML-escaped +or percent-encoded, with dot segments or a trailing dot on the host, through +``/research/`` or ``/blog/`` or a dated post slug. Tracking links and link +shorteners are refused, because where they lead cannot be checked. An edition is a file whose name starts with its date, ``YYYY-MM-DD`` (``editions/2026-11-02-uk.html``, ``editions/2026-11-02.html``, and the -``.md`` social posts beside them). ``upload-newsletter`` refuses an undated -file and judges freshness on the later of the edition's date and the day it -runs; CI checks the editions a change adds or modifies, on their dates. Only -pushes on or after ``PROMOTION_CUTOVER_DATE`` are checked. The registries and -the scorecard's index are downloaded from their repositories' main branches -only when an edition is due; a failed download, or an index of the wrong -shape, fails the check. +``.md`` social posts beside them, ``editions/--social.md``). +``upload-newsletter`` refuses an undated file and judges freshness on the +later of the edition's date and the day it runs; CI checks the editions a +change adds or modifies, on their dates. Only pushes on or after +``PROMOTION_CUTOVER_DATE`` are checked. The registries, the route map and the +scorecard's index are read from their repositories' main branches, at the +commit main points to when the check runs (so a cached copy is never read), +only when an edition is due; a failed download, or data of the wrong shape, +fails the check. python -m newsletter_uploader.comparables editions python -m newsletter_uploader.comparables --require-dates editions/ @@ -50,48 +56,52 @@ import datetime as dt import html import json +import os +import posixpath import re import sys import time import urllib.parse from pathlib import Path -from typing import Dict, List, Optional, Sequence, Tuple +from typing import Dict, List, Optional, Sequence, Set, Tuple import requests PROMOTION_CUTOVER_DATE = "2026-10-26" +APP_V2 = "PolicyEngine/policyengine-app-v2" +SCORECARD = "PolicyEngine/policyengine-scorecard" APPS_URL = ( - "https://raw.githubusercontent.com/PolicyEngine/policyengine-app-v2/main/" - "app/src/data/apps/apps.json" + f"https://raw.githubusercontent.com/{APP_V2}/main/app/src/data/apps/apps.json" ) POSTS_URL = ( - "https://raw.githubusercontent.com/PolicyEngine/policyengine-app-v2/main/" - "app/src/data/posts/posts.json" + f"https://raw.githubusercontent.com/{APP_V2}/main/app/src/data/posts/posts.json" +) +ROUTES_URL = ( + f"https://raw.githubusercontent.com/{APP_V2}/main/" + "website/src/data/analysisRoutes.json" ) INDEX_URL = ( - "https://raw.githubusercontent.com/PolicyEngine/policyengine-scorecard/main/" - "data/comparables/index.json" + f"https://raw.githubusercontent.com/{SCORECARD}/main/data/comparables/index.json" ) INDEX_SCHEMA = "comparables/index/1" +ROUTES_SCHEMA = "analysis-routes/1" # The scorecard's policy keeps its window in this range too. MAX_WINDOW_DAYS = 90 +STATUSES = ("comparable", "constructed", "concept_mismatch", "pe_gap", "pending") EDITION_SUFFIXES = (".html", ".htm", ".md") +SITE_HOST = "policyengine.org" _EDITION = re.compile(r"^(\d{4}-\d{2}-\d{2})(?=[-_.]|$)") _DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") _DATED_SLUG = re.compile(r"^\d{4}-\d{2}-\d{2}-(.+)$") -_SITE = re.compile( - r"(?)\]]*)?", +# a host (a trailing dot allowed), with or without a scheme, and its path +_LINK = re.compile( + r"(? str: @@ -161,46 +157,39 @@ def _decode(text: str) -> str: return text -def _segments(path: str) -> List[str]: - return [s for s in path.split("/") if s] - +def normalize_host(host: str) -> str: + host = host.lower().rstrip(".") + return host[4:] if host.startswith("www.") else host -def _clean_slug(segment: str) -> Optional[str]: - match = re.match(r"^[a-z0-9_-]+", segment.lower()) - if not match: - return None - slug = match.group(0).rstrip("-_").replace("_", "-") - return slug or None - -def site_paths(text: str) -> List[List[str]]: - """The path segments of every policyengine.org link, lowercased, once each.""" - seen: List[List[str]] = [] - for match in _SITE.finditer(_decode(text)): - segments = [s.lower() for s in _segments(match.group(1) or "")] - if segments not in seen: - seen.append(segments) - return seen +def normalize_path(path: str) -> str: + """A path as the site routes it: lowercase, underscores as hyphens, + repeated slashes and dot segments resolved, trailing punctuation and + slash dropped.""" + path = re.sub(r"/+", "/", "/" + (path or "").lower()) + path = posixpath.normpath(path) + segments = [s.strip(".-_").replace("_", "-") for s in path.split("/")] + path = "/".join(s for s in segments if s) + return f"/{path}" if path else "/" -def other_links(text: str) -> List[Tuple[str, str]]: - """(host, path) of every link to another host, lowercased, once each.""" +def links(text: str) -> List[Tuple[str, str]]: + """(host, path) of every link in the text, normalised, once each.""" seen: List[Tuple[str, str]] = [] - for match in _URL.finditer(_decode(text)): - host = match.group(1).lower().rstrip(".") - if host == "policyengine.org" or host.endswith(".policyengine.org"): - continue - link = (host, (match.group(2) or "/").split("?")[0].split("#")[0].lower()) + for match in _LINK.finditer(_decode(text)): + link = (normalize_host(match.group(1)), normalize_path(match.group(2) or "/")) if link not in seen: seen.append(link) return seen -def _host(url: str) -> Tuple[str, str]: +def _host_path(url: str) -> Tuple[str, str]: parsed = urllib.parse.urlsplit(url) - return (parsed.hostname or "").lower(), (parsed.path or "/").rstrip( - "/" - ).lower() or "/" + return normalize_host(parsed.hostname or ""), normalize_path(parsed.path) + + +def _under(path: str, prefix: str) -> bool: + return prefix == "/" or path == prefix or path.startswith(prefix + "/") class Registry: @@ -209,28 +198,21 @@ class Registry: def __init__(self, apps: Sequence[dict], posts: Sequence[dict], rules: dict): self.rules = rules - countries = rules["country_tags"] - self.countries = set(countries) + self.countries = set(rules["country_tags"]) self.entries: Dict[Key, dict] = {} - self.hosts: Dict[str, List[Key]] = {} for app in apps: tags = app.get("tags") or [] if not (app.get("slug") and app.get("countryId")): continue if not (app.get("displayWithResearch") or "policy" in tags): continue - key: Key = ("app", app["countryId"], app["slug"]) - self.entries[key] = { + self.entries[("app", app["countryId"], app["slug"])] = { "kind": "app", "countries": [app["countryId"]], "slug": app["slug"], "tags": tags, "comparables": app.get("comparables"), } - source = app.get("source") or "" - host, _ = _host(source) - if host.endswith(".vercel.app") or host.endswith(".github.io"): - self.hosts.setdefault(host, []).append(key) for post in posts: tags = post.get("tags") or [] slug = post_slug(post["filename"]) @@ -243,50 +225,6 @@ def __init__(self, apps: Sequence[dict], posts: Sequence[dict], rules: dict): "comparables": post.get("comparables"), } - def stale_aliases(self) -> List[str]: - return [ - f"{c}/{path} -> {tc}/{ts}" - for (c, path), (tc, ts) in ALIASES.items() - if ("app", tc, ts) not in self.entries - ] - - def from_path(self, segments: List[str]) -> Tuple[Optional[Key], Optional[str]]: - """The entry a policyengine.org path names, or why none is known.""" - if len(segments) < 2 or segments[0] not in self.countries: - return None, None - country = segments[0] - if segments[1] in ("research", "blog"): - slug = _clean_slug(segments[2]) if len(segments) > 2 else None - if slug is None: - return None, None - alias = ALIASES.get((country, f"research/{slug}")) - if alias: - return ("app",) + alias, None - if ("post", "", slug) in self.entries: - return ("post", "", slug), None - dated = _DATED_SLUG.match(slug) - if dated and ("post", "", dated.group(1)) in self.entries: - return ("post", "", dated.group(1)), None - return None, f"policyengine.org/{country}/research/{slug} is not a post" - slug = _clean_slug(segments[1]) - alias = ALIASES.get((country, slug or "")) - if alias: - return ("app",) + alias, None - if slug and ("app", country, slug) in self.entries: - return ("app", country, slug), None - return None, None - - def from_host(self, host: str) -> List[Key]: - """The research apps served from a host: the host in an app's source, - or one of its Vercel preview deployments (-git--...).""" - if host in self.hosts: - return self.hosts[host] - for known, keys in self.hosts.items(): - project = known.split(".")[0] - if host.endswith(".vercel.app") and host.startswith(f"{project}-git-"): - return keys - return [] - def may_declare_no_estimate(self, entry: dict) -> bool: tags = set(entry["tags"]) return ( @@ -304,17 +242,90 @@ def publishes_estimate(self, entry: dict) -> bool: ) -def linked_analyses(text: str, registry: Registry) -> Tuple[List[Key], List[str]]: - """Every research entry an edition links, and every link the check refuses.""" - keys: List[Key] = [] +class Routes: + """Where links lead: policyengine-app-v2's route map, and the pages and + deployments of every scorecard record.""" + + def __init__(self, routes: dict, registry: Registry, index: dict): + self.registry = registry + self.paths: List[Tuple[str, bool, Target]] = [] + self.hosts: List[Tuple[str, str, List[Target]]] = [] + for route in routes["paths"]: + key = self._entry(route["entry"]) + if key: + self.paths.append((route["path"], route["prefix"], ("entry", key))) + for route in routes["hosts"]: + keys = [k for k in (self._entry(e) for e in route["entries"]) if k] + if keys: + self.hosts.append( + (route["host"], route["pathPrefix"], [("entry", k) for k in keys]) + ) + # a record's own page and deployments lead to it, so a dashboard + # taken off the registry is still held to its record + for source, record in index["analyses"].items(): + target: Target = ("record", source) + host, path = _host_path(record.get("url") or "") + if host == SITE_HOST and path != "/": + self.paths.append((path, True, target)) + for url in record.get("deployments") or []: + d_host, d_path = _host_path(url) + project = d_path.split("/")[1] if d_path != "/" else "" + prefix = ( + f"/{project}" if d_host.endswith(".github.io") and project else "/" + ) + self.hosts.append((d_host, prefix, [target])) + + def _entry(self, key: str) -> Optional[Key]: + match = re.match(r"^app ([a-z]{2})/([a-z0-9-]+)$", key) + if not match: + return None + entry = ("app", match.group(1), match.group(2)) + return entry if entry in self.registry.entries else None + + def on_site(self, path: str) -> Tuple[List[Target], Optional[str]]: + """What a policyengine.org path leads to, or why it is refused.""" + segments = [s for s in path.split("/") if s] + if ( + len(segments) >= 3 + and segments[0] in self.registry.countries + and segments[1] in ("research", "blog") + ): + slug = segments[2] + for candidate in (slug, (_DATED_SLUG.match(slug) or [None, None])[1]): + if candidate and ("post", "", candidate) in self.registry.entries: + return [("entry", ("post", "", candidate))], None + found: List[Target] = [] + for route_path, prefix, target in self.paths: + if path == route_path or (prefix and path.startswith(route_path + "/")): + if target not in found: + found.append(target) + if found: + return found, None + if len(segments) >= 3 and segments[1] in ("research", "blog"): + return [], f"policyengine.org{path} is not a post" + return [], None + + def off_site(self, host: str, path: str) -> List[Target]: + """What a link to another host leads to: a deployment in the route map + or a record, or a Vercel preview of one (-git--...).""" + found: List[Target] = [] + for route_host, prefix, targets in self.hosts: + preview = ( + route_host.endswith(".vercel.app") + and host.endswith(".vercel.app") + and host.startswith(route_host.split(".")[0] + "-git-") + ) + if (host == route_host or preview) and _under(path, prefix): + found.extend(t for t in targets if t not in found) + return found + + +def linked(text: str, routes: Routes) -> Tuple[List[Target], List[str]]: + """Everything an edition links that the check applies to, and every link + it refuses.""" + targets: List[Target] = [] refused: List[str] = [] - for segments in site_paths(text): - key, problem = registry.from_path(segments) - if problem: - refused.append(problem) - if key and key not in keys: - keys.append(key) - for host, path in other_links(text): + for host, path in links(text): tracked = any( (host == h or host.endswith(f".{h}")) and path.startswith(prefix) for h, prefix in TRACKING @@ -327,10 +338,16 @@ def linked_analyses(text: str, registry: Registry) -> Tuple[List[Key], List[str] "link the policyengine.org page itself" ) continue - for key in registry.from_host(host): - if key not in keys: - keys.append(key) - return keys, refused + if host == SITE_HOST or host.endswith(f".{SITE_HOST}"): + if host != SITE_HOST: + continue # another PolicyEngine site (the legacy app, the API) + found, problem = routes.on_site(path) + if problem: + refused.append(problem) + else: + found = routes.off_site(host, path) + targets.extend(t for t in found if t not in targets) + return targets, refused def fresh(index_entry: dict, on: str, window_days: int) -> List[str]: @@ -367,6 +384,14 @@ def fresh(index_entry: dict, on: str, window_days: int) -> List[str]: return reasons +def _string_list(value: object, nonempty: bool = False) -> bool: + return ( + isinstance(value, list) + and all(isinstance(v, str) and v for v in value) + and (bool(value) or not nonempty) + ) + + def check_index(index: object) -> List[str]: """Every reason the downloaded index cannot be trusted.""" if not isinstance(index, dict): @@ -390,12 +415,12 @@ def check_index(index: object) -> List[str]: if ( not isinstance(rules, dict) or not all( - isinstance(rules.get(k), list) + _string_list(rules.get(k), nonempty=True) for k in ("analysis_tags", "no_estimate_tags", "country_tags") ) - or not isinstance(rules.get("default_country"), str) + or rules.get("default_country") not in (rules.get("country_tags") or []) ): - errors.append("the scorecard index publishes no entry_rules") + errors.append("the scorecard index publishes no complete entry_rules") analyses = index.get("analyses") if not isinstance(analyses, dict): return errors + ["the scorecard index has no analyses object"] @@ -411,22 +436,68 @@ def check_index(index: object) -> List[str]: for s in searches: if not isinstance(s, dict) or not is_date(s.get("date")): errors.append(f"{where} has a search with no real date") - elif not isinstance(s.get("results_pins"), list): + elif not _string_list(s.get("results_pins")): errors.append(f"{where} has a search with no results_pins") - if not isinstance(entry.get("current_pins"), list): + if not _string_list(entry.get("current_pins")): errors.append(f"{where} has no current_pins") - comparisons = entry.get("comparisons", []) - if not isinstance(comparisons, list) or not all( - isinstance(c, dict) and isinstance(c.get("status"), str) - for c in comparisons + if not isinstance(entry.get("url"), str) or not _string_list( + entry.get("deployments", []) ): - errors.append(f"{where} has a comparison with no status") + errors.append(f"{where} has no page url or a malformed deployments list") + comparisons = entry.get("comparisons", []) + if not isinstance(comparisons, list): + errors.append(f"{where} has no comparisons list") + continue + for c in comparisons: + if ( + not isinstance(c, dict) + or c.get("status") not in STATUSES + or not isinstance(c.get("claim"), str) + or ( + "found_on" in c + and c["found_on"] is not None + and not is_date(c["found_on"]) + ) + ): + errors.append( + f"{where} has a comparison with no claim, a status not in " + f"{STATUSES}, or a found_on that is not a date" + ) + break + return errors + + +def check_routes(routes: object) -> List[str]: + """Every reason the downloaded route map cannot be trusted.""" + if not isinstance(routes, dict) or routes.get("schema") != ROUTES_SCHEMA: + return [f"policyengine-app-v2's route map is not {ROUTES_SCHEMA}"] + errors = [] + paths, hosts = routes.get("paths"), routes.get("hosts") + if ( + not isinstance(paths, list) + or not paths + or not all( + isinstance(p, dict) + and isinstance(p.get("path"), str) + and p["path"].startswith("/") + and isinstance(p.get("prefix"), bool) + and isinstance(p.get("entry"), str) + for p in paths + ) + ): + errors.append("policyengine-app-v2's route map has malformed paths") + if not isinstance(hosts, list) or not all( + isinstance(h, dict) + and isinstance(h.get("host"), str) + and isinstance(h.get("pathPrefix"), str) + and _string_list(h.get("entries"), nonempty=True) + for h in hosts + ): + errors.append("policyengine-app-v2's route map has malformed hosts") return errors -def record_for( - key: Key, entry: dict, index: dict -) -> Tuple[Optional[str], Optional[dict]]: +def record_for(entry: dict, index: dict) -> Tuple[str, Optional[dict]]: """The scorecard record an entry names, or the one its name implies.""" analyses = index["analyses"] block = entry.get("comparables") @@ -445,51 +516,43 @@ def record_for( return candidates[0], None -def _page(key: Key) -> str: - kind, country, slug = key - return ( - f"policyengine.org/{country}/{slug}" - if kind == "app" - else f"policyengine.org/research/{slug}" - ) +def _page(target: Target, index: dict) -> str: + kind, ref = target + if kind == "record": + return str(index["analyses"][ref].get("url") or ref) + entry_kind, country, slug = ref # type: ignore[misc] + if entry_kind == "app": + return f"policyengine.org/{country}/{slug}" + return f"policyengine.org/research/{slug}" def check_edition( - name: str, text: str, on: str, registry: Registry, index: dict + name: str, text: str, on: str, routes: Routes, index: dict ) -> List[str]: """Every reason this edition may not go out on ``on``.""" window = index["promotion_window_days"] - keys, refused = linked_analyses(text, registry) + registry = routes.registry + targets, refused = linked(text, routes) errors = [f"{name}: {problem}" for problem in refused] - for key in keys: - entry = registry.entries[key] - if not registry.publishes_estimate(entry): - continue - source, recorded = record_for(key, entry, index) - if recorded is None: - errors.append( - f"{name}: {_page(key)} has no comparables record in the scorecard; " - f"run the comparables review ({source}) before sending" - ) - continue - for reason in fresh(recorded, on, window): - errors.append(f"{name}: {_page(key)}: {reason}") - # A page the registry no longer lists (taken down, or not yet listed) - # that still has a scorecard record is held to its record. - for segments in site_paths(text): - if len(segments) < 2 or segments[0] not in registry.countries: - continue - slug = _clean_slug(segments[1]) - source = f"analysis-{segments[0]}-{slug}" - recorded = index["analyses"].get(source) - if ( - slug is None - or ("app", segments[0], slug) in keys - or recorded is None - or recorded.get("kind") != "app" - ): + checked: Set[str] = set() + for target in targets: + page = _page(target, index) + if target[0] == "record": + source, recorded = str(target[1]), index["analyses"][target[1]] + else: + entry = registry.entries[target[1]] # type: ignore[index] + if not registry.publishes_estimate(entry): + continue + source, recorded = record_for(entry, index) + if recorded is None: + errors.append( + f"{name}: {page} has no comparables record in the scorecard; " + f"run the comparables review ({source}) before sending" + ) + continue + if source in checked: continue - page = f"policyengine.org/{segments[0]}/{slug}" + checked.add(source) for reason in fresh(recorded, on, window): errors.append(f"{name}: {page}: {reason}") return errors @@ -533,19 +596,49 @@ def due_editions( return due, errors -def _load(where: str) -> object: - if not where.startswith("https://"): - return json.loads(Path(where).read_text(encoding="utf-8")) +def _get(url: str, accept: Optional[str] = None) -> requests.Response: + headers = {"Accept": accept} if accept else {} + token = os.environ.get("GITHUB_TOKEN") + if token and url.startswith("https://api.github.com/"): + headers["Authorization"] = f"Bearer {token}" last_error: Optional[Exception] = None for attempt in range(4): try: - response = requests.get(where, timeout=30) + response = requests.get(url, timeout=30, headers=headers) response.raise_for_status() - return response.json() + return response except Exception as error: # noqa: BLE001 - every failure retries, then fails last_error = error time.sleep(2**attempt) - raise SystemExit(f"could not read {where}: {last_error}") + raise SystemExit(f"could not read {url}: {last_error}") + + +def _pinned(url: str, commits: Dict[str, str]) -> str: + """A raw main-branch URL at the commit main points to now, so a cached + copy of the file is never read.""" + match = _RAW_MAIN.match(url) + if not match: + return url + repo, path = match.groups() + if repo not in commits: + sha = _get( + f"https://api.github.com/repos/{repo}/commits/main", + accept="application/vnd.github.sha", + ).text.strip() + if not re.fullmatch(r"[0-9a-f]{40}", sha): + raise SystemExit(f"could not resolve {repo}'s main branch: {sha[:80]!r}") + commits[repo] = sha + return f"https://raw.githubusercontent.com/{repo}/{commits[repo]}/{path}" + + +def _load(where: str, commits: Optional[Dict[str, str]] = None) -> object: + if not where.startswith("https://"): + return json.loads(Path(where).read_text(encoding="utf-8")) + url = _pinned(where, commits if commits is not None else {}) + try: + return _get(url).json() + except ValueError as error: + raise SystemExit(f"{url} is not JSON: {error}") from error def run_check( @@ -556,43 +649,41 @@ def run_check( cutover: Optional[str] = None, today: Optional[str] = None, require_dates: bool = False, + routes_source: Optional[str] = None, ) -> Tuple[List[Path], List[str]]: """Check every edition due on or after the cutover. The sources and the cutover default to the module's ``APPS_URL``, ``POSTS_URL``, - ``INDEX_URL`` and ``PROMOTION_CUTOVER_DATE``, read when called. Returns - the editions checked and every error; downloads nothing when none is - due.""" - apps = apps or APPS_URL - posts = posts or POSTS_URL - index_source = index_source or INDEX_URL + ``ROUTES_URL``, ``INDEX_URL`` and ``PROMOTION_CUTOVER_DATE``, read when + called. Returns the editions checked and every error; downloads nothing + when none is due.""" due, errors = due_editions( files, cutover or PROMOTION_CUTOVER_DATE, today, require_dates ) if not due: return [], errors - index = _load(index_source) + commits: Dict[str, str] = {} + index = _load(index_source or INDEX_URL, commits) problems = check_index(index) + route_map = _load(routes_source or ROUTES_URL, commits) + problems += check_routes(route_map) if problems: return [path for path, _ in due], errors + problems - registry = Registry(_load(apps), _load(posts), index["entry_rules"]) - stale = registry.stale_aliases() - if stale: - return [path for path, _ in due], errors + [ - "the redirect table in comparables.py names an analysis the " - f"registry no longer lists: {a}" - for a in stale - ] + registry = Registry( + _load(apps or APPS_URL, commits), + _load(posts or POSTS_URL, commits), + index["entry_rules"], + ) + routes = Routes(route_map, registry, index) for path, on in due: text = path.read_text(encoding="utf-8") - errors.extend(check_edition(path.name, text, on, registry, index)) + errors.extend(check_edition(path.name, text, on, routes, index)) return [path for path, _ in due], errors HOW_TO_FIX = ( "Run the comparables review for each analysis above (the " "policyengine-comparables-review skill, search kind prepromotion), merge its " - "scorecard PR, then check again; raw.githubusercontent.com caches the index " - "for up to 5 minutes after a merge." + "scorecard PR, then check again." ) @@ -601,6 +692,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.add_argument("paths", nargs="*", default=["editions"]) parser.add_argument("--apps", default=None) parser.add_argument("--posts", default=None) + parser.add_argument("--routes", default=None) parser.add_argument("--index", default=None) parser.add_argument("--cutover", default=None) parser.add_argument( @@ -618,6 +710,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: args.index, args.cutover, require_dates=args.require_dates, + routes_source=args.routes, ) cutover = args.cutover or PROMOTION_CUTOVER_DATE if not due and not errors: diff --git a/tests/conftest.py b/tests/conftest.py index d58c364..61e5872 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,7 +38,17 @@ def offline_comparables(tmp_path_factory, monkeypatch): ) monkeypatch.setattr(comparables, "APPS_URL", str(root / "apps.json")) monkeypatch.setattr(comparables, "POSTS_URL", str(root / "posts.json")) + (root / "routes.json").write_text( + json.dumps( + { + "schema": comparables.ROUTES_SCHEMA, + "paths": [{"path": "/us/none", "prefix": True, "entry": "app us/none"}], + "hosts": [], + } + ) + ) monkeypatch.setattr(comparables, "INDEX_URL", str(root / "index.json")) + monkeypatch.setattr(comparables, "ROUTES_URL", str(root / "routes.json")) @pytest.fixture diff --git a/tests/test_comparables.py b/tests/test_comparables.py index e2b7729..ad22d90 100644 --- a/tests/test_comparables.py +++ b/tests/test_comparables.py @@ -2,15 +2,22 @@ Invariants: - every way of writing a link to a research entry finds it (Hypothesis): with - or without a scheme or www., in any case, escaped or encoded, with any - trailing punctuation, through /research/ or /blog/ or a dated slug, through - a redirect in ALIASES, or to the app's own host; + or without a scheme or www., in any case, escaped or encoded, with dot + segments, a trailing dot on the host or trailing punctuation, through + /research/ or /blog/ or a dated slug, through a path or redirect in + policyengine-app-v2's route map, or to a deployment host in it; - a post is found by its slug under any country; +- a scorecard record's own page and deployments lead to it, so an analysis + taken off the registry is still held to its record; +- a shared host (GitHub Pages, one deployment serving two countries) leads + only to the analyses under the linked path; - every research entry is checked unless it is a post that is plainly not an analysis and declares no_estimate; - a linked analysis passes exactly when the scorecard's fresh says so, on the cases the scorecard checks its own fresh against; -- an index of the wrong shape fails with a message, never a traceback; +- an index or route map of the wrong shape fails with a message, never a + traceback or a pass; +- sources are read at the commit their main branch points to; - an undated file fails wherever dates are required; an upload is judged on the later of its date and today, before Mailchimp is touched; - tracking links and shorteners are refused. @@ -23,7 +30,7 @@ import pytest from click.testing import CliRunner -from hypothesis import given, settings +from hypothesis import assume, given, settings from hypothesis import strategies as st from newsletter_uploader import comparables as cmp @@ -41,43 +48,56 @@ "search_kinds": ["prepublish", "backfill", "reharvest", "prepromotion"], } + +def app(country, slug, source="", listed=True, tags=("policy",), **extra): + return { + "slug": slug, + "countryId": country, + "displayWithResearch": listed, + "tags": [country, *tags], + "source": source, + **extra, + } + + APPS = [ - { - "slug": "childcare-100k-threshold", - "countryId": "uk", - "displayWithResearch": True, - "tags": ["uk", "featured", "policy", "interactives"], - "source": "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold", - }, - { - "slug": "scottish-budget-2026-27", - "countryId": "uk", - "displayWithResearch": True, - "tags": ["uk", "policy"], - "source": "https://post-scottish-budget-dashboard.vercel.app/", - }, - { - "slug": "obbba-households", - "countryId": "us", - "displayWithResearch": True, - "tags": ["us", "policy"], - "source": "https://obbba-household-by-household.vercel.app/us/obbba-households", - }, - {"slug": "aca-calc", "countryId": "us", "tags": ["us", "policy"]}, - { - "slug": "cliffwatch", - "countryId": "us", - "displayWithResearch": True, - "tags": ["us"], - }, - { - "slug": "marriage", - "countryId": "uk", - "displayWithResearch": True, - "tags": ["uk", "policy"], - "comparables": {"status": "no_estimate", "reason": "a household calculator"}, - }, - {"slug": "year-in-review", "countryId": "us", "tags": ["us", "featured"]}, + app( + "uk", + "childcare-100k-threshold", + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold", + ), + app( + "uk", + "scottish-budget-2026-27", + "https://post-scottish-budget-dashboard.vercel.app/", + ), + app( + "us", + "obbba-households", + "https://obbba-household-by-household.vercel.app/us/obbba-households", + ), + app("us", "aca-calc", listed=False), + app("us", "cliffwatch", "https://cliff-watch.vercel.app/"), + app( + "us", + "bill-tracker", + "https://policyengine--state-legislative-tracker.modal.run/", + ), + app("us", "california-wealth-tax", "/us/california-wealth-tax/embed"), + app("us", "tanf-calculator", "https://policyengine.github.io/tanf-calculator"), + app( + "uk", + "uk-salary-sacrifice-tool", + "https://policyengine.github.io/uk-salary-sacrifice-analysis", + ), + app( + "uk", + "marriage", + "https://marriage-zeta-beryl.vercel.app/us/marriage?country=uk", + comparables={"status": "no_estimate", "reason": "a household calculator"}, + ), + app("us", "marriage", "https://marriage-zeta-beryl.vercel.app/us/marriage"), + app("us", "year-in-review", listed=False, tags=("featured",)), ] POSTS = [ {"filename": "UK_Two_Child_Limit.md", "tags": ["uk", "policy"]}, @@ -108,18 +128,84 @@ ] -def shift(date, days): - return (dt.date.fromisoformat(date) + dt.timedelta(days=days)).isoformat() +def route(path, entry, prefix=True): + return {"path": path, "prefix": prefix, "entry": entry} + + +def host(name, entries, prefix="/"): + return {"host": name, "pathPrefix": prefix, "entries": entries} + +# policyengine-app-v2's route map for these apps, as its build would write it +ROUTE_MAP = { + "schema": cmp.ROUTES_SCHEMA, + "paths": [ + *( + route(f"/{a['countryId']}/{a['slug']}", f"app {a['countryId']}/{a['slug']}") + for a in APPS + if a["displayWithResearch"] or "policy" in a["tags"] + ), + route("/us/ob3-households", "app us/obbba-households"), + route("/us/obbba-household-explorer", "app us/obbba-households"), + route( + "/us/research/obbba-household-explorer", "app us/obbba-households", False + ), + route("/us/aca-reforms-calculator", "app us/aca-calc"), + route("/us/cliff-watch", "app us/cliffwatch"), + route("/uk/cliff-watch", "app us/cliffwatch"), + route("/uk/cliffwatch", "app us/cliffwatch"), + route("/us/state-legislative-tracker", "app us/bill-tracker"), + route("/uk/state-legislative-tracker", "app us/bill-tracker"), + route("/uk/bill-tracker", "app us/bill-tracker"), + ], + "hosts": [ + host( + "uk-childcare-100k-threshold.vercel.app", + ["app uk/childcare-100k-threshold"], + ), + host( + "post-scottish-budget-dashboard.vercel.app", + ["app uk/scottish-budget-2026-27"], + ), + host("obbba-household-by-household.vercel.app", ["app us/obbba-households"]), + host("cliff-watch.vercel.app", ["app us/cliffwatch"]), + host( + "policyengine--state-legislative-tracker.modal.run", ["app us/bill-tracker"] + ), + host("california-wealth-tax.vercel.app", ["app us/california-wealth-tax"]), + host("policyengine.github.io", ["app us/tanf-calculator"], "/tanf-calculator"), + host( + "policyengine.github.io", + ["app uk/uk-salary-sacrifice-tool"], + "/uk-salary-sacrifice-analysis", + ), + host( + "marriage-zeta-beryl.vercel.app", + ["app uk/marriage", "app us/marriage"], + "/us/marriage", + ), + ], +} -def registry(): - return cmp.Registry(APPS, POSTS, RULES) + +def shift(date, days): + return (dt.date.fromisoformat(date) + dt.timedelta(days=days)).isoformat() -def record(kind, slug, search_dates, pending_found=(), pins_match=True): +def record( + kind, + slug, + search_dates, + pending_found=(), + pins_match=True, + country="uk", + deployments=(), +): return { "kind": kind, "slug": slug, + "url": f"https://policyengine.org/{country}/{slug}", + "deployments": list(deployments), "current_pins": [PIN], "searches": [ { @@ -140,7 +226,7 @@ def index(analyses): return { "schema": cmp.INDEX_SCHEMA, "promotion_window_days": WINDOW, - "entry_rules": RULES, + "entry_rules": copy.deepcopy(RULES), "analyses": copy.deepcopy(analyses), } @@ -158,11 +244,20 @@ def index(analyses): } -def keys(text): - found, _ = cmp.linked_analyses(text, registry()) +def routes_for(apps=APPS, posts=POSTS, analyses=None): + registry = cmp.Registry(apps, posts, RULES) + return cmp.Routes(ROUTE_MAP, registry, index(analyses or {})) + + +def targets(text, **kwargs): + found, _ = cmp.linked(text, routes_for(**kwargs)) return found +def entry(key): + return ("entry", key) + + # --- recognising a link --- SCHEMES = ["https://", "http://", "//", "", "HTTPS://"] @@ -171,6 +266,7 @@ def keys(text): "www.policyengine.org", "PolicyEngine.org", "policyengine.org:443", + "policyengine.org.", ] TRAILS = [ "", @@ -182,8 +278,8 @@ def keys(text): ")", "**", "_", - "—", - "’", + "\u2014", + "\u2019", " ", "&utm=1", "|", @@ -192,8 +288,8 @@ def keys(text): ] -def written(path, scheme, host, trail, encode): - url = f"{scheme}{host}{path}{trail}" +def written(path, scheme, host_, trail, encode): + url = f"{scheme}{host_}{path}{trail}" if encode == "html": url = url.replace("/", "/").replace("-", "-") elif encode == "percent": @@ -201,13 +297,15 @@ def written(path, scheme, host, trail, encode): return f"

Read this now

" -@settings(max_examples=300, deadline=None) +@settings(max_examples=400, deadline=None) @given( target=st.sampled_from( [ ("/uk/childcare-100k-threshold", CHILDCARE), ("/UK/Childcare-100k-Threshold", CHILDCARE), ("//uk/childcare-100k-threshold", CHILDCARE), + ("/uk/./childcare-100k-threshold", CHILDCARE), + ("/uk/research/../childcare-100k-threshold", CHILDCARE), ("/uk/research/uk-two-child-limit", ("post", "", "uk-two-child-limit")), ("/uk/research/UK_Two_Child_Limit", ("post", "", "uk-two-child-limit")), ("/uk/blog/uk-two-child-limit", ("post", "", "uk-two-child-limit")), @@ -227,34 +325,103 @@ def written(path, scheme, host, trail, encode): ), ("/us/aca-reforms-calculator", ("app", "us", "aca-calc")), ("/us/cliff-watch", ("app", "us", "cliffwatch")), + ( + "/us/state-legislative-tracker/MN/mn-hf4621", + ("app", "us", "bill-tracker"), + ), + ("/uk/bill-tracker", ("app", "us", "bill-tracker")), + ("/uk/cliffwatch", ("app", "us", "cliffwatch")), ] ), scheme=st.sampled_from(SCHEMES), - host=st.sampled_from(HOSTS), + host_=st.sampled_from(HOSTS), trail=st.sampled_from(TRAILS), encode=st.sampled_from(["none", "html", "percent"]), ) def test_every_way_of_writing_a_link_finds_its_entry( - target, scheme, host, trail, encode + target, scheme, host_, trail, encode ): path, key = target - assert key in keys(written(path, scheme, host, trail, encode)) + # the site redirects that old research path exactly, not paths under it + assume(not (path.startswith("/us/research/obbba") and trail == "/deep/path")) + assert entry(key) in targets(written(path, scheme, host_, trail, encode)) def test_the_scottish_budget_social_post_link_is_found(): text = "Explore it at policyengine.org/uk/scottish-budget-2026-27 today." - assert keys(text) == [("app", "uk", "scottish-budget-2026-27")] + assert targets(text) == [entry(("app", "uk", "scottish-budget-2026-27"))] + + +@pytest.mark.parametrize( + "link, key", + [ + ( + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold", + CHILDCARE, + ), + ("uk-childcare-100k-threshold.vercel.app", CHILDCARE), + ( + "https://uk-childcare-100k-threshold-git-dev-policy-engine.vercel.app/", + CHILDCARE, + ), + ( + "https://policyengine--state-legislative-tracker.modal.run/", + ("app", "us", "bill-tracker"), + ), + ( + "https://california-wealth-tax.vercel.app/", + ("app", "us", "california-wealth-tax"), + ), + ( + "https://policyengine.github.io/tanf-calculator/", + ("app", "us", "tanf-calculator"), + ), + ], +) +def test_a_deployment_link_finds_its_app(link, key): + assert targets(link) == [entry(key)] + + +def test_a_shared_host_leads_only_to_the_analyses_under_the_path(): + assert targets("https://policyengine.github.io/uk-salary-sacrifice-analysis/") == [ + entry(("app", "uk", "uk-salary-sacrifice-tool")) + ] + assert targets("https://policyengine.github.io/plugin-blog/") == [] + assert targets("https://marriage-zeta-beryl.vercel.app/us/marriage") == [ + entry(("app", "uk", "marriage")), + entry(("app", "us", "marriage")), + ] + assert targets("https://unrelated.vercel.app/") == [] -def test_an_apps_own_host_and_its_previews_are_found(): - assert keys( - "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold" - ) == [CHILDCARE] - assert keys("uk-childcare-100k-threshold.vercel.app") == [CHILDCARE] - assert keys( - "https://uk-childcare-100k-threshold-git-dev-policy-engine.vercel.app/" - ) == [CHILDCARE] - assert keys("https://unrelated.vercel.app/") == [] +def test_an_analysis_off_the_registry_is_held_to_its_record_on_every_link(): + apps = [a for a in APPS if a["slug"] != "childcare-100k-threshold"] + deployed = { + "analysis-uk-childcare-100k-threshold": record( + "app", + "childcare-100k-threshold", + ["2026-01-01"], + deployments=[ + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold" + ], + ) + } + routes = cmp.Routes( + { + **ROUTE_MAP, + "hosts": [h for h in ROUTE_MAP["hosts"] if "childcare" not in h["host"]], + "paths": [p for p in ROUTE_MAP["paths"] if "childcare" not in p["path"]], + }, + cmp.Registry(apps, POSTS, RULES), + index(deployed), + ) + for link in ( + "policyengine.org/uk/childcare-100k-threshold", + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold", + "https://uk-childcare-100k-threshold.vercel.app/", + ): + errors = cmp.check_edition("e.html", link, PUSH, routes, index(deployed)) + assert errors and "no comparables search between" in errors[0], link def test_other_pages_are_not_analyses(): @@ -264,30 +431,36 @@ def test_other_pages_are_not_analyses(): "legacy.policyengine.org/uk/x " "policyengine.org/us/year-in-review" ) - assert keys(text) == [] + assert targets(text) == [] def test_tracking_links_and_shorteners_are_refused(): - _, refused = cmp.linked_analyses( - "https://policyengine.us5.list-manage.com/track/click?u=1 bit.ly/abc https://t.co/x", - registry(), + routes = routes_for() + _, refused = cmp.linked( + "https://policyengine.us5.list-manage.com/track/click?u=1 bit.ly/abc " + "https://t.co/x https://policyengine.us5.list-manage.com./track/click?u=2", + routes, ) - assert len(refused) == 3 - _, refused = cmp.linked_analyses( + assert len(refused) == 3 # the trailing-dot host is the same link + _, refused = cmp.linked( "https://policyengine.us5.list-manage.com/unsubscribe?u=1 " "https://policyengine.us5.list-manage.com/profile?u=1", - registry(), + routes, ) assert refused == [] def test_a_research_link_to_no_post_is_refused(): - _, refused = cmp.linked_analyses( - "policyengine.org/uk/research/no-such-post", registry() - ) + _, refused = cmp.linked("policyengine.org/uk/research/no-such-post", routes_for()) assert refused == ["policyengine.org/uk/research/no-such-post is not a post"] +def test_paths_normalise_like_the_site(): + assert cmp.normalize_path("/uk/research/../x.") == "/uk/x" + assert cmp.normalize_path("//UK/./Two_Child") == "/uk/two-child" + assert cmp.normalize_host("WWW.PolicyEngine.org.") == "policyengine.org" + + def test_post_slug_is_the_websites(): assert cmp.post_slug("UK_Two_Child_Limit.md") == "uk-two-child-limit" assert cmp.post_slug("a.b.md") == "a" @@ -297,7 +470,7 @@ def test_post_slug_is_the_websites(): def test_every_entry_is_checked_unless_it_may_declare_no_estimate(): - reg = registry() + reg = cmp.Registry(APPS, POSTS, RULES) checked = {k for k, e in reg.entries.items() if reg.publishes_estimate(e)} assert ("post", "", "fuel-duty-freeze") in checked # an untagged costing assert ("post", "", "giving-and-policy") in checked # policy wins @@ -309,17 +482,19 @@ def test_every_entry_is_checked_unless_it_may_declare_no_estimate(): def test_a_post_with_no_country_tag_is_filed_under_the_default(): - assert registry().entries[("post", "", "api-v2")]["countries"] == ["us"] + reg = cmp.Registry(APPS, POSTS, RULES) + assert reg.entries[("post", "", "api-v2")]["countries"] == ["us"] def test_a_post_is_checked_against_the_record_its_block_names(): - reg = registry() - key = ("post", "", "joint-post") - entry = reg.entries[key] analyses = {"analysis-uk-joint-post": record("post", "joint-post", [PUSH])} - assert cmp.record_for(key, entry, index(analyses))[0] == "analysis-uk-joint-post" + routes = routes_for(analyses=analyses) errors = cmp.check_edition( - "e.html", "policyengine.org/us/research/joint-post", PUSH, reg, index(analyses) + "e.html", + "policyengine.org/us/research/joint-post", + PUSH, + routes, + index(analyses), ) assert errors == [] @@ -329,7 +504,7 @@ def test_a_post_linked_under_another_country_is_still_checked(): "e.html", "https://policyengine.org/us/research/childcare-expansion-2023", PUSH, - registry(), + routes_for(), index({}), ) assert errors and "analysis-uk-childcare-expansion-2023" in errors[0] @@ -343,29 +518,14 @@ def test_a_record_of_another_kind_or_slug_does_not_count(): } errors = cmp.check_edition( "e.html", - "policyengine.org/uk/childcare-100k-threshold", + "https://uk-childcare-100k-threshold.vercel.app/", PUSH, - registry(), + routes_for(analyses={}), index(analyses), ) assert errors and "no comparables record" in errors[0] -def test_a_page_off_the_registry_is_held_to_its_scorecard_record(): - reg = cmp.Registry( - [a for a in APPS if a["slug"] != "childcare-100k-threshold"], POSTS, RULES - ) - link = "policyengine.org/uk/childcare-100k-threshold" - assert cmp.check_edition("e.html", link, PUSH, reg, index(STALE_CHILDCARE)) - assert cmp.check_edition("e.html", link, PUSH, reg, index(FRESH_CHILDCARE)) == [] - assert cmp.check_edition("e.html", link, PUSH, reg, index({})) == [] - - -def test_a_stale_alias_table_fails_the_check(): - reg = cmp.Registry([a for a in APPS if a["slug"] != "aca-calc"], POSTS, RULES) - assert reg.stale_aliases() == ["us/aca-reforms-calculator -> us/aca-calc"] - - # --- freshness: the scorecard's rule --- @@ -379,9 +539,9 @@ def test_fresh_agrees_with_the_scorecards_cases(): def test_a_pending_gap_with_no_found_on_counts_as_found(): - entry = record("app", "x", [PUSH]) - entry["comparisons"] = [{"claim": "c", "status": "pending"}] - assert cmp.fresh(entry, PUSH, WINDOW) + rec = record("app", "x", [PUSH]) + rec["comparisons"] = [{"claim": "c", "status": "pending"}] + assert cmp.fresh(rec, PUSH, WINDOW) @settings(max_examples=200, deadline=None) @@ -398,12 +558,14 @@ def test_an_edition_passes_exactly_when_its_analysis_is_fresh(offsets, window, p dates, pending_found=() if pending is None else [shift(PUSH, pending)], ) - idx = dict( - index({"analysis-uk-childcare-100k-threshold": rec}), - promotion_window_days=window, - ) + analyses = {"analysis-uk-childcare-100k-threshold": rec} + idx = dict(index(analyses), promotion_window_days=window) errors = cmp.check_edition( - "e.html", "policyengine.org/uk/childcare-100k-threshold", PUSH, registry(), idx + "e.html", + "policyengine.org/uk/childcare-100k-threshold", + PUSH, + routes_for(analyses=analyses), + idx, ) in_window = any(-window <= o <= 0 for o in offsets) blocked = pending is not None and pending <= 0 @@ -412,23 +574,27 @@ def test_an_edition_passes_exactly_when_its_analysis_is_fresh(offsets, window, p def test_a_search_against_old_results_is_not_fresh(): rec = record("app", "childcare-100k-threshold", [PUSH], pins_match=False) + analyses = {"analysis-uk-childcare-100k-threshold": rec} errors = cmp.check_edition( "e.html", "policyengine.org/uk/childcare-100k-threshold", PUSH, - registry(), - index({"analysis-uk-childcare-100k-threshold": rec}), + routes_for(analyses=analyses), + index(analyses), ) assert errors and "compared the results the page reads now" in errors[0] -# --- the index's shape --- +# --- the index's and route map's shape --- def good_index(): return index(FRESH_CHILDCARE) +A = "analysis-uk-childcare-100k-threshold" + + @pytest.mark.parametrize( "breaks", [ @@ -442,23 +608,25 @@ def good_index(): lambda i: i.pop("analyses"), lambda i: i.update(analyses=[]), lambda i: i.pop("entry_rules"), - lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].update( - searches=None - ), - lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].update( - searches=[] + lambda i: i["entry_rules"].update(country_tags=[]), + lambda i: i["entry_rules"].update(default_country="xx"), + lambda i: i["analyses"][A].update(searches=None), + lambda i: i["analyses"][A].update(searches=[]), + lambda i: i["analyses"][A]["searches"][0].update(date="soon"), + lambda i: i["analyses"][A]["searches"][0].pop("results_pins"), + lambda i: i["analyses"][A].pop("current_pins"), + lambda i: i["analyses"][A].update(current_pins=[{}, "x"]), + lambda i: i["analyses"][A].pop("url"), + lambda i: i["analyses"][A].update(deployments="x"), + lambda i: i["analyses"][A].update(comparisons=[{"claim": "c"}]), + lambda i: i["analyses"][A].update( + comparisons=[{"status": "pending", "found_on": "2026-11-02"}] ), - lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"]["searches"][ - 0 - ].update(date="soon"), - lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"]["searches"][ - 0 - ].pop("results_pins"), - lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].pop( - "current_pins" + lambda i: i["analyses"][A].update( + comparisons=[{"claim": "c", "status": "PENDING"}] ), - lambda i: i["analyses"]["analysis-uk-childcare-100k-threshold"].update( - comparisons=[{"claim": "c"}] + lambda i: i["analyses"][A].update( + comparisons=[{"claim": "c", "status": "pending", "found_on": "9999"}] ), ], ) @@ -466,11 +634,10 @@ def test_an_index_of_the_wrong_shape_fails_with_a_message(breaks, tmp_path): idx = good_index() breaks(idx) assert cmp.check_index(idx) - path = tmp_path / "index.json" - path.write_text(json.dumps(idx)) - edition = tmp_path / f"{PUSH}-uk.html" - edition.write_text("policyengine.org/uk/childcare-100k-threshold", encoding="utf-8") - due, errors = cmp.run_check([edition], index_source=str(path), cutover="2026-10-26") + sources = write_sources(tmp_path, FRESH_CHILDCARE) + Path(sources["index_source"]).write_text(json.dumps(idx)) + edition = make_edition(tmp_path, f"{PUSH}-uk.html") + due, errors = cmp.run_check([edition], **sources, cutover="2026-10-26") assert due == [edition] and errors @@ -478,6 +645,60 @@ def test_a_good_index_passes(): assert cmp.check_index(good_index()) == [] +@pytest.mark.parametrize( + "routes", + [ + {"schema": "x", "paths": [], "hosts": []}, + {**ROUTE_MAP, "paths": []}, + {**ROUTE_MAP, "paths": [{"path": "uk/x", "prefix": True, "entry": "app uk/x"}]}, + { + **ROUTE_MAP, + "hosts": [{"host": "a.vercel.app", "pathPrefix": "/", "entries": []}], + }, + [], + ], +) +def test_a_route_map_of_the_wrong_shape_fails(routes): + assert cmp.check_routes(routes) + + +def test_sources_are_read_at_the_commit_main_points_to(monkeypatch): + calls = [] + + class Response: + def __init__(self, text): + self.text = text + + def raise_for_status(self): + pass + + def json(self): + return {"ok": True} + + def get(url, timeout, headers): + calls.append((url, headers)) + return Response("c" * 40 if "api.github.com" in url else "{}") + + monkeypatch.setattr(cmp.requests, "get", get) + monkeypatch.setenv("GITHUB_TOKEN", "t") + commits = {} + url = ( + "https://raw.githubusercontent.com/PolicyEngine/policyengine-app-v2/main/a.json" + ) + cmp._load(url, commits) + cmp._load(url.replace("a.json", "b.json"), commits) + api = [c for c in calls if "api.github.com" in c[0]] + assert len(api) == 1 and api[0][1]["Authorization"] == "Bearer t" + assert calls[-1][0].endswith(f"/{'c' * 40}/b.json") + + monkeypatch.setattr(cmp.time, "sleep", lambda _s: None) + monkeypatch.setattr( + cmp.requests, "get", lambda url, timeout, headers: Response("not a sha") + ) + with pytest.raises(SystemExit, match="could not resolve"): + cmp._load(url, {}) + + # --- editions and their dates --- @@ -503,12 +724,14 @@ def write_sources(tmp_path, analyses, apps=APPS, posts=POSTS): ("apps.json", apps), ("posts.json", posts), ("index.json", index(analyses)), + ("routes.json", ROUTE_MAP), ): (tmp_path / name).write_text(json.dumps(data)) return dict( apps=str(tmp_path / "apps.json"), posts=str(tmp_path / "posts.json"), index_source=str(tmp_path / "index.json"), + routes_source=str(tmp_path / "routes.json"), ) @@ -583,6 +806,8 @@ def test_main_passes_and_fails(tmp_path, capsys): sources["posts"], "--index", sources["index_source"], + "--routes", + sources["routes_source"], ] assert cmp.main(args) == 0 assert f"ok {path.name}" in capsys.readouterr().out @@ -605,6 +830,8 @@ def test_main_prints_no_verdicts_when_the_index_is_bad(tmp_path, capsys): sources["posts"], "--index", sources["index_source"], + "--routes", + sources["routes_source"], ] assert cmp.main(args) == 1 out = capsys.readouterr().out @@ -658,6 +885,7 @@ def test_upload_refuses_a_stale_edition_before_mailchimp( monkeypatch.setattr(cmp, "APPS_URL", sources["apps"]) monkeypatch.setattr(cmp, "POSTS_URL", sources["posts"]) monkeypatch.setattr(cmp, "INDEX_URL", sources["index_source"]) + monkeypatch.setattr(cmp, "ROUTES_URL", sources["routes_source"]) path = make_edition(tmp_path, "2026-11-02-uk.html") result, client = upload(path, mocker, campaign) assert result.exit_code == 1 @@ -682,6 +910,7 @@ def test_upload_judges_an_old_edition_on_today(tmp_path, mocker, monkeypatch): monkeypatch.setattr(cmp, "APPS_URL", sources["apps"]) monkeypatch.setattr(cmp, "POSTS_URL", sources["posts"]) monkeypatch.setattr(cmp, "INDEX_URL", sources["index_source"]) + monkeypatch.setattr(cmp, "ROUTES_URL", sources["routes_source"]) monkeypatch.setattr(cmp, "today_utc", lambda: "2026-11-30") path = make_edition(tmp_path, "2026-10-01-uk.html") result, client = upload(path, mocker) From 8f6c0408b822fa15c1114caa3dc15fe7c862143e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 14 Nov 2026 01:08:24 -0500 Subject: [PATCH 4/5] Close routing and artifact validation gaps before promotion Preserve changed filenames through CI, resolve complete destination URLs, and scope shared deployments by path and country. Validate all consumed source artifacts so malformed data cannot remove coverage, with regressions for every round-three finding. Co-Authored-By: Codex --- .github/workflows/ci.yml | 19 +- scripts/select_changed_editions.py | 41 +++ src/newsletter_uploader/comparables.py | 450 ++++++++++++++++++++++--- tests/conftest.py | 12 +- tests/test_comparables.py | 4 +- tests/test_comparables_routing.py | 201 +++++++++++ tests/test_comparables_validation.py | 244 ++++++++++++++ tests/test_comparables_workflow.py | 193 +++++++++++ uv.lock | 307 ++++++++++++++++- 9 files changed, 1388 insertions(+), 83 deletions(-) create mode 100644 scripts/select_changed_editions.py create mode 100644 tests/test_comparables_routing.py create mode 100644 tests/test_comparables_validation.py create mode 100644 tests/test_comparables_workflow.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ec22fc..4b2b93b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,21 +60,16 @@ jobs: echo "::error::cannot read the base commit $BASE_SHA" exit 1 fi - git diff --name-only --diff-filter=AMR "$BASE_SHA" HEAD > "$RUNNER_TEMP/all-changed.txt" - # Social posts live in editions/, named --social.md; - # one anywhere else could not be checked. - if grep -iE '(^|/)[^/]*social[^/]*\.md$' "$RUNNER_TEMP/all-changed.txt" | grep -v '^editions/' > "$RUNNER_TEMP/misplaced.txt"; then - echo "::error::social posts go in editions/ as --social.md, so their comparables can be checked:" - cat "$RUNNER_TEMP/misplaced.txt" - exit 1 - fi - grep -iE '^editions/.*\.(html?|md)$' "$RUNNER_TEMP/all-changed.txt" > "$RUNNER_TEMP/changed.txt" || true - echo "Changed editions:"; cat "$RUNNER_TEMP/changed.txt" - if [ ! -s "$RUNNER_TEMP/changed.txt" ]; then + git diff --name-only -z --diff-filter=AMR "$BASE_SHA" HEAD > "$RUNNER_TEMP/all-changed" + # Keep Git's NUL delimiters through selection and checker invocation, + # so Unicode and control characters cannot hide a changed edition. + # The selector also rejects social posts outside editions/. + python scripts/select_changed_editions.py "$RUNNER_TEMP/all-changed" "$RUNNER_TEMP/changed" + if [ ! -s "$RUNNER_TEMP/changed" ]; then echo "No edition changed; nothing to check." exit 0 fi - xargs -d '\n' python -m newsletter_uploader.comparables --require-dates < "$RUNNER_TEMP/changed.txt" + xargs -0 python -m newsletter_uploader.comparables --require-dates < "$RUNNER_TEMP/changed" test: runs-on: ubuntu-latest diff --git a/scripts/select_changed_editions.py b/scripts/select_changed_editions.py new file mode 100644 index 0000000..2913945 --- /dev/null +++ b/scripts/select_changed_editions.py @@ -0,0 +1,41 @@ +"""Select changed editions without decoding Git's NUL-delimited filenames.""" + +import os +import re +import sys +from pathlib import Path + + +def main(): + """Reject misplaced social posts and write selected names as NUL-delimited bytes.""" + names = [name for name in Path(sys.argv[1]).read_bytes().split(b"\0") if name] + misplaced = [ + name + for name in names + if re.search(rb"(^|/)[^/]*social[^/]*\.md$", name, re.IGNORECASE) + and not name.startswith(b"editions/") + ] + if misplaced: + print( + "::error::social posts go in editions/ as --social.md, " + "so their comparables can be checked:" + ) + for name in misplaced: + print(repr(os.fsdecode(name))) + return 1 + + selected = [ + name + for name in names + if name.lower().startswith(b"editions/") + and name.lower().endswith((b".html", b".htm", b".md")) + ] + Path(sys.argv[2]).write_bytes(b"".join(name + b"\0" for name in selected)) + print("Changed editions:") + for name in selected: + print(repr(os.fsdecode(name))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/newsletter_uploader/comparables.py b/src/newsletter_uploader/comparables.py index aa2e584..7e6e3a2 100644 --- a/src/newsletter_uploader/comparables.py +++ b/src/newsletter_uploader/comparables.py @@ -25,7 +25,10 @@ deployment host that leads to a research app, built from the site's own routing) and from the scorecard's records (each analysis's page and the ``deployments`` it is also served from, which keep a dashboard taken off the -site from being promoted through its own URL). A link is recognised in any +site from being promoted through its own URL). For a shared deployment, the +pinned registry's source URLs narrow the map's candidates by path and country +query, including the default country. Ambiguous destinations are refused. +A link is recognised in any common form: with or without a scheme or ``www.``, in any case, HTML-escaped or percent-encoded, with dot segments or a trailing dot on the host, through ``/research/`` or ``/blog/`` or a dated post slug. Tracking links and link @@ -55,6 +58,7 @@ import argparse import datetime as dt import html +import ipaddress import json import os import posixpath @@ -67,6 +71,8 @@ import requests +from .hosting_links import ATTRIBUTE, URL_ATTRIBUTES + PROMOTION_CUTOVER_DATE = "2026-10-26" APP_V2 = "PolicyEngine/policyengine-app-v2" @@ -95,10 +101,15 @@ _EDITION = re.compile(r"^(\d{4}-\d{2}-\d{2})(?=[-_.]|$)") _DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") _DATED_SLUG = re.compile(r"^\d{4}-\d{2}-\d{2}-(.+)$") -# a host (a trailing dot allowed), with or without a scheme, and its path +# Complete absolute links must be parsed before scanning bare domains: userinfo +# and backslashes affect their browser destination, rather than ending a link. +_ABSOLUTE_LINK = re.compile( + r"(?:https?(?::|%(?:25){0,2}3a)[^\s<>\"']+|//[^\s<>\"']+)", + re.IGNORECASE, +) _LINK = re.compile( - r"(?\"']*)?", re.IGNORECASE, ) _RAW_MAIN = re.compile(r"^https://raw\.githubusercontent\.com/([^/]+/[^/]+)/main/(.+)$") @@ -158,7 +169,7 @@ def _decode(text: str) -> str: def normalize_host(host: str) -> str: - host = host.lower().rstrip(".") + host = host.encode("idna").decode("ascii").lower().rstrip(".") return host[4:] if host.startswith("www.") else host @@ -173,19 +184,87 @@ def normalize_path(path: str) -> str: return f"/{path}" if path else "/" -def links(text: str) -> List[Tuple[str, str]]: - """(host, path) of every link in the text, normalised, once each.""" - seen: List[Tuple[str, str]] = [] - for match in _LINK.finditer(_decode(text)): - link = (normalize_host(match.group(1)), normalize_path(match.group(2) or "/")) +def _link_url(url: str) -> Tuple[str, str, str]: + """Parse the browser destination of an HTTP(S) URL, preserving its query. + + Browsers treat backslashes before the query as slashes for these schemes. + urllib alone does not, so apply that rule before parsing its authority. + """ + url = html.unescape(url).strip().rstrip(".,;:!?)\u2014\u2019|*") + # Unwrap an encoded whole URL only until its structural delimiters are + # visible. Decoding a normal URL before splitting turns encoded ? or # + # inside its path into a false query or fragment boundary. + for _ in range(3): + if re.match(r"^(?:https?:|//)", url, re.IGNORECASE): + break + decoded = urllib.parse.unquote(url) + if decoded == url: + break + url = decoded + before_query, separator, query = url.partition("?") + url = before_query.replace("\\", "/") + separator + query + if url.startswith("//"): + url = "https:" + url + try: + parsed = urllib.parse.urlsplit(url) + if parsed.scheme.lower() not in ("http", "https") or not parsed.netloc: + raise ValueError("an absolute HTTP(S) URL needs a hostname") + host = parsed.hostname + parsed.port # Reject invalid ports instead of losing a destination. + if not host or any(c.isspace() for c in host): + raise ValueError("the hostname is missing or invalid") + host = normalize_host(_decode(host)) + if not _valid_hostname(host): + raise ValueError("the hostname is invalid") + except (ValueError, UnicodeError) as exc: + raise ValueError(f"unsupported absolute link {url!r}: {exc}") from exc + # Keep the complete path through dot-segment resolution, including Unicode + # segments. Truncating at an unfamiliar character can hide the destination + # of a later /../ segment. + path = normalize_path(_decode(parsed.path)) + # Some newsletter prose appends tracking fields without a question mark. + # Read that suffix only after resolving the complete browser path. + path = re.sub(r"&[a-z_][a-z0-9_-]*=.*$", "", path) + return host, path, parsed.query + + +def links(text: str) -> List[Tuple[str, str, str]]: + """Normalised (host, path, query) links, once each. + + Unsupported absolute URLs raise ValueError so the edition fails closed. + """ + seen: List[Tuple[str, str, str]] = [] + # Read complete URL attribute values before decoding percent-escapes. + # Encoded whitespace or quotes must not split a path before a later /../. + for attribute in ATTRIBUTE.finditer(text): + name, _, double, single, bare = attribute.groups() + if name.lower() not in URL_ATTRIBUTES: + continue + raw = next(value for value in (double, single, bare) if value is not None) + value = re.sub("[\t\n\r]", "", html.unescape(raw)).strip() + if re.match(r"^(?:https?(?::|%(?:25){0,2}3a)|//)", value, re.IGNORECASE): + link = _link_url(value) + if link not in seen: + seen.append(link) + text = html.unescape(text) + consumed: List[Tuple[int, int]] = [] + for match in _ABSOLUTE_LINK.finditer(text): + consumed.append(match.span()) + link = _link_url(match.group(0)) + if link not in seen: + seen.append(link) + for match in _LINK.finditer(text): + if any(start <= match.start() < end for start, end in consumed): + continue + link = _link_url("https://" + match.group(0)) if link not in seen: seen.append(link) return seen def _host_path(url: str) -> Tuple[str, str]: - parsed = urllib.parse.urlsplit(url) - return normalize_host(parsed.hostname or ""), normalize_path(parsed.path) + host, path, _ = _link_url(url) + return host, path def _under(path: str, prefix: str) -> bool: @@ -212,6 +291,7 @@ def __init__(self, apps: Sequence[dict], posts: Sequence[dict], rules: dict): "slug": app["slug"], "tags": tags, "comparables": app.get("comparables"), + "source": app.get("source"), } for post in posts: tags = post.get("tags") or [] @@ -243,13 +323,25 @@ def publishes_estimate(self, entry: dict) -> bool: class Routes: - """Where links lead: policyengine-app-v2's route map, and the pages and - deployments of every scorecard record.""" + """Where links lead: the route map supplies app membership; registry + sources disambiguate shared deployments; scorecard pages retain delisted + analyses. A shared host never implies ownership of every path on it. + """ def __init__(self, routes: dict, registry: Registry, index: dict): self.registry = registry + self.index = index self.paths: List[Tuple[str, bool, Target]] = [] - self.hosts: List[Tuple[str, str, List[Target]]] = [] + self.hosts: List[Tuple[str, str, bool, List[Target]]] = [] + self.sources: Dict[str, List[str]] = {} + owners: Dict[str, Set[str]] = {} + bindings: List[Tuple[str, str, List[Target], bool]] = [] + for key, research in registry.entries.items(): + source = research.get("source") + if isinstance(source, str) and source.startswith(("http://", "https://")): + self.sources.setdefault(self._identity(("entry", key)), []).append( + source + ) for route in routes["paths"]: key = self._entry(route["entry"]) if key: @@ -257,23 +349,63 @@ def __init__(self, routes: dict, registry: Registry, index: dict): for route in routes["hosts"]: keys = [k for k in (self._entry(e) for e in route["entries"]) if k] if keys: - self.hosts.append( - (route["host"], route["pathPrefix"], [("entry", k) for k in keys]) + bindings.append( + ( + route["host"], + route["pathPrefix"], + [("entry", k) for k in keys], + True, + ) ) - # a record's own page and deployments lead to it, so a dashboard - # taken off the registry is still held to its record + # Primary pages and additional deployments use the same registration + # path, including a standalone primary page with no deployments. for source, record in index["analyses"].items(): target: Target = ("record", source) - host, path = _host_path(record.get("url") or "") - if host == SITE_HOST and path != "/": - self.paths.append((path, True, target)) - for url in record.get("deployments") or []: - d_host, d_path = _host_path(url) - project = d_path.split("/")[1] if d_path != "/" else "" - prefix = ( - f"/{project}" if d_host.endswith(".github.io") and project else "/" + urls = [record["url"], *record.get("deployments", [])] + self.sources.setdefault(source, []).extend(urls) + for url in urls: + host, path = _host_path(url) + if host == SITE_HOST: + if path != "/": + self.paths.append((path, True, target)) + else: + bindings.append((host, path, [target], False)) + for host, _, targets, _ in bindings: + owners.setdefault(host, set()).update(self._identity(t) for t in targets) + for host, prefix, targets, from_map in bindings: + shared = len(owners[host]) > 1 + for target in targets: + boundary = prefix + if shared and from_map and prefix == "/": + # The map's broad host ownership becomes unsafe when a + # second source identifies another analysis on that host. + # Its pinned registry source supplies a narrower boundary. + paths = [p for _, p, _ in self._source_links(target, host)] + nonroot = [p for p in paths if p != "/"] + if nonroot: + for path in nonroot: + self.hosts.append((host, path, True, [target])) + continue + if host.endswith(".github.io") and boundary != "/": + boundary = "/" + boundary.split("/")[1] + elif not shared and not from_map and not host.endswith(".github.io"): + boundary = "/" + self.hosts.append( + (host, boundary, not (shared and boundary == "/"), [target]) ) - self.hosts.append((d_host, prefix, [target])) + + def _identity(self, target: Target) -> str: + if target[0] == "record": + return str(target[1]) + return record_for(self.registry.entries[target[1]], self.index)[0] + + def _source_links(self, target: Target, host: str) -> List[Tuple[str, str, str]]: + result = [] + for source in self.sources.get(self._identity(target), []): + link = _link_url(source) + if link[0] == host: + result.append(link) + return result def _entry(self, key: str) -> Optional[Key]: match = re.match(r"^app ([a-z]{2})/([a-z0-9-]+)$", key) @@ -305,18 +437,86 @@ def on_site(self, path: str) -> Tuple[List[Target], Optional[str]]: return [], f"policyengine.org{path} is not a post" return [], None - def off_site(self, host: str, path: str) -> List[Target]: - """What a link to another host leads to: a deployment in the route map - or a record, or a Vercel preview of one (-git--...).""" - found: List[Target] = [] - for route_host, prefix, targets in self.hosts: + def _country(self, target: Target) -> Optional[str]: + if target[0] == "entry": + countries = self.registry.entries[target[1]]["countries"] + return countries[0] if len(countries) == 1 else None + record = self.index["analyses"][target[1]] + _, path = _host_path(record["url"]) + country = path.split("/")[1] if path != "/" else "" + return country if country in self.registry.countries else None + + def off_site(self, host: str, path: str, query: str = "") -> List[Target]: + """Resolve a deployment, refusing ambiguous country/default routing.""" + matches: List[Tuple[int, Target, str]] = [] + for route_host, prefix, subtree, targets in self.hosts: preview = ( route_host.endswith(".vercel.app") and host.endswith(".vercel.app") and host.startswith(route_host.split(".")[0] + "-git-") ) - if (host == route_host or preview) and _under(path, prefix): - found.extend(t for t in targets if t not in found) + if (host == route_host or preview) and ( + path == prefix or (subtree and _under(path, prefix)) + ): + matches.extend((len(prefix), target, route_host) for target in targets) + if not matches: + return [] + # An explicit nested project path takes precedence over its parent. + most_specific = max(m[0] for m in matches) + candidates = [(t, h) for length, t, h in matches if length == most_specific] + identities = {self._identity(t) for t, _ in candidates} + if len(identities) > 1: + requested = urllib.parse.parse_qs(query, keep_blank_values=True).get( + "country" + ) + source_countries: Dict[str, Set[Optional[str]]] = {} + explicit = False + for target, route_host in candidates: + identity = self._identity(target) + countries = source_countries.setdefault(identity, set()) + for _, source_path, source_query in self._source_links( + target, route_host + ): + if not _under(path, source_path): + continue + values = urllib.parse.parse_qs( + source_query, keep_blank_values=True + ).get("country") + if values: + explicit = True + countries.update(values) + else: + countries.add(None) + # An absent source query identifies the default country. Its app + # country also recognises an explicit request for that default. + selected: Set[str] = set() + if ( + explicit + and all(source_countries.values()) + and (requested is None or len(set(requested)) == 1) + ): + wanted = requested[0] if requested else None + for target, _ in candidates: + identity = self._identity(target) + countries = source_countries[identity] + if wanted in countries or ( + wanted is not None + and None in countries + and wanted == self._country(target) + ): + selected.add(identity) + if len(selected) != 1: + raise ValueError( + f"{host}{path} has ambiguous deployment routing; " + "link its policyengine.org page" + ) + candidates = [ + (t, h) for t, h in candidates if self._identity(t) in selected + ] + found: List[Target] = [] + for target, _ in candidates: + if target not in found: + found.append(target) return found @@ -325,7 +525,11 @@ def linked(text: str, routes: Routes) -> Tuple[List[Target], List[str]]: it refuses.""" targets: List[Target] = [] refused: List[str] = [] - for host, path in links(text): + try: + destinations = links(text) + except ValueError as exc: + return [], [str(exc)] + for host, path, query in destinations: tracked = any( (host == h or host.endswith(f".{h}")) and path.startswith(prefix) for h, prefix in TRACKING @@ -333,19 +537,25 @@ def linked(text: str, routes: Routes) -> Tuple[List[Target], List[str]]: if tracked or any( host == h or host.endswith(f".{h}") for h in UNCHECKABLE_HOSTS ): - refused.append( + problem = ( f"{host}{path} hides where it leads (a tracking link or a shortener); " "link the policyengine.org page itself" ) + if problem not in refused: + refused.append(problem) continue if host == SITE_HOST or host.endswith(f".{SITE_HOST}"): if host != SITE_HOST: - continue # another PolicyEngine site (the legacy app, the API) + continue found, problem = routes.on_site(path) if problem: refused.append(problem) else: - found = routes.off_site(host, path) + try: + found = routes.off_site(host, path, query) + except ValueError as exc: + refused.append(str(exc)) + continue targets.extend(t for t in found if t not in targets) return targets, refused @@ -392,6 +602,107 @@ def _string_list(value: object, nonempty: bool = False) -> bool: ) +def _valid_hostname(value: object) -> bool: + if not isinstance(value, str) or not value: + return False + try: + ipaddress.ip_address(value) + return True + except ValueError: + pass + try: + hostname = value.rstrip(".").encode("idna").decode("ascii") + except UnicodeError: + return False + return len(hostname) <= 253 and all( + re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label, re.I) + for label in hostname.split(".") + ) + + +def _valid_page_url(value: object) -> bool: + """A usable absolute HTTPS page URL, before any routing consumes it.""" + if ( + not isinstance(value, str) + or not value + or re.search(r"[\s\\\x00-\x1f\x7f]", value) + ): + return False + try: + parsed = urllib.parse.urlsplit(value) + # Reading the port also validates invalid and out-of-range ports. + parsed.port + return ( + parsed.scheme == "https" + and bool(parsed.netloc) + and parsed.username is None + and parsed.password is None + and _valid_hostname(parsed.hostname) + ) + except ValueError: + return False + + +def check_registries(apps: object, posts: object) -> List[str]: + """Validate every registry field this checker consumes before loading it.""" + errors = [] + for name, items in (("apps", apps), ("posts", posts)): + where = f"policyengine-app-v2's {name} registry" + if not isinstance(items, list): + errors.append(f"{where} is not a JSON array") + continue + for position, item in enumerate(items): + at = f"{where} item {position}" + if not isinstance(item, dict): + errors.append(f"{at} is not an object") + continue + if "tags" in item and not _string_list(item["tags"]): + errors.append(f"{at} has malformed tags") + block = item.get("comparables") + if "comparables" in item and ( + not isinstance(block, dict) + or any( + key in block + and (not isinstance(block[key], str) or not block[key].strip()) + for key in ("status", "scorecardSource") + ) + ): + errors.append(f"{at} has malformed comparables metadata") + if name == "apps": + if ( + not isinstance(item.get("slug"), str) + or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", item["slug"]) + or not isinstance(item.get("countryId"), str) + or not re.fullmatch(r"[a-z]{2}", item["countryId"]) + ): + errors.append(f"{at} has no canonical slug or countryId") + if "displayWithResearch" in item and not isinstance( + item["displayWithResearch"], bool + ): + errors.append(f"{at} has a malformed displayWithResearch flag") + if ( + "source" in item + and item["source"] != "" + and not ( + _valid_page_url(item["source"]) + or ( + isinstance(item["source"], str) + and item["source"].startswith("/") + and not item["source"].startswith("//") + and not re.search(r"[\s\\\x00-\x1f\x7f]", item["source"]) + ) + ) + ): + errors.append(f"{at} has a malformed source URL") + elif ( + not isinstance(item.get("filename"), str) + or not item["filename"].strip() + or not post_slug(item["filename"]) + ): + errors.append(f"{at} has no usable filename") + return errors + + def check_index(index: object) -> List[str]: """Every reason the downloaded index cannot be trusted.""" if not isinstance(index, dict): @@ -440,10 +751,13 @@ def check_index(index: object) -> List[str]: errors.append(f"{where} has a search with no results_pins") if not _string_list(entry.get("current_pins")): errors.append(f"{where} has no current_pins") - if not isinstance(entry.get("url"), str) or not _string_list( - entry.get("deployments", []) + if not _valid_page_url(entry.get("url")): + errors.append(f"{where} has no valid absolute HTTPS page url") + deployments = entry.get("deployments", []) + if not _string_list(deployments) or not all( + _valid_page_url(url) for url in deployments ): - errors.append(f"{where} has no page url or a malformed deployments list") + errors.append(f"{where} has a malformed deployments list of HTTPS URLs") comparisons = entry.get("comparisons", []) if not isinstance(comparisons, list): errors.append(f"{where} has no comparisons list") @@ -467,7 +781,22 @@ def check_index(index: object) -> List[str]: return errors -def check_routes(routes: object) -> List[str]: +def _canonical_route_path(value: object) -> bool: + return ( + isinstance(value, str) + and re.fullmatch(r"/[a-z0-9/-]*", value) is not None + and normalize_path(value) == value + ) + + +def _route_entry(value: object) -> Optional[Key]: + if not isinstance(value, str): + return None + match = re.fullmatch(r"app ([a-z]{2})/([a-z0-9]+(?:-[a-z0-9]+)*)", value) + return ("app", match.group(1), match.group(2)) if match else None + + +def check_routes(routes: object, registry: Optional[Registry] = None) -> List[str]: """Every reason the downloaded route map cannot be trusted.""" if not isinstance(routes, dict) or routes.get("schema") != ROUTES_SCHEMA: return [f"policyengine-app-v2's route map is not {ROUTES_SCHEMA}"] @@ -478,10 +807,9 @@ def check_routes(routes: object) -> List[str]: or not paths or not all( isinstance(p, dict) - and isinstance(p.get("path"), str) - and p["path"].startswith("/") + and _canonical_route_path(p.get("path")) and isinstance(p.get("prefix"), bool) - and isinstance(p.get("entry"), str) + and _route_entry(p.get("entry")) is not None for p in paths ) ): @@ -489,11 +817,25 @@ def check_routes(routes: object) -> List[str]: if not isinstance(hosts, list) or not all( isinstance(h, dict) and isinstance(h.get("host"), str) - and isinstance(h.get("pathPrefix"), str) + and _valid_hostname(h["host"]) + and normalize_host(h["host"]) == h["host"] + and _canonical_route_path(h.get("pathPrefix")) and _string_list(h.get("entries"), nonempty=True) + and all(_route_entry(e) is not None for e in h["entries"]) for h in hosts ): errors.append("policyengine-app-v2's route map has malformed hosts") + if not errors and registry is not None: + entries = [p["entry"] for p in paths] + entries.extend(e for h in hosts for e in h["entries"]) + missing = sorted( + {e for e in entries if _route_entry(e) not in registry.entries} + ) + if missing: + errors.append( + "policyengine-app-v2's route map references missing registry entries: " + + ", ".join(missing) + ) return errors @@ -668,11 +1010,15 @@ def run_check( problems += check_routes(route_map) if problems: return [path for path, _ in due], errors + problems - registry = Registry( - _load(apps or APPS_URL, commits), - _load(posts or POSTS_URL, commits), - index["entry_rules"], - ) + app_items = _load(apps or APPS_URL, commits) + post_items = _load(posts or POSTS_URL, commits) + problems = check_registries(app_items, post_items) + if problems: + return [path for path, _ in due], errors + problems + registry = Registry(app_items, post_items, index["entry_rules"]) + problems = check_routes(route_map, registry) + if problems: + return [path for path, _ in due], errors + problems routes = Routes(route_map, registry, index) for path, on in due: text = path.read_text(encoding="utf-8") diff --git a/tests/conftest.py b/tests/conftest.py index 61e5872..ddef658 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,11 +9,15 @@ @pytest.fixture(autouse=True) def offline_comparables(tmp_path_factory, monkeypatch): - """The comparables check reads local, empty registries and a valid empty - index, so no test reaches the network; tests that need data pass their - own sources.""" + """Use consistent local registries and an empty index without networking. + + The route map's sentinel app exists in its registry; tests that need + analyses pass their own sources. + """ root = tmp_path_factory.mktemp("comparables-sources") - (root / "apps.json").write_text("[]") + (root / "apps.json").write_text( + json.dumps([{"slug": "none", "countryId": "us", "displayWithResearch": True}]) + ) (root / "posts.json").write_text("[]") (root / "index.json").write_text( json.dumps( diff --git a/tests/test_comparables.py b/tests/test_comparables.py index ad22d90..eb82ef5 100644 --- a/tests/test_comparables.py +++ b/tests/test_comparables.py @@ -388,9 +388,11 @@ def test_a_shared_host_leads_only_to_the_analyses_under_the_path(): ] assert targets("https://policyengine.github.io/plugin-blog/") == [] assert targets("https://marriage-zeta-beryl.vercel.app/us/marriage") == [ - entry(("app", "uk", "marriage")), entry(("app", "us", "marriage")), ] + assert targets("https://marriage-zeta-beryl.vercel.app/us/marriage?country=uk") == [ + entry(("app", "uk", "marriage")), + ] assert targets("https://unrelated.vercel.app/") == [] diff --git a/tests/test_comparables_routing.py b/tests/test_comparables_routing.py new file mode 100644 index 0000000..8afb9e1 --- /dev/null +++ b/tests/test_comparables_routing.py @@ -0,0 +1,201 @@ +"""Regression cases for primary pages, browser URLs and shared deployments.""" + +import copy + +import pytest + +from newsletter_uploader import comparables as cmp +from tests.test_comparables import ( + APPS, + POSTS, + PUSH, + ROUTE_MAP, + RULES, + app, + host, + index, + record, + route, +) + + +def resolver(apps=APPS, analyses=None, route_map=ROUTE_MAP): + return cmp.Routes( + route_map, cmp.Registry(apps, POSTS, RULES), index(analyses or {}) + ) + + +def edition_errors(url, routes, analyses): + return cmp.check_edition( + "2026-11-02-us.html", + f'Analysis', + PUSH, + routes, + index(analyses), + ) + + +def test_external_primary_page_with_no_deployments_retains_its_stale_record(): + url = "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold" + analyses = { + "analysis-uk-childcare-100k-threshold": { + **record("app", "childcare-100k-threshold", ["2026-10-21"]), + "url": url, + "deployments": [], + } + } + apps = [a for a in APPS if a["slug"] != "childcare-100k-threshold"] + routes = resolver(apps, analyses, {"paths": [], "hosts": []}) + errors = edition_errors(url, routes, analyses) + assert len(errors) == 1 + assert "no comparables search between" in errors[0] + + +@pytest.mark.parametrize( + "url", + [ + "https://reader@policyengine.org/uk/scottish-budget-2026-27", + r"https://policyengine.org\uk\scottish-budget-2026-27", + r"https://policyengine.org/uk\scottish-budget-2026-27", + "https://policyengine.org/uk/\nscottish-budget-2026-27", + "https://policyengine.org/uk/\tscottish-budget-2026-27", + "https://policyengine.org/uk/\rscottish-budget-2026-27", + "https://policyengine.org/uk/💡/../scottish-budget-2026-27", + "https://policyengine。org/uk/scottish-budget-2026-27", + "https://POLICYENGINE.org/uk/scottish-budget-2026-27", + "https://policyengine.org/uk/%22/../scottish-budget-2026-27", + "https://policyengine.org/uk/%20/../scottish-budget-2026-27", + "https://policyengine.org/uk/%3f/../scottish-budget-2026-27", + "https://policyengine.org/uk/%23/../scottish-budget-2026-27", + "https%3A%2F%2Fpolicyengine.org%2Fuk%2Fscottish-budget-2026-27", + "https%253A%252F%252Fpolicyengine.org%252Fuk%252Fscottish-budget-2026-27", + ], +) +def test_browser_equivalent_anchor_destinations_require_the_missing_record(url): + errors = edition_errors(url, resolver(), {}) + assert len(errors) == 1 + assert "analysis-uk-scottish-budget-2026-27" in errors[0] + assert "has no comparables record" in errors[0] + + +@pytest.mark.parametrize("url", ["https://[", "https:/policyengine.org/uk/marriage"]) +def test_unsupported_absolute_urls_are_explicitly_refused(url): + errors = edition_errors(url, resolver(), {}) + assert len(errors) == 1 + assert "unsupported absolute link" in errors[0] + + +@pytest.mark.parametrize( + "suffix, expected_country", + [("", "us"), ("?country=us", "us"), ("?country=uk", "uk")], +) +@pytest.mark.parametrize("record_deployments", [False, True]) +def test_country_queries_and_the_default_only_judge_the_selected_country( + suffix, expected_country, record_deployments +): + analyses = { + "analysis-us-marriage": record("app", "marriage", [PUSH], country="us"), + "analysis-uk-marriage": record("app", "marriage", ["2026-01-01"]), + } + if record_deployments: + for country, record_query in (("us", ""), ("uk", "?country=uk")): + analyses["analysis-" + country + "-marriage"]["deployments"] = [ + "https://marriage-zeta-beryl.vercel.app/us/marriage" + record_query + ] + url = "https://marriage-zeta-beryl.vercel.app/us/marriage" + suffix + routes = resolver(analyses=analyses) + errors = edition_errors(url, routes, analyses) + canonical = edition_errors( + "https://policyengine.org/" + expected_country + "/marriage", routes, analyses + ) + assert errors == canonical + assert bool(errors) == (expected_country == "uk") + + +@pytest.mark.parametrize("query", ["country=ca", "country=us&country=uk", "country="]) +def test_unknown_or_conflicting_shared_country_selection_is_refused(query): + _, refused = cmp.linked( + "https://marriage-zeta-beryl.vercel.app/us/marriage?" + query, resolver() + ) + assert len(refused) == 1 + assert "ambiguous deployment routing" in refused[0] + + +def shared_sources(delisted=False): + analyses = { + "analysis-uk-a": record( + "app", "a", ["2026-01-01"], deployments=["https://shared.example/uk/a"] + ), + "analysis-us-b": record( + "app", + "b", + [PUSH], + country="us", + deployments=["https://shared.example/us/b"], + ), + } + apps = [app("uk", "a", "https://shared.example/uk/a")] + routes = { + "paths": [route("/uk/a", "app uk/a")], + "hosts": [host("shared.example", ["app uk/a"], "/uk/a")], + } + if not delisted: + apps.append(app("us", "b", "https://shared.example/us/b")) + routes["paths"].append(route("/us/b", "app us/b")) + routes["hosts"].append(host("shared.example", ["app us/b"], "/us/b")) + return analyses, resolver(apps, analyses, routes) + + +@pytest.mark.parametrize("delisted", [False, True]) +def test_record_deployments_preserve_shared_host_project_boundaries(delisted): + analyses, routes = shared_sources(delisted) + assert edition_errors("https://shared.example/us/b", routes, analyses) == [] + assert edition_errors("https://shared.example/us/b/details", routes, analyses) == [] + assert edition_errors("https://shared.example/unrelated", routes, analyses) == [] + errors = edition_errors("https://shared.example/uk/a", routes, analyses) + assert len(errors) == 1 + assert "no comparables search between" in errors[0] + + +def test_record_ownership_narrows_an_artifact_whole_host_mapping(): + analyses, _ = shared_sources(delisted=True) + routes = resolver( + [app("uk", "a", "https://shared.example/uk/a")], + analyses, + { + "paths": [route("/uk/a", "app uk/a")], + "hosts": [host("shared.example", ["app uk/a"])], + }, + ) + assert edition_errors("https://shared.example/us/b", routes, analyses) == [] + assert edition_errors("https://shared.example/unrelated", routes, analyses) == [] + + +def test_an_app_and_its_record_count_as_one_host_owner(): + analyses = { + "analysis-uk-childcare-100k-threshold": record( + "app", + "childcare-100k-threshold", + ["2026-01-01"], + deployments=[ + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold" + ], + ) + } + errors = edition_errors( + "https://uk-childcare-100k-threshold.vercel.app/", + resolver(analyses=analyses), + analyses, + ) + assert len(errors) == 1 + assert "no comparables search between" in errors[0] + + +def test_registry_sources_only_disambiguate_artifact_membership(): + route_map = copy.deepcopy(ROUTE_MAP) + route_map["hosts"] = [h for h in route_map["hosts"] if "marriage" not in h["host"]] + found, refused = cmp.linked( + "https://marriage-zeta-beryl.vercel.app/us/marriage?country=uk", + resolver(route_map=route_map), + ) + assert found == refused == [] diff --git a/tests/test_comparables_validation.py b/tests/test_comparables_validation.py new file mode 100644 index 0000000..b875b02 --- /dev/null +++ b/tests/test_comparables_validation.py @@ -0,0 +1,244 @@ +"""Downloaded source artifacts must fail closed before routing editions.""" + +import copy +import json + +import pytest + +from newsletter_uploader import comparables as cmp + +PUSH = "2026-11-02" +CHILDCARE = "analysis-uk-childcare-100k-threshold" +RULES = { + "analysis_tags": ["policy"], + "no_estimate_tags": ["event"], + "country_tags": ["uk", "us"], + "default_country": "us", +} +APPS = [ + { + "slug": "bill-tracker", + "countryId": "us", + "tags": ["us", "policy"], + "displayWithResearch": True, + "source": "https://tracker.example/us/bill-tracker?embedded=true", + } +] +POSTS = [{"filename": "example-post.md", "tags": ["us", "policy"]}] +ROUTES = { + "schema": cmp.ROUTES_SCHEMA, + "paths": [ + {"path": "/us/bill-tracker", "prefix": True, "entry": "app us/bill-tracker"} + ], + "hosts": [ + { + "host": "tracker.example", + "pathPrefix": "/", + "entries": ["app us/bill-tracker"], + } + ], +} +INDEX = { + "schema": cmp.INDEX_SCHEMA, + "promotion_window_days": 7, + "entry_rules": RULES, + "analyses": { + CHILDCARE: { + "kind": "app", + "slug": "childcare-100k-threshold", + "url": "https://policyengine.org/uk/childcare-100k-threshold", + "deployments": [ + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold" + ], + "current_pins": [], + "searches": [{"date": "2026-10-21", "results_pins": []}], + "comparisons": [], + } + }, +} + + +def source_files(tmp_path): + sources = {} + for name, value in ( + ("apps", APPS), + ("posts", POSTS), + ("routes", ROUTES), + ("index", INDEX), + ): + path = tmp_path / f"{name}.json" + path.write_text(json.dumps(value), encoding="utf-8") + sources[name] = path + return sources + + +def cli_args(tmp_path, sources, link="policyengine.org/us/bill-tracker"): + edition = tmp_path / f"{PUSH}-us.html" + edition.write_text(f'Analysis', encoding="utf-8") + return [ + str(edition), + "--require-dates", + *(part for name, path in sources.items() for part in (f"--{name}", str(path))), + ] + + +@pytest.mark.parametrize( + "field,value,problem,link", + [ + ( + "url", + "", + "page url", + "policyengine.org/uk/childcare-100k-threshold", + ), + ( + "deployments", + ["not-a-url"], + "deployments", + "https://uk-childcare-100k-threshold.vercel.app/uk/childcare-100k-threshold", + ), + ("url", "https://[", "page url", "https://example.com"), + ("url", "http://policyengine.org/uk/childcare", "page url", "example.com"), + ("url", "https://", "page url", "example.com"), + ("url", "https://bad host.example/childcare", "page url", "example.com"), + ("url", "https://example.com:invalid/childcare", "page url", "example.com"), + ("url", "https://example.com:70000/childcare", "page url", "example.com"), + ("deployments", ["https://["], "deployments", "example.com"), + ], +) +def test_invalid_index_urls_fail_the_cli_before_routes_disappear_or_raise( + tmp_path, capsys, field, value, problem, link +): + sources = source_files(tmp_path) + index = copy.deepcopy(INDEX) + index["analyses"][CHILDCARE][field] = value + assert cmp.check_index(index) + sources["index"].write_text(json.dumps(index), encoding="utf-8") + + assert cmp.main(cli_args(tmp_path, sources, link)) == 1 + output = capsys.readouterr().out + assert "the scorecard index" in output and problem in output + assert "ok " not in output and "Traceback" not in output + + +@pytest.mark.parametrize( + "change", + [ + {"entry": "bill-tracker"}, + {"entry": "app us/bill-traker"}, + {"path": "/US/BILL-TRACKER"}, + {"path": "/us/%62ill-tracker"}, + {"path": "/us/./bill-tracker"}, + {"path": "/us/bill-tracker;"}, + {"path": "/us/bill:tracker"}, + ], +) +def test_invalid_route_paths_and_targets_fail_loading_instead_of_losing_an_app( + tmp_path, capsys, change +): + sources = source_files(tmp_path) + routes = copy.deepcopy(ROUTES) + routes["paths"][0].update(change) + registry = cmp.Registry(APPS, POSTS, RULES) + assert cmp.check_routes(routes, registry) + sources["routes"].write_text(json.dumps(routes), encoding="utf-8") + + assert cmp.main(cli_args(tmp_path, sources)) == 1 + output = capsys.readouterr().out + assert "route map" in output + assert "ok " not in output and "no comparables record" not in output + + +@pytest.mark.parametrize( + "change", + [ + {"host": "TRACKER.example"}, + {"host": "https://tracker.example"}, + {"host": "bad host.example"}, + {"pathPrefix": "us/bill-tracker"}, + {"pathPrefix": "/US/BILL-TRACKER"}, + {"pathPrefix": "/us/bill-tracker;"}, + {"pathPrefix": "/us/bill:tracker"}, + {"entries": ["bill-tracker"]}, + {"entries": ["app us/bill-traker"]}, + ], +) +def test_invalid_route_hosts_and_prefixes_fail_loading(tmp_path, capsys, change): + sources = source_files(tmp_path) + routes = copy.deepcopy(ROUTES) + routes["hosts"][0].update(change) + assert cmp.check_routes(routes, cmp.Registry(APPS, POSTS, RULES)) + sources["routes"].write_text(json.dumps(routes), encoding="utf-8") + + assert cmp.main(cli_args(tmp_path, sources)) == 1 + assert "route map" in capsys.readouterr().out + + +@pytest.mark.parametrize( + "name,value", + [ + ("apps", {}), + ("posts", {}), + ("apps", [None]), + ("posts", ["example-post.md"]), + ("apps", [{**APPS[0], "slug": None}]), + ("apps", [{**APPS[0], "countryId": ["us"]}]), + ("apps", [{**APPS[0], "tags": "policy"}]), + ("apps", [{**APPS[0], "displayWithResearch": "false"}]), + ("apps", [{**APPS[0], "source": "https://["}]), + ("apps", [{**APPS[0], "comparables": []}]), + ("posts", [{**POSTS[0], "filename": None}]), + ("posts", [{**POSTS[0], "tags": ["us", 1]}]), + ("posts", [{**POSTS[0], "comparables": {"scorecardSource": []}}]), + ], +) +def test_wrong_registry_shapes_fail_before_constructing_the_registry( + tmp_path, capsys, name, value +): + sources = source_files(tmp_path) + sources[name].write_text(json.dumps(value), encoding="utf-8") + + assert cmp.main(cli_args(tmp_path, sources)) == 1 + output = capsys.readouterr().out + assert f"{name} registry" in output + assert "ok " not in output and "Traceback" not in output + + +def test_a_valid_empty_apps_array_cannot_erase_route_references(tmp_path, capsys): + sources = source_files(tmp_path) + sources["apps"].write_text("[]", encoding="utf-8") + + assert cmp.main(cli_args(tmp_path, sources)) == 1 + output = capsys.readouterr().out + assert "missing registry entries: app us/bill-tracker" in output + + +def test_valid_registry_fields_and_consistent_references_pass_validation(): + apps = copy.deepcopy(APPS) + apps.append( + { + "slug": "embedded-app", + "countryId": "us", + "source": "/us/embedded-app/embed", + "tags": ["policy"], + } + ) + apps.append({"slug": "no-source-app", "countryId": "uk", "tags": []}) + assert cmp.check_registries(apps, POSTS) == [] + assert cmp.check_index(INDEX) == [] + assert cmp.check_routes(ROUTES, cmp.Registry(apps, POSTS, RULES)) == [] + + +@pytest.mark.parametrize( + "url", + [ + "https://example.com/page?country=uk", + "https://example.com:443/page", + "https://[2001:db8::1]/page", + "https://xn--bcher-kva.example/page", + ], +) +def test_valid_https_pages_pass_index_validation(url): + index = copy.deepcopy(INDEX) + index["analyses"][CHILDCARE].update(url=url, deployments=[url]) + assert cmp.check_index(index) == [] diff --git a/tests/test_comparables_workflow.py b/tests/test_comparables_workflow.py new file mode 100644 index 0000000..6950565 --- /dev/null +++ b/tests/test_comparables_workflow.py @@ -0,0 +1,193 @@ +"""Exercise the comparables workflow's actual shell block with Git filenames.""" + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" +SELECTOR = ROOT / "scripts" / "select_changed_editions.py" +GIT = shutil.which("git") +BASH = shutil.which("bash") + +pytestmark = pytest.mark.skipif(not GIT or not BASH, reason="needs git and bash") + + +def workflow_script(): + """Read the checked-in step, so tests cannot drift from CI selection.""" + step = WORKFLOW.read_text(encoding="utf-8").split( + " - name: Every analysis a changed edition links has a fresh " + "comparables search\n", + 1, + )[1] + body = step.split(" run: |\n", 1)[1] + lines = [] + for line in body.splitlines(): + if line and not line.startswith(" "): + break + lines.append(line[10:]) + return "\n".join(lines) + + +def git(repo, *args): + return subprocess.check_output([GIT, *args], cwd=repo, text=True).strip() + + +def run_workflow(tmp_path, names, base=None, fail=None): + """Commit synthetic changes and capture every argument reaching the checker.""" + repo = tmp_path / "repo" + repo.mkdir() + git(repo, "init", "-q") + git(repo, "config", "user.name", "Workflow test") + git(repo, "config", "user.email", "workflow@example.com") + git(repo, "config", "core.quotePath", "true") + (repo / "README.md").write_text("Baseline\n", encoding="utf-8") + git(repo, "add", ".") + git(repo, "commit", "-qm", "Baseline") + baseline = git(repo, "rev-parse", "HEAD") + for name in names: + path = repo / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("Changed\n", encoding="utf-8") + if names: + git(repo, "add", ".") + git(repo, "commit", "-qm", "Changed files") + + scripts = repo / "scripts" + scripts.mkdir(exist_ok=True) + shutil.copyfile(SELECTOR, scripts / SELECTOR.name) + package = tmp_path / "fake" / "newsletter_uploader" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "comparables.py").write_text( + "import json, os, sys\n" + "with open(os.environ['CHECKER_LOG'], 'a', encoding='utf-8') as output:\n" + " output.write(json.dumps(sys.argv[1:]) + '\\n')\n" + "sys.exit(int(os.environ.get('CHECKER_EXIT', '0')))\n", + encoding="utf-8", + ) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + python = bin_dir / "python" + python.write_text( + f"#!{sys.executable}\n" + "import os, sys\n" + "if os.environ.get('FAIL_SELECTOR') and " + "sys.argv[1].endswith('select_changed_editions.py'):\n" + " print('selector failed', file=sys.stderr)\n" + " sys.exit(24)\n" + f"os.execv({sys.executable!r}, [{sys.executable!r}] + sys.argv[1:])\n", + encoding="utf-8", + ) + python.chmod(0o755) + if fail == "diff": + wrapper = bin_dir / "git" + wrapper.write_text( + f"#!{sys.executable}\n" + "import os, sys\n" + "if sys.argv[1] == 'diff':\n" + " print('diff failed', file=sys.stderr)\n" + " sys.exit(23)\n" + f"os.execv({GIT!r}, [{GIT!r}] + sys.argv[1:])\n", + encoding="utf-8", + ) + wrapper.chmod(0o755) + runner_temp = tmp_path / "runner" + runner_temp.mkdir() + checker_log = tmp_path / "checker.jsonl" + env = dict( + os.environ, + BASE_SHA=baseline if base is None else base, + RUNNER_TEMP=str(runner_temp), + CHECKER_LOG=str(checker_log), + CHECKER_EXIT="7" if fail == "checker" else "0", + FAIL_SELECTOR="1" if fail == "selector" else "", + PYTHONPATH=str(package.parent), + PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], + ) + result = subprocess.run( + [BASH, "-c", workflow_script()], + cwd=repo, + env=env, + capture_output=True, + text=True, + ) + calls = ( + [ + json.loads(line) + for line in checker_log.read_text(encoding="utf-8").splitlines() + ] + if checker_log.exists() + else [] + ) + return result, calls + + +def test_git_quoted_names_reach_the_checker_without_changes(tmp_path): + names = [ + "editions/2026-11-03-uk-é.html", + "editions/2026-11-03-us-\ttab.htm", + "editions/2026-11-03-us-\nline-social.md", + "editions/nested/2026-11-03-us space.HTML", + ] + result, calls = run_workflow(tmp_path, names + ["notes.txt"]) + assert result.returncode == 0, result.stderr + assert len(calls) == 1 + assert calls[0][0] == "--require-dates" + assert set(calls[0][1:]) == set(names) + + +@pytest.mark.parametrize("base", ["", "0" * 40]) +def test_first_push_checks_editions_against_the_empty_tree(tmp_path, base): + name = "editions/2026-11-03-uk-é.html" + result, calls = run_workflow(tmp_path, [name], base=base) + assert result.returncode == 0, result.stderr + assert calls == [["--require-dates", name]] + + +@pytest.mark.parametrize("names", [[], ["docs/notes.md"]]) +def test_no_edition_change_skips_only_the_checker(tmp_path, names): + result, calls = run_workflow(tmp_path, names) + assert result.returncode == 0, result.stderr + assert "No edition changed; nothing to check." in result.stdout + assert calls == [] + + +@pytest.mark.parametrize( + "name", + ["social-posts-é.md", "posts/2026-11-03-us-\tsocial\n.md"], +) +def test_misplaced_social_names_fail_even_when_git_quotes_them(tmp_path, name): + result, calls = run_workflow(tmp_path, [name]) + assert result.returncode != 0 + assert "social posts go in editions/" in result.stdout + assert calls == [] + + +def test_missing_base_commit_fails_before_selection(tmp_path): + result, calls = run_workflow(tmp_path, [], base="f" * 40) + assert result.returncode != 0 + assert "cannot read the base commit" in result.stdout + assert "No edition changed" not in result.stdout + assert calls == [] + + +@pytest.mark.parametrize("failure", ["diff", "selector"]) +def test_failed_diff_or_selection_does_not_become_a_successful_noop(tmp_path, failure): + result, calls = run_workflow(tmp_path, [], fail=failure) + assert result.returncode != 0 + assert f"{failure} failed" in result.stderr + assert "No edition changed" not in result.stdout + assert calls == [] + + +def test_checker_failure_fails_the_job(tmp_path): + name = "editions/2026-11-03-uk-é.html" + result, calls = run_workflow(tmp_path, [name], fail="checker") + assert result.returncode != 0 + assert calls == [["--require-dates", name]] diff --git a/uv.lock b/uv.lock index 10e2282..dd32f72 100644 --- a/uv.lock +++ b/uv.lock @@ -2,11 +2,36 @@ version = 1 revision = 3 requires-python = ">=3.8" resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", "python_full_version < '3.9'", ] +[[package]] +name = "attrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "black" version = "24.8.0" @@ -53,7 +78,8 @@ name = "black" version = "25.9.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] dependencies = [ @@ -243,7 +269,8 @@ name = "click" version = "8.3.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", ] dependencies = [ { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, @@ -473,7 +500,8 @@ name = "coverage" version = "7.11.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/1c/38/ee22495420457259d2f3390309505ea98f98a5eed40901cf62196abad006/coverage-7.11.0.tar.gz", hash = "sha256:167bd504ac1ca2af7ff3b81d245dfea0292c5032ebef9d66cc08a7d28c1b8050", size = 811905, upload-time = "2025-10-15T15:15:08.542Z" } wheels = [ @@ -581,13 +609,242 @@ version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] +[[package]] +name = "hypothesis" +version = "6.113.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +dependencies = [ + { name = "attrs", version = "25.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.9'" }, + { name = "sortedcontainers", marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/32/6513cd7256f38c19a6c8a1d5ce9792bcd35c7f11651989994731f0e97672/hypothesis-6.113.0.tar.gz", hash = "sha256:5556ac66fdf72a4ccd5d237810f7cf6bdcd00534a4485015ef881af26e20f7c7", size = 408897, upload-time = "2024-10-09T03:51:05.707Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/fa/4acb477b86a94571958bd337eae5baf334d21b8c98a04b594d0dad381ba8/hypothesis-6.113.0-py3-none-any.whl", hash = "sha256:d539180eb2bb71ed28a23dfe94e67c851f9b09f3ccc4125afad43f17e32e2bad", size = 469790, upload-time = "2024-10-09T03:51:02.629Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.141.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +dependencies = [ + { name = "attrs", version = "26.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "exceptiongroup", marker = "python_full_version == '3.9.*'" }, + { name = "sortedcontainers", marker = "python_full_version == '3.9.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/20/8aa62b3e69fea68bb30d35d50be5395c98979013acd8152d64dc927e4cdb/hypothesis-6.141.1.tar.gz", hash = "sha256:8ef356e1e18fbeaa8015aab3c805303b7fe4b868e5b506e87ad83c0bf951f46f", size = 467389, upload-time = "2025-10-15T19:12:25.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/9a/f901858f139694dd669776983781b08a7c1717911025da6720e526bd8ce3/hypothesis-6.141.1-py3-none-any.whl", hash = "sha256:a5b3c39c16d98b7b4c3c5c8d4262e511e3b2255e6814ced8023af49087ad60b3", size = 535000, upload-time = "2025-10-15T19:12:21.659Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.168.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, + { name = "sortedcontainers", marker = "python_full_version == '3.10.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/a8/bd70d7c2966e561228b9fdc075ee77c0ba577dcbbfbf921edf614db14f6a/hypothesis-6.168.5.tar.gz", hash = "sha256:76b9226962fe11d40858253a967eda95bb65811365286317e0118f4ec8f808c7", size = 511115, upload-time = "2026-10-05T23:26:35.416Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/0c/7f04c8d277dfc828ba584b7d9d10dbac5e91fce673fa5328f7bd5bf64609/hypothesis-6.168.5-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ca43a751410a9c6685f029fd5126cc5507664cafaa76017922aa8ae2e17b6620", size = 791539, upload-time = "2026-10-05T23:24:25.544Z" }, + { url = "https://files.pythonhosted.org/packages/11/5c/660906d83db74eb86feda715d0f2df14836205b14a183332116676733e6f/hypothesis-6.168.5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c8b98707cbe9f430d100a945bbe17612fd3aa44eac1b0ac5299669fe3b8e4128", size = 787208, upload-time = "2026-10-05T23:25:14.028Z" }, + { url = "https://files.pythonhosted.org/packages/01/85/36e19492bc4ff354c2be9c8fa7c6ace0c65f9d2c7116656b741680c6ca55/hypothesis-6.168.5-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4dde52a0b696c642e7f988a03026c7c29f90daf21e74507b6f865c3ccc9d536e", size = 1115200, upload-time = "2026-10-05T23:25:53.064Z" }, + { url = "https://files.pythonhosted.org/packages/d4/82/3273fb0a3567c09b767bb8fe2824d65e16ae2abb92cf1f43762df723df94/hypothesis-6.168.5-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42f02e4541fe0c17a1320617effc0ab8a8aca2a9af15e3358d4150acf3bbdc00", size = 1145113, upload-time = "2026-10-05T23:25:17.502Z" }, + { url = "https://files.pythonhosted.org/packages/74/59/5c5904555a0bbd4b2898d73ea90c6d03f5be0d8ff0756ac1d519ace6ae66/hypothesis-6.168.5-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bf6dd7e537a12763c9afa017f7a6159e5cda608e98670621fa44596a1e8e9288", size = 1139564, upload-time = "2026-10-05T23:25:56.681Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ce/55654ff9575587a401e304f08ad1d43b7e6318f81c66bd866fdc5ab4665b/hypothesis-6.168.5-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df2c04cd30abf42c52580184216162a75b5508b214a472b86670f6dd50659a3b", size = 1192142, upload-time = "2026-10-05T23:26:06.565Z" }, + { url = "https://files.pythonhosted.org/packages/48/91/4cc9d6e8a950473e07e3ebf00cbb8ee0d76b14d193f94c3de20f1c09e2b1/hypothesis-6.168.5-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:278662eb21aaec9eaae71ea4dabd4fe390c2af11ec58a6a0606687cf6d7689b0", size = 1157262, upload-time = "2026-10-05T23:24:59.229Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3a/4b8aa3be788ea81b9a7bc6b673ed89edd72fd0645c6aa691d4c159ff971a/hypothesis-6.168.5-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:6bcedc4ab8ab92dd0f3af0cfe24dce184d225751d7bc870a9cddb9a557de847f", size = 1113779, upload-time = "2026-10-05T23:24:12.327Z" }, + { url = "https://files.pythonhosted.org/packages/f9/98/2eb4c79d1851195e6a083568b065235680ab984e984bbd472f2a7d02ba33/hypothesis-6.168.5-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b58097cc3b98d8616f635ac73888fc9f859311875f2adc043f1544c40c3c466", size = 1152639, upload-time = "2026-10-05T23:25:43.635Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9c/68f7e99b43c6f37c077669a4d3bd88f48c042444ced9e7cff0eaf44bc70a/hypothesis-6.168.5-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f8a387d9ee7f804e830b31f2e2e339ab5731665e922cfda4f6f6fbdb05e191b4", size = 1290068, upload-time = "2026-10-05T23:25:28.45Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/d4f87164a0d028ab102cea345b601d9dafb3196358df5448caa88ac3c1e2/hypothesis-6.168.5-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:326f6383fdf2e37ac69773589a8238a3bf396ca8ac8efacb0fb9ed42dd08e426", size = 1418551, upload-time = "2026-10-05T23:24:51.25Z" }, + { url = "https://files.pythonhosted.org/packages/a8/32/6b518a25514f0e643f95610c77e279bfbf0e0b3bd423aac0187d6f039b9a/hypothesis-6.168.5-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d33fc74e43bbd7c3a8f6f7161a8b93b676924286e97e70e828c6e0dcee5c01f", size = 1372130, upload-time = "2026-10-05T23:25:32.359Z" }, + { url = "https://files.pythonhosted.org/packages/48/c2/32538e14e63193ca894ba584696805d1eb45cfc27e15fccd47acfb87531c/hypothesis-6.168.5-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:1994923cf5e5220ae6bf19645302504b27c0289d83e5d8690df71dcae63d8416", size = 1268950, upload-time = "2026-10-05T23:25:02.544Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/e50e7e98af9489aa05203c2ab38c95d891dd8d1ed08fad972dcdb6955332/hypothesis-6.168.5-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:501038fd24d3bc95239cfd093a23cf1151f29dd82382a3554dac5dfdab9729ae", size = 1283907, upload-time = "2026-10-05T23:24:29.909Z" }, + { url = "https://files.pythonhosted.org/packages/71/46/41c460a7d2148a04b212b2d594d39992fb52e0b844e13bf6784573fc8dea/hypothesis-6.168.5-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e2292ddc24fe6d04b7d30fa6a7e2c9e280ad5078fe671d0bf4aa6df6e143b5ac", size = 1323762, upload-time = "2026-10-05T23:24:18.984Z" }, + { url = "https://files.pythonhosted.org/packages/68/4f/37a7fc1fe445e3589e0f56ff4573c28de1d6e6a03009cba2f99f04e46ffa/hypothesis-6.168.5-cp310-abi3-win32.whl", hash = "sha256:925d67c69b719d416334aa961c0cdfc4a58a471af1ebd2d7101bd515a70f4e5f", size = 677903, upload-time = "2026-10-05T23:25:07.129Z" }, + { url = "https://files.pythonhosted.org/packages/81/e6/7b25ca7845a60522ebc5f8054f6bba68d47126fb5d940c784fc528a4be4a/hypothesis-6.168.5-cp310-abi3-win_amd64.whl", hash = "sha256:2311590eccba452de863dfe3466daa86a05c25f072ab31ed8bb4d3313ee68439", size = 684616, upload-time = "2026-10-05T23:25:04.028Z" }, + { url = "https://files.pythonhosted.org/packages/c3/00/40e7c36b46c8788eddc7a322ad324e6db53c8ab9a8b9a95d6535ee7bdaaf/hypothesis-6.168.5-cp310-abi3-win_arm64.whl", hash = "sha256:222a6d23a2a824b0f9f73761c2fb9cd2aca96cf3e5b441617625bce4f7eb4fd4", size = 683061, upload-time = "2026-10-05T23:25:19.403Z" }, + { url = "https://files.pythonhosted.org/packages/04/0a/3b3414124055ac49c2478cb49add90eb3b727508b2aa54a4fc50de88f98a/hypothesis-6.168.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:8dfead3a6b2e2ceb6165505885b81396b0e3fe8a556bd941d88fa43cd8daff2f", size = 792187, upload-time = "2026-10-05T23:25:51.287Z" }, + { url = "https://files.pythonhosted.org/packages/a1/60/90ccc9e18d831480920dc0f1d33a9af142e796d67dbe6a760e93d0122587/hypothesis-6.168.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:658563b8f2782a0577a4d8d195e31f29b18f3f3b61ba58c4dcbd8e6ac502d14d", size = 788228, upload-time = "2026-10-05T23:24:57.84Z" }, + { url = "https://files.pythonhosted.org/packages/53/1b/8257699b8456241b8348fe0071c29912aeeaf5d16ef97a45e9c1d3170ca6/hypothesis-6.168.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f40be9b9c6b7b058ff56b0b18a91ff4cfa57a7c7756043eabaa094a0a162c9", size = 1115527, upload-time = "2026-10-05T23:24:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/42/42/31e66ce21aa6ea030ace8874269e5a169b0c69d8a3043042e315bd64c6ad/hypothesis-6.168.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30208c44364b6fe1f70c74b45f3f1f8a173a749d876294a80fe88c9cf16ab6d0", size = 1157678, upload-time = "2026-10-05T23:25:54.904Z" }, + { url = "https://files.pythonhosted.org/packages/cc/2a/b46ea00cb1cb9930b9cf7f844673913bf8bfc34f38c031d39ede6f649c59/hypothesis-6.168.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:09ca5b2f45786feb93ab41c16de602de4a54f42f35985565423417f4ed9d5b6b", size = 1290397, upload-time = "2026-10-05T23:25:34.184Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e8/eb50f72257f8b00f950da99c7ee444aae5f7c6364fce4ffbe82dd550ffdf/hypothesis-6.168.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:257175b2800cb3073f21041d174e67db7613dc64cc79f3f09f93cfecf7cfeb68", size = 1324447, upload-time = "2026-10-05T23:26:32.767Z" }, + { url = "https://files.pythonhosted.org/packages/35/88/cbb53055091323c186752b437024ff6cd95564af4389bfd1b36900aa459d/hypothesis-6.168.5-cp310-cp310-win_amd64.whl", hash = "sha256:3cacf8e84badb92e34336a6b6b95e2135ad248f870382daf56fe471d6c6e794a", size = 684598, upload-time = "2026-10-05T23:24:40.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/95/f1149d913d685809c016b2a3ae9d727741ae22f52376c6d0ed51eecb5ac8/hypothesis-6.168.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:8c35e5d4a85d0d6071cc267a6cbb8fd7ae23ca8a0f745ea5a52c0064d7c1c4b8", size = 792108, upload-time = "2026-10-05T23:25:12.323Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/7e5ffb6bbfc033c85746243dc4d1541876082e136ee44c02f843bb77427e/hypothesis-6.168.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:244a8d14c0a8a3be0345ad0b120deafb94517cc1d74a961d14b5b5eb041b4c0c", size = 788035, upload-time = "2026-10-05T23:26:26.557Z" }, + { url = "https://files.pythonhosted.org/packages/38/df/022129d3e16d19a84e7a5a35ebf7baca07d3482fb34f0faaab865b14fe66/hypothesis-6.168.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e68e1d43b7c9c7a1aa659dfe1c0ecc2de79391b20db853c1e18ea7e3d2ce31f", size = 1115438, upload-time = "2026-10-05T23:24:52.639Z" }, + { url = "https://files.pythonhosted.org/packages/da/09/b3e45b0386d8f643a304105883c5bfce79fd530b2dfe3a70564e1d7aa0bd/hypothesis-6.168.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01a4d3773f285e75551eeef12df058e6316b666bcc3ec187c5eb52a893fbb015", size = 1157503, upload-time = "2026-10-05T23:25:05.609Z" }, + { url = "https://files.pythonhosted.org/packages/ee/4a/aba5a74ddb20c9f41ba5b8f2918c5a12660146cab2120f14122122715060/hypothesis-6.168.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cc327005f2fbb55db81d132948ee7c6cec0589694bed04b1e45fc8fc317e12bd", size = 1290235, upload-time = "2026-10-05T23:25:08.982Z" }, + { url = "https://files.pythonhosted.org/packages/34/f4/7204aa6117a38085e6f1dbefd5cd98050a58c847f2bdecc917422cdb2b1c/hypothesis-6.168.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:62f21c74ad83fe77abc72e82c54114148fb01396769c234e26c9b9dbc21344a9", size = 1324204, upload-time = "2026-10-05T23:26:16.239Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4b/15a46ced6d999148d1b718c5488c243bd56dfcd687a61404fe371192dfd5/hypothesis-6.168.5-cp311-cp311-win_amd64.whl", hash = "sha256:bd3ff6e53e29b86ec6078f123284e65e1c678fe7b30c2b52512244faf266502c", size = 684414, upload-time = "2026-10-05T23:26:18.231Z" }, + { url = "https://files.pythonhosted.org/packages/90/43/a04a727578cbef9f75c11fa6fbad66d13aaffc354f4f979506219814c7d4/hypothesis-6.168.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ddee1ef4bab47e315b705e42d2f4354e789973d11f9620d2df242aef4cfa42b2", size = 793156, upload-time = "2026-10-05T23:25:49.433Z" }, + { url = "https://files.pythonhosted.org/packages/f4/91/55de4e2a12fe98ebd5bc8f35e59870c897ab360cbfe5aa63862cdbef56ad/hypothesis-6.168.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:81ceb49b0dc3a4b6126cd0d3bf2b634af4e91513c8f1e2daee16041414ed8e3d", size = 784788, upload-time = "2026-10-05T23:26:20.188Z" }, + { url = "https://files.pythonhosted.org/packages/f4/61/230abc6320540bdf73baf9a1c025fb0aa27cfd5a3791a2e0c95114239a70/hypothesis-6.168.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a09caa95d2d7e6546f727f703de606145835d9ca215fb3134a21353c69afaac", size = 1113853, upload-time = "2026-10-05T23:25:30.593Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4d/3bf0a7806b3fa12ed076f2daeb3db0e6f9738994e879432ffd8dbcffd634/hypothesis-6.168.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97ac1d516a42a3b1f13b36a1aa6a5f842e43d67e69d4dc664a9645b28de411ef", size = 1156695, upload-time = "2026-10-05T23:24:28.607Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a0/603f918fcf8f74f81ea593b04e3a9a9fcd426bbf389ed52cb340249bdc14/hypothesis-6.168.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e4819fba78c6cbaa6e2f9fd5a69a413817446943f286763819b5ac52391bff3e", size = 1288792, upload-time = "2026-10-05T23:25:36.354Z" }, + { url = "https://files.pythonhosted.org/packages/69/7c/711ef5be6e889dcd40d9b03cdd85cd42ae39af75835bced3c374730291a9/hypothesis-6.168.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87334b95dfbc101652fa48a427a742b0715b814506d9a10f621c29e476b4a2c1", size = 1323090, upload-time = "2026-10-05T23:25:58.753Z" }, + { url = "https://files.pythonhosted.org/packages/66/66/0377d7d13ff3e2c16efd141942649edcdb568caec4576f86ac779545dd85/hypothesis-6.168.5-cp312-cp312-win_amd64.whl", hash = "sha256:2fcec23ff4eb526ee85d3510f564b938ca74f6011f1eec1050e4eb55280b0468", size = 681999, upload-time = "2026-10-05T23:25:41.86Z" }, + { url = "https://files.pythonhosted.org/packages/7b/b3/1f7f72cd28d02a5ca99c432fbffe4b750a375df2284af9d916943dd3aa4f/hypothesis-6.168.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:714337b25ca9137bc359c570b868269462307e120999412ca1946f997f4b9db5", size = 793075, upload-time = "2026-10-05T23:25:15.905Z" }, + { url = "https://files.pythonhosted.org/packages/8f/ba/5b0874828695c4d49e3858d0967254f783e563cd0e211a6db27d11d48a1f/hypothesis-6.168.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f1c3617155fcf5b5259a1f2e4c775d3eec7bfa80b162b2f6f145b08f871ab08", size = 784693, upload-time = "2026-10-05T23:24:16.559Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5f/ca777becba5251b0d778bb9d83d15524c559a07e4b5d4e6211473855bae2/hypothesis-6.168.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebee70b7a026210bb47c86c89e5bfb42effd5bd630080e76bc084f29c01c7f7a", size = 1113693, upload-time = "2026-10-05T23:24:44.262Z" }, + { url = "https://files.pythonhosted.org/packages/a7/e7/5a74bf329e405db3edc5639a2595eccf33ad6f5aaa191019e9f824d630f4/hypothesis-6.168.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cfb06b31cca005345b8ad63f88986d21fd359a7dc3dba2965dd3515b720e5c9", size = 1156530, upload-time = "2026-10-05T23:24:47.153Z" }, + { url = "https://files.pythonhosted.org/packages/34/7d/e79cf67f03f212a1394abac21053bd6887aa70f557be1da3f9c9c73e58ae/hypothesis-6.168.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4a4c244d7ab64963fb575f0ec2d813630e1d14cefc39e7c460d5d778e5af4118", size = 1288713, upload-time = "2026-10-05T23:25:22.763Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/df452159ac8d7b278071a3e81fafc69da833ec4302b8c85f5b6e530aea21/hypothesis-6.168.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e59d519f6fb38b3fa4fcde046767b03a24740fe827d261ee7ff9a721c06169b", size = 1323134, upload-time = "2026-10-05T23:26:02.485Z" }, + { url = "https://files.pythonhosted.org/packages/af/fb/f07d8d09fb57eb14555cad64dfbe29bfdcecff3806f1e01268258088e741/hypothesis-6.168.5-cp313-cp313-win_amd64.whl", hash = "sha256:c103f655644afa4ef6bf7efbf86e44b78ee475fd0691da2db86e2cfe72c07234", size = 682019, upload-time = "2026-10-05T23:24:22.888Z" }, + { url = "https://files.pythonhosted.org/packages/de/e9/7c3c2262b8cfa825c4c1764d62aa15e628bae257ccfd2ee4f3ffa4f81eaa/hypothesis-6.168.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c4dc037d8001bc6eccb8636f4a38d16ea6b250d6bf0a89075aaa5e5069f751cc", size = 793253, upload-time = "2026-10-05T23:26:14.331Z" }, + { url = "https://files.pythonhosted.org/packages/3a/a6/7909ed7d29302491e9b7bc0e7ac3287c20736c05a0cc35bae65024aeec3b/hypothesis-6.168.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c90743321f29b65491d146adfc2ece85869bacb71ce18b47674795e896c81ee3", size = 784837, upload-time = "2026-10-05T23:25:24.728Z" }, + { url = "https://files.pythonhosted.org/packages/91/8c/57742c459349052e6a3e0c011855840f8cbbbadca91079d5b591f08b25ae/hypothesis-6.168.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09debb7f7f0f229da5f7e2ad515a5be7a8dc607ec204074775f8ab6731a447f0", size = 1114268, upload-time = "2026-10-05T23:24:27.35Z" }, + { url = "https://files.pythonhosted.org/packages/55/80/07bd2449f91f9426f705fb689429bab6e26d1365f8ac4ef7d7c1cec9055e/hypothesis-6.168.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d227f8ac497eca0bde4e8562d32dd4e82fc9566526020bbd567f76b833b923b0", size = 1156838, upload-time = "2026-10-05T23:24:35.211Z" }, + { url = "https://files.pythonhosted.org/packages/fe/75/7f3dda517e5134f73e2ae41821bf40b3fd3ac6551a9a43ea9287471738a1/hypothesis-6.168.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:cc6ebd35601c72c842e5899c3f760f9ed26c69e786ee40a9a64fb5a4a3058315", size = 1288864, upload-time = "2026-10-05T23:24:42.645Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cd/4b1364140642cf3f1431ca59b5841fc322872dfa7197b2facb97692da234/hypothesis-6.168.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:503e103ad49e702bad200157d82778eebbc14d3045e9700a8e8fe5db40912953", size = 1323300, upload-time = "2026-10-05T23:24:14.826Z" }, + { url = "https://files.pythonhosted.org/packages/20/e7/47d7cffcaf15318a4308516b6b3d2fd0db599f18eacc0f2dc553be2206a7/hypothesis-6.168.5-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:bc5cc310f9f86ec62f0d0dd7eea5a4788f18ec793b70ee2c7163b916768e1057", size = 624239, upload-time = "2026-10-05T23:24:48.453Z" }, + { url = "https://files.pythonhosted.org/packages/97/6e/2ca0f68150be175b7cfa7bfb6692260638d86aeb9313478ba82e198186e6/hypothesis-6.168.5-cp314-cp314-win_amd64.whl", hash = "sha256:71ce0599e806ce3a68f9f118edf450bf091e11b134f6bcc5f8dd706b42c91ebc", size = 681884, upload-time = "2026-10-05T23:25:00.757Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4b/4fc2b5970df0c27668ec08abc505f1d01314a69953f89dc0edc6528ff5a0/hypothesis-6.168.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f66b02c9e95e916a2c58f725a92377ec988146ed7b5aeccd5e78ceecac1eae6f", size = 791636, upload-time = "2026-10-05T23:26:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/04/b2/03cdf5f052dcb441e045be1fd0aa531e85cde1a1cbabab60968625c570a3/hypothesis-6.168.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bab27926e1d1575fb43b70d4aeece05b74a5e477af0509b56cb6fd778070dd93", size = 783280, upload-time = "2026-10-05T23:26:24.333Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d8/615557af244e2f3ce4763029c03a62ed82dcbbd646b72a8c479ef0408b33/hypothesis-6.168.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edeb42c3009b5652dc1c44907ec91bfe9284100ad5e57993dfebabb76f2961a1", size = 1112149, upload-time = "2026-10-05T23:24:13.526Z" }, + { url = "https://files.pythonhosted.org/packages/9f/67/a6707fcd51dc5f2531bf88ac072e99f31ab9d8020488b01349a6d2981081/hypothesis-6.168.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8977456328147c521a16a089325017b2c728fddc23351693a4fd924cc7fc7001", size = 1155637, upload-time = "2026-10-05T23:24:24.047Z" }, + { url = "https://files.pythonhosted.org/packages/85/d4/ac2e852d2f163afd398854662bcbb0b849a767abbf2f95a75de6f685f821/hypothesis-6.168.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:36ecf7ac351f9c0b5489ba800884b607da754e88ef40713fbfcc170d2151e6eb", size = 1287040, upload-time = "2026-10-05T23:25:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/23/07/f77b1602704bda6ff3d9d0817120bd7fb94fd792bd25508b36ce4b8bd2a2/hypothesis-6.168.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0333aa5129ba3019a83fb81a7f0fc238180e415a9edddd9a15101f8deaaa517e", size = 1322166, upload-time = "2026-10-05T23:25:45.39Z" }, + { url = "https://files.pythonhosted.org/packages/6d/2e/94138a73e0906b31cb5968d20be58688f582a09e5958f2c75d45a7049545/hypothesis-6.168.5-cp314-cp314t-win_amd64.whl", hash = "sha256:2fcb87341d76ae0183e8219c9a14d55957c50d14973879db5fea3e81da45ba1a", size = 681599, upload-time = "2026-10-05T23:24:37.827Z" }, + { url = "https://files.pythonhosted.org/packages/92/13/92cb8092b680be2b6ec5ffe83b9f1a98dbf414117566f9e3ba4e8b569214/hypothesis-6.168.5-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:453ab7d0a1fadbaa54ae8722d22463cc2046fa8ef25b9b88715d28279bf79fc1", size = 790950, upload-time = "2026-10-05T23:24:45.852Z" }, + { url = "https://files.pythonhosted.org/packages/e6/22/78aea12694e3d1177e2980d44798b6d93e191faf59155b18bf5ae315f6a2/hypothesis-6.168.5-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:bbdbc43d1f9dad595b249b7bbe8ee5102bc94a4fcb0a79ff76d20e41fcfe342a", size = 782939, upload-time = "2026-10-05T23:25:39.961Z" }, + { url = "https://files.pythonhosted.org/packages/bd/12/5ef9947b2d149f773428e555bdf66688405aa5167510bdbe97c8ec5c6090/hypothesis-6.168.5-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bc36194d7b6083591060836c7872711a6820217b325bf432dd7e10b3d4af5cb", size = 1111380, upload-time = "2026-10-05T23:24:39.087Z" }, + { url = "https://files.pythonhosted.org/packages/e1/65/7e668e203fb2659c6214dc0c24cc09b7dea8a02c7c8d0ad338f644a054c4/hypothesis-6.168.5-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:22425e2b1543a43c157a81472c713ba8f291cbaf054c70ffe128e2cacc294f65", size = 1142644, upload-time = "2026-10-05T23:26:30.697Z" }, + { url = "https://files.pythonhosted.org/packages/c0/77/b112978676e795658d58c4294bf90cdbb8cb56cb8292c8c4874650468cf9/hypothesis-6.168.5-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eea0bc513d0e38d1d5ddfb581132928871cd02dc54dfe4511a5396727c48e9d0", size = 1135966, upload-time = "2026-10-05T23:24:49.806Z" }, + { url = "https://files.pythonhosted.org/packages/e6/27/cd3bf01e8246c4318ec3df15f5eeeee3214f444df0129a6c7f9a62859ee8/hypothesis-6.168.5-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb142bc70bbf6645e15c7ca72de3f7c8dae198aa2743a609f4f3e3bb4f9c3a52", size = 1189646, upload-time = "2026-10-05T23:26:04.471Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a6/4d3e882f31c289e432dfec34dbb9029296038a8c69e8b28cebb0a5fb7ea8/hypothesis-6.168.5-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a27b758707bd37f5a1759cca6eef83fe1a212c38dc4ca0a203434004c5647d15", size = 1154770, upload-time = "2026-10-05T23:25:10.814Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/96f5455e1b3d0409cbbb1434c98e792bcceefd614a4b11e600072520b487/hypothesis-6.168.5-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:77a111cb50c330fa7098f65852fa17a01ecd781a85be3cf5e5871bdeeeb0ecbc", size = 1109925, upload-time = "2026-10-05T23:25:26.369Z" }, + { url = "https://files.pythonhosted.org/packages/3d/64/0758985d9d36f0c5ec981a1457ea1c8173f62d46a94531417aec117df4d7/hypothesis-6.168.5-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cdd0afc13e86ec76cae3d3659569c1f601f4e9ca52b5cf91c1685979eae64d7b", size = 1148476, upload-time = "2026-10-05T23:24:54.552Z" }, + { url = "https://files.pythonhosted.org/packages/43/5c/a9b8953e1d8aefcd3c22cf8d10dd8acf93e602b903278e2e51cf8544ccea/hypothesis-6.168.5-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:5fefb02035864c3d322e3b0969b296250923fdcfb574ea1ad4374f1a6333f663", size = 1285856, upload-time = "2026-10-05T23:25:38.239Z" }, + { url = "https://files.pythonhosted.org/packages/bf/37/66098444dc832523ddc4f2e05723662834e5f99bba3c759615d059f6420e/hypothesis-6.168.5-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:9db8aa1f5529e1b577ec18b775c2fb4225821712e946f7762b90c966604faf83", size = 1415500, upload-time = "2026-10-05T23:24:21.682Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d3/e971b6fe20ef8d7c2019cbf24b4f6149468efc88c42a744f5bc99e6ca0ed/hypothesis-6.168.5-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:59e07d2f62b5ff573b0059959ae9cef9edfb0f5393fdb35ea81fce1ee77b27ac", size = 1368623, upload-time = "2026-10-05T23:26:11.292Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ac/b279dfbd2c06cdb3030ba7eea042cb2cf0171d0013563d103d5207dde63b/hypothesis-6.168.5-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:8a03ca128bea29d6826fc545f1f6289fb1ea2e83a5bb811321761b2d515ca575", size = 1264457, upload-time = "2026-10-05T23:25:21.073Z" }, + { url = "https://files.pythonhosted.org/packages/55/57/16ac9f8ddfada1cd278bd2185234d0d36ebd304926b69ad0497c210c6fed/hypothesis-6.168.5-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:5c03f2d3f84f626f3fd07f54573ab40455e1a1996e98a4f4971caf8b7e796afe", size = 1281744, upload-time = "2026-10-05T23:24:31.263Z" }, + { url = "https://files.pythonhosted.org/packages/3b/d1/99a44430b82998fdef0ffd7d353f64ee5f078c2805ff70f8677ee102cb6c/hypothesis-6.168.5-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2bdf8ce9b72a620cd5ec4dd6b1c1837ff6971489a863851d11d9b0f58dd4062a", size = 1321504, upload-time = "2026-10-05T23:26:00.583Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6a/58ef2564d1985a5c1a1dc57906b8363a767094abca180e80a0aca4cb635f/hypothesis-6.168.5-cp315-abi3.abi3t-win32.whl", hash = "sha256:5c3abbef7b17571fd713b0922407d9cd8cbc652254c0f462875f15199fcb29f7", size = 674959, upload-time = "2026-10-05T23:24:36.482Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/153414f55eb0c85bd9d891bd7811d746978c7ad3de81ea79eeb4e62e088b/hypothesis-6.168.5-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:38172199abab94a04bc017613e055faa796d7175fbc6221aac504d406c960b60", size = 681329, upload-time = "2026-10-05T23:26:08.897Z" }, + { url = "https://files.pythonhosted.org/packages/6d/63/117c82f08ab3ba1dcfbf6562ac43b8deb8efa8106646494fadd15122cc1b/hypothesis-6.168.5-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:0600ddc24c32dab5ca8e780630ab6e2561df6d7f594f781d0608b38e04c4da91", size = 679632, upload-time = "2026-10-05T23:24:33.753Z" }, + { url = "https://files.pythonhosted.org/packages/73/25/5c38b739fb778d4de48aab6509b9cf0afd0317bb0459741afdcd0ad44aed/hypothesis-6.168.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6786049db92275e0c5cfac7dfcda6d4bbc80bdf84cbc8c9c7171ca17f47b5aac", size = 792028, upload-time = "2026-10-05T23:24:56.365Z" }, + { url = "https://files.pythonhosted.org/packages/7b/3f/91071d53240f5f13ab1dda286e3ddb33177537dbf55cede76e7f4a3856db/hypothesis-6.168.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ffbde24430dcd73231fd03324a934e0f638f7c0899fc566f3ef8c851534f8030", size = 787324, upload-time = "2026-10-05T23:24:17.822Z" }, + { url = "https://files.pythonhosted.org/packages/10/ef/eb262e50d7741de6c49d27923e2c282d079273b8bcdacde33165ea39488d/hypothesis-6.168.5-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea967baaedfd532f1a521aaedafc66bb9de09795071492b0e7252139df38479f", size = 1115118, upload-time = "2026-10-05T23:24:20.43Z" }, + { url = "https://files.pythonhosted.org/packages/87/67/a655a8666164aa896516f919af272fa3a3a00d2786be880c31bb638e79e2/hypothesis-6.168.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b2f98289a5da876c08b9eeb68d1cfdfbd0fcc110cf364d33c3cc32cf229ffe8", size = 1157894, upload-time = "2026-10-05T23:26:28.641Z" }, + { url = "https://files.pythonhosted.org/packages/57/4d/71c422a29446c03e9a052f10b8ee527044242e71e3c0139100991f721e16/hypothesis-6.168.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e313a01ce580180dc3bb8fa98ddd0ffb20e51e108d9fa747ba6c1596790dc3fa", size = 685030, upload-time = "2026-10-05T23:24:09.964Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.170.15" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", +] +dependencies = [ + { name = "sortedcontainers", marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/ec/59f14eb701087710888b66bfef2dc8bee9dd4a364c69a732db0cc1d3d2dc/hypothesis-6.170.15.tar.gz", hash = "sha256:a4a57e5fb106ecb29cad37e22e411d957e20d933bfb9fb029e1999b1deb8a0b6", size = 521997, upload-time = "2026-11-06T09:54:57.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/cf/3b56a81685915f6bf0d67b3a6c1139fd9bd4583066161823834e42b9eaa7/hypothesis-6.170.15-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:db8211e0a56fa29bf9e5d9ff9cd969a91ff4e5ffa8fb4983d3dd788253aeee2c", size = 800537, upload-time = "2026-11-06T09:53:36.823Z" }, + { url = "https://files.pythonhosted.org/packages/b6/09/477b3233cca8c447a6dc5fcc076a7294a4fdacb615d960695f6970d706dc/hypothesis-6.170.15-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:34e0859a53f20200f39db3f9cd92112eb6034c31dc670e88d56d8725f453562d", size = 795693, upload-time = "2026-11-06T09:54:14.601Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7a/34e042f1545472abe29ecf9eebac6f748096580f6fb831817f097addbe29/hypothesis-6.170.15-cp311-abi3-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:5c5980821384206939c1c9d519b7adfb80b54d07e669ab0d36fe877f08cff07a", size = 1162432, upload-time = "2026-11-06T09:53:55.461Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7f/95b7b04d2b7e45669db446ee10d97f5d04b143bba380559a7f3a7e0c5b1f/hypothesis-6.170.15-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f29339b5cd4c2a221885e8b7ddbd447b4dc7ee14f763a6f5aaff726d4d3fd5ed", size = 1124598, upload-time = "2026-11-06T09:54:26.015Z" }, + { url = "https://files.pythonhosted.org/packages/33/8e/896172d9706bfe2e09c9154fecb11c7b76e50014442783dd9d5b404c8c84/hypothesis-6.170.15-cp311-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:d836a61c3dd94c6fc4677f4d0d95d30b92eb0ceab6f2b5c58e35c32545583170", size = 1155097, upload-time = "2026-11-06T09:54:47.123Z" }, + { url = "https://files.pythonhosted.org/packages/29/33/b8791c6a9b4232c3cc8f592d3e36175bcb4bad6203f7e461db0a4f5fa11b/hypothesis-6.170.15-cp311-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d18ffd069360589a66ed125db62f12b490b3b9f6d47e5fd2505f1089d63a0f17", size = 1148988, upload-time = "2026-11-06T09:54:33.101Z" }, + { url = "https://files.pythonhosted.org/packages/87/2c/3ee87dd6fdec16ce55a4aeb5f72ae24756855cdc4ee87637fb68c4daeadc/hypothesis-6.170.15-cp311-abi3-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:7636424c7b1477fdd279fce5a210b67687a3024ac7759fe2e46a767a76f8761c", size = 1201778, upload-time = "2026-11-06T09:53:49.736Z" }, + { url = "https://files.pythonhosted.org/packages/6b/6a/f2757fde7d1b3b5dda6df3441d346b59c60627f2c216d966739289149903/hypothesis-6.170.15-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:99a1a4711b5276e5bfc3aa0cca034d0e45f8696a46ac8dce008308f580c18234", size = 1168007, upload-time = "2026-11-06T09:54:34.331Z" }, + { url = "https://files.pythonhosted.org/packages/8d/79/b6ff2a2840e4bcdea5c4cf1bcdd3803954a632beec16cb9d77a9ee8ddbb8/hypothesis-6.170.15-cp311-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:f545a04342a53008fa696a0592f0a026278ee61fab04f3e9e395d0c7eba42fef", size = 1124154, upload-time = "2026-11-06T09:53:24.354Z" }, + { url = "https://files.pythonhosted.org/packages/52/d0/19bc4a98e9583936dbe6e754014ad1133d3a43c9068a880c4e91ce384fad/hypothesis-6.170.15-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1f11d685322a56ee73c2a7566ec9d1195740313c3b965bad3121c340ec0e657b", size = 1299503, upload-time = "2026-11-06T09:53:12.86Z" }, + { url = "https://files.pythonhosted.org/packages/ee/06/fad2d1546c08eb4133f9478716bf6bc5f881e0d06b5b4f1faf34fd515a5a/hypothesis-6.170.15-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f6d5d34fa6b56f236ad9659043123efb85391a8a5640f067c9de82b0be0bff73", size = 1428191, upload-time = "2026-11-06T09:54:44.43Z" }, + { url = "https://files.pythonhosted.org/packages/ca/cd/71d43284cca9c5145ceae04ddac1adfc57e9da57cf8efd30c0876844c9bf/hypothesis-6.170.15-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:2ff9983a58647b03754fa3d1005e59a9c563e15256340fc062b4a34c7969d52a", size = 1382322, upload-time = "2026-11-06T09:54:06.405Z" }, + { url = "https://files.pythonhosted.org/packages/0f/56/5754189cd709d4eb34601f8ebb57f1d2381b2dd74f62352ba6b84bce4a12/hypothesis-6.170.15-cp311-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:6e461d7b392a630593134611ebb1fce6baba88dbbf4f07661a37ff58ff4e1b6a", size = 1278225, upload-time = "2026-11-06T09:52:59.772Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/5224ea74356981f01222c42f0acc55821fb5aadf3f1f15adc797c7eefbd2/hypothesis-6.170.15-cp311-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:62c0675a837bbe0f3f7c7a89b701bee159759840c23b9bec9d565d3434fb0b3a", size = 1294018, upload-time = "2026-11-06T09:54:29.127Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c5/297f16660b15eacb2872af9fb127e7fe83f48f6f9caf223cb5abd5a25bd8/hypothesis-6.170.15-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:124b0b9962ff6b25786692b7b6cb0065f8bb7870081ddd9baf4745b0ff4a5c7a", size = 1333737, upload-time = "2026-11-06T09:54:03.427Z" }, + { url = "https://files.pythonhosted.org/packages/19/18/7e35ba1bc1f0e3ec55d9d97864db42b5ffeea92766ae4bd7ced0f77cc930/hypothesis-6.170.15-cp311-abi3-win32.whl", hash = "sha256:d34e54f485777eb217c85c425ddbe7b278ef56540feedb5de4097a5abc9f5719", size = 687717, upload-time = "2026-11-06T09:54:40.259Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d6/c16f5abdb56baa2d401de38c2f758090796c1716fed9aa690179772e6adf/hypothesis-6.170.15-cp311-abi3-win_amd64.whl", hash = "sha256:de9fe573a4955b0cf0354822fbfe9b79882e8de02f26287313fe1ad47ba3e69f", size = 693782, upload-time = "2026-11-06T09:53:17.157Z" }, + { url = "https://files.pythonhosted.org/packages/13/7a/ae3b7fdd44c30b662f27344ccb810d080d11b7f557c2cffa88d01e677295/hypothesis-6.170.15-cp311-abi3-win_arm64.whl", hash = "sha256:de5e11c568fd411a3cf07e4e34d4f8cf07d638de2f254c8bd87c1879679cdd9a", size = 692109, upload-time = "2026-11-06T09:53:52.599Z" }, + { url = "https://files.pythonhosted.org/packages/22/fa/4f61c0e5487d75d9ef20bb6d7b9e13b5aabf89d249896d9af0a648578be7/hypothesis-6.170.15-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:679635cd52626790214f879a8bdbb877923333fbd18905c591af41bd74d2f8db", size = 801251, upload-time = "2026-11-06T09:53:45.982Z" }, + { url = "https://files.pythonhosted.org/packages/51/c2/8eb51b3b243248fc7289b37aa78caeff847b98e7a35ba6f6e12a846d68ec/hypothesis-6.170.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f95b532f23bed43ef3c45fbd19baf4717687a4993fa6f42fc704fbde18bd744c", size = 796937, upload-time = "2026-11-06T09:54:31.637Z" }, + { url = "https://files.pythonhosted.org/packages/af/fe/80d94b0366d3408078c32758f7149592fdf963d431e52188728e5380d806/hypothesis-6.170.15-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9f5937e70c636ee46bd2242d9ef120d19744e63ce7d17a92fd4c9f57ef62b1c3", size = 1125542, upload-time = "2026-11-06T09:54:16.018Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b4/c9dfacdfa074ebcee35ad8d9e6436480a45b543bd42ab92fa556e3bee113/hypothesis-6.170.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ecbd6294a636594580689debc3011a0ca3eb76577770f661224314202f107e14", size = 1168312, upload-time = "2026-11-06T09:53:22.993Z" }, + { url = "https://files.pythonhosted.org/packages/66/dd/29996ba7a8ef7eaea1493dffda4bfb04e85180a2f45d3d9ab7793ada3722/hypothesis-6.170.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efaee8533a06b6a2382ff7c336ea5a747c1b22a4e97c19bfe825a02388f56d2d", size = 1300279, upload-time = "2026-11-06T09:54:30.502Z" }, + { url = "https://files.pythonhosted.org/packages/57/2d/b4be9bbb76ac5079af9cd5cf484e3e0ad91f60a7e98c69d56ac623d55573/hypothesis-6.170.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4af2d31c00306707166b4f39e5208d2585f9ef4accb10586534101eb32f243fb", size = 1334296, upload-time = "2026-11-06T09:53:08.948Z" }, + { url = "https://files.pythonhosted.org/packages/69/55/f0677edf64249c5d91ae9387678bfd43ae0f54905fd2a4dbede9cbaf1e0e/hypothesis-6.170.15-cp311-cp311-win_amd64.whl", hash = "sha256:b48cb85180255ec3eb3b3e5d49068c9ad447b25e34fccafdabae63d2c3d854ad", size = 693893, upload-time = "2026-11-06T09:54:49.687Z" }, + { url = "https://files.pythonhosted.org/packages/91/db/84d418c965fe889d21e7791f946f7c893bf492bcedb38f3f5b8eb405a035/hypothesis-6.170.15-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:29223a2d180e463f1205b7e5eb397db0c4827de06f2ae571b00c392d355c410e", size = 802327, upload-time = "2026-11-06T09:54:20.994Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8f/63d8644162728ba8b2c8a8a8b6a34476663c964cc0f6885c19f84ead40d6/hypothesis-6.170.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7953ad8f123b4289801b833666aaab757ba52e91ed687d954eb26f4af7328e5b", size = 794302, upload-time = "2026-11-06T09:54:24.828Z" }, + { url = "https://files.pythonhosted.org/packages/ec/2e/d479c69c08d5aabd7e6499f3dddc22b241db42922d746d05df9548213229/hypothesis-6.170.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1f37920547eafa77814bc9b638e47299562123023bdf3c9cb002795bb90fc92", size = 1123978, upload-time = "2026-11-06T09:54:36.215Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e2/824c524d04152e5c26111247d2d4dbf210d43df7c6612f7ff22c20de5abc/hypothesis-6.170.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e20ec17d2095e9314ea232b4e34b5510def0d9e7a7e764cc70cae88fe4b0c4", size = 1167677, upload-time = "2026-11-06T09:54:55.123Z" }, + { url = "https://files.pythonhosted.org/packages/ce/49/4aab201f1fd0109560d985076a4c852c1abda14984470369bf29ea89ade3/hypothesis-6.170.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:11c541cf103c784306f6037e5ab962cbe3d5891d36fd98df993f38d7ddbcd360", size = 1298862, upload-time = "2026-11-06T09:53:51.237Z" }, + { url = "https://files.pythonhosted.org/packages/db/a7/83fc4ef9484deefd955949a9a72a14d9435b6ace339feb27de25bf286e29/hypothesis-6.170.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eecdcd38395facf6ea3af90320a574f5e63c310c194d2a0eb75f121223942386", size = 1333300, upload-time = "2026-11-06T09:53:39.58Z" }, + { url = "https://files.pythonhosted.org/packages/d5/18/5f1a5ce07a9e6b7a2cbfb6a8bceb9b5161de5477abb8d99d659a898b7554/hypothesis-6.170.15-cp312-cp312-win_amd64.whl", hash = "sha256:7dfb3d92df4faa67d3c975d1ecdefabc5a2481b789048f52cd5fb775c6c517b8", size = 691422, upload-time = "2026-11-06T09:54:05.043Z" }, + { url = "https://files.pythonhosted.org/packages/16/16/d75485989f250dd5017d1627902961c2f985686355f50d10170a2490d970/hypothesis-6.170.15-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e3f478d4d9e13d528e352a4d938dea10b25793663ac752d774d7d35b008a1b45", size = 802290, upload-time = "2026-11-06T09:53:33.892Z" }, + { url = "https://files.pythonhosted.org/packages/24/45/54e42ac718e1164090c04b588024e4818a8e2a28b9002faa21e9fd59ab67/hypothesis-6.170.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c92af3f0c8844d9eba793a4371035a45e88bb59579a1408ce5d3cceb9e2b1da", size = 794293, upload-time = "2026-11-06T09:54:09.14Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ee/1a81a7d544e321165159a07e666f6c302ec109beccca3b37f8d0de54e0cd/hypothesis-6.170.15-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dce4d989f6b9c0a589b944591f13de3dff34e186c14401e11b819fed81d70a5b", size = 1123881, upload-time = "2026-11-06T09:53:01.454Z" }, + { url = "https://files.pythonhosted.org/packages/c9/07/4d506b2e37601ec8b38a0c4b8f547dd344bd8eccd577dbd0543a744fce03/hypothesis-6.170.15-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f25261d1656f830a78863844be5fb55cd0208154dc9ddec85d44fe691488c557", size = 1167605, upload-time = "2026-11-06T09:53:32.309Z" }, + { url = "https://files.pythonhosted.org/packages/a5/77/a88824d02272f149870e36aaad0197dc5a186f53a1896ea3117f250b8414/hypothesis-6.170.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5ae92fa40b57c1cd331d63fe6f95aecfdb6aa32f15b4c418c108c41fcd4aef84", size = 1298841, upload-time = "2026-11-06T09:53:15.809Z" }, + { url = "https://files.pythonhosted.org/packages/68/b9/6e1b55b6b32926cebb7ef37076d8fda9d0f87045e90e4b75e93eeb9a56ac/hypothesis-6.170.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b829dc3cbd73fe08c02151afc438448180907da62c154cfa68abd178eafd4d48", size = 1333319, upload-time = "2026-11-06T09:52:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/c8/77/7d2009c663eb26b329c622091f7593ee6c90eac663f389fdf7311d44b035/hypothesis-6.170.15-cp313-cp313-win_amd64.whl", hash = "sha256:1f7ede44b875dddd8f3e7562e6702c363e82c6fb680b0fe079536bae662987f8", size = 691393, upload-time = "2026-11-06T09:54:56.287Z" }, + { url = "https://files.pythonhosted.org/packages/96/85/977d43b789fe4ca530e76020bbd2db07a9af3c17a4bf27a276416b6540e5/hypothesis-6.170.15-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:ffac67d06f765fd000a132e71117ee92dbd59863cea18403220ca4e39b50397f", size = 802549, upload-time = "2026-11-06T09:54:17.455Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/51e637155a8c897774463389f1341ec8d466a9860cf5e004689866c557b2/hypothesis-6.170.15-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82b613ee882068e781d810eb73a8195b59edd6cc47e8acbe4971562044a99c83", size = 794333, upload-time = "2026-11-06T09:54:27.915Z" }, + { url = "https://files.pythonhosted.org/packages/24/c1/ea255465b482ba5fc5f678e4347d6c6bbe878c827ca652d634328be96534/hypothesis-6.170.15-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb6fe3d1cb91d4f14fd307f10ae969ef7e866c16e4010a750d1cade145dfd764", size = 1124402, upload-time = "2026-11-06T09:53:02.773Z" }, + { url = "https://files.pythonhosted.org/packages/7a/69/32a05f0ae41f699f30ff872be74fe46ee1d45ead7abfb114afe3a4b4a3ee/hypothesis-6.170.15-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f91f27a89e8d55c7f043cb1e10705096236d4f7bcb35f3ac4f821e0c9b9bcd05", size = 1167795, upload-time = "2026-11-06T09:54:00.357Z" }, + { url = "https://files.pythonhosted.org/packages/c2/fb/c16886cea3a7ae3045093480775eb058eed549f41aa0e21a8d96aa7d51b1/hypothesis-6.170.15-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e0e4ab687c0f2bd62b7963dc8c3c8d29211cdc6db7ce05d37ef2a20d21219c18", size = 1299361, upload-time = "2026-11-06T09:53:04.441Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f3/ad57518f76503276c896388affeccca1ffb1cccc529d063dac588c4bb652/hypothesis-6.170.15-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b01d130c103fdf78a4a748cae7d095edfe5d37534828413a509936fb3567db61", size = 1333515, upload-time = "2026-11-06T09:53:18.642Z" }, + { url = "https://files.pythonhosted.org/packages/2b/08/500fbe05990d886d4502fa334f6cf03607be473b2a2efffb660238135f05/hypothesis-6.170.15-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:1138c03edf4c4401afb28e50255644b4ab4e1ad52864456cd4e84458ba8c671a", size = 634104, upload-time = "2026-11-06T09:54:22.317Z" }, + { url = "https://files.pythonhosted.org/packages/41/53/11e3260a72024f41246061ad204627f8336bfbf0ec2b180648d01a05491f/hypothesis-6.170.15-cp314-cp314-win_amd64.whl", hash = "sha256:57ae2617424f5bb55af0146b47e0aff64a9d6677a4d5216cf2a66bd3bda0d457", size = 691115, upload-time = "2026-11-06T09:53:41.223Z" }, + { url = "https://files.pythonhosted.org/packages/ba/41/655fca0f45b7ea6df6b5bb70ab52b740cc6bc30b971366036eddd36a77c9/hypothesis-6.170.15-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c0514f3f8ab5d56103b756317dedc8476945b0633e51112312247e3caecd0eb2", size = 800640, upload-time = "2026-11-06T09:53:05.61Z" }, + { url = "https://files.pythonhosted.org/packages/a5/89/6bf52da8b15846c7263eac9fcffb89cc1d1c82f96ca5b839a0d00bb3aa3a/hypothesis-6.170.15-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cc5434cde3b22c900dfe01f891804b48f6e2aa93e891ca98a14c07dd04fa6d54", size = 792915, upload-time = "2026-11-06T09:53:11.508Z" }, + { url = "https://files.pythonhosted.org/packages/11/f9/a0c2a356c7a3a4b65ec2f80382c9ea5acc1a86247ac64ecb1b7dfc38ee95/hypothesis-6.170.15-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f22900eec7df2696073c05f60c666654eaa93f17aebe8964f751d75b131dcd90", size = 1122841, upload-time = "2026-11-06T09:53:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f1/d4fa867fae5a733e4041720622417804bcfb79c00d265c5451243a5aba20/hypothesis-6.170.15-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7288412dfba6b5a6f875279a25409eda6b25d097c26ea710540f904287c55d06", size = 1166715, upload-time = "2026-11-06T09:54:42.908Z" }, + { url = "https://files.pythonhosted.org/packages/04/5d/cbf8acf14caa5a2e4d34dd088a507cc84da49a660b6f532104d8afdb2903/hypothesis-6.170.15-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7d546a5d6a5549a6b34304f7315d20aff7afd206996592329877da529b9d9153", size = 1297608, upload-time = "2026-11-06T09:53:29.22Z" }, + { url = "https://files.pythonhosted.org/packages/9c/a0/912a0ebf3ab19b39235233f9e1664147f6e842aa6915828d68c1d1e13f27/hypothesis-6.170.15-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bd8b36d0403eff6473686c0e4d8b7704780ebf548b0b2cc86cb9cbbbe64f8c2f", size = 1332521, upload-time = "2026-11-06T09:53:47.664Z" }, + { url = "https://files.pythonhosted.org/packages/37/36/d9286c792272651db4059a3497a19cc7f4f60e0452c862274d506b0132fd/hypothesis-6.170.15-cp314-cp314t-win_amd64.whl", hash = "sha256:67eb006925a3a49ddc4a48d9af8314349291a7eee9b2baea6beb93bc48211ead", size = 691229, upload-time = "2026-11-06T09:54:45.888Z" }, + { url = "https://files.pythonhosted.org/packages/91/42/b152b66db2061b47ff928efc7162c2dcf4591d92dab19b3c574af3552220/hypothesis-6.170.15-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:e007ff1dfb8bfa228b97ed6695325cfa1ba6487c7e1175c884e8795e71fb41a5", size = 800291, upload-time = "2026-11-06T09:54:07.899Z" }, + { url = "https://files.pythonhosted.org/packages/67/6f/49fe04f0b4cfd142d474e08a1855ea093c7367e6ff33e7c5d4a64592f743/hypothesis-6.170.15-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:742846b0e2e31640acf8bba7c2b8b6eb9cecd6817a906af6d4b5ced9db697660", size = 792763, upload-time = "2026-11-06T09:54:51.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/e6/61ec7fb3b84dd8d8644d6a322d5dba4501e2f32d0b10b8b5467e7d6defd4/hypothesis-6.170.15-cp315-abi3.abi3t-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:508488dd4e92e03ecd16ea086f33d766855ed1febd322a439ec231c920b1740a", size = 1158761, upload-time = "2026-11-06T09:54:02.105Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ae/c9eb63eb1b2d935551494aa42ecf91bb214d89c52631854dfbb09ccb6d39/hypothesis-6.170.15-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ccc6a9ce18dced7f0d2cba36f334f44697c1c6e692949c1bbd0bc67a7065c86a", size = 1121659, upload-time = "2026-11-06T09:54:12.041Z" }, + { url = "https://files.pythonhosted.org/packages/0a/44/b99e644480a31153de52489dc4c062df5f82a10fbb2b699d0f17ecb08171/hypothesis-6.170.15-cp315-abi3.abi3t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:b8a0b5b3e21a1d2e235f42a813fbf04872196ad53151fdd4cfe02723db699f8d", size = 1152560, upload-time = "2026-11-06T09:54:41.478Z" }, + { url = "https://files.pythonhosted.org/packages/f8/0a/bfc1c697f34e395d635f49d2a29f1bfc41a0eccdcf9fba32fb33601325de/hypothesis-6.170.15-cp315-abi3.abi3t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b59187094929ce6d5351a6f92e7f3831cd463fe69591def6ebd0023a3073af2e", size = 1146287, upload-time = "2026-11-06T09:53:42.894Z" }, + { url = "https://files.pythonhosted.org/packages/bb/69/90fac02af55ccbea1d1b3bbc72512c4f998e3fabc6b375c8c0afd0e6003f/hypothesis-6.170.15-cp315-abi3.abi3t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:0ecf4e2948c3de57ea12d66f6cf6c0e3ad88b5eae0d216cfa2922800ba5522a7", size = 1200042, upload-time = "2026-11-06T09:54:37.409Z" }, + { url = "https://files.pythonhosted.org/packages/f8/eb/6810206ee4864b39da6e81b2318516773d5a89b0e2132c00dfc9aa760179/hypothesis-6.170.15-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:55e3dcd5d8fbfd5a97018ff2712f375148b62ef537490c281a198977f3588578", size = 1165971, upload-time = "2026-11-06T09:53:25.72Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c8/ebfb57d277e571c207d690efa56462f1f582721e5170bb86af42c3f97b07/hypothesis-6.170.15-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:417fe54775f846fa4903bb2094ce3092a0ecc107db6676b21fd8069fcc328cb9", size = 1120269, upload-time = "2026-11-06T09:53:56.767Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2b/69dff10b56fbc21dc0d240de503d741ec14cb6819f67530bca0b6ff7c5af/hypothesis-6.170.15-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:47438186039fcfba3a245bb407490d442ce2c9cce4370565f1bfcdcb887e91e2", size = 1296436, upload-time = "2026-11-06T09:53:54.24Z" }, + { url = "https://files.pythonhosted.org/packages/5f/43/42b037bb17dca205f5953b11792ecc2c73dbf7ec743d5dc490d010edb9b0/hypothesis-6.170.15-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:372b05075d4a259c5ebf3fcf43dc689ef5d4335bed9d0c9dd25d14997842c0c0", size = 1425535, upload-time = "2026-11-06T09:53:27.106Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/bb3050f958b0e3156909011afdcc07812b58a47eba50f53a39d931c1256b/hypothesis-6.170.15-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:2161fb65416bf8da3b1c3119aa92ea63cc177fd88249bf7dc4ca64b4872e0bd1", size = 1379162, upload-time = "2026-11-06T09:54:23.523Z" }, + { url = "https://files.pythonhosted.org/packages/a5/db/30a2291c92cd437b0c8184bcead204add93d174a1317894dc50ddd3a681d/hypothesis-6.170.15-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:b3db4c8aecd13e043c419d90ccb683fdf19711b547ca9d23e685a35157d4657f", size = 1274757, upload-time = "2026-11-06T09:54:38.796Z" }, + { url = "https://files.pythonhosted.org/packages/a5/0a/5e1bf0060a1f9486f6cad2b512b5101526dc376565eb669c20964ccee730/hypothesis-6.170.15-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:3b81f08d6052ac74f479215029e253d88fb80e9867e8bc034252d345abfa94ce", size = 1292003, upload-time = "2026-11-06T09:53:14.207Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b9/9d18977c7ec9fa0d0fb22f0e52b3a270fef7b8251579d46660fca285e289/hypothesis-6.170.15-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:a51297c26ab637a97ec3283a39de163efce0a926d0ccd4378faabf5994adb01f", size = 1331925, upload-time = "2026-11-06T09:54:53.632Z" }, + { url = "https://files.pythonhosted.org/packages/ff/af/a86d51e4550c92a794777a59a9332d135c975c6f6c25b7506c447c3be27e/hypothesis-6.170.15-cp315-abi3.abi3t-win32.whl", hash = "sha256:51246e702c458b42a4b5079c3e6904c9dfb5193de920ed91fdbfa09fa694c55b", size = 685060, upload-time = "2026-11-06T09:53:35.48Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3d/ef64b33ef8c5eceb7448860b77bc1d28e3e3b3e92b7d662456fee227f4b0/hypothesis-6.170.15-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:a951de05b61b7c1235a732543dc6c3b105b6c894e736f21cb891449589aaeba7", size = 691031, upload-time = "2026-11-06T09:53:10.201Z" }, + { url = "https://files.pythonhosted.org/packages/e8/3d/f875573e232550be631b6932f93bd6b2234d880891394fe4d70d57e292b7/hypothesis-6.170.15-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:9dc31166ccb95928c6eb05817684b2dd1c2a3698a6daad0450e5f449e44bb633", size = 689008, upload-time = "2026-11-06T09:54:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/4d/f4/34e1ca38f88cbf478f970b7743ea022e0d70d5b4e943eea7c06532aea6b0/hypothesis-6.170.15-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bfe10cd8c8138ba84cdd0f72d72e0a85f0f9adea8bc730f4ab4676cd41a730af", size = 801079, upload-time = "2026-11-06T09:53:19.956Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8b/3438b357b3f6e076ff3f68aee5a62c1319573c4858b9f9db6e12e2893fca/hypothesis-6.170.15-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c1331070d8688d31b9ce99a100593bc6afdada964689b5e1a5f2ecffefa2455", size = 795993, upload-time = "2026-11-06T09:54:52.505Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a7/f25cc6ec59659ef59093ffa743a2494cc153bb8a8eeeb78e1c1e58720888/hypothesis-6.170.15-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b56ea17542eabcb9e2c1d574fcabc9df7e14a7376cfce324e2fa05e9c836fff8", size = 1125066, upload-time = "2026-11-06T09:54:19.213Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c3/4b58275fe5771859bd03fe5ceccee64968874fdb999c549abf7ae7f716b9/hypothesis-6.170.15-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1c3b98713318639a6c852f3fff60bddd466096528eb2cd5853ab077ef04f74c", size = 1168451, upload-time = "2026-11-06T09:53:38.214Z" }, + { url = "https://files.pythonhosted.org/packages/dd/b8/c935e11303f94a6e280e059aedc7c5a6458207f7039aa241d2504e178c53/hypothesis-6.170.15-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7ea75bc4c55f0ebb80179ae7e8fe1c8ab09b553005d1aadcec14e97e1306a82a", size = 694365, upload-time = "2026-11-06T09:53:59.021Z" }, + { url = "https://files.pythonhosted.org/packages/3b/54/ab7c9a7581b0acaeb58850629f4772aab054f2fd6a77bd3264262dfb1156/hypothesis-6.170.15-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:45c267cb5bcd7b73b1ca6cedb039de423b8695b34a31cc87c29a7787058e7e59", size = 801399, upload-time = "2026-11-06T09:54:10.562Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a6/a7dfce075d7371c9951e2a69e78d93bff5099d1b7b0ca207497eed68428b/hypothesis-6.170.15-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:3eb9428e0ba92993b61cab0289b8450f469b1e4c56cbdbdeaad258f0fb4f107e", size = 793309, upload-time = "2026-11-06T09:53:07.629Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b4/ce76691a588dca99c73116593b2514af2d0095e7e72d3bcdfb7dcecf2170/hypothesis-6.170.15-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9c30c911be51aa4fa84706f1d79dcd9ad6b9728207e69667a3fed2d4ab711434", size = 1123240, upload-time = "2026-11-06T09:53:44.286Z" }, + { url = "https://files.pythonhosted.org/packages/52/a7/998ba78de4842158178f6d96eca52ec1c786548b3e08902d527469329c04/hypothesis-6.170.15-pp312-pypy312_pp80-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2b68004bd5a238320569f7c03322b22815c2c145b1f7bbf52a319662570ee5c", size = 1167454, upload-time = "2026-11-06T09:54:48.426Z" }, + { url = "https://files.pythonhosted.org/packages/03/f5/410ff24a49dd645f651732f45e466cbb73f4bb73d1c54eaa5a07b832b0a2/hypothesis-6.170.15-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:d25e9a3f92e9f824bb110679506ece2b8fd7d093f1fa7d819ca46278a681a1f1", size = 692539, upload-time = "2026-11-06T09:53:30.812Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -615,7 +872,8 @@ name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ @@ -678,7 +936,8 @@ name = "platformdirs" version = "4.5.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/61/33/9611380c2bdb1225fdef633e2a9610622310fed35ab11dac9620972ee088/platformdirs-4.5.0.tar.gz", hash = "sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312", size = 21632, upload-time = "2025-10-08T17:44:48.791Z" } wheels = [ @@ -702,7 +961,8 @@ name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } @@ -725,6 +985,10 @@ dependencies = [ dev = [ { name = "black", version = "24.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "black", version = "25.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "hypothesis", version = "6.113.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "hypothesis", version = "6.141.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "hypothesis", version = "6.168.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "hypothesis", version = "6.170.15", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pytest", version = "8.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "pytest-cov", version = "5.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, @@ -739,6 +1003,7 @@ dev = [ requires-dist = [ { name = "black", marker = "extra == 'dev'", specifier = ">=23.0.0" }, { name = "click", specifier = ">=8.1.0" }, + { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1.0" }, { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.11.0" }, @@ -782,7 +1047,8 @@ name = "pytest" version = "8.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] dependencies = [ @@ -821,7 +1087,8 @@ name = "pytest-cov" version = "7.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] dependencies = [ @@ -855,7 +1122,8 @@ name = "pytest-mock" version = "3.15.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] dependencies = [ @@ -978,7 +1246,8 @@ name = "requests" version = "2.32.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] dependencies = [ @@ -1034,6 +1303,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2e/5d/aa883766f8ef9ffbe6aa24f7192fb71632f31a30e77eb39aa2b0dc4290ac/ruff-0.14.2-py3-none-win_arm64.whl", hash = "sha256:ea9d635e83ba21569fbacda7e78afbfeb94911c9434aff06192d9bc23fd5495a", size = 12554956, upload-time = "2025-10-23T19:36:58.714Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "tomli" version = "2.3.0" @@ -1100,7 +1378,7 @@ name = "typing-extensions" version = "4.15.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } @@ -1125,7 +1403,8 @@ name = "urllib3" version = "2.5.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.10'", + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } From 74272241d53fb4079db5e5cbe6bd969e59395f17 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 14 Nov 2026 01:11:04 -0500 Subject: [PATCH 5/5] Keep the supported Python matrix executable in CI Pin the test runner to Ubuntu 24.04 because ubuntu-latest now resolves to 26.04, where setup-python cannot install Python 3.8 or 3.9. Co-Authored-By: Codex --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b2b93b..58910d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,8 @@ jobs: xargs -0 python -m newsletter_uploader.comparables --require-dates < "$RUNNER_TEMP/changed" test: - runs-on: ubuntu-latest + # Ubuntu 26.04 has no setup-python builds for Python 3.8 or 3.9. + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11"]