From d1116172c3a03879a4e40b0a4e1826e6206a3146 Mon Sep 17 00:00:00 2001 From: robfrank Date: Thu, 12 Nov 2026 16:56:22 +0100 Subject: [PATCH] ci: ping IndexNow with the URLs each deploy changed Snapshot the live sitemaps before the Pages deploy, then submit the URLs that are new, removed, or whose git-based lastmod changed. The key file is published at the site root. Non-blocking: continue-on-error, waits for the key to be live, retries 429/5xx, logs every HTTP status. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/cloudflare-deploy.yml | 16 ++ .pre-commit-config.yaml | 2 + CLAUDE.md | 4 + package.json | 2 +- scripts/indexnow.py | 165 ++++++++++++++++++ .../d83c04689e7d448be13934efdf3c9f4d.txt | 1 + 6 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 scripts/indexnow.py create mode 100644 scripts/templates/d83c04689e7d448be13934efdf3c9f4d.txt diff --git a/.github/workflows/cloudflare-deploy.yml b/.github/workflows/cloudflare-deploy.yml index 2ee31a7d..b4f11e94 100644 --- a/.github/workflows/cloudflare-deploy.yml +++ b/.github/workflows/cloudflare-deploy.yml @@ -48,10 +48,26 @@ jobs: # which rebuilds the full-text search index on every commit. run: npm run build + # IndexNow (Bing, Yandex, ...): snapshot the live sitemaps before the + # deploy, then submit only the URLs whose lastmod changed, plus new and + # removed ones. Never fails the deploy. + - name: Snapshot live sitemaps for IndexNow + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && github.repository == 'ArcadeData/arcadedb-docs' + continue-on-error: true + timeout-minutes: 2 + run: python3 scripts/indexnow.py snapshot "$RUNNER_TEMP/live-sitemaps" + - name: Deploy to Cloudflare Pages (production) + id: deploy if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' uses: cloudflare/wrangler-action@953926a2e2182532811c01a25e53647d93bf07c0 # v4.1.3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} command: pages deploy build/site --project-name=arcadedb-docs --branch=main + + - name: Submit changed URLs to IndexNow + if: steps.deploy.outcome == 'success' && github.repository == 'ArcadeData/arcadedb-docs' + continue-on-error: true + timeout-minutes: 10 + run: python3 scripts/indexnow.py submit "$RUNNER_TEMP/live-sitemaps" build/site diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0fe5d1eb..176c6b6b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,8 @@ repos: - id: fix-byte-order-marker - id: trailing-whitespace - id: end-of-file-fixer + # The IndexNow key file must hold only the key, no trailing newline. + exclude: ^scripts/templates/[0-9a-f]{32}\.txt$ - id: mixed-line-ending - id: detect-aws-credentials args: diff --git a/CLAUDE.md b/CLAUDE.md index 16fe9213..f4c75278 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,6 +137,10 @@ Bing flags descriptions much shorter than 150-160 characters, so the generator k - `head-title.hbs` drops the ` :: ArcadeDB Documentation` suffix when the page title equals the site title (the docs home). - `npm run seo` (part of `npm run build`) runs `scripts/sitemap-lastmod.py`, which replaces Antora's build-time `` with each page source's last git commit date (it needs full history, hence `fetch-depth: 0` in the deploy workflow, and skips itself on a shallow clone), and `scripts/generate-llms-txt.py`, which writes `/llms.txt` from the nav. It links the HTML pages: Cloudflare's WAF currently 403s every `*.md` path. +### IndexNow + +The deploy workflow snapshots the live sitemaps before `wrangler pages deploy`, then `scripts/indexnow.py submit` posts only the URLs whose `lastmod` changed (plus new and removed URLs) to `api.indexnow.org`. Both steps are `continue-on-error`. The key is public: `scripts/templates/.txt` (no trailing newline), copied to the site root by `npm run redirects`. + ### URLs (no .html) Cloudflare Pages serves pages without `.html` and 308s `/page.html` to `/page`. The playbook sets `urls.html_extension_style: drop` so canonical tags, the sitemap and xrefs use the extensionless URL, and `scripts/drop-html-links.py` (part of `npm run build`) strips `.html` from relative links written by hand in passthrough HTML. Keep writing `page.html` in hand-written links; the build fixes them. `scripts/templates/robots.txt` adds the `Sitemap:` line and the `Content-Signal:` line (Cloudflare serves the file as is; it does not prepend a managed block). diff --git a/package.json b/package.json index b54058df..9e47edd2 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "links": "python3 scripts/drop-html-links.py build/site", "seo": "python3 scripts/sitemap-lastmod.py build/site && python3 scripts/generate-llms-txt.py build/site", "index": "pagefind --site build/site", - "redirects": "cp scripts/templates/_redirects scripts/templates/robots.txt build/site/", + "redirects": "cp scripts/templates/_redirects scripts/templates/robots.txt scripts/templates/d83c04689e7d448be13934efdf3c9f4d.txt build/site/", "serve": "npx -y http-server build/site -p 8080 -c-1" }, "devDependencies": { diff --git a/scripts/indexnow.py b/scripts/indexnow.py new file mode 100644 index 00000000..83898313 --- /dev/null +++ b/scripts/indexnow.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +""" +Tell IndexNow (Bing, Yandex, Seznam, Naver, ...) which docs URLs changed in +a deploy. Google does not use IndexNow. + +Two steps, run by .github/workflows/cloudflare-deploy.yml around the deploy: + + indexnow.py snapshot DIR before the deploy: save the live sitemaps + indexnow.py submit DIR SITE after the deploy: diff SITE's sitemaps + against DIR and submit what changed + +A URL counts as changed when it is new, when its differs (lastmod +is the page source's last git commit date, see sitemap-lastmod.py), or when +it left the sitemap (a removed or redirected page, so engines recrawl it and +see the 404 or 301). Without a snapshot nothing is submitted: resubmitting +every URL on every deploy is what IndexNow asks sites not to do. + +The key is public by design: scripts/templates/.txt holds it and +`npm run redirects` copies it to the site root. Exit status is 0 unless the +command line is wrong; the workflow step is non-blocking anyway. +""" +from __future__ import annotations + +import json +import re +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +HOST = "docs.arcadedb.com" +SITE_URL = f"https://{HOST}" +ENDPOINT = "https://api.indexnow.org/indexnow" +KEY_DIR = Path(__file__).resolve().parent / "templates" +MAX_URLS = 10_000 +UA = "arcadedb-docs-deploy" + +URL_RE = re.compile(r"(.*?)", re.S) +LOC_RE = re.compile(r"([^<]+)") +LASTMOD_RE = re.compile(r"([^<]+)") + + +def key() -> str: + keys = [p.stem for p in KEY_DIR.glob("*.txt") if re.fullmatch(r"[a-f0-9]{32}", p.stem)] + if len(keys) != 1: + sys.exit(f"expected one IndexNow key file in {KEY_DIR}, found {len(keys)}") + return keys[0] + + +def get(url: str) -> tuple[int, str]: + req = urllib.request.Request(url, headers={"User-Agent": UA}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.read().decode("utf-8") + except urllib.error.HTTPError as e: + return e.code, "" + + +def parse(xml: str) -> dict[str, str]: + entries = {} + for body in URL_RE.findall(xml): + loc = LOC_RE.search(body) + if loc: + mod = LASTMOD_RE.search(body) + entries[loc.group(1).strip()] = mod.group(1).strip() if mod else "" + return entries + + +def snapshot(out: Path) -> int: + out.mkdir(parents=True, exist_ok=True) + status, index = get(f"{SITE_URL}/sitemap.xml") + if status != 200: + print(f"indexnow: live sitemap.xml returned {status}, no snapshot") + return 0 + for loc in re.findall(r"([^<]+)", index): + name = loc.rsplit("/", 1)[-1] + status, xml = get(loc) + if status == 200: + (out / name).write_text(xml, encoding="utf-8") + print(f"indexnow: snapshot {name}: {status}") + return 0 + + +def load(directory: Path) -> dict[str, str]: + entries: dict[str, str] = {} + for f in sorted(directory.glob("sitemap-*.xml")): + entries.update(parse(f.read_text(encoding="utf-8"))) + return entries + + +def changed(old: dict[str, str], new: dict[str, str]) -> list[str]: + urls = [u for u, mod in new.items() if old.get(u) != mod] + urls += [u for u in old if u not in new] + return sorted(set(urls)) + + +def wait_for(url: str, expect: str, attempts: int = 10, delay: int = 15) -> bool: + for i in range(attempts): + status, body = get(url) + if status == 200 and body.strip() == expect: + return True + print(f"indexnow: {url} -> {status}, waiting ({i + 1}/{attempts})") + time.sleep(delay) + return False + + +def post(urls: list[str], k: str) -> int: + payload = json.dumps({ + "host": HOST, + "key": k, + "keyLocation": f"{SITE_URL}/{k}.txt", + "urlList": urls, + }).encode("utf-8") + req = urllib.request.Request(ENDPOINT, data=payload, method="POST", + headers={"Content-Type": "application/json; charset=utf-8", "User-Agent": UA}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status + except urllib.error.HTTPError as e: + return e.code + except urllib.error.URLError as e: + print(f"indexnow: {e.reason}") + return 0 + + +def submit(previous: Path, site: Path) -> int: + old = load(previous) if previous.is_dir() else {} + if not old: + print("indexnow: no snapshot of the previous sitemaps, nothing submitted") + return 0 + urls = changed(old, load(site)) + print(f"indexnow: {len(urls)} changed URLs") + for u in urls: + print(f" {u}") + if not urls: + return 0 + k = key() + if not wait_for(f"{SITE_URL}/{k}.txt", k): + print("indexnow: key file not live, nothing submitted") + return 0 + for start in range(0, len(urls), MAX_URLS): + batch = urls[start:start + MAX_URLS] + for attempt in range(3): + status = post(batch, k) + print(f"indexnow: POST {len(batch)} URLs -> HTTP {status}") + if status in (200, 202): + break + if status in (400, 403, 422): # bad request or key: retrying will not help + break + time.sleep(30 * (attempt + 1)) + return 0 + + +def main(argv: list[str]) -> int: + if len(argv) == 2 and argv[0] == "snapshot": + return snapshot(Path(argv[1])) + if len(argv) == 3 and argv[0] == "submit": + return submit(Path(argv[1]), Path(argv[2])) + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/templates/d83c04689e7d448be13934efdf3c9f4d.txt b/scripts/templates/d83c04689e7d448be13934efdf3c9f4d.txt new file mode 100644 index 00000000..c1ff731b --- /dev/null +++ b/scripts/templates/d83c04689e7d448be13934efdf3c9f4d.txt @@ -0,0 +1 @@ +d83c04689e7d448be13934efdf3c9f4d \ No newline at end of file