From ab4ddbb2992be37bf196a285dab6a182d28669e5 Mon Sep 17 00:00:00 2001 From: Julkar Naen Nahian Date: Mon, 7 Sep 2026 00:07:30 +0600 Subject: [PATCH 1/3] feat(web): make the site discoverable to agents and crawlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pages were already static HTML with titles, descriptions, canonicals and OG tags — readable, but with nothing telling a machine what is here. Nothing enumerated the 16 docs pages, and nothing typed the claims the prose makes. Four additions, none of them a list to maintain by hand: - @astrojs/sitemap, since `site` was already configured. 18 URLs, 404 excluded. It hard-codes `-index.xml`, so a build hook renames the index to the /sitemap.xml a crawler actually guesses; renaming the index rather than the chunk keeps the split intact. - public/robots.txt, pointing at /sitemap.xml. - llms.txt (llmstxt.org), generated from the docs collection — so a new page under the repo's docs/ lists itself, for the same reason it appears on the /docs hub. A static file in public/ would drift on the next page added, which is what plugins/docs-pages.mjs exists to prevent. - JSON-LD, as a `schema` prop on Base.astro passed by the page that knows what it is describing: SoftwareApplication from index.astro (softwareVersion off latestVersion, so it tracks releases), FAQPage from docs/[...slug].astro. The FAQ's questions are collected by rehype-faq-accordion as it builds the accordion and handed over as render()'s remarkPluginFrontmatter, so the visible answers and the structured ones cannot disagree — 30 questions, no second parse of the markdown. That coupling has a silent failure mode: a stale Astro content cache re-emits the old HTML after a plugin edit and the JSON-LD just isn't there, build still green. Recorded in web/CLAUDE.md under Machine readers and Verify, alongside the FAQPage schema's assumption that every h3 in docs/faq.md is a real question. The description moves to site.ts, which llms.txt needs verbatim too. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JGMQ49hvA936om3GQfNTAn --- web/CLAUDE.md | 28 +++++++++++++ web/README.md | 2 + web/astro.config.mjs | 18 +++++++++ web/package-lock.json | 60 ++++++++++++++++++++++++++++ web/package.json | 1 + web/plugins/rehype-faq-accordion.mjs | 25 +++++++++--- web/public/robots.txt | 5 +++ web/src/data/site.ts | 4 ++ web/src/layouts/Base.astro | 19 ++++++++- web/src/pages/docs/[...slug].astro | 24 ++++++++++- web/src/pages/index.astro | 24 ++++++++++- web/src/pages/llms.txt.ts | 58 +++++++++++++++++++++++++++ 12 files changed, 257 insertions(+), 11 deletions(-) create mode 100644 web/public/robots.txt create mode 100644 web/src/pages/llms.txt.ts diff --git a/web/CLAUDE.md b/web/CLAUDE.md index 604aef7..4b7dd7e 100644 --- a/web/CLAUDE.md +++ b/web/CLAUDE.md @@ -102,8 +102,36 @@ shipped belongs in the app's `CHANGELOG.md` only. a fallback for republishing without a commit, per [DEPLOYMENT.md](./DEPLOYMENT.md). +## Machine readers + +Three files exist for crawlers and agents rather than people, and all three are +generated — none is a list to keep up to date by hand: + +- `sitemap.xml` (a sitemap index over `sitemap-0.xml`), from `@astrojs/sitemap`. + `public/robots.txt` points at the index. +- `llms.txt` ([llmstxt.org](https://llmstxt.org)), from `src/pages/llms.txt.ts`, + built off the `docs` collection — so a new page under the repo's `docs/` + appears in it for the same reason it appears on the `/docs` hub. +- JSON-LD, passed to `Base.astro` as a `schema` prop by the page that knows what + it is describing: `SoftwareApplication` from `index.astro`, `FAQPage` from + `docs/[...slug].astro`. + +The `FAQPage` one has a **silent** failure mode. Its questions are collected by +`rehype-faq-accordion.mjs` as it builds the accordion and handed over as +`render()`'s `remarkPluginFrontmatter` — so the markup and the structured data +can't disagree, but a stale Astro content cache drops the JSON-LD with the build +still exiting 0 (see Verify below). It also treats every `### heading` under a +`## section` of `docs/faq.md` as a real question, so a `###` that isn't one lands +in `mainEntity` as a question with whatever prose follows it. + ## Verify `npm run build` is the check that matters — `src/data/*.ts` is typed, so a malformed entry fails the build rather than rendering wrong. Run it after any data edit. + +**After editing a remark or rehype plugin, `rm -rf .astro node_modules/.astro` +first.** The content cache keys on the markdown, not on the plugins, so a plugin +edit alone re-emits the previous HTML — a passing build proving nothing. Confirm +the output rather than the exit code: `grep -rl 'ld+json' dist` should list +`dist/index.html` and `dist/docs/faq/index.html`. diff --git a/web/README.md b/web/README.md index e8e4979..30d95b8 100644 --- a/web/README.md +++ b/web/README.md @@ -38,6 +38,7 @@ web/ │ └─ remark-docs-assets.mjs rewrites .md links + screenshot paths at build time ├─ public/ │ ├─ icon.png app icon (favicon + OG image) +│ ├─ robots.txt crawlers welcome; points at the generated sitemap │ └─ screenshots/ GENERATED — `prebuild` copies ../docs/assets/screenshots ├─ src/ │ ├─ styles/global.css design tokens, base styles, keyframes, shared utilities @@ -54,6 +55,7 @@ web/ │ ├─ docs.astro docs hub — card grid built from the collection │ ├─ docs/[...slug].astro every docs page, rendered from ../docs/*.md │ ├─ changelog.astro release notes +│ ├─ llms.txt.ts llmstxt.org index, built from the docs collection │ └─ 404.astro ``` diff --git a/web/astro.config.mjs b/web/astro.config.mjs index fe587fa..26c650d 100644 --- a/web/astro.config.mjs +++ b/web/astro.config.mjs @@ -1,5 +1,7 @@ // @ts-check +import { rename } from "node:fs/promises"; import { defineConfig } from "astro/config"; +import sitemap from "@astrojs/sitemap"; import { rehypeHeadingIds } from "@astrojs/markdown-remark"; import { remarkDocsAssets } from "./plugins/remark-docs-assets.mjs"; import { rehypeFaqAccordion } from "./plugins/rehype-faq-accordion.mjs"; @@ -7,6 +9,22 @@ import { rehypeFaqAccordion } from "./plugins/rehype-faq-accordion.mjs"; // https://astro.build/config export default defineConfig({ site: "https://reader-md.jnahian.me", + // Every page is static and listed here; robots.txt points crawlers at it. + // 404 is excluded by the integration itself. + integrations: [ + sitemap(), + // The integration hard-codes `-index.xml`, but /sitemap.xml is + // the name a crawler guesses. Renaming the index rather than the chunk keeps + // the split: sitemap.xml stays an index over sitemap-0.xml, so a site that + // outgrew one chunk would still be described correctly. + { + name: "sitemap-at-the-conventional-name", + hooks: { + "astro:build:done": ({ dir }) => + rename(new URL("sitemap-index.xml", dir), new URL("sitemap.xml", dir)), + }, + }, + ], markdown: { remarkPlugins: [remarkDocsAssets], // rehypeHeadingIds is listed explicitly so it runs before the accordion diff --git a/web/package-lock.json b/web/package-lock.json index 746ca97..fc70614 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "@astrojs/markdown-remark": "^7.2.1", + "@astrojs/sitemap": "^3.7.4", "astro": "^7.0.7" }, "engines": { @@ -257,6 +258,16 @@ "node": ">=22.12.0" } }, + "node_modules/@astrojs/sitemap": { + "version": "3.7.4", + "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.7.4.tgz", + "integrity": "sha512-LbKNC24bdUWcQf/pThB6qLlSqHojxGjZDURIzFocY8rlWnAn2t74nnhnK6S5x0NHriHoAduLEpVjRykmeGiVvA==", + "license": "MIT", + "dependencies": { + "sitemap": "^9.0.0", + "zod": "^4.3.6" + } + }, "node_modules/@astrojs/telemetry": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.3.3.tgz", @@ -1947,6 +1958,24 @@ "@types/unist": "*" } }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/sax": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz", + "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -1996,6 +2025,12 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -4740,6 +4775,25 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "license": "MIT" }, + "node_modules/sitemap": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-9.0.1.tgz", + "integrity": "sha512-S6hzjGJSG3d6if0YoF5kTyeRJvia6FSTBroE5fQ0bu1QNxyJqhhinfUsXi9fH3MgtXODWvwo2BDyQSnhPQ88uQ==", + "license": "MIT", + "dependencies": { + "@types/node": "^24.9.2", + "@types/sax": "^1.2.1", + "arg": "^5.0.0", + "sax": "^1.4.1" + }, + "bin": { + "sitemap": "dist/esm/cli.js" + }, + "engines": { + "node": ">=20.19.5", + "npm": ">=10.8.2" + } + }, "node_modules/smol-toml": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.0.tgz", @@ -4895,6 +4949,12 @@ "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==", "license": "MIT" }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, "node_modules/unified": { "version": "11.0.5", "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", diff --git a/web/package.json b/web/package.json index a18f9c5..97127e9 100644 --- a/web/package.json +++ b/web/package.json @@ -15,6 +15,7 @@ }, "dependencies": { "@astrojs/markdown-remark": "^7.2.1", + "@astrojs/sitemap": "^3.7.4", "astro": "^7.0.7" } } diff --git a/web/plugins/rehype-faq-accordion.mjs b/web/plugins/rehype-faq-accordion.mjs index cd9a0c6..2ab27d5 100644 --- a/web/plugins/rehype-faq-accordion.mjs +++ b/web/plugins/rehype-faq-accordion.mjs @@ -37,8 +37,10 @@ function text(node) { return (node.children ?? []).map(text).join(""); } -const searchable = (nodes) => - nodes.map(text).join(" ").replace(/\s+/g, " ").trim().toLowerCase(); +const flatten = (nodes) => + nodes.map(text).join(" ").replace(/\s+/g, " ").trim(); + +const searchable = (nodes) => flatten(nodes).toLowerCase(); // The field, the live count, and the empty state. Emitted here rather than in // the page component so everything that knows the FAQ is special stays in this @@ -100,6 +102,11 @@ export function rehypeFaqAccordion() { return (tree, file) => { if (!/(^|\/)docs\/faq\.md$/.test(file?.path?.replace(/\\/g, "/") ?? "")) return; + // The same questions, as plain text, for the page's FAQPage JSON-LD. Read + // off the tree here because this is where the answer's extent is already + // known — a second parse in the page would have to re-derive it. + const questionsAndAnswers = []; + visit(tree, "root", (root) => { const { lead, groups } = sections(root.children, "h2"); @@ -109,15 +116,16 @@ export function rehypeFaqAccordion() { ...groups.map(([heading, ...body]) => { const { lead: prose, groups: questions } = sections(body, "h3"); - const list = questions.map(([q, ...answer]) => - el("details", { + const list = questions.map(([q, ...answer]) => { + questionsAndAnswers.push({ q: flatten([q]), a: flatten(answer) }); + return el("details", { className: ["faq-q"], dataSearch: searchable([q, ...answer]), }, [ el("summary", { className: ["faq-q__q"] }, [q]), el("div", { className: ["faq-q__a"] }, answer), - ]) - ); + ]); + }); return el("section", { className: ["faq-group"], @@ -136,5 +144,10 @@ export function rehypeFaqAccordion() { }), ]; }); + + // Reaches the page as render()'s remarkPluginFrontmatter, which is outside + // the collection schema — so this doesn't have to be declared as content. + const frontmatter = file?.data?.astro?.frontmatter; + if (frontmatter) frontmatter.faq = questionsAndAnswers; }; } diff --git a/web/public/robots.txt b/web/public/robots.txt new file mode 100644 index 0000000..e74db4e --- /dev/null +++ b/web/public/robots.txt @@ -0,0 +1,5 @@ +# Everything here is public documentation for a free, open-source app. +User-agent: * +Allow: / + +Sitemap: https://reader-md.jnahian.me/sitemap.xml diff --git a/web/src/data/site.ts b/web/src/data/site.ts index 33f30b7..f01f8d2 100644 --- a/web/src/data/site.ts +++ b/web/src/data/site.ts @@ -9,6 +9,10 @@ export const brewTap = "brew tap jnahian/reader.md https://github.com/jnahian/re export const brewTrust = "brew trust --cask jnahian/reader.md/reader.md"; export const brewInstall = "brew install --cask reader-md"; export const author = "Julkar Naen Nahian"; +// The site's one-sentence description: the default , +// and the summary line of llms.txt. +export const description = + "Reader.md opens plans, specs and READMEs in a native macOS reading window — outline, search across every folder, highlights, live reload, Mermaid diagrams and LaTeX math."; export type Page = "home" | "docs" | "changelog"; diff --git a/web/src/layouts/Base.astro b/web/src/layouts/Base.astro index 6f258ca..7c31a88 100644 --- a/web/src/layouts/Base.astro +++ b/web/src/layouts/Base.astro @@ -2,18 +2,22 @@ import "../styles/global.css"; import Nav from "../components/Nav.astro"; import Footer from "../components/Footer.astro"; -import type { Page } from "../data/site"; +import { description as siteDescription, type Page } from "../data/site"; interface Props { title: string; description?: string; page?: Page; + // schema.org JSON-LD for this page, if it has any worth stating. Built by the + // page, not here — only the page knows what it is describing. + schema?: Record; } const { title, - description = "Reader.md opens plans, specs and READMEs in a native macOS reading window — outline, search across every folder, highlights, live reload, Mermaid diagrams and LaTeX math.", + description = siteDescription, page = "home", + schema, } = Astro.props; // Home nav starts transparent and turns solid on scroll; inner pages are solid. @@ -39,6 +43,17 @@ const wideFooter = page === "docs"; + { + /* `<` is escaped rather than written raw: the FAQ's answers become part of + this payload, and one of them contains a shell command. */ + schema && ( +