Make the site discoverable to agents, and serve the docs as markdown

web/CLAUDE.md

@@ -102,8 +102,49 @@ shipped belongs in the app's `CHANGELOG.md` only.
   a fallback for republishing without a commit, per
   [DEPLOYMENT.md](./DEPLOYMENT.md).
 
+## Machine readers
+
+Some of what the site serves is addressed to crawlers and agents rather than
+people. All of it is generated — none of it is a list to keep up to date by hand:
+
+- `sitemap.xml` (a sitemap index over `sitemap-0.xml`), from `@astrojs/sitemap`.
+  `public/robots.txt` points at the index.
+- `llms.txt` ([llmstxt.org](https://llmstxt.org)), from `src/pages/llms.txt.ts`,
+  built off the `docs` collection — so a new page under the repo's `docs/`
+  appears in it for the same reason it appears on the `/docs` hub. It links the
+  `.md` twins below, not the pages: what follows an entry there is an agent.
+- **A `.md` twin of every docs page**, from `src/pages/docs/[slug].md.ts`:
+  `/docs/reading` is the page, `/docs/reading.md` is the markdown it was
+  rendered from. The "Copy markdown" button on each page fetches its own twin,
+  so the HTML never carries a second copy of the prose. The body is served
+  verbatim except for its links, which are resolved to absolute URLs through the
+  same `rewriteLink` the rendered page uses — exported from
+  `remark-docs-assets.mjs` so a link cannot mean two things. It relies on the
+  docs corpus having no link titles and no reference definitions; a parser would
+  reformat the prose, and serving the source byte for byte is the point.
+- JSON-LD, passed to `Base.astro` as a `schema` prop by the page that knows what
+  it is describing: `SoftwareApplication` from `index.astro`, `FAQPage` from
+  `docs/[...slug].astro`.
+
+So the FAQ's answers now reach a reader three ways — the accordion, the JSON-LD,
+and `/docs/faq.md` — all three derived from `docs/faq.md` alone.
+
+The `FAQPage` one has a **silent** failure mode. Its questions are collected by
+`rehype-faq-accordion.mjs` as it builds the accordion and handed over as
+`render()`'s `remarkPluginFrontmatter` — so the markup and the structured data
+can't disagree, but a stale Astro content cache drops the JSON-LD with the build
+still exiting 0 (see Verify below). It also treats every `### heading` under a
+`## section` of `docs/faq.md` as a real question, so a `###` that isn't one lands
+in `mainEntity` as a question with whatever prose follows it.
+
 ## Verify
 
 `npm run build` is the check that matters — `src/data/*.ts` is typed, so a
 malformed entry fails the build rather than rendering wrong. Run it after any
 data edit.
+
+**After editing a remark or rehype plugin, `rm -rf .astro node_modules/.astro`
+first.** The content cache keys on the markdown, not on the plugins, so a plugin
+edit alone re-emits the previous HTML — a passing build proving nothing. Confirm
+the output rather than the exit code: `grep -rl 'ld+json' dist` should list
+`dist/index.html` and `dist/docs/faq/index.html`.

web/README.md

@@ -38,6 +38,7 @@ web/
 │  └─ remark-docs-assets.mjs  rewrites .md links + screenshot paths at build time
 ├─ public/
 │  ├─ icon.png                app icon (favicon + OG image)
+│  ├─ robots.txt              crawlers welcome; points at the generated sitemap
 │  └─ screenshots/            GENERATED — `prebuild` copies ../docs/assets/screenshots
 ├─ src/
 │  ├─ styles/global.css       design tokens, base styles, keyframes, shared utilities
@@ -53,7 +54,9 @@ web/
 │     ├─ index.astro          landing
 │     ├─ docs.astro           docs hub — card grid built from the collection
 │     ├─ docs/[...slug].astro every docs page, rendered from ../docs/*.md
+│     ├─ docs/[slug].md.ts    the same pages as raw markdown, at /docs/<x>.md
 │     ├─ changelog.astro      release notes
+│     ├─ llms.txt.ts          llmstxt.org index, built from the docs collection
 │     └─ 404.astro
 ```
 

web/astro.config.mjs

@@ -1,12 +1,30 @@
 // @ts-check
+import { rename } from "node:fs/promises";
 import { defineConfig } from "astro/config";
+import sitemap from "@astrojs/sitemap";
 import { rehypeHeadingIds } from "@astrojs/markdown-remark";
 import { remarkDocsAssets } from "./plugins/remark-docs-assets.mjs";
 import { rehypeFaqAccordion } from "./plugins/rehype-faq-accordion.mjs";
 
 // https://astro.build/config
 export default defineConfig({
   site: "https://reader-md.jnahian.me",
+  // Every page is static and listed here; robots.txt points crawlers at it.
+  // 404 is excluded by the integration itself.
+  integrations: [
+    sitemap(),
+    // The integration hard-codes `<filenameBase>-index.xml`, but /sitemap.xml is
+    // the name a crawler guesses. Renaming the index rather than the chunk keeps
+    // the split: sitemap.xml stays an index over sitemap-0.xml, so a site that
+    // outgrew one chunk would still be described correctly.
+    {
+      name: "sitemap-at-the-conventional-name",
+      hooks: {
+        "astro:build:done": ({ dir }) =>
+          rename(new URL("sitemap-index.xml", dir), new URL("sitemap.xml", dir)),
+      },
+    },
+  ],
   markdown: {
     remarkPlugins: [remarkDocsAssets],
     // rehypeHeadingIds is listed explicitly so it runs before the accordion

web/package.json

@@ -15,6 +15,7 @@
   },
   "dependencies": {
     "@astrojs/markdown-remark": "^7.2.1",
+    "@astrojs/sitemap": "^3.7.4",
     "astro": "^7.0.7"
   }
 }

web/plugins/rehype-faq-accordion.mjs

@@ -37,8 +37,10 @@ function text(node) {
   return (node.children ?? []).map(text).join("");
 }
 
-const searchable = (nodes) =>
-  nodes.map(text).join(" ").replace(/\s+/g, " ").trim().toLowerCase();
+const flatten = (nodes) =>
+  nodes.map(text).join(" ").replace(/\s+/g, " ").trim();
+
+const searchable = (nodes) => flatten(nodes).toLowerCase();
 
 // The field, the live count, and the empty state. Emitted here rather than in
 // the page component so everything that knows the FAQ is special stays in this
@@ -100,6 +102,11 @@ export function rehypeFaqAccordion() {
   return (tree, file) => {
     if (!/(^|\/)docs\/faq\.md$/.test(file?.path?.replace(/\\/g, "/") ?? "")) return;
 
+    // The same questions, as plain text, for the page's FAQPage JSON-LD. Read
+    // off the tree here because this is where the answer's extent is already
+    // known — a second parse in the page would have to re-derive it.
+    const questionsAndAnswers = [];
+
     visit(tree, "root", (root) => {
       const { lead, groups } = sections(root.children, "h2");
 
@@ -109,15 +116,16 @@ export function rehypeFaqAccordion() {
         ...groups.map(([heading, ...body]) => {
           const { lead: prose, groups: questions } = sections(body, "h3");
 
-          const list = questions.map(([q, ...answer]) =>
-            el("details", {
+          const list = questions.map(([q, ...answer]) => {
+            questionsAndAnswers.push({ q: flatten([q]), a: flatten(answer) });
+            return el("details", {
               className: ["faq-q"],
               dataSearch: searchable([q, ...answer]),
             }, [
               el("summary", { className: ["faq-q__q"] }, [q]),
               el("div", { className: ["faq-q__a"] }, answer),
-            ])
-          );
+            ]);
+          });
 
           return el("section", {
             className: ["faq-group"],
@@ -136,5 +144,10 @@ export function rehypeFaqAccordion() {
         }),
       ];
     });
+
+    // Reaches the page as render()'s remarkPluginFrontmatter, which is outside
+    // the collection schema — so this doesn't have to be declared as content.
+    const frontmatter = file?.data?.astro?.frontmatter;
+    if (frontmatter) frontmatter.faq = questionsAndAnswers;
   };
 }

web/plugins/remark-docs-assets.mjs

@@ -25,7 +25,9 @@ const BLOB = "https://github.com/jnahian/reader.md/blob/main/";
 // Absolute URLs, site-absolute paths, and bare fragments are already correct.
 const ABSOLUTE = /^(?:[a-z][a-z0-9+.-]*:|\/|#)/i;
 
-function rewriteLink(url, from) {
+// Exported: pages/docs/[slug].md.ts rewrites the same links in the raw markdown
+// it serves, and the two readings of a link must not disagree.
+export function rewriteLink(url, from) {
   if (!from || ABSOLUTE.test(url)) return null;
   const cut = url.indexOf("#");
   const target = cut === -1 ? url : url.slice(0, cut);

web/public/robots.txt

@@ -0,0 +1,5 @@
+# Everything here is public documentation for a free, open-source app.
+User-agent: *
+Allow: /
+
+Sitemap: https://reader-md.jnahian.me/sitemap.xml

web/src/components/Icon.astro

@@ -1,7 +1,7 @@
 ---
 // Inline Lucide icons (https://lucide.dev) — no dependency, just the paths we use.
 interface Props {
-  name: "grid" | "book" | "help" | "history" | "download" | "star" | "menu" | "close" | "chevron";
+  name: "grid" | "book" | "help" | "history" | "download" | "star" | "menu" | "close" | "chevron" | "copy" | "check";
   size?: number;
 }
 const { name, size = 16 } = Astro.props;
@@ -16,6 +16,8 @@ const paths: Record<Props["name"], string> = {
   menu: '<path d="M4 6h16"/><path d="M4 12h16"/><path d="M4 18h16"/>',
   close: '<path d="M18 6 6 18"/><path d="m6 6 12 12"/>',
   chevron: '<path d="m6 9 6 6 6-6"/>',
+  copy: '<rect width="14" height="14" x="8" y="8" rx="2"/><path d="M4 16c-1.1 0-2-.9-2-2V4c0-1.1.9-2 2-2h10c1.1 0 2 .9 2 2"/>',
+  check: '<path d="M20 6 9 17l-5-5"/>',
 };
 ---
 

web/src/data/site.ts

@@ -9,6 +9,10 @@ export const brewTap = "brew tap jnahian/reader.md https://github.com/jnahian/re
 export const brewTrust = "brew trust --cask jnahian/reader.md/reader.md";
 export const brewInstall = "brew install --cask reader-md";
 export const author = "Julkar Naen Nahian";
+// The site's one-sentence description: the default <meta name="description">,
+// and the summary line of llms.txt.
+export const description =
+  "Reader.md opens plans, specs and READMEs in a native macOS reading window — outline, search across every folder, highlights, live reload, Mermaid diagrams and LaTeX math.";
 
 export type Page = "home" | "docs" | "changelog";
 

web/src/layouts/Base.astro

@@ -2,18 +2,22 @@
 import "../styles/global.css";
 import Nav from "../components/Nav.astro";
 import Footer from "../components/Footer.astro";
-import type { Page } from "../data/site";
+import { description as siteDescription, type Page } from "../data/site";
 
 interface Props {
   title: string;
   description?: string;
   page?: Page;
+  // schema.org JSON-LD for this page, if it has any worth stating. Built by the
+  // page, not here — only the page knows what it is describing.
+  schema?: Record<string, unknown>;
 }
 
 const {
   title,
-  description = "Reader.md opens plans, specs and READMEs in a native macOS reading window — outline, search across every folder, highlights, live reload, Mermaid diagrams and LaTeX math.",
+  description = siteDescription,
   page = "home",
+  schema,
 } = Astro.props;
 
 // Home nav starts transparent and turns solid on scroll; inner pages are solid.
@@ -39,6 +43,17 @@ const wideFooter = page === "docs";
     <meta property="og:url" content={new URL(Astro.url.pathname, Astro.site)} />
     <meta property="og:type" content="website" />
     <meta name="twitter:card" content="summary_large_image" />
+    {
+      /* `<` is escaped rather than written raw: the FAQ's answers become part of
+         this payload, and one of them contains a shell command. */
+      schema && (
+        <script
+          type="application/ld+json"
+          is:inline
+          set:html={JSON.stringify(schema).replace(/</g, "\\u003c")}
+        />
+      )
+    }
   </head>
   <body>
     <Nav page={page} solid={solidNav} />