docs(seo): JSON-LD, Open Graph, git-dated sitemap lastmod, llms.txt, robots.txt

.github/workflows/cloudflare-deploy.yml

@@ -23,6 +23,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # Full history: scripts/sitemap-lastmod.py dates each sitemap URL
+          # from the last commit that touched the page's source.
+          fetch-depth: 0
 
       - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
         with:

CLAUDE.md

@@ -129,9 +129,15 @@ Every page gets `<meta name="description">` from its `:description:` header attr
 
 Bing flags descriptions much shorter than 150-160 characters, so the generator keeps adding intro text up to ~120 characters and tops up a short intro with the section titles. Pages with no usable intro (chapter aggregators, SQL reference pages that open on a syntax block) get a generic "ArcadeDB X: A, B, C." summary; give those a hand-written `:description:` of 140-160 characters. For a chapter file with no heading in the source, put it on the first line.
 
+### Structured data, social tags, sitemap dates, llms.txt
+
+- `docs/ui/supplemental/partials/head-meta.hbs` writes Open Graph and Twitter tags plus one JSON-LD `@graph` per page (WebSite, BreadcrumbList, TechArticle, or WebPage on a component home). The object is built in `docs/ui/supplemental/helpers/jsonld.js` and serialized with `JSON.stringify`, never templated by hand; Organization and SoftwareApplication are referenced by `@id` on arcadedb.com only. `@id`/`url`/`og:url` use `page.canonicalUrl`, so they stay extensionless. Antora loads each helper from a string, so a helper cannot `require()` a sibling helper.
+- `head-title.hbs` drops the ` :: ArcadeDB Documentation` suffix when the page title equals the site title (the docs home).
+- `npm run seo` (part of `npm run build`) runs `scripts/sitemap-lastmod.py`, which replaces Antora's build-time `<lastmod>` with each page source's last git commit date (it needs full history, hence `fetch-depth: 0` in the deploy workflow, and skips itself on a shallow clone), and `scripts/generate-llms-txt.py`, which writes `/llms.txt` from the nav. It links the HTML pages: Cloudflare's WAF currently 403s every `*.md` path.
+
 ### URLs (no .html)
 
-Cloudflare Pages serves pages without `.html` and 308s `/page.html` to `/page`. The playbook sets `urls.html_extension_style: drop` so canonical tags, the sitemap and xrefs use the extensionless URL, and `scripts/drop-html-links.py` (part of `npm run build`) strips `.html` from relative links written by hand in passthrough HTML. Keep writing `page.html` in hand-written links; the build fixes them. `scripts/templates/robots.txt` adds the `Sitemap:` line (Cloudflare prepends its content-signal block).
+Cloudflare Pages serves pages without `.html` and 308s `/page.html` to `/page`. The playbook sets `urls.html_extension_style: drop` so canonical tags, the sitemap and xrefs use the extensionless URL, and `scripts/drop-html-links.py` (part of `npm run build`) strips `.html` from relative links written by hand in passthrough HTML. Keep writing `page.html` in hand-written links; the build fixes them. `scripts/templates/robots.txt` adds the `Sitemap:` line and the `Content-Signal:` line (Cloudflare serves the file as is; it does not prepend a managed block).
 
 ### Full-text search (Pagefind)
 

docs/ui/supplemental/helpers/jsonld.js

@@ -0,0 +1,105 @@
+'use strict'
+
+// Schema.org JSON-LD for every page: the docs WebSite, a BreadcrumbList built
+// from the nav, and a TechArticle (WebPage on a component home). The
+// Organization and SoftwareApplication nodes live on arcadedb.com and are
+// referenced by @id only. Building the object in JS and serialising it with
+// JSON.stringify keeps quotes and entities in titles from breaking the script
+// block. Use as {{{jsonld}}} inside <script type="application/ld+json">.
+
+// Same as helpers/plain.js: Antora loads each helper from a string, so a
+// relative require() of a sibling helper does not resolve.
+const TAG_RX = /<[^>]+>/g
+const ENTITY_RX = /&(#\d+|#x[0-9a-f]+|[a-z]+);/gi
+const NAMED = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }
+
+function plain (value) {
+  if (value == null) return ''
+  return String(value)
+    .replace(TAG_RX, '')
+    .replace(ENTITY_RX, (match, ref) => {
+      if (ref[0] === '#') {
+        const code = ref[1] === 'x' || ref[1] === 'X' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10)
+        return Number.isFinite(code) ? String.fromCodePoint(code) : match
+      }
+      const named = NAMED[ref.toLowerCase()]
+      return named === undefined ? match : named
+    })
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+const ORG = 'https://arcadedb.com/#organization'
+const SOFTWARE = 'https://arcadedb.com/#software'
+const MAIN_SITE = 'https://arcadedb.com/#website'
+const IMAGE = 'https://arcadedb.com/assets/images/arcadedb-logo.png'
+
+module.exports = ({ data: { root } }) => {
+  const { page = {}, site = {} } = root
+  const siteUrl = (site.url || '').replace(/\/$/, '')
+  const pageUrl = page.canonicalUrl
+  if (!siteUrl || !pageUrl) return ''
+  const abs = (url) => (/^https?:/.test(url) ? url : siteUrl + url)
+  const websiteId = siteUrl + '/#website'
+
+  const website = {
+    '@type': 'WebSite',
+    '@id': websiteId,
+    url: siteUrl + '/',
+    name: plain(site.title) || 'ArcadeDB Documentation',
+    inLanguage: 'en',
+    publisher: { '@id': ORG },
+    isPartOf: { '@id': MAIN_SITE },
+  }
+
+  // Component home first, then the nav trail, then the page itself. Crumbs
+  // without a link (group labels) or pointing at a fragment are skipped:
+  // Google allows a missing item only on the last element.
+  const crumbs = []
+  const add = (name, url) => {
+    if (!name || !url || url.includes('#')) return
+    const item = abs(url)
+    if (crumbs.length && crumbs[crumbs.length - 1].item === item) return
+    crumbs.push({ name: plain(name), item })
+  }
+  const cv = page.componentVersion
+  if (cv) add(cv.title, cv.url)
+  for (const crumb of page.breadcrumbs || []) {
+    if (crumb.urlType === 'internal') add(crumb.content, crumb.url)
+  }
+  if (!crumbs.length || crumbs[crumbs.length - 1].item !== pageUrl) {
+    crumbs.push({ name: plain(page.title), item: pageUrl })
+  }
+
+  const graph = [website]
+  if (crumbs.length > 1) {
+    graph.push({
+      '@type': 'BreadcrumbList',
+      '@id': pageUrl + '#breadcrumb',
+      itemListElement: crumbs.map((c, i) => ({ '@type': 'ListItem', position: i + 1, name: c.name, item: c.item })),
+    })
+  }
+
+  const isHome = cv && page.url === cv.url
+  const article = {
+    '@type': isHome ? 'WebPage' : 'TechArticle',
+    '@id': pageUrl + (isHome ? '#webpage' : '#article'),
+    url: pageUrl,
+    mainEntityOfPage: pageUrl,
+    [isHome ? 'name' : 'headline']: plain(page.title),
+    inLanguage: 'en',
+    isPartOf: { '@id': websiteId },
+    about: { '@id': SOFTWARE },
+    author: { '@id': ORG },
+    publisher: { '@id': ORG },
+    image: IMAGE,
+  }
+  if (page.description) article.description = plain(page.description)
+  if (crumbs.length > 1) article.breadcrumb = { '@id': pageUrl + '#breadcrumb' }
+  graph.push(article)
+
+  return JSON.stringify({ '@context': 'https://schema.org', '@graph': graph }, null, 2)
+    .replace(/</g, '\\u003c')
+    .replace(/>/g, '\\u003e')
+    .replace(/&/g, '\\u0026')
+}

docs/ui/supplemental/helpers/plain.js

@@ -0,0 +1,28 @@
+'use strict'
+
+// Plain text from an Antora title or description: drop HTML tags and decode
+// character references (`—`, `&`, ...) so meta tags and JSON-LD
+// carry the characters a reader sees. Output with {{plain x}} (double stash)
+// so Handlebars escapes it again for the attribute it lands in.
+
+const TAG_RX = /<[^>]+>/g
+const ENTITY_RX = /&(#\d+|#x[0-9a-f]+|[a-z]+);/gi
+const NAMED = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }
+
+function plain (value) {
+  if (value == null) return ''
+  return String(value)
+    .replace(TAG_RX, '')
+    .replace(ENTITY_RX, (match, ref) => {
+      if (ref[0] === '#') {
+        const code = ref[1] === 'x' || ref[1] === 'X' ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10)
+        return Number.isFinite(code) ? String.fromCodePoint(code) : match
+      }
+      const named = NAMED[ref.toLowerCase()]
+      return named === undefined ? match : named
+    })
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+module.exports = (value) => plain(value)

docs/ui/supplemental/partials/head-meta.hbs

@@ -0,0 +1,19 @@
+    {{#if page.canonicalUrl}}
+    <meta property="og:type" content="{{#if (eq page.url page.componentVersion.url)}}website{{else}}article{{/if}}">
+    <meta property="og:site_name" content="{{plain site.title}}">
+    <meta property="og:title" content="{{plain page.title}}">
+    {{#with page.description}}
+    <meta property="og:description" content="{{plain this}}">
+    {{/with}}
+    <meta property="og:url" content="{{page.canonicalUrl}}">
+    <meta property="og:image" content="https://arcadedb.com/assets/images/arcadedb-logo.png">
+    <meta property="og:locale" content="en_US">
+    <meta name="twitter:card" content="summary">
+    <meta name="twitter:title" content="{{plain page.title}}">
+    {{#with page.description}}
+    <meta name="twitter:description" content="{{plain this}}">
+    {{/with}}
+    <script type="application/ld+json">
+{{{jsonld}}}
+    </script>
+    {{/if}}

docs/ui/supplemental/partials/head-title.hbs

@@ -0,0 +1,2 @@
+    {{!-- The docs home is titled after the site; don't repeat it as a suffix. --}}
+    <title>{{{detag (or page.title defaultPageTitle)}}}{{#with site.title}}{{#unless (eq (detag ../page.title) this)}} :: {{this}}{{/unless}}{{/with}}</title>

package.json

@@ -3,9 +3,10 @@
   "private": true,
   "description": "ArcadeDB documentation built with Antora",
   "scripts": {
-    "build": "antora --fetch antora-playbook.yml && npm run links && npm run redirects && npm run index",
-    "build:local": "antora antora-playbook.yml && npm run links && npm run redirects && npm run index",
+    "build": "antora --fetch antora-playbook.yml && npm run links && npm run redirects && npm run seo && npm run index",
+    "build:local": "antora antora-playbook.yml && npm run links && npm run redirects && npm run seo && npm run index",
     "links": "python3 scripts/drop-html-links.py build/site",
+    "seo": "python3 scripts/sitemap-lastmod.py build/site && python3 scripts/generate-llms-txt.py build/site",
     "index": "pagefind --site build/site",
     "redirects": "cp scripts/templates/_redirects scripts/templates/robots.txt build/site/",
     "serve": "npx -y http-server build/site -p 8080 -c-1"

scripts/generate-llms-txt.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""
+Write build/site/llms.txt (https://llmstxt.org) from the Antora nav.
+
+Sections follow the sidebar: one H2 per nav title and per top-level nav
+group, a link per page with the page's meta description. Links point at the
+HTML pages: Markdown copies of the pages are not published yet.
+
+Run after Antora, so the built pages (for descriptions) and the generated
+nav files (docs/modules/ROOT/nav*.adoc, written by migrate.sh) both exist.
+
+Usage: scripts/generate-llms-txt.py build/site
+"""
+from __future__ import annotations
+
+import html
+import re
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+NAV_DIR = REPO / "docs" / "modules" / "ROOT"
+NAV_FILES = ["nav.adoc", "nav-query.adoc"]
+SITE_URL = "https://docs.arcadedb.com"
+COMPONENT = "arcadedb"
+
+NAV_TITLE_RE = re.compile(r"^\.(\S.*)$")
+ITEM_RE = re.compile(r"^(\*+)\s+(.*)$")
+XREF_RE = re.compile(r"^xref:([^\[#]+)\.adoc(#[^\[]*)?\[(.*)\]\s*$")
+DESC_RE = re.compile(r'<meta name="description" content="([^"]*)"')
+TITLE_RE = re.compile(r"<title>(.*?)(?: :: [^<]*)?</title>", re.S)
+
+HEADER = """# ArcadeDB Documentation
+
+> ArcadeDB is an open-source multi-model database: graph, document, key/value, time-series, vector and geospatial data in one engine, queried with SQL, Cypher, Gremlin, GraphQL, MongoDB QL or Redis commands, embedded in Java or run as a server.
+
+The documentation is organized like its sidebar: getting started and examples, language drivers, data modeling and operations guides, then the query language and API reference. The current release is listed on the home page.
+"""
+
+
+def page_url(path: str) -> str:
+    if path == "index":
+        return f"{SITE_URL}/{COMPONENT}/"
+    return f"{SITE_URL}/{COMPONENT}/{path}"
+
+
+def description(site: Path, path: str) -> str:
+    f = site / COMPONENT / (path + ".html")
+    if not f.exists():
+        return ""
+    m = DESC_RE.search(f.read_text(encoding="utf-8"))
+    return html.unescape(m.group(1)).strip() if m else ""
+
+
+def link(label: str, url: str, desc: str) -> str:
+    label = label.replace("[", "(").replace("]", ")")
+    return f"- [{label}]({url})" + (f": {desc}" if desc else "")
+
+
+def main() -> int:
+    site = Path(sys.argv[1] if len(sys.argv) > 1 else "build/site")
+    out = [HEADER]
+    seen: set[str] = set()
+    section: list[str] = []
+
+    def flush(title: str | None) -> None:
+        if title and section:
+            out.append(f"## {title}\n")
+            out.extend(section)
+            out.append("")
+        section.clear()
+
+    title = None
+    groups: list[str] = []  # label path of unlinked parent groups
+    for nav in NAV_FILES:
+        for raw in (NAV_DIR / nav).read_text(encoding="utf-8").splitlines():
+            if m := NAV_TITLE_RE.match(raw):
+                flush(title)
+                title, groups = m.group(1), []
+                continue
+            m = ITEM_RE.match(raw)
+            if not m:
+                continue
+            depth, text = len(m.group(1)), m.group(2).strip()
+            groups = groups[: depth - 1]
+            x = XREF_RE.match(text)
+            if depth == 1:
+                flush(title)
+                title = x.group(3) if x else text
+            if not x:
+                groups.append(text if depth > 1 else "")
+                continue
+            groups.append("")
+            path, frag, label = x.group(1), x.group(2) or "", x.group(3)
+            url = page_url(path) + frag
+            if url in seen:
+                continue
+            seen.add(url)
+            prefix = ": ".join(g for g in groups[1:-1] if g)
+            section.append(link(f"{prefix}: {label}" if prefix else label, url,
+                                "" if frag else description(site, path)))
+    flush(title)
+
+    # The Python bindings component has its own nav in another repository;
+    # list its pages from the sitemap Antora wrote for it.
+    py_sitemap = site / "sitemap-arcadedb-python.xml"
+    if py_sitemap.exists():
+        out.append("## Python Bindings (arcadedb-python)\n")
+        for loc in re.findall(r"<loc>([^<]+)</loc>", py_sitemap.read_text(encoding="utf-8")):
+            rel = loc[len(SITE_URL) + 1:]
+            f = site / (rel + "index.html" if rel.endswith("/") else rel + ".html")
+            if not f.exists():
+                continue
+            text = f.read_text(encoding="utf-8")
+            t = TITLE_RE.search(text)
+            d = DESC_RE.search(text)
+            out.append(link(html.unescape(t.group(1)).strip() if t else rel, loc,
+                            html.unescape(d.group(1)).strip() if d else ""))
+        out.append("")
+
+    (site / "llms.txt").write_text("\n".join(out).rstrip() + "\n", encoding="utf-8")
+    print(f"llms.txt: {sum(1 for l in out if l.startswith('- ['))} links")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())