From 21b0e3c383254874712b2650840f75e424e17fce Mon Sep 17 00:00:00 2001 From: Nik Cubrilovic Date: Fri, 17 Jul 2026 16:57:11 +1000 Subject: [PATCH] fix(build): sitemap emits served route for index pages, not /index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root index.mdx renders at `/` but slugifies to the bare string `index` (scan-pages' collapse regex needs a leading `/`, so only nested foo/index.mdx collapses). The sitemap/llms.txt emitters derived URLs straight from the slug, publishing `/index` — which 307s to `/`. Google classifies redirecting sitemap URLs as "Page with redirect" and drops them. Add pageRouteForSlug/pagePathForSlug to @tanglydocs/schema (the shared leaf, same rationale as resolve-site) and route sitemap, llms.txt, llms-full.txt, agent .md preambles and Seo.astro's canonical through it, so published URLs can't diverge from the canonical again. og card filenames stay slug-keyed (dist/og/.png, home => index.png) — deliberately the inverse mapping, kept separate. --- packages/schema/package.json | 4 ++ packages/schema/src/index.ts | 1 + packages/schema/src/page-path.test.ts | 44 +++++++++++++++++ packages/schema/src/page-path.ts | 33 +++++++++++++ .../tangly/src/build-outputs/index.test.ts | 47 +++++++++++++++++++ packages/tangly/src/build-outputs/index.ts | 7 +-- .../tangly/src/build-outputs/page-markdown.ts | 5 +- packages/theme-ui/src/Seo.astro | 13 ++--- 8 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 packages/schema/src/page-path.test.ts create mode 100644 packages/schema/src/page-path.ts diff --git a/packages/schema/package.json b/packages/schema/package.json index 6402019..d0fecf5 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -33,6 +33,10 @@ "types": "./dist/resolve-site.d.ts", "import": "./dist/resolve-site.js" }, + "./page-path": { + "types": "./dist/page-path.d.ts", + "import": "./dist/page-path.js" + }, "./docs.json": "./dist/docs.json" }, "publishConfig": { diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts index 8db96b3..f852147 100644 --- a/packages/schema/src/index.ts +++ b/packages/schema/src/index.ts @@ -18,6 +18,7 @@ export { } from "./format-error.js"; export { generateDocsJsonSchema } from "./json-schema.js"; export { convertMintToDocs, type MintJson } from "./mint-json.js"; +export { pagePathForSlug, pageRouteForSlug } from "./page-path.js"; export { GlobalNavSchema, NavigationSchema, diff --git a/packages/schema/src/page-path.test.ts b/packages/schema/src/page-path.test.ts new file mode 100644 index 0000000..def754c --- /dev/null +++ b/packages/schema/src/page-path.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "vitest"; +import { pagePathForSlug, pageRouteForSlug } from "./page-path.js"; + +describe("pageRouteForSlug", () => { + test("root index.mdx maps to /", () => { + expect(pageRouteForSlug("index")).toBe("/"); + expect(pageRouteForSlug("")).toBe("/"); + }); + + test("nested index.mdx maps to its directory route", () => { + // scan-pages already collapses `cli/index` → `cli`; both spellings must + // resolve to the same served route regardless of which caller we get. + expect(pageRouteForSlug("cli")).toBe("/cli"); + expect(pageRouteForSlug("cli/index")).toBe("/cli"); + expect(pageRouteForSlug("guides/deploying/index")).toBe("/guides/deploying"); + }); + + test("'index' inside a segment name is not stripped", () => { + expect(pageRouteForSlug("indexing")).toBe("/indexing"); + expect(pageRouteForSlug("api/index-tuning")).toBe("/api/index-tuning"); + }); + + test("ordinary pages keep their slug", () => { + expect(pageRouteForSlug("introduction")).toBe("/introduction"); + expect(pageRouteForSlug("guides/deploying")).toBe("/guides/deploying"); + }); + + test("stray slashes are trimmed", () => { + expect(pageRouteForSlug("/introduction/")).toBe("/introduction"); + expect(pageRouteForSlug("/index")).toBe("/"); + }); +}); + +describe("pagePathForSlug", () => { + test("no base", () => { + expect(pagePathForSlug("index")).toBe("/"); + expect(pagePathForSlug("cli")).toBe("/cli"); + }); + + test("base prefix, home collapses to the base itself", () => { + expect(pagePathForSlug("index", "/docs")).toBe("/docs"); + expect(pagePathForSlug("cli", "/docs")).toBe("/docs/cli"); + }); +}); diff --git a/packages/schema/src/page-path.ts b/packages/schema/src/page-path.ts new file mode 100644 index 0000000..1adae82 --- /dev/null +++ b/packages/schema/src/page-path.ts @@ -0,0 +1,33 @@ +/** + * Map a page slug to the route it is actually served at. + * + * Slugs come from the source file path, and `index` files are special: a + * nested `cli/index.mdx` already collapses to `cli` during the page scan, but + * a root `index.mdx` slugifies to the bare string `index` (the collapse regex + * needs a leading `/`). That page renders at `/` and is only *also* routable + * at `/index`, which redirects. Anything published as a URL — canonical tags, + * sitemap entries, llms.txt links — must use the served route, or search + * engines see a redirecting duplicate. + * + * Lives in `@tanglydocs/schema` (the shared leaf both `tangly` and the themes + * depend on) so theme components can resolve routes without importing `tangly` + * — keeping the package graph acyclic. Same rationale as `resolve-site`. + */ + +/** Route for a page slug, leading slash, no trailing slash. Home → "/". */ +export function pageRouteForSlug(slug: string): string { + // The trailing-`/index` collapse duplicates what `scanPages` already does to + // nested slugs; kept here so any caller holding a raw file-derived slug + // resolves the same route. + const trimmed = slug.replace(/^\/+|\/+$/g, "").replace(/(^|\/)index$/, ""); + if (trimmed === "") return "/"; + return `/${trimmed}`; +} + +/** + * Public URL path for a page slug under a deploy `base` (e.g. "/docs"). + * `base` is expected pre-normalized: "" for root, "/docs" otherwise. + */ +export function pagePathForSlug(slug: string, base = ""): string { + return `${base}${pageRouteForSlug(slug)}`.replace(/\/+$/, "") || "/"; +} diff --git a/packages/tangly/src/build-outputs/index.test.ts b/packages/tangly/src/build-outputs/index.test.ts index bdc717f..5e7569f 100644 --- a/packages/tangly/src/build-outputs/index.test.ts +++ b/packages/tangly/src/build-outputs/index.test.ts @@ -12,6 +12,25 @@ function fakeManifest(): Manifest { sidebar: [], draft: false, }); + // Root index.mdx: scan-pages leaves the slug as the bare string "index" + // (only a nested foo/index.mdx collapses). It is served at "/". + pages.set("index", { + slug: "index", + file: "/x/index.mdx", + frontmatter: { title: "Home", description: "Start here" }, + breadcrumbs: [], + sidebar: [], + draft: false, + }); + // Subdirectory cli/index.mdx: already collapsed to "cli", served at "/cli". + pages.set("cli", { + slug: "cli", + file: "/x/cli/index.mdx", + frontmatter: { title: "CLI", description: "Commands" }, + breadcrumbs: [], + sidebar: [], + draft: false, + }); pages.set("hidden", { slug: "hidden", file: "/x/hidden.mdx", @@ -103,6 +122,34 @@ describe("build-outputs", () => { expect(llms).toContain("- [Introduction](/docs/introduction): Welcome"); }); + test("sitemap emits the served route for index pages, never /index", () => { + const xml = generateSitemap({ + manifest, + outDir: "/tmp", + siteUrl: "https://example.com", + }); + // Root index.mdx → "/", not "/index" (which 307s to "/"). + expect(xml).toContain("https://example.com/"); + expect(xml).not.toContain("https://example.com/index<"); + // Subdirectory cli/index.mdx → "/cli", not "/cli/index". + expect(xml).toContain("https://example.com/cli"); + expect(xml).not.toContain("/cli/index"); + }); + + test("sitemap index routes honor the base prefix", () => { + const xml = generateSitemap({ manifest, outDir: "/tmp", base: "/docs" }); + expect(xml).toContain("/docs"); + expect(xml).not.toContain("/docs/index"); + expect(xml).toContain("/docs/cli"); + }); + + test("llms.txt links the served route for index pages", () => { + const txt = generateLlmsTxt({ manifest, outDir: "/tmp" }); + expect(txt).toContain("- [Home](/): Start here"); + expect(txt).not.toContain("](/index)"); + expect(txt).toContain("- [CLI](/cli): Commands"); + }); + test("base of '/' or empty is treated as root", () => { const root = generateLlmsTxt({ manifest, outDir: "/tmp", base: "/" }); expect(root).toContain("- [Introduction](/introduction): Welcome"); diff --git a/packages/tangly/src/build-outputs/index.ts b/packages/tangly/src/build-outputs/index.ts index d03ff79..237db9c 100644 --- a/packages/tangly/src/build-outputs/index.ts +++ b/packages/tangly/src/build-outputs/index.ts @@ -1,5 +1,6 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { pagePathForSlug } from "@tanglydocs/schema"; import pc from "picocolors"; import type { Manifest, PageEntry } from "../manifest/index.js"; import { writePageMarkdown } from "./page-markdown.js"; @@ -27,7 +28,7 @@ export function generateSitemap(opts: BuildOutputsOptions): string { for (const page of opts.manifest.pages.values()) { if (page.draft) continue; if (page.frontmatter.noindex) continue; - const path = `${base}/${page.slug}`; + const path = pagePathForSlug(page.slug, base); const loc = site ? `${site}${path}` : path; urls.push(` ${escapeXml(loc)}`); } @@ -62,7 +63,7 @@ export function generateLlmsTxt(opts: BuildOutputsOptions): string { if (page.frontmatter.noindex) continue; const title = page.frontmatter.title ?? page.slug; const desc = page.frontmatter.description ? `: ${page.frontmatter.description}` : ""; - lines.push(`- [${title}](${base}/${page.slug})${desc}`); + lines.push(`- [${title}](${pagePathForSlug(page.slug, base)})${desc}`); } lines.push(""); return lines.join("\n"); @@ -81,7 +82,7 @@ export function generateLlmsFullTxt(opts: BuildOutputsOptions): string { const title = page.frontmatter.title ?? page.slug; lines.push(`\n---\n\n# ${title}\n`); if (page.frontmatter.description) lines.push(`${page.frontmatter.description}\n`); - lines.push(`\nURL: ${base}/${page.slug}\n\n`); + lines.push(`\nURL: ${pagePathForSlug(page.slug, base)}\n\n`); try { const raw = readFileSync(page.file, "utf8"); const body = raw.replace(/^---[\s\S]*?---\n/, ""); diff --git a/packages/tangly/src/build-outputs/page-markdown.ts b/packages/tangly/src/build-outputs/page-markdown.ts index 45c8f74..293b7f4 100644 --- a/packages/tangly/src/build-outputs/page-markdown.ts +++ b/packages/tangly/src/build-outputs/page-markdown.ts @@ -1,5 +1,6 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; +import { pagePathForSlug } from "@tanglydocs/schema"; import type { Manifest, PageEntry } from "../manifest/index.js"; export interface PageMarkdownOptions { @@ -43,7 +44,9 @@ export function writePageMarkdown(opts: PageMarkdownOptions): { written: number if (page.frontmatter.noindex) continue; const dest = join(opts.outDir, `${page.slug}.md`); mkdirSync(dirname(dest), { recursive: true }); - const urlPath = `${base}/${page.slug}`; + // `dest` stays slug-derived (the file lives at `index.md`); the URL + // preamble must be the served route, not the file path. + const urlPath = pagePathForSlug(page.slug, base); writeFileSync(dest, generatePageMarkdown(page, urlPath), "utf8"); written += 1; } diff --git a/packages/theme-ui/src/Seo.astro b/packages/theme-ui/src/Seo.astro index 7349c9c..aacdad9 100644 --- a/packages/theme-ui/src/Seo.astro +++ b/packages/theme-ui/src/Seo.astro @@ -14,6 +14,7 @@ * 5. logo — last-resort fallback (legacy behavior) */ import type { DocsJson } from "@tanglydocs/schema"; +import { pageRouteForSlug } from "@tanglydocs/schema/page-path"; import { resolveSite } from "@tanglydocs/schema/site"; import type { PageEntry } from "tangly"; import { withBase } from "./with-base.ts"; @@ -46,13 +47,13 @@ const ogBase = site.ogUrl; const canonicalBase = site.canonicalUrl; const robotsNoindex = site.noindex || page.frontmatter.noindex === true; +// Card filenames are slug-keyed (dist/og/.png), NOT route-keyed — the +// home card is /og/index.png. Keep this separate from `pagePath` below. const slug = page.slug.replace(/\/+$/, ""); -// The root index page is the home: it renders at `/` and is also routable at -// `/index`. Canonicalize both to `/` so they don't compete as duplicate -// content. (Only a root index.mdx slugifies to "index"; nested foo/index.mdx -// collapses to "foo".) -const isHome = slug === "" || slug === "index"; -const pagePath = withBase(isHome ? "/" : `/${slug}`)?.replace(/\/+$/, "") || "/"; +// Canonicalize to the served route: a root index.mdx renders at `/` but is +// also routable at `/index`, which must not compete as duplicate content. +// Shared with the sitemap/llms.txt emitters so published URLs never diverge. +const pagePath = withBase(pageRouteForSlug(page.slug))?.replace(/\/+$/, "") || "/"; const canonicalUrl = canonicalBase ? `${canonicalBase}${pagePath}` : ""; /** Absolutize a relative/external path against `base` (or "" if it can't be). */