diff --git a/.changeset/README.md b/.changeset/README.md index 13e865c..c5aa3ab 100644 --- a/.changeset/README.md +++ b/.changeset/README.md @@ -1,7 +1,7 @@ # Changesets This directory tracks pending version + changelog entries between -releases. Each PR that user-visibly changes the package adds one +releases. Each PR that user-visibly changes a public package adds one markdown file here describing the change. ## Adding a changeset @@ -12,13 +12,16 @@ pnpm changeset …which interactively asks: -- **Which packages are affected?** Just `@cloudflare/polystella` for most - changes. The `polystella-docs` site is ignored (`ignore` list in - `config.json`). +- **Which packages are affected?** Select every package changed by the PR: + `@cloudflare/polystella-core`, `@cloudflare/polystella-adapters`, + `@cloudflare/polystella-providers`, `@cloudflare/polystella`, and/or + `@cloudflare/polystella-astro`. Astro and the compatibility package form a + fixed group; core, adapters, and providers are versioned independently. The + private root and `polystella-docs` are not released. - **Is the change major / minor / patch?** Pre-1.0, "major" stays reserved for 1.0; bump minor for breaking changes within 0.x, patch otherwise. -- **A summary.** One-liner that lands in `CHANGELOG.md`. +- **A summary.** One-liner that lands in the affected package changelogs. The result is a small markdown file in this directory. Commit it with the PR. @@ -28,12 +31,10 @@ with the PR. Pending changesets are consumed by the versioning step for a release. That step: -- Bumps `package.json`'s version per the changeset severities. -- Updates `CHANGELOG.md` with each changeset's summary. +- Bumps affected independent packages and both packages in the Astro fixed group. +- Updates the package-local `CHANGELOG.md` files. - Deletes the consumed changeset files. Publishing is automated from `.github/workflows/release.yml`. Merging the generated release PR back to `main` publishes to npm via Trusted -Publishing, so routine releases must not use an npm token secret. The -tag-based `.github/workflows/publish.yml` workflow remains a manual -fallback for `v*` tags. +Publishing, so routine releases must not use an npm token secret. diff --git a/.changeset/clean-stars-extract.md b/.changeset/clean-stars-extract.md new file mode 100644 index 0000000..10bf8e2 --- /dev/null +++ b/.changeset/clean-stars-extract.md @@ -0,0 +1,9 @@ +--- +"@cloudflare/polystella-core": minor +"@cloudflare/polystella-adapters": minor +"@cloudflare/polystella-providers": minor +"@cloudflare/polystella": minor +"@cloudflare/polystella-astro": minor +--- + +Extract PolyStella into independently versioned portable core, adapters, and providers packages, plus the canonical `@cloudflare/polystella-astro` integration and its fixed-version `@cloudflare/polystella` compatibility package. diff --git a/.changeset/config.json b/.changeset/config.json index c4fc267..40a8955 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -7,9 +7,10 @@ } ], "commit": false, - "fixed": [], + "fixed": [["@cloudflare/polystella", "@cloudflare/polystella-astro"]], "linked": [], "access": "public", + "privatePackages": false, "baseBranch": "main", "updateInternalDependencies": "patch", "ignore": ["polystella-docs"] diff --git a/.changeset/document-package-boundaries.md b/.changeset/document-package-boundaries.md new file mode 100644 index 0000000..a8cd3b9 --- /dev/null +++ b/.changeset/document-package-boundaries.md @@ -0,0 +1,9 @@ +--- +"@cloudflare/polystella": patch +"@cloudflare/polystella-astro": patch +"@cloudflare/polystella-core": patch +"@cloudflare/polystella-adapters": patch +"@cloudflare/polystella-providers": patch +--- + +Document post-migration package ownership, dependency boundaries, key files, and package relationships for contributors. diff --git a/.changeset/lazy-maps-prompt.md b/.changeset/lazy-maps-prompt.md new file mode 100644 index 0000000..530fdbe --- /dev/null +++ b/.changeset/lazy-maps-prompt.md @@ -0,0 +1,7 @@ +--- +"@cloudflare/polystella-astro": patch +"@cloudflare/polystella-core": patch +"@cloudflare/polystella-adapters": patch +--- + +Keep format-specific prompt guidance in its owning adapter, enforce segment grouping invariants, and reject duplicate segment IDs in the format-neutral core. diff --git a/.changeset/tidy-adapter-layout.md b/.changeset/tidy-adapter-layout.md new file mode 100644 index 0000000..4f116ea --- /dev/null +++ b/.changeset/tidy-adapter-layout.md @@ -0,0 +1,6 @@ +--- +"@cloudflare/polystella-astro": minor +"@cloudflare/polystella-adapters": minor +--- + +Colocate the Markdown and MDX adapter implementation, move MDX-specific options out of generic contracts, protect nested inline JSX, and reject malformed key paths. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a858e36..0bcf531 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -37,7 +37,7 @@ Closes # - [ ] I have added a changeset, or this change does not affect the published package - [ ] I have updated docs, or this change does not affect public behavior - [ ] `pnpm test` passes, or I have explained why it was not run -- [ ] `pnpm exec tsc --noEmit` passes, or I have explained why it was not run +- [ ] `pnpm typecheck` passes, or I have explained why it was not run - [ ] Docs checks/builds pass if docs or public APIs changed ## AI-generated code disclosure diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 334b77e..5a7c023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: test: name: Test runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -30,8 +30,18 @@ jobs: node-version: 24.x cache: pnpm - run: pnpm install --frozen-lockfile - - run: pnpm test - - run: pnpm exec tsc --noEmit + - name: Audit production dependencies + run: pnpm audit:prod + - name: Test reusable packages under Node + run: pnpm test:packages + - name: Test Astro package under Node + run: pnpm test:node + - name: Test reusable packages under workerd + run: pnpm test:workerd + - name: Check package boundaries + run: pnpm test:boundaries + - name: Typecheck all public packages + run: pnpm typecheck format: name: Prettier check @@ -74,9 +84,9 @@ jobs: fi build: - name: Library + CLI build + name: Topological package build runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -87,23 +97,32 @@ jobs: node-version: 24.x cache: pnpm - run: pnpm install --frozen-lockfile - - run: pnpm build + - name: Build all public packages in dependency order + run: pnpm build + - name: Check packed packages in a clean consumer + run: pnpm check:packages - name: Verify dist tree # Smoke-check every exports entry resolves to an emitted file. run: | - test -f dist/cli.js - test -x dist/cli.js - test -f dist/index.js - test -f dist/index.d.ts - test -f dist/runtime/index.js - test -f dist/runtime/middleware.js - test -f dist/runtime/locals.d.ts - test -f dist/content/index.js - test -f dist/i18n/index.js - test -f dist/react/index.js + test -f packages/astro/dist/cli.js + test -x packages/astro/dist/cli.js + test -f packages/astro/dist/index.js + test -f packages/astro/dist/index.d.ts + test -f packages/astro/dist/runtime/index.js + test -f packages/astro/dist/runtime/middleware.js + test -f packages/astro/dist/runtime/locals.d.ts + test -f packages/astro/dist/content/index.js + test -f packages/astro/dist/i18n/index.js + test -f packages/astro/dist/react/index.js + test -f packages/polystella/dist/cli.js + test -x packages/polystella/dist/cli.js + test -f packages/polystella/dist/index.js + test -f packages/polystella/dist/index.d.ts - name: Smoke-test the CLI bundle under bare Node # No tsx, no Vite — exactly the environment a downstream # consumer's standalone `polystella ` invocation # uses. Regression guard for the Node 24 # `ERR_UNSUPPORTED_NODE_MODULES_TYPESCRIPT` class of bug. - run: node dist/cli.js --version + run: | + node packages/astro/dist/cli.js --version + node packages/polystella/dist/cli.js --version diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c027d4c..f097563 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,16 +6,40 @@ on: - main paths: - "docs/**" - - "src/config/options.ts" + - "packages/astro/src/config/options.ts" + - "packages/core/package.json" + - "packages/adapters/package.json" + - "packages/providers/package.json" + - "packages/astro/package.json" + - "packages/polystella/package.json" - "package.json" + - "AGENTS.md" + - "PACKAGE_ARCHITECTURE.md" + - "ARCHITECTURE.md" + - "skills/polystella-consumer/SKILL.md" + - "skills/polystella-contributor/SKILL.md" + - "scripts/build-llms-full.mjs" + - "llms-full.txt" - ".github/workflows/docs.yml" pull_request: branches: - main paths: - "docs/**" - - "src/config/options.ts" + - "packages/astro/src/config/options.ts" + - "packages/core/package.json" + - "packages/adapters/package.json" + - "packages/providers/package.json" + - "packages/astro/package.json" + - "packages/polystella/package.json" - "package.json" + - "AGENTS.md" + - "PACKAGE_ARCHITECTURE.md" + - "ARCHITECTURE.md" + - "skills/polystella-consumer/SKILL.md" + - "skills/polystella-contributor/SKILL.md" + - "scripts/build-llms-full.mjs" + - "llms-full.txt" - ".github/workflows/docs.yml" permissions: @@ -40,9 +64,13 @@ jobs: node-version: 24.x cache: pnpm - run: pnpm install --frozen-lockfile + - name: Verify generated agent context is current + run: | + pnpm build:llms + git diff --exit-code llms-full.txt - name: Build docs (includes config-reference generation) run: pnpm --filter polystella-docs build - name: Verify every public export is documented run: pnpm --filter polystella-docs check-exports - - name: Verify every example builds (currently no-op) + - name: Verify examples compile run: pnpm --filter polystella-docs check-examples diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml index 4422040..f8cb856 100644 --- a/.github/workflows/pr-triage.yml +++ b/.github/workflows/pr-triage.yml @@ -66,19 +66,29 @@ jobs: } const areaMap = { - "area/catalog": (file) => file.startsWith("src/catalog/") || file.includes("CATALOG_ONLY_PLAN"), - "area/cli": (file) => file === "src/cli.ts" || file.startsWith("src/cli/"), - "area/config": (file) => file.startsWith("src/config/"), - "area/content": (file) => file.startsWith("src/content/"), - "area/docs": (file) => file.startsWith("docs/") || file === "README.md" || file === "CHANGELOG.md", + "area/catalog": (file) => file.startsWith("packages/astro/src/catalog/") || file.includes("CATALOG_ONLY_PLAN"), + "area/cli": (file) => file === "packages/astro/src/cli.ts" || file.startsWith("packages/astro/src/cli/"), + "area/config": (file) => file.startsWith("packages/astro/src/config/"), + "area/content": (file) => file.startsWith("packages/astro/src/content/"), + "area/docs": (file) => + file.startsWith("docs/") || + file === "README.md" || + /^packages\/(astro|core|adapters|providers)\/(README|CHANGELOG)\.md$/.test(file), "area/github": (file) => file.startsWith(".github/"), - "area/i18n": (file) => file.startsWith("src/i18n/"), - "area/parsing": (file) => file.startsWith("src/parsing/"), - "area/routing": (file) => file.startsWith("src/routing/"), - "area/runtime": (file) => file.startsWith("src/runtime/") || file.startsWith("src/react/"), - "area/storage": (file) => file.startsWith("src/storage/"), - "area/tests": (file) => file.startsWith("tests/"), - "area/translation": (file) => file.startsWith("src/translation/") || file.startsWith("src/glossary/"), + "area/i18n": (file) => file.startsWith("packages/astro/src/i18n/"), + "area/parsing": (file) => + file.startsWith("packages/adapters/") || file.startsWith("packages/astro/src/parsing/"), + "area/routing": (file) => file.startsWith("packages/astro/src/routing/"), + "area/runtime": (file) => + file.startsWith("packages/astro/src/runtime/") || file.startsWith("packages/astro/src/react/"), + "area/storage": (file) => file.startsWith("packages/astro/src/storage/"), + "area/tests": (file) => + /^packages\/(astro|core|adapters|providers)\/tests\//.test(file) || file.startsWith("tests/"), + "area/translation": (file) => + file.startsWith("packages/core/") || + file.startsWith("packages/providers/") || + file.startsWith("packages/astro/src/translation/") || + file.startsWith("packages/astro/src/glossary/"), }; for (const file of files) { diff --git a/.github/workflows/preview-releases.yml b/.github/workflows/preview-releases.yml index 1e2c957..acd0da0 100644 --- a/.github/workflows/preview-releases.yml +++ b/.github/workflows/preview-releases.yml @@ -35,4 +35,4 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm build - name: Publish preview - run: pnpm exec pkg-pr-new publish --pnpm --commentWithSha . + run: pnpm exec pkg-pr-new publish --pnpm --commentWithSha './packages/*' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index daffde8..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Publish - -on: - push: - tags: - - "v*" - -permissions: - contents: read - id-token: write - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -jobs: - publish: - name: Publish to npm - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 24.x - cache: pnpm - registry-url: "https://registry.npmjs.org" - - run: npm install -g "npm@^11.5.1" - - run: pnpm install --frozen-lockfile - - run: pnpm test - - run: pnpm exec tsc --noEmit - - run: pnpm build - - name: Publish package - run: npm publish --access public diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8ef8a29..f3f16df 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,7 +4,6 @@ on: push: branches: - main - workflow_dispatch: permissions: {} @@ -34,9 +33,12 @@ jobs: registry-url: "https://registry.npmjs.org" - run: npm install -g "npm@^11.5.1" - run: pnpm install --frozen-lockfile + - run: pnpm audit:prod - run: pnpm test - - run: pnpm exec tsc --noEmit + - run: pnpm typecheck - run: pnpm build + - run: pnpm check:baseline + - run: pnpm check:packages - name: Create release PR or publish uses: changesets/action@a45c4d594aa4e2c509dc14a9f2b3b67ba3780d0d # v1 with: diff --git a/.prettierignore b/.prettierignore index 4b8ee48..f87c3b3 100644 --- a/.prettierignore +++ b/.prettierignore @@ -15,4 +15,4 @@ docs/src/content/docs/configuration/reference.md # variants (e.g. `***` / `___` / `---` thematic breaks, inline vs. # block footnote definitions). Prettier would normalise them away # and defeat the test corpus. -tests/fixtures/parsing/round-trip/ +packages/astro/tests/fixtures/parsing/round-trip/ diff --git a/AGENTS.md b/AGENTS.md index d8f695b..5e0ad5c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,10 @@ additional locales at build time using AI, caches translations in Cloudflare R2, and injects locale-prefixed routes. This file is the entry point for coding agents working **on the -PolyStella package itself**. Three companion docs: +PolyStella package itself**. Four companion docs: + +- [`PACKAGE_ARCHITECTURE.md`](./PACKAGE_ARCHITECTURE.md) — post-migration + package ownership, dependency direction, key files, and boundary checks. - [`ARCHITECTURE.md`](./ARCHITECTURE.md) — system design, invariants, glossary, per-subsystem reference. The "why" answers. @@ -23,18 +26,19 @@ section numbers. Inserting new sections never breaks links. ## Commands -| Command | What it does | -| :----------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `pnpm test` | Run vitest (1104 tests / 56 files / ~1.2s at time of writing). | -| `pnpm test:watch` | Vitest in watch mode. | -| `pnpm build` | Compile `src/` → `dist/` via `tsc -p tsconfig.build.json` (mirrored layout, `.js` + `.d.ts` + sourcemaps + declaration maps). Produces the standalone `polystella` CLI at `dist/cli.js` and library entries. | -| `pnpm exec tsc --noEmit` | Typecheck against the root `tsconfig.json` (which includes tests). The build config (`tsconfig.build.json`) sets `noEmit: false` and narrows `include` to `src/**`. | -| `pnpm changeset` | Add a Changesets entry for package-affecting work. Use `pnpm changeset add --empty` only for changes that intentionally do not need a package release. | +| Command | What it does | +| :-------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- | +| `pnpm test` | Run all package, Astro, workerd, and boundary tests. | +| `pnpm test:watch` | Run the Astro package tests in watch mode. | +| `pnpm build` | Build all five public packages. Astro emits its standalone CLI and library entries under `packages/astro/dist/`. | +| `pnpm typecheck` | Build all five packages, then typecheck every public package against its package-local `tsconfig.json`. | +| `pnpm check:packages` | Pack all five public packages and exercise every export from a clean temporary consumer. | +| `pnpm changeset` | Add a Changesets entry for package-affecting work. Use `pnpm changeset add --empty` only for changes that intentionally do not need a package release. | No lint step yet. > Test counts age. The authoritative count is `pnpm test`'s output; -> the number here is a snapshot pinned by [`tests/docs.test.ts`](./tests/docs.test.ts). +> the number here is a snapshot pinned by [`packages/astro/tests/docs.test.ts`](./packages/astro/tests/docs.test.ts). --- @@ -44,24 +48,29 @@ Task → entry-point file(s) → key contract → deep-dive link. | Task | Entry point | Contract | See | | :----------------------------------------------------- | :----------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------- | -| Add a file-format adapter | `src/parsing/adapters/.ts`; register in `src/parsing/registry.ts` | `FileTypeAdapter` in `src/parsing/adapter.ts` | [#adapter-contract](./ARCHITECTURE.md#adapter-contract); recipe in [contributor SKILL](./skills/polystella-contributor/SKILL.md#add-adapter) | -| Add a CLI subcommand | Handler in `src/cli/.ts`; register in `src/cli.ts` (`parseSubcommand` + switch) | Argv parser + `run(args, deps)` | Recipe in [contributor SKILL](./skills/polystella-contributor/SKILL.md#add-cli-subcommand) | -| Add a translation provider | New branch in `createTranslator` (`src/translation/provider.ts`) | `Translator` interface; permanent vs retriable error classification | [#translator-contract](./ARCHITECTURE.md#translator-contract); recipe in [contributor SKILL](./skills/polystella-contributor/SKILL.md#add-provider) | -| Change cache key formula | `src/storage/hash.ts` | **Invariant 1** — cache-wide invalidation | [#cache-key](./ARCHITECTURE.md#cache-key) | -| Edit translation batching | `src/translation/batch.ts`, `src/translation/translate-segments.ts` | **Invariant 2** — `flat(groups) === segments` | [#translation-batching](./ARCHITECTURE.md#translation-batching) | -| Modify cache/storage behaviour | `src/storage/{cache,r2,prune,local-cache,report}.ts` | Apply-before-PUT (**Invariant 3**); index isolation (**Invariant 4**) | [#cache-write-order](./ARCHITECTURE.md#cache-write-order), [#local-staging-index](./ARCHITECTURE.md#local-staging-index) | -| Modify runtime APIs (entry/collection/href/middleware) | `src/runtime/*` | Bridge timing (**Invariant 5**); per-locale closures | [#runtime-bridge](./ARCHITECTURE.md#runtime-bridge) | -| Modify routing shims | `src/routing/{shim,expand-routes,walk-pages}.ts` | Stale shims nuked per build; CSS via `routesImports` | [#routing-shims](./ARCHITECTURE.md#routing-shims) | -| Edit UI-string handling | `src/i18n/*`, `src/cli/{check,sync,translate}-ui.ts` | Three drift modes; layout-aware writer; `{{token}}` preservation | [#ui-strings](./ARCHITECTURE.md#ui-strings) | -| Edit catalog-only adoption | `src/catalog/*`, `src/i18n/{translate,drift,sync}.ts` | Pure imports; middleware binds only `t` + `lhref` | [`CATALOG_ONLY_PLAN.md`](./CATALOG_ONLY_PLAN.md); [consumer SKILL](./skills/polystella-consumer/SKILL.md#catalog-only-adoption) | -| Edit content-collection wiring | `src/content/*` | Sibling collections; custom-loader wrapper; bridge timing | [#runtime-bridge](./ARCHITECTURE.md#runtime-bridge) | +| Add a file-format adapter | `packages/adapters/src/adapters/.ts`; wrap/register in `packages/astro/src/parsing/` | `FileAdapter` in `packages/adapters/src/adapter.ts` | [#adapter-contract](./ARCHITECTURE.md#adapter-contract); recipe in [contributor SKILL](./skills/polystella-contributor/SKILL.md#add-adapter) | +| Add a CLI subcommand | Handler in `packages/astro/src/cli/.ts`; register in `packages/astro/src/cli.ts` | Argv parser + `run(args, deps)` | Recipe in [contributor SKILL](./skills/polystella-contributor/SKILL.md#add-cli-subcommand) | +| Add a translation provider | Factory in `packages/providers/src/`; map config in `packages/astro/src/translation/provider.ts` | `Translator` in `packages/core/src/translator.ts` | [#translator-contract](./ARCHITECTURE.md#translator-contract); recipe in [contributor SKILL](./skills/polystella-contributor/SKILL.md#add-provider) | +| Change cache key formula | `packages/astro/src/storage/hash.ts` | **Invariant 1** — cache-wide invalidation | [#cache-key](./ARCHITECTURE.md#cache-key) | +| Edit translation batching | `packages/core/src/{batch,translate-segments}.ts` | **Invariant 2** — `flat(groups) === segments` | [#translation-batching](./ARCHITECTURE.md#translation-batching) | +| Modify cache/storage behaviour | `packages/astro/src/storage/{cache,r2,prune,local-cache,report}.ts` | Apply-before-PUT (**Invariant 3**); index isolation (**Invariant 4**) | [#cache-write-order](./ARCHITECTURE.md#cache-write-order), [#local-staging-index](./ARCHITECTURE.md#local-staging-index) | +| Modify runtime APIs (entry/collection/href/middleware) | `packages/astro/src/runtime/*` | Bridge timing (**Invariant 5**); per-locale closures | [#runtime-bridge](./ARCHITECTURE.md#runtime-bridge) | +| Modify routing shims | `packages/astro/src/routing/{shim,expand-routes,walk-pages}.ts` | Stale shims nuked per build; CSS via `routesImports` | [#routing-shims](./ARCHITECTURE.md#routing-shims) | +| Edit UI-string handling | `packages/astro/src/i18n/*`, `packages/astro/src/cli/{check,sync,translate}-ui.ts` | Three drift modes; layout-aware writer; `{{token}}` preservation | [#ui-strings](./ARCHITECTURE.md#ui-strings) | +| Edit catalog-only adoption | `packages/astro/src/catalog/*`, `packages/astro/src/i18n/{translate,drift,sync}.ts` | Pure imports; middleware binds only `t` + `lhref` | [`CATALOG_ONLY_PLAN.md`](./CATALOG_ONLY_PLAN.md); [consumer SKILL](./skills/polystella-consumer/SKILL.md#catalog-only-adoption) | +| Edit content-collection wiring | `packages/astro/src/content/*` | Sibling collections; custom-loader wrapper; bridge timing | [#runtime-bridge](./ARCHITECTURE.md#runtime-bridge) | | Debug a translation that's wrong | Start: `pnpm translate --dry-run` to inspect planned R2 keys; `LOG_LEVEL=debug` for batch detail | — | Recipe in [contributor SKILL](./skills/polystella-contributor/SKILL.md#debug-translation) | | Tune cold-cache build performance | `r2.bulkListOnStart`, `concurrency`, `batchInputTokenBudget` knobs | — | [#bulk-prelist](./ARCHITECTURE.md#bulk-prelist), [#translation-batching](./ARCHITECTURE.md#translation-batching) | -If your task isn't on this list, the answer is in `src//` +If your task isn't on this list, the answer is in `packages/astro/src//` matching one of the subsystem sections in [ARCHITECTURE.md](./ARCHITECTURE.md). +The root is private. Reusable package code uses standard Web APIs and must +work without `nodejs_compat`, although consumers may enable it. Import +low-level contracts from core, formats from adapters, and transports from +providers; do not add compatibility shims to the Astro package. + --- ## Invariants @@ -92,10 +101,12 @@ to the explanatory section when adding code that touches one. `pnpm changeset add --empty` only when the change deliberately does not require a package release (for example, docs-site-only or CI-only maintenance). -- Bump the package version in `package.json` only — `POLYSTELLA_VERSION` - (in `src/version.ts`) reads it at module-load time via a JSON - import attribute, so the constant flows automatically through to - `dist/version.js` after `pnpm build`. [→ #version-constant](./ARCHITECTURE.md#version-constant) +- Let Changesets version core, adapters, and providers independently while + keeping Astro and its compatibility package in one fixed group; do not + manually bump individual manifests. `POLYSTELLA_VERSION` in + `packages/astro/src/version.ts` reads the Astro manifest at module-load time + and flows to `packages/astro/dist/version.js` after `pnpm build`. + [→ #version-constant](./ARCHITECTURE.md#version-constant) - Mirror filesystem path semantics across OS: forward slashes for R2 keys, `path.sep` for local I/O. - Forward `signal: AbortSignal` when adding a new async function on @@ -156,7 +167,8 @@ Tiered so you can scan the ones that matter for your change. Before pushing: - `pnpm test` must pass. -- `pnpm exec tsc --noEmit` must pass (strict mode). +- `pnpm typecheck` must pass (strict mode). +- `pnpm check:packages` must pass for packaging or export changes. - For changes to the translation pipeline, run end-to-end against a real consumer's fixtures: `polystella translate --dry-run` walks the full pipeline without hitting AI/R2. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e6dcfe4..7664d75 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -7,6 +7,8 @@ canonical home for the longer-form context. For task recipes ("I want to add an adapter / a provider / a CLI subcommand"), see [`AGENTS.md`](./AGENTS.md). For consumer-side wiring, see [`skills/polystella-consumer/SKILL.md`](./skills/polystella-consumer/SKILL.md). +For the post-migration package graph and file map, see +[`PACKAGE_ARCHITECTURE.md`](./PACKAGE_ARCHITECTURE.md). All cross-references in this doc and in `AGENTS.md` use **stable slug anchors** (e.g. `#cache-key`), not section numbers. Inserting a new @@ -17,6 +19,7 @@ section never invalidates an existing link. ## Contents - [Overview](#overview) +- [Package boundaries](#package-boundaries) - [Glossary](#glossary) - [Invariants](#invariants) - [Pipeline](#pipeline) @@ -52,58 +55,72 @@ Cloudflare R2, and injects locale-prefixed routes for the translated pages. The same orchestrator powers a standalone `polystella` CLI so operators can run the pipeline outside `astro build`. +The direct in-process flow is exact: + +```text +source/record -> adapter -> core -> provider -> core -> adapter -> output ``` - ┌────────────────────────────────────────────────────────┐ - │ astro:config:setup (or CLI) │ - └────────────────────────────────────────────────────────┘ - │ - ▼ - walk sources ──► per (file × locale) worker pool ◄── R2 bulk pre-list - │ - ┌─────────────────┴─────────────────┐ - ▼ ▼ - cache key matches? override file present? - │ │ - no ┌────┴────┐ yes ▼ - ▼ ▼ read verbatim, rewrite URLs - parse + return cached - extract bytes, rewrite │ - │ URLs, stage │ - ▼ │ - translate (token-aware batches, │ - heading-anchored grouping, │ - document-context preamble) │ - │ │ - ▼ │ - apply translations + AI marker │ - │ │ - ▼ │ - PUT to R2 (post-apply bytes) │ - │ │ - ▼ ▼ - rewrite URLs → stage under /.astro/i18n-staging// - │ - ▼ - polystellaCollections (in user's content.config.ts) - reads staged bytes via the runtime bridge → Astro content layer - │ - ▼ - routing shims under /polystella-shims/ - inject `/[lang]/...` routes pointing at staged content - │ - ▼ - astro build → dist/ + +There is no hosted core service or required network hop beyond the +selected provider transport. + +```mermaid +flowchart TD + setup["astro:config:setup or CLI"] --> workers["walk sources and process file-locale pairs"] + prelist["R2 bulk pre-list"] --> workers + workers --> override{"override present?"} + override -- yes --> overrideStage["read verbatim, rewrite URLs, stage"] + override -- no --> local{"local index and staged file match?"} + local -- yes --> ready["keep staged bytes"] + local -- no --> r2{"R2 cache hit?"} + r2 -- yes --> cached["return cached bytes, rewrite URLs, stage"] + r2 -- no --> translate["parse, extract, batch, and translate"] + translate --> apply["apply translations and AI marker"] + apply --> put["PUT post-apply bytes to R2"] + put --> stage["rewrite URLs and stage"] + overrideStage --> content["regular sibling collections read staged files"] + ready --> content + cached --> content + stage --> content + content --> routes["route shims inject /lang routes"] + routes --> dist["Astro build emits dist"] ``` -Two entry points share `runTranslationPass` in `src/translation/run.ts`: +Two entry points share `runTranslationPass` in `packages/astro/src/translation/run.ts`: -- **Astro integration** (`src/index.ts`) — registers hooks, runs the +- **Astro integration** (`packages/astro/src/index.ts`) — registers hooks, runs the pass, publishes the runtime bridge. -- **CLI** (`src/cli.ts`) — verb-style dispatcher routing to - `src/cli/.ts`. `translate` reuses `runTranslationPass`; +- **CLI** (`packages/astro/src/cli.ts`) — verb-style dispatcher routing to + `packages/astro/src/cli/.ts`. `translate` reuses `runTranslationPass`; `check-ui`, `sync-ui`, `translate-ui` operate on UI-string JSONs and don't touch the markdown pipeline or R2. +## Package boundaries + + + +- `@cloudflare/polystella-core` owns `Segment`, glossaries, the + `Translator` contract, prompt construction/response parsing, batching, + and retries. +- `@cloudflare/polystella-adapters` owns portable format parsing, + extraction, grouping, and translation application. +- `@cloudflare/polystella-providers` owns Workers AI HTTP/binding and + Anthropic transports. +- `@cloudflare/polystella-astro` owns Astro hooks, filesystem and R2 access, + cache/marker/URL policy, routing, runtime APIs, and the CLI. + +The three reusable packages depend only on standard Web APIs at runtime. +They execute in Workers without `nodejs_compat`; enabling the flag in a +consumer remains supported. Providers use package-owned structural types +for bindings and do not import generated Cloudflare types. + +Low-level exports moved to their owning packages during extraction. In +particular, `Segment`, `Glossary`, `Translator`, +`PermanentProviderError`, prompt helpers, and batching moved from the +Astro root to core; portable parsing/application moved to adapters; and +provider factories moved to providers. No compatibility shims preserve +the old low-level imports. + --- ## Glossary @@ -112,7 +129,7 @@ Two entry points share `runTranslationPass` in `src/translation/run.ts`: | Term | Meaning | | :------------------------ | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Adapter** | A `FileTypeAdapter` implementation owning one file format (markdown, TOML, JSON, YAML). Parses, extracts segments, applies translations. See [adapter contract](#adapter-contract). | +| **Adapter** | A reusable `FileAdapter` implementation owning one file format (markdown, TOML, JSON, YAML). Astro wraps it with host policy as `FileTypeAdapter`. See [adapter contract](#adapter-contract). | | **Segment** | The atomic translatable unit emitted by `adapter.extractSegments`. Has a stable per-file ID (`body:N`, `fm:key`, `fm:key[i]`, or a dotted key path). | | **Group** | An ordered list of segments inside one section (e.g. all paragraphs under one heading). Produced by `adapter.groupSegments`; the batcher packs groups into batches under a token budget. | | **Batch** | One prompt round-trip's worth of segments. Produced by `packGroupsIntoBatches`. Each batch carries its own document-context block. | @@ -123,7 +140,7 @@ Two entry points share `runTranslationPass` in `src/translation/run.ts`: | **Drift** | A non-default-locale UI-string JSON disagreeing with the default-locale source: missing keys, extra keys, or `""` placeholders where the source is non-empty. | | **Miss path** | The code branch in the cache layer when an R2 GET returns nothing. Triggers translator + apply + PUT. | | **Live phase / live run** | A run that actually translates (provider configured, `dryRun: false`). The pipeline only walks sources once in this mode. | -| **Bridge** | `src/runtime/custom-loader-runtime.ts` — the module-scoped singleton holding live JS objects (R2 client, translators, glossaries) shared between `astro:config:setup` and the sibling content collections registered at content-sync time. | +| **Bridge** | `packages/astro/src/runtime/custom-loader-runtime.ts` — the symbol-keyed `globalThis` value holding live JS objects (R2 client, translators, glossaries) across `astro:config:setup` and sibling content collections at content-sync time. | | **Sibling collection** | A per-locale content collection (`publications__pt-BR`, etc.) auto-registered by `polystellaCollections` alongside the user's source collection. | | **Branch dispatch** | The three-mode R2 prefix selection (local / CI main / CI preview) driven by `WORKERS_CI_BRANCH` and `POLYSTELLA_CLI`. See [R2 dispatch](#r2-dispatch). | @@ -141,7 +158,7 @@ adding new code that touches one. 2. **Group flattening.** `flat(adapter.groupSegments(...)) === segments` (reference-equal, order-preserved). Asserted at runtime. See [#translation-batching](#translation-batching). 3. **Apply before PUT.** `adapter.applyTranslations` must produce the exact bytes that get PUT to R2; any AI-translation marker is woven in inside `apply`, never after. Cache hits return the PUT bytes verbatim. See [#cache-write-order](#cache-write-order). 4. **Local cache index write isolation.** Pool workers read from `localCacheIndex` (immutable for the run) and write to `nextLocalCacheIndex` (accumulated, persisted at end). A worker MUST NOT read from `nextLocalCacheIndex`. See [#local-staging-index](#local-staging-index). -5. **Bridge timing.** `setRuntimeBridge` is called inside `astro:config:setup`; `polystellaCollections` reads it at content-sync time, which happens between `config:setup` and `build:start`. Translation MUST run in `config:setup` so staged files exist before sibling loaders execute. See [#hook-timing](#hook-timing). +5. **Bridge timing.** `setRuntimeBridge` is called inside `astro:config:setup`; custom-loader siblings read it at content-sync time, which happens between `config:setup` and `build:start`. Translation MUST run in `config:setup` so staged files exist before regular sibling loaders execute. See [#hook-timing](#hook-timing). 6. **URL-rewrite idempotence.** Both layers (`adapter.rewriteUrls` + `rewriteInternalLinks`) must be safe to apply twice. Overrides re-rewrite without doubling prefixes. See [#url-rewriting](#url-rewriting). 7. **Path separator convention.** R2 keys use forward slashes (POSIX); local filesystem paths use `path.sep`. Sources are walked relative to project root, then joined per target. 8. **Permanent vs retriable provider errors.** Only `PermanentProviderError` short-circuits `translateBatch`'s retry loop. Wrap 4xx (400/401/403/404/422); everything else retries with exponential backoff. See [#translator-contract](#translator-contract). @@ -153,21 +170,21 @@ adding new code that touches one. -`runTranslationPass` (in `src/translation/run.ts`) is the +`runTranslationPass` (in `packages/astro/src/translation/run.ts`) is the orchestrator. Same function powers both the Astro integration and the `polystella translate` CLI subcommand. Zero direct dependency on Astro's types so the CLI can run without Astro on the import path. Sequenced steps: -1. Load glossaries (`src/glossary/`). -2. Walk sources (`src/source/walk.ts`) — respects `include` / `exclude`. +1. Load glossaries (`packages/astro/src/glossary/`). +2. Walk sources (`packages/astro/src/source/walk.ts`) — respects `include` / `exclude`. 3. Bulk pre-list R2 once per (prefix × locale) — populates an in-memory existence predicate (see [#bulk-prelist](#bulk-prelist)). 4. Read the local staging index (see [#local-staging-index](#local-staging-index)). 5. Run a worker pool over (file, locale) pairs with - `runWithConcurrency` (`src/source/pool.ts`). -6. Per pair: short-circuit on local-skip → check override → check R2 → + `runWithConcurrency` (`packages/astro/src/source/pool.ts`). +6. Per pair: check override → short-circuit on local-skip → check R2 → translate → apply → PUT → rewrite URLs → stage. 7. Persist `nextLocalCacheIndex` to disk. 8. Prune R2 keys not touched this run (within the configured prefix @@ -221,22 +238,21 @@ location. -`polystellaCollections` runs **after** the integration's `config:setup` -returns (it's called from the user's `content.config.ts`, at -content-sync time). The two halves need to share live JS objects (R2 -client, translators, glossaries) that can't be serialised through the +Custom-loader sibling collections run after the integration's `config:setup` +returns, at content-sync time. Unlike regular file and glob siblings, which +read staged files directly, custom-loader siblings need live JS objects (R2 +client, translators, glossaries) that cannot be serialised through the `polystella:runtime-config` virtual module. -The bridge in `src/runtime/custom-loader-runtime.ts` is a module-scoped -singleton populated by `setRuntimeBridge` during `config:setup` and -read by the sibling loaders at sync time. Module-scoped state is fine -because Astro runs both halves in the same Node process. +The bridge in `packages/astro/src/runtime/custom-loader-runtime.ts` is a +symbol-keyed `globalThis` value populated by `setRuntimeBridge` during +`config:setup` and read by sibling loaders at sync time. The global survives +Vite module-graph reloads while Astro runs both halves in one process. -`publishRuntimeBridge` in `index.ts` re-loads glossaries and constructs -translators that `runTranslationPass` already built internally. The -duplication is deliberate (and cheap — one extra FS read per locale) -to keep `runTranslationPass`'s signature focused on the file-based -pipeline. A future consolidation can extract a shared dep builder. +`publishRuntimeBridge` in `index.ts` reuses glossary maps returned by +`runTranslationPass` when available and loads them from disk when the file +pass did not run. It constructs the translators and R2 client needed by +custom-loader siblings. --- @@ -374,7 +390,7 @@ tests must not produce different output. -The cache layer (`src/storage/cache.ts`) is format-agnostic. On a miss +The cache layer (`packages/astro/src/storage/cache.ts`) is format-agnostic. On a miss it: 1. calls the translator, @@ -416,11 +432,13 @@ exemptions apply uniformly. Both layers are idempotent -Every file format implements `FileTypeAdapter` in -`src/parsing/adapter.ts` and registers in `src/parsing/registry.ts`. -No changes to `run.ts` or the cache layer required. +Every reusable file format implements `FileAdapter` in +`packages/adapters/src/adapter.ts`. Astro wraps it with cache selection, +`noTranslate`, URL, document-context, marker, and parser policies in +`packages/astro/src/parsing/`, then registers it in +`packages/astro/src/parsing/registry.ts`. -Abbreviated shape: +The Astro wrapper's abbreviated superset shape: ```ts interface FileTypeAdapter { @@ -465,7 +483,9 @@ call `resetRegistry()` before registering. One `Translator` per (provider, locale). Two concrete providers ship: Workers AI and Anthropic. Both speak the same prompt-and-JSON-back -contract enforced by `src/translation/prompt.ts`. +contract enforced by `packages/core/src/prompt.ts`. +`packages/astro/src/translation/provider.ts` only maps validated Astro configuration +to the concrete factories in `packages/providers`. ```ts interface Translator { @@ -511,7 +531,7 @@ single-group default in `translateSegments`, which then packs by token budget alone. **Token-aware packing.** `packGroupsIntoBatches` (in -`src/translation/batch.ts`) is a pure function over `Segment[][]`. +`packages/core/src/batch.ts`) is a pure function over `Segment[][]`. Greedy-fills batches under a soft input-token budget (`provider.batchInputTokenBudget`, default `4000`) using `Math.ceil((id + text + 8) / 4)` per segment — the `+8` covers the @@ -594,26 +614,27 @@ source of truth; non-default locales must match its key set. Three CLI subcommands maintain the invariant: -- **`check-ui`** (`src/cli/check-ui.ts`) — pure drift detection. Zero +- **`check-ui`** (`packages/astro/src/cli/check-ui.ts`) — pure drift detection. Zero writes, zero network. Pre-commit hook target. Catches three failure modes: missing keys, extra keys, and **empty-placeholder values** (a key shared with the source dict but with `""` in the locale where the source value is non-empty). The build's own drift check at `astro:config:setup` uses the same predicate. -- **`sync-ui`** (`src/cli/sync-ui.ts` + `src/i18n/sync.ts`) — +- **`sync-ui`** (`packages/astro/src/cli/sync-ui.ts` + + `packages/astro/src/i18n/sync.ts`) — mechanical key reconciliation. Adds missing keys as empty strings, drops extras, preserves existing values (empty or not), re-emits files in source-file key order with blank-line section breaks preserved. -- **`translate-ui`** (`src/cli/translate-ui.ts` + - `src/i18n/ui-translate.ts`) — runs sync, skips fully translated +- **`translate-ui`** (`packages/astro/src/cli/translate-ui.ts` + + `packages/astro/src/i18n/ui-translate.ts`) — runs sync, skips fully translated locale JSONs before provider setup, then translates each queued locale in small sequential request batches capped by `provider.batchInputTokenBudget` and 25 UI strings per request. Queued locales run in parallel via `runWithConcurrency` with a hard max locale concurrency of 3. -**Layout-aware writer.** `formatLocaleFile` in `src/i18n/sync.ts` +**Layout-aware writer.** `formatLocaleFile` in `packages/astro/src/i18n/sync.ts` parses the source file's text (not just its JSON) to recover top-level key order AND which keys start a new "section" (blank line immediately before). The output then mirrors that layout for every @@ -623,7 +644,7 @@ keys alphabetically. **`{{token}}` preservation.** Validated post-translation by extracting `{{\w+}}` tokens from both source and translation and comparing the sets. Validator lives _outside_ `translateBatch` (in -`src/i18n/ui-translate.ts`) because `translateBatch` doesn't expose +`packages/astro/src/i18n/ui-translate.ts`) because `translateBatch` doesn't expose a post-parse hook. The orchestrator runs its own per-request-batch retry wrapper with `maxRetries: 0` passed to `translateBatch` so the retry loop is single-layer. A token-invalid translation after all @@ -714,19 +735,24 @@ boundaries that could otherwise run indefinitely. -`POLYSTELLA_VERSION` lives in `src/version.ts` as a JSON import from +`POLYSTELLA_VERSION` lives in `packages/astro/src/version.ts` as a JSON import from `package.json` (`import pkg from "../package.json" with { type: -"json" }`). Re-exported from `src/index.ts`, consumed directly by -`src/cli.ts`. - -Both the CLI (`dist/cli.js`) and the library entries (`dist/index.js`, -`dist/runtime/index.js`, …) are produced by `tsc -p -tsconfig.build.json` (`pnpm build`). tsc preserves the `with { type: -"json" }` import attribute, so `dist/version.js` resolves +"json" }`). Re-exported from `packages/astro/src/index.ts`, consumed directly by +`packages/astro/src/cli.ts`. + +Both the CLI (`packages/astro/dist/cli.js`) and the library entries +(`packages/astro/dist/index.js`, `packages/astro/dist/runtime/index.js`, …) +are produced by `tsc -p packages/astro/tsconfig.build.json` (`pnpm build`). +tsc preserves the `with { type: "json" }` import attribute, so +`packages/astro/dist/version.js` resolves `../package.json` (i.e. the package root) at module-load time inside -the consumer's `node_modules/polystella/`. No version inlining; one +the consumer's `node_modules/@cloudflare/polystella-astro/`. No version inlining; one source of truth. -Bump `package.json` only; both surfaces follow. The constant is baked -into R2 metadata and the build report but is NOT in the cache key -formula, so a version bump doesn't re-translate. +Changesets versions the canonical Astro package and +`@cloudflare/polystella` compatibility package as a fixed group. Core, +adapters, and providers are versioned independently and consumed through +compatible ranges. The Astro manifest remains this constant's source, so the +canonical package and compatibility alias report the same generated version. +The constant is baked into R2 metadata and the build report but is NOT in the +cache key formula, so a version bump doesn't re-translate. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3b463e..a9834ec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,17 +10,22 @@ easier to evaluate when maintainer time is available. ## Repository overview -PolyStella is a pnpm workspace with two members: +PolyStella is a pnpm workspace with five public packages under `packages/`: -- **The package itself** at the repo root (`package.json` → - `@cloudflare/polystella`). -- **The docs site** under `docs/` (`docs/package.json` → - `polystella-docs`). +- `packages/astro/` — `@cloudflare/polystella-astro`. +- `packages/polystella/` — `@cloudflare/polystella`, a forwarding compatibility package. +- `packages/core/` — platform-neutral translation orchestration. +- `packages/adapters/` — Markdown, MDX, JSON, YAML, and TOML adapters. +- `packages/providers/` — Workers AI and Anthropic transports. + +The private root coordinates those packages, the `docs/` site, and the +`playgrounds/` fixtures. The agent-facing context lives in [`AGENTS.md`](./AGENTS.md). The -system-level design rationale lives in -[`ARCHITECTURE.md`](./ARCHITECTURE.md). Read those before working -on anything non-trivial; they save a lot of back-and-forth. +post-migration package map lives in +[`PACKAGE_ARCHITECTURE.md`](./PACKAGE_ARCHITECTURE.md), and the system-level +design rationale lives in [`ARCHITECTURE.md`](./ARCHITECTURE.md). Read those +before working on anything non-trivial; they save a lot of back-and-forth. ## Development setup @@ -32,16 +37,16 @@ pnpm install Required: -- Node 20+ (24 LTS recommended). +- Node 22.12+ (24 recommended). - pnpm 9+ (the lockfile is `pnpm-lock.yaml`). ## Commands | Command | Purpose | | ------------------------------------- | --------------------------------------------------------------- | -| `pnpm test` | Run the package's unit + smoke tests (vitest). | -| `pnpm exec tsc --noEmit` | Typecheck the package. | -| `pnpm build` | Compile `src/` → `dist/` (library + CLI, JS + `.d.ts`). | +| `pnpm test` | Run package, Astro, workerd, and boundary tests. | +| `pnpm typecheck` | Build and typecheck all five public packages. | +| `pnpm build` | Build all five public packages. | | `pnpm build:llms` | Regenerate `llms-full.txt` from canonical agent docs. | | `pnpm --filter polystella-docs dev` | Run the Nimbus docs site locally. | | `pnpm --filter polystella-docs build` | Build the docs site (includes auto-generated config reference). | @@ -69,7 +74,7 @@ Required: - **TypeScript strict mode**, including `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes`, `noImplicitReturns`, `noFallthroughCasesInSwitch`. Tooling configured in - `tsconfig.json`. + `tsconfig.base.json` and each package's local config. - **No `any`, no `!`.** Use `unknown` + type guards; use destructure-and-check instead of non-null assertions. See [`AGENTS.md`](./AGENTS.md) for the rationale. @@ -79,35 +84,34 @@ Required: - **Comments document the "why", not the "what".** Long-form rationale belongs in `ARCHITECTURE.md`. Inline comments are for non-obvious decisions and known footguns. -- **Tests are integration-heavy.** We use vitest with - `singleThread: true` (faster than multi-worker at our scale). - Tests live under `tests//.test.ts` mirroring - the source structure. A 9-test smoke suite under `tests/smoke.test.ts` - exercises the integration end-to-end against a temp project. +- **Tests are integration-heavy.** Package tests live under each + `packages/*/tests/` directory. Astro tests under `packages/astro/tests/` + mirror `packages/astro/src/` and include an end-to-end smoke suite. ## Adding new APIs Before adding to the public surface: -- **Is it covered by an existing export path?** Check the - `exports` field in `package.json`. Eight subpaths are exposed; - unless your addition needs its own namespace, it should fit in - one of them. +- **Is it covered by an existing export path?** Check the `exports` field in + the owning `packages/*/package.json`; avoid adding a namespace when an + existing entry already fits. - **Does it have a documentation page?** `pnpm --filter polystella-docs check-exports` asserts every `exports` path is mentioned on `docs/src/content/docs/reference/exports.md`. CI fails if not. - **Does the schema reference need updating?** The `docs/scripts/generate-config-ref.ts` script auto-walks - `src/config/options.ts`'s zod schema. If your change adds a new + `packages/astro/src/config/options.ts` zod schema. If your change adds a new config field, regenerate the page locally with `pnpm --filter polystella-docs prebuild` and verify the output reads cleanly. ## Adding a new adapter -Adapters implement the `FileTypeAdapter` interface in -`src/parsing/adapter.ts` and register via `parsing/registry.ts`. +Portable adapters implement `FileAdapter` in +`packages/adapters/src/adapter.ts`; Astro policy wrappers implement +`FileTypeAdapter` in `packages/astro/src/parsing/adapter.ts` and register via +`packages/astro/src/parsing/registry.ts`. See [`ARCHITECTURE.md`](./ARCHITECTURE.md) `#adapter-contract` for the full contract. The Markdown adapter is the reference implementation. @@ -115,8 +119,9 @@ implementation. ## Adding a new provider Providers implement the `Translator` interface in -`src/translation/provider.ts`. Throw `PermanentProviderError` on -4xx HTTP responses that retries can't fix (401/403/404/422); throw +`packages/core/src/translator.ts` and expose transports from +`packages/providers/src/`. Throw `PermanentProviderError` on +4xx HTTP responses that retries can't fix (400/401/403/404/422); throw plain `Error` on anything retriable. See [`ARCHITECTURE.md`](./ARCHITECTURE.md) `#translator-contract` for the detail. @@ -128,7 +133,7 @@ Bug reports against PolyStella are most useful when they include: that exhibits the issue). - Source file(s) that trigger the issue (or a synthetic example with the same shape). -- The build report from `dist/i18n-r2-report.json`, if relevant. +- The `i18n-r2-report.json` build report, if relevant. - The PolyStella version (visible in the report; or `polystella --version`). ## License diff --git a/PACKAGE_ARCHITECTURE.md b/PACKAGE_ARCHITECTURE.md new file mode 100644 index 0000000..19c051c --- /dev/null +++ b/PACKAGE_ARCHITECTURE.md @@ -0,0 +1,251 @@ +# PolyStella Package Architecture + +This guide explains the repository structure after the package migration. It +is the starting point for deciding where a change belongs. For detailed +pipeline behavior and hard correctness contracts, use +[`ARCHITECTURE.md`](./ARCHITECTURE.md). + +## Package Graph + +PolyStella publishes five packages. The canonical Astro package and its +compatibility package form one fixed version group. Core, adapters, and +providers are versioned independently. Arrows mean "depends on." + +```mermaid +flowchart TD + compat["@cloudflare/polystella
compatibility only"] --> astro["@cloudflare/polystella-astro
canonical Astro package"] + astro --> adapters["@cloudflare/polystella-adapters
portable formats"] + astro --> providers["@cloudflare/polystella-providers
portable transports"] + astro --> core["@cloudflare/polystella-core
translation protocol"] + adapters --> core + providers --> core +``` + +Dependencies point toward reusable code. Core never imports adapters, +providers, or Astro. Adapters and providers do not import each other. The +compatibility package contains no implementation and points only to the +canonical Astro package. + +Published dependencies on independently versioned packages use compatible +caret ranges. The compatibility package pins the exact Astro version because +it forwards that package's API and CLI unchanged. + +| Directory | Published package | Responsibility | +| :----------------------------------------------- | :--------------------------------- | :--------------------------------------------------------------------------------- | +| [`packages/core/`](./packages/core/) | `@cloudflare/polystella-core` | Translation contracts, prompts, batching, retries, and response parsing. | +| [`packages/adapters/`](./packages/adapters/) | `@cloudflare/polystella-adapters` | Portable parsing, extraction, grouping, and translation application. | +| [`packages/providers/`](./packages/providers/) | `@cloudflare/polystella-providers` | Workers AI and Anthropic implementations of the core translator contract. | +| [`packages/astro/`](./packages/astro/) | `@cloudflare/polystella-astro` | Canonical Astro integration, host policy, storage, routing, runtime APIs, and CLI. | +| [`packages/polystella/`](./packages/polystella/) | `@cloudflare/polystella` | Temporary compatibility forwarding to `@cloudflare/polystella-astro`. | + +## Direct Translation Flow + +The three reusable packages form an in-process translation pipeline: + +```mermaid +flowchart LR + source[Source bytes or record] --> parse[Adapter parses and extracts segments] + parse --> orchestrate[Core groups, batches, prompts, and validates] + orchestrate --> transport[Provider calls the selected model] + transport --> orchestrate + orchestrate --> apply[Adapter applies translations] + apply --> output[Translated output] +``` + +The adapter owns source syntax. Core owns the translation protocol. The +provider owns transport-specific I/O. None of these layers needs Astro. + +## Package Responsibilities + +### Core + +[`@cloudflare/polystella-core`](./packages/core/) is the lowest internal layer. +It owns: + +- `Segment`, `Glossary`, `Logger`, and `Translator` contracts. +- Prompt construction and provider-response parsing. +- Token estimation, grouping validation, and batch packing. +- Translation execution, retries, cancellation, and permanent provider errors. + +Start at [`packages/core/src/index.ts`](./packages/core/src/index.ts). The main +implementations are `translator.ts`, `prompt.ts`, `batch.ts`, +`translate-batch.ts`, and `translate-segments.ts`. + +Core does not know about file formats, R2, the filesystem, Astro, or any +specific AI transport. + +### Adapters + +[`@cloudflare/polystella-adapters`](./packages/adapters/) owns portable format +handling: + +- The [`FileAdapter`](./packages/adapters/src/adapter.ts) contract. +- Markdown, MDX, JSON, YAML, and TOML adapters. +- Segment extraction, grouping, and translation application. +- Structured key paths, MDX rules, and placeholder handling. + +Start at [`packages/adapters/src/index.ts`](./packages/adapters/src/index.ts) +and [`packages/adapters/src/adapters/`](./packages/adapters/src/adapters/). +Adapters depend on core for `Segment` and related contracts. + +Astro-specific defaults, configuration, staging, and URL policy do not belong +here. Those wrappers live under +[`packages/astro/src/parsing/`](./packages/astro/src/parsing/). + +### Providers + +[`@cloudflare/polystella-providers`](./packages/providers/) implements the +core `Translator` contract for external model APIs: + +- Workers AI over HTTP. +- Workers AI through a binding described with package-owned structural types. +- Anthropic over HTTP. +- Transport error normalization and permanent/retriable classification. + +Start at [`packages/providers/src/index.ts`](./packages/providers/src/index.ts), +`workers-ai.ts`, `anthropic.ts`, and `http-error.ts`. Providers depend only on +core internally. + +Provider configuration belongs to the Astro package. The mapping from Astro +options to provider factories is +[`packages/astro/src/translation/provider.ts`](./packages/astro/src/translation/provider.ts). + +### Astro + +[`@cloudflare/polystella-astro`](./packages/astro/) is the canonical product +package. It composes the reusable packages and owns host-specific behavior: + +- Astro hooks, option validation, and virtual modules. +- Source walking, translation-pass orchestration, and local staging. +- R2 keys, reads, writes, metadata, local indexes, reports, and pruning. +- Overrides, AI markers, and URL rewriting policy. +- Content collections, custom-loader support, runtime lookup, and middleware. +- Route shims, UI strings, catalog-only mode, React hooks, recipes, and CLI. + +The primary entry points are: + +| Area | Start here | +| :------------------ | :--------------------------------------------------------------------------------- | +| Integration hooks | [`packages/astro/src/index.ts`](./packages/astro/src/index.ts) | +| Configuration | [`packages/astro/src/config/options.ts`](./packages/astro/src/config/options.ts) | +| Translation pass | [`packages/astro/src/translation/run.ts`](./packages/astro/src/translation/run.ts) | +| Storage and cache | [`packages/astro/src/storage/`](./packages/astro/src/storage/) | +| Format policy | [`packages/astro/src/parsing/`](./packages/astro/src/parsing/) | +| Content collections | [`packages/astro/src/content/`](./packages/astro/src/content/) | +| Runtime APIs | [`packages/astro/src/runtime/`](./packages/astro/src/runtime/) | +| Routing | [`packages/astro/src/routing/`](./packages/astro/src/routing/) | +| UI strings | [`packages/astro/src/i18n/`](./packages/astro/src/i18n/) | +| Catalog-only mode | [`packages/astro/src/catalog/`](./packages/astro/src/catalog/) | +| CLI dispatch | [`packages/astro/src/cli.ts`](./packages/astro/src/cli.ts) | + +Its public export map is declared in +[`packages/astro/package.json`](./packages/astro/package.json). The standalone +`polystella` executable is emitted from `src/cli.ts`. The `./client` export is +types-only and comes from [`packages/astro/client.d.ts`](./packages/astro/client.d.ts). + +### Compatibility Package + +[`@cloudflare/polystella`](./packages/polystella/) is not an implementation +layer. It exists so projects using the old package name can migrate without an +immediate import rewrite. + +- Every source entry re-exports the matching `@cloudflare/polystella-astro` + entry. +- `client.d.ts` references the canonical client declarations. +- Its CLI launches the canonical package's CLI. +- It must not gain independent behavior or restore low-level exports moved to + core, adapters, or providers. + +When adding an Astro public export, update the canonical manifest, add the +matching forwarding file and export in `packages/polystella/`, update the +public export reference, and run `pnpm check:packages`. + +## Astro Build Flow + +The integration and standalone CLI share +[`runTranslationPass`](./packages/astro/src/translation/run.ts). Astro-specific +setup remains in the integration entry. + +```mermaid +sequenceDiagram + participant Astro + participant Integration as astro/src/index.ts + participant Run as translation/run.ts + participant Cache as local index and R2 + participant Content as Astro content layer + + Astro->>Integration: astro:config:setup + Integration->>Run: runTranslationPass() + Run->>Cache: override, local index, R2, or provider + Run-->>Integration: staged files and run metadata + Integration->>Integration: publish custom-loader bridge + Astro->>Content: content sync + Content->>Content: regular siblings read staged files + Content->>Integration: custom-loader siblings use bridge + Astro->>Integration: astro:build:done + Integration->>Integration: emit build report +``` + +Translation must run during `astro:config:setup` because Astro syncs content +before `build:start`. See [`ARCHITECTURE.md#hook-timing`](./ARCHITECTURE.md#hook-timing). + +## Where Changes Belong + +| Change | Package | +| :-------------------------------------------------------------------- | :-------------------- | +| Change prompts, batching, retry behavior, or translator contracts | Core | +| Parse or reconstruct a portable content format | Adapters | +| Add or change an external AI transport | Providers | +| Change Astro options, files, R2, routing, middleware, content, or CLI | Astro | +| Mirror a canonical Astro export under the old package name | Compatibility package | + +If a change requires Node, Astro, filesystem, or R2 APIs, it does not belong in +core, adapters, or providers. If a provider implementation starts importing +Astro config types, move that mapping back to the Astro package instead. + +## Boundary Enforcement + +The boundaries are executable, not only documented: + +| Contract | Enforcement | +| :---------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------- | +| Package dependency graph, exports, tarball contents, CLIs, and compatibility parity | [`scripts/check-packages.mjs`](./scripts/check-packages.mjs) | +| Reusable packages avoid Node and host imports | [`tests/boundaries/reusable-packages.test.ts`](./tests/boundaries/reusable-packages.test.ts) | +| Reusable packages execute under Workerd without `nodejs_compat` | [`tests/workerd/`](./tests/workerd/) and [`scripts/check-workerd-portability.mjs`](./scripts/check-workerd-portability.mjs) | +| End-to-end extraction behavior remains stable | [`scripts/check-monorepo-baseline.mjs`](./scripts/check-monorepo-baseline.mjs) | +| Public export documentation matches manifests | [`docs/scripts/check-exports.ts`](./docs/scripts/check-exports.ts) | +| All packages version together | [`.changeset/config.json`](./.changeset/config.json) | + +The root [`package.json`](./package.json) builds in dependency order: core, +adapters, providers, canonical Astro, then compatibility forwarding. + +## Contributor Checklist + +Before opening a package-affecting change: + +1. Put the change in the lowest package that can own it without importing a + higher layer. +2. Export it only if downstream consumers need it; package manifests are the + public API source of truth. +3. Mirror new Astro exports in the compatibility package while that package is + supported. +4. Add or update the smallest boundary or behavior test that protects the + contract. +5. Add a Changesets entry and run `pnpm test`, `pnpm typecheck`, and + `pnpm check:packages`. + +## Deeper Design References + +Use these stable sections for implementation details: + +- [Package boundaries](./ARCHITECTURE.md#package-boundaries) +- [Pipeline](./ARCHITECTURE.md#pipeline) +- [Invariants](./ARCHITECTURE.md#invariants) +- [Adapter contract](./ARCHITECTURE.md#adapter-contract) +- [Translator contract](./ARCHITECTURE.md#translator-contract) +- [Translation batching](./ARCHITECTURE.md#translation-batching) +- [Cache key](./ARCHITECTURE.md#cache-key) +- [Cache write order](./ARCHITECTURE.md#cache-write-order) +- [Runtime bridge](./ARCHITECTURE.md#runtime-bridge) +- [Routing shims](./ARCHITECTURE.md#routing-shims) +- [UI strings](./ARCHITECTURE.md#ui-strings) diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md new file mode 100644 index 0000000..7c693cc --- /dev/null +++ b/PR_DESCRIPTION.md @@ -0,0 +1,78 @@ +## What does this PR do? + +Extracts PolyStella from a single Astro package into five packages with explicit dependency boundaries. Astro and its compatibility package share a version; core, adapters, and providers are versioned independently: + +```text +@cloudflare/polystella -> @cloudflare/polystella-astro + |-> @cloudflare/polystella-adapters -> core + |-> @cloudflare/polystella-providers -> core + `-> @cloudflare/polystella-core +``` + +- Makes `@cloudflare/polystella-astro` the canonical Astro integration. +- Keeps `@cloudflare/polystella` as a forwarding compatibility package. +- Extracts platform-neutral translation orchestration, format adapters, and provider transports into reusable packages that work without `nodejs_compat`. +- Adds package boundary, clean-consumer, tarball, workerd, and baseline checks. +- Updates CI, release workflows, documentation, examples, contributor guidance, and the MDX playground for the new package graph. +- Documents package ownership and dependency direction in `PACKAGE_ARCHITECTURE.md`. + +The extraction also hardens the new package boundaries: + +- Moves format-specific prompt guidance from core to the Markdown adapter. +- Enforces segment grouping and unique-ID invariants in core. +- Colocates Markdown and MDX implementation files and moves MDX options out of generic adapter contracts. +- Protects recursively nested inline MDX JSX and restores nested placeholders inside-out. +- Rejects malformed structured-data key paths after array indexes. +- Consolidates duplicated private MDX AST utilities. + +### Migration notes + +- Astro consumers should import the integration from `@cloudflare/polystella-astro`. +- Existing `@cloudflare/polystella` integration imports continue through the compatibility package. +- Low-level consumers should import contracts from `@cloudflare/polystella-core`, formats from `@cloudflare/polystella-adapters`, and transports from `@cloudflare/polystella-providers`. +- Markdown adapter callers should use `MarkdownAdapterExtractOptions` and `MarkdownAdapterApplyOptions` for `mdxRules`; the generic adapter options are now format-neutral. + +## Type of change + +- [x] Bug fix +- [ ] Feature +- [x] Refactor (no behavior change) +- [x] Documentation +- [x] Tests +- [x] CI / tooling +- [x] Chore + +## PolyStella invariants touched + +- [ ] None +- [ ] Cache key formula +- [x] Translation batching / segment grouping +- [ ] Apply-before-PUT cache write order +- [ ] Local cache index isolation +- [ ] Runtime bridge timing +- [ ] URL-rewrite idempotence +- [ ] Provider permanent vs retriable errors +- [ ] R2 key / local path separator handling + +## Checklist + +- [x] I have read `CONTRIBUTING.md` +- [x] I have added or updated tests, or this change does not need tests +- [x] I have added a changeset, or this change does not affect the published package +- [x] I have updated docs, or this change does not affect public behavior +- [x] `pnpm test` passes +- [x] `pnpm typecheck` passes +- [x] Docs checks and builds pass + +## AI-generated code disclosure + +- [x] This PR includes AI-generated code - model/tool: OpenCode (GPT-5.6 Sol) + +## Screenshots / logs / test output + +- `pnpm test`: 1,299 tests passed, including package, Astro, workerd, and boundary suites. +- `pnpm typecheck`: all five public packages passed strict typechecking. +- `pnpm check:packages`: five tarballs, 17 runtime imports, clean Astro consumers, and both CLIs passed. +- `pnpm check:baseline`: passed. +- `pnpm format:check` and `git diff --check`: passed. +- Documentation export checks, example checks, and production build passed. diff --git a/README.md b/README.md index f2c5870..5841542 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,46 @@ PolyStella is an [Astro](https://astro.build) integration that translates content into additional locales at build time using AI, caches translations in Cloudflare R2, and injects locale-prefixed routes for the translated pages. +The repository publishes five packages. The canonical Astro package and its +compatibility package share a version; core, adapters, and providers are +versioned independently: + +| Package | Directory | Role | Internal dependencies | +| ---------------------------------- | ---------------------- | ------------------------------------------------------------------ | ------------------------- | +| `@cloudflare/polystella-core` | `packages/core/` | Platform-neutral prompts, batching, retries, and shared contracts. | None | +| `@cloudflare/polystella-adapters` | `packages/adapters/` | Portable Markdown, MDX, JSON, YAML, and TOML adapters. | Core | +| `@cloudflare/polystella-providers` | `packages/providers/` | Workers AI HTTP/binding and Anthropic transports. | Core | +| `@cloudflare/polystella-astro` | `packages/astro/` | Canonical Astro integration, CLI, R2, routing, and host policy. | Core, adapters, providers | +| `@cloudflare/polystella` | `packages/polystella/` | Temporary compatibility forwarding to the Astro package. | Astro | + +Dependencies point toward reusable code: + +```text +@cloudflare/polystella --> @cloudflare/polystella-astro + ├──> @cloudflare/polystella-adapters --> core + ├──> @cloudflare/polystella-providers --> core + └──> @cloudflare/polystella-core +``` + +Core, adapters, and providers are portable and use standard Web APIs. The +Astro package composes them and owns all host-specific behavior. The generic +package contains forwarding files only; new projects should use +`@cloudflare/polystella-astro`. + +Contributors should start with +[`PACKAGE_ARCHITECTURE.md`](./PACKAGE_ARCHITECTURE.md) for package boundaries, +key files, dependency rules, and enforcement checks. Detailed subsystem +invariants remain in [`ARCHITECTURE.md`](./ARCHITECTURE.md). + +Direct low-level use stays in-process: + +```text +source/record -> adapter -> core -> provider -> core -> adapter -> output +``` + +The reusable packages work in Workers without `nodejs_compat`; consumers may +still enable it. + ## What it does - **Build-time translation.** Translates `.md`, `.mdx`, and `.toml` content into additional locales during `astro build`. Visitors get static bytes; no runtime AI calls. @@ -18,10 +58,17 @@ PolyStella is an [Astro](https://astro.build) integration that translates conten Install from npm: ```bash -pnpm add @cloudflare/polystella +pnpm add @cloudflare/polystella-astro ``` -Peer dependencies: `astro ^7.0.0`, optionally `react ^17 || ^18 || ^19`. +Peer dependencies: `astro ^7.0.10`, optionally `react ^17 || ^18 || ^19`. + +Install the owning package for low-level APIs. `Segment`, `Glossary`, +`Translator`, `PermanentProviderError`, prompt helpers, and batching moved +to `@cloudflare/polystella-core`; portable format helpers moved to +`@cloudflare/polystella-adapters`; provider factories moved to +`@cloudflare/polystella-providers`. The Astro package does not provide +compatibility shims for those old low-level imports. ## Quick start @@ -31,7 +78,7 @@ Four files participate in a typical setup. ```js import { defineConfig } from "astro/config"; -import polystella from "@cloudflare/polystella"; +import polystella from "@cloudflare/polystella-astro"; import polystellaConfig from "./polystella.config.mjs"; export default defineConfig({ @@ -49,8 +96,8 @@ export default defineConfig({ ```ts import { defineCollection } from "astro:content"; -import { polystellaCollections } from "@cloudflare/polystella/content"; -import { i18nLoader, i18nSchema } from "@cloudflare/polystella/i18n"; +import { polystellaCollections } from "@cloudflare/polystella-astro/content"; +import { i18nLoader, i18nSchema } from "@cloudflare/polystella-astro/i18n"; import { blog, authors } from "./content-schemas"; @@ -65,7 +112,7 @@ export const collections = { **4. `src/env.d.ts`** — pick up types for PolyStella's virtual modules: ```ts -/// +/// ``` ## Catalog-Only Usage @@ -74,7 +121,7 @@ Projects that already handle localized content and routing can adopt only PolyStella's JSON catalog flow: ```ts -import catalogAstro from "@cloudflare/polystella/catalog/astro"; +import catalogAstro from "@cloudflare/polystella-astro/catalog/astro"; export default defineConfig({ i18n: { defaultLocale: "en-US", locales: ["en-US", "pt-BR"] }, @@ -102,7 +149,9 @@ Full documentation lives at the Nimbus docs site (under `docs/` in this repo): Contributions are welcome, but PolyStella is maintained by a small team and review is not guaranteed. See [`CONTRIBUTING.md`](./CONTRIBUTING.md). The agent-facing context is in -[`AGENTS.md`](./AGENTS.md) and [`ARCHITECTURE.md`](./ARCHITECTURE.md). +[`AGENTS.md`](./AGENTS.md), +[`PACKAGE_ARCHITECTURE.md`](./PACKAGE_ARCHITECTURE.md), and +[`ARCHITECTURE.md`](./ARCHITECTURE.md). ## License diff --git a/docs/astro.config.ts b/docs/astro.config.ts index f6696c8..65c63c5 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -1,6 +1,7 @@ import { defineConfig } from "astro/config"; import icon from "astro-icon"; import nimbus, { defineConfig as defineNimbusConfig } from "nimbus-docs"; +import polystellaPackage from "../packages/astro/package.json"; const nimbusConfig = defineNimbusConfig({ site: "https://polystella-docs.pcx-team.workers.dev", @@ -10,6 +11,7 @@ const nimbusConfig = defineNimbusConfig({ github: "https://github.com/cloudflare/polystella", editPattern: "https://github.com/cloudflare/polystella/edit/main/docs/{path}", socialImageAlt: "PolyStella documentation preview", + versions: { current: polystellaPackage.version, others: [] }, sidebar: { items: [ { @@ -83,7 +85,7 @@ export default defineConfig({ defaultStrategy: "hover", }, integrations: [ - icon(), + icon({ iconDir: "src/components" }), nimbus(nimbusConfig, { rules: { "nimbus/frontmatter-shape": "error", diff --git a/docs/examples/README.md b/docs/examples/README.md index b38a75b..6b46e7c 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -1,31 +1,21 @@ # PolyStella docs — runnable examples -This directory will hold minimal Astro projects paired with the -cookbook recipes in `docs/src/content/docs/cookbook/`. +This directory holds compile-only documentation fixtures. ## Status -Empty for v0.x. The cookbook pages are written but don't yet have -companion projects. +`direct-packages/` typechecks the direct core/adapters/providers flow and +the package-owned Workers AI binding types without generated Cloudflare +types. ## Adding an example -When you add a cookbook recipe, drop a runnable Astro project here -under a slug matching the recipe filename: +Add a focused fixture under a descriptive slug: ``` -docs/examples/monorepo/ # paired with cookbook/monorepo.md -docs/examples/custom-loader/ # paired with cookbook/custom-loader.md (planned) +docs/examples/direct-packages/ ``` -Each example project must: - -- Have its own `package.json` declaring `@cloudflare/polystella` as a - workspace dependency (`"@cloudflare/polystella": "workspace:*"`). -- Be buildable in isolation via `pnpm --filter polystella-example- build`. -- Include a `README.md` pointing back to the cookbook recipe it - illustrates. - -`docs/scripts/check-examples.ts` walks this directory and runs -`astro check` against each example. The CI build's `docs:build` -step depends on this passing. +`docs/scripts/check-examples.ts` runs `tsc` against the fixture. Keep it +dependency-free and map public workspace imports to package sources in its +local `tsconfig.json`. diff --git a/docs/examples/direct-packages/index.ts b/docs/examples/direct-packages/index.ts new file mode 100644 index 0000000..a61da77 --- /dev/null +++ b/docs/examples/direct-packages/index.ts @@ -0,0 +1,29 @@ +import { jsonAdapter } from "@cloudflare/polystella-adapters"; +import { EMPTY_GLOSSARY, translateSegments, type Translator } from "@cloudflare/polystella-core"; +import { createWorkersAIBindingTranslator, type WorkersAIInput } from "@cloudflare/polystella-providers/workers-ai"; + +interface WorkersAIBinding { + run(modelId: string, input: WorkersAIInput): Promise; +} + +export async function translateRecord(ai: WorkersAIBinding): Promise { + const source = JSON.stringify({ title: "Hello" }); + const parsed = jsonAdapter.parse(source, "record.json"); + const segments = jsonAdapter.extractSegments(parsed, source, { + sourcePath: "record.json", + translatableKeys: { "record.json": ["title"] }, + }); + const translator: Translator = createWorkersAIBindingTranslator({ + modelId: "@cf/meta/llama-3.1-8b-instruct", + maxTokens: 8192, + run: (modelId, input) => ai.run(modelId, input), + }); + const { translations } = await translateSegments({ + translator, + segments, + glossary: EMPTY_GLOSSARY, + sourceLocale: "en-US", + targetLocale: "pt-BR", + }); + return jsonAdapter.applyTranslations(parsed, source, translations); +} diff --git a/docs/examples/direct-packages/tsconfig.json b/docs/examples/direct-packages/tsconfig.json new file mode 100644 index 0000000..02e2300 --- /dev/null +++ b/docs/examples/direct-packages/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "paths": { + "@cloudflare/polystella-core": ["../../../packages/core/src/index.ts"], + "@cloudflare/polystella-adapters": ["../../../packages/adapters/src/index.ts"], + "@cloudflare/polystella-providers": ["../../../packages/providers/src/index.ts"], + "@cloudflare/polystella-providers/*": ["../../../packages/providers/src/*.ts"] + } + }, + "include": ["index.ts"] +} diff --git a/docs/package.json b/docs/package.json index bb19bb2..c2aa3b5 100644 --- a/docs/package.json +++ b/docs/package.json @@ -20,24 +20,24 @@ "@fontsource-variable/inter": "^5.2.8", "@fontsource-variable/jetbrains-mono": "^5.2.8", "@iconify-json/ph": "^1.2.0", - "astro": "^7.0.0", - "astro-icon": "^1.1.0", + "astro": "^7.2.9", + "astro-icon": "^1.2.0", "astro-og-canvas": "^0.13.0", "canvaskit-wasm": "^0.41.1", "clsx": "^2.1.1", - "nimbus-docs": "^0.1.22", + "nimbus-docs": "^0.5.0", + "sharp": "^0.35.3", "shiki": "^4.2.0", - "tailwind-merge": "^3.5.0", - "sharp": "^0.35.3" + "tailwind-merge": "^3.5.0" }, "devDependencies": { "@astrojs/check": "^0.9.9", "@tailwindcss/postcss": "^4.1.4", + "@types/node": "^26.1.0", "pagefind": "^1.5.2", "tailwindcss": "^4.1.4", - "@types/node": "^26.1.0", "tsx": "^4.19.0", "typescript": "^6.0.3", - "wrangler": "^4.0.0" + "wrangler": "^4.127.0" } } diff --git a/docs/scripts/check-examples.ts b/docs/scripts/check-examples.ts index 5e04ec6..3e7d565 100644 --- a/docs/scripts/check-examples.ts +++ b/docs/scripts/check-examples.ts @@ -1,98 +1,34 @@ #!/usr/bin/env tsx -/** - * Walk `docs/examples//` and run `astro check` against each - * example project. Catches the "we changed PolyStella's API in a - * way the documented examples no longer compile against" class of - * regression. - * - * For v0.x the examples directory is empty (the cookbook recipes - * exist but don't have companion projects yet). The script reports - * "no examples" and exits cleanly so CI doesn't fail prematurely. - * - * Exit codes: - * 0 no examples present, OR every example checks clean - * 1 one or more examples failed `astro check` - */ +/** Compile the local public-package example with the workspace compiler. */ -import { readdir, stat } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { spawnSync } from "node:child_process"; const DOCS_ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url))); -const EXAMPLES_DIR = path.join(DOCS_ROOT, "examples"); +const REPOSITORY_ROOT = path.join(DOCS_ROOT, ".."); +const EXAMPLE_CONFIG = path.join(DOCS_ROOT, "examples", "direct-packages", "tsconfig.json"); +const tsc = fileURLToPath(import.meta.resolve("typescript/bin/tsc")); -async function isDirectory(p: string): Promise { - try { - return (await stat(p)).isDirectory(); - } catch { - return false; - } -} - -async function listExamples(): Promise { - const entries = await readdir(EXAMPLES_DIR); - const out: string[] = []; - for (const entry of entries) { - if (entry.startsWith(".") || entry === "README.md") continue; - const full = path.join(EXAMPLES_DIR, entry); - if (!(await isDirectory(full))) continue; - // An example must have a package.json — anything else is - // probably not a runnable project. - if (!(await isDirectory(full))) continue; - try { - await stat(path.join(full, "package.json")); - out.push(entry); - } catch { - console.warn(`[check-examples] skipping ${entry} (no package.json)`); - } - } - return out; -} - -async function checkExample(slug: string): Promise<{ slug: string; ok: boolean; output: string }> { - const dir = path.join(EXAMPLES_DIR, slug); - const result = spawnSync("pnpm", ["exec", "astro", "check"], { - cwd: dir, +function main(): void { + const result = spawnSync(process.execPath, [tsc, "--noEmit", "-p", EXAMPLE_CONFIG], { + cwd: REPOSITORY_ROOT, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], }); - const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; - return { slug, ok: result.status === 0, output }; -} - -async function main(): Promise { - if (!(await isDirectory(EXAMPLES_DIR))) { - console.log("[check-examples] no examples directory; nothing to check."); - return; + if (result.error !== undefined) { + throw new Error("failed to start TypeScript for direct-packages compilation", { cause: result.error }); } - - const examples = await listExamples(); - if (examples.length === 0) { - console.log("[check-examples] examples directory is empty; nothing to check."); - return; - } - - console.log(`[check-examples] checking ${examples.length} example${examples.length === 1 ? "" : "s"}…`); - - let failures = 0; - for (const slug of examples) { - const result = await checkExample(slug); - if (result.ok) { - console.log(` ✓ ${slug}`); - } else { - failures++; - console.error(` ✗ ${slug}`); - console.error(result.output); - } - } - - if (failures > 0) { - process.exitCode = 1; + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + if (result.status !== 0) { + throw new Error(`direct-packages failed to compile:\n${output}`); } + console.log("[check-examples] direct-packages compiled with tsc."); } -main().catch((err) => { +try { + main(); +} catch (err) { console.error("[check-examples] failed:", err); process.exitCode = 1; -}); +} diff --git a/docs/scripts/check-exports.ts b/docs/scripts/check-exports.ts index fe050ab..6bf6ac8 100644 --- a/docs/scripts/check-exports.ts +++ b/docs/scripts/check-exports.ts @@ -1,23 +1,18 @@ #!/usr/bin/env tsx -/** - * Sanity-check that every public export path in `package.json` is - * mentioned at least once on the `reference/exports` docs page. - * - * Catches the "new export, forgot to document it" regression that - * would otherwise surface only when someone tries to import from - * the path and finds nothing in the docs. - * - * Exit codes: - * 0 every exports entry is mentioned in reference/exports.md - * 1 one or more entries are missing — print the list and exit - */ +/** Check that the exports table exactly matches all public manifests. */ import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; const DOCS_ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url))); -const PACKAGE_JSON = path.join(DOCS_ROOT, "..", "package.json"); +const PACKAGE_MANIFESTS = [ + { owner: "Core", directory: "core" }, + { owner: "Adapters", directory: "adapters" }, + { owner: "Providers", directory: "providers" }, + { owner: "Astro", directory: "astro" }, + { owner: "Astro", directory: "polystella" }, +] as const; const EXPORTS_PAGE = path.join(DOCS_ROOT, "src", "content", "docs", "reference", "exports.md"); interface PackageJson { @@ -25,45 +20,75 @@ interface PackageJson { exports?: Record; } -async function main(): Promise { - const pkg = JSON.parse(await readFile(PACKAGE_JSON, "utf8")) as PackageJson; - const exportsField = pkg.exports; - if (!exportsField || typeof exportsField !== "object") { - console.error("[check-exports] package.json has no `exports` map."); - process.exitCode = 1; - return; - } +interface DocumentedExport { + owner: string; + importPath: string; + example: string; +} - // Build the list of import paths a consumer can write. The map's - // keys are subpaths starting with `.` (e.g. `.`, `./content`). - // Translate to the consumer-facing form: `@cloudflare/polystella`, - // `@cloudflare/polystella/content`, etc. - const importPaths = Object.keys(exportsField).map((key) => { - if (key === ".") return pkg.name; - return `${pkg.name}${key.slice(1)}`; +function parseExportsTable(content: string): DocumentedExport[] { + const lines = content.split("\n"); + const headerIndex = lines.findIndex((line) => { + const cells = line + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()); + return cells.join("|") === "Owner|Path|Purpose|Example"; }); + if (headerIndex < 0) throw new Error("exports page has no Owner/Path/Purpose/Example table"); - const exportsPageContent = await readFile(EXPORTS_PAGE, "utf8"); - - const missing: string[] = []; - for (const importPath of importPaths) { - if (!exportsPageContent.includes(`\`${importPath}\``)) { - missing.push(importPath); - } + const rows: DocumentedExport[] = []; + for (const line of lines.slice(headerIndex + 2)) { + if (!line.startsWith("|")) break; + const cells = line + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()); + if (cells.length !== 4) throw new Error(`invalid exports table row: ${line}`); + const [owner, pathCell, , exampleCell] = cells; + const importPath = pathCell?.match(/^`([^`]+)`$/)?.[1]; + const example = exampleCell?.match(/^`([^`]+)`$/)?.[1]; + if (!owner || !importPath || !example) throw new Error(`invalid exports table row: ${line}`); + rows.push({ owner, importPath, example }); } + return rows; +} - if (missing.length === 0) { - console.log(`[check-exports] ${importPaths.length} export paths, all documented.`); - return; +async function main(): Promise { + const actual = new Map(); + for (const packageInfo of PACKAGE_MANIFESTS) { + const manifestPath = path.join(DOCS_ROOT, "..", "packages", packageInfo.directory, "package.json"); + const pkg = JSON.parse(await readFile(manifestPath, "utf8")) as PackageJson; + if (!pkg.exports || typeof pkg.exports !== "object") throw new Error(`${pkg.name} has no \`exports\` map`); + for (const key of Object.keys(pkg.exports)) { + const importPath = key === "." ? pkg.name : `${pkg.name}${key.slice(1)}`; + actual.set(importPath, packageInfo.owner); + } } - console.error("[check-exports] the following export paths are NOT mentioned in reference/exports.md:"); - for (const m of missing) console.error(` - ${m}`); - console.error( - `\nAdd a row for each missing path to ${path.relative(process.cwd(), EXPORTS_PAGE)} ` + - "(typically as a row in the path/purpose table, with the path wrapped in backticks).", + const documented = parseExportsTable(await readFile(EXPORTS_PAGE, "utf8")); + const documentedPaths = documented.map(({ importPath }) => importPath); + const duplicates = documentedPaths.filter((importPath, index) => documentedPaths.indexOf(importPath) !== index); + const missing = [...actual.keys()].filter((importPath) => !documentedPaths.includes(importPath)); + const extra = documentedPaths.filter((importPath) => !actual.has(importPath)); + const badOwners = documented.filter(({ owner, importPath }) => actual.get(importPath) !== owner); + const badExamples = documented.filter( + ({ importPath, example }) => + !example.includes(`from "${importPath}"`) && + !example.includes(`import "${importPath}"`) && + !example.includes(`types="${importPath}"`), ); - process.exitCode = 1; + + const errors = [ + ...duplicates.map((value) => `duplicate: ${value}`), + ...missing.map((value) => `missing: ${value}`), + ...extra.map((value) => `extra: ${value}`), + ...badOwners.map(({ importPath, owner }) => `wrong owner: ${importPath} (${owner})`), + ...badExamples.map(({ importPath }) => `missing import/reference example: ${importPath}`), + ]; + if (errors.length > 0) throw new Error(`exports table does not match manifests:\n${errors.map((error) => ` - ${error}`).join("\n")}`); + + console.log(`[check-exports] ${documented.length} export paths exactly documented with examples.`); } main().catch((err) => { diff --git a/docs/scripts/generate-config-ref.ts b/docs/scripts/generate-config-ref.ts index 2f92a19..5eac4e9 100644 --- a/docs/scripts/generate-config-ref.ts +++ b/docs/scripts/generate-config-ref.ts @@ -1,7 +1,7 @@ #!/usr/bin/env tsx /** * Auto-generate the configuration reference markdown page from the - * canonical zod schema in `src/config/options.ts`. Runs in + * canonical zod schema in `packages/astro/src/config/options.ts`. Runs in * `pnpm --filter polystella-docs prebuild` so any `astro dev` / * `astro build` cycle picks up schema drift automatically. * @@ -20,7 +20,7 @@ import { fileURLToPath } from "node:url"; import { z } from "astro/zod"; -import { polystellaOptionsSchema } from "../../src/config/options.js"; +import { polystellaOptionsSchema } from "../../packages/astro/src/config/options.js"; const DOCS_ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url))); const OUTPUT_PATH = path.join(DOCS_ROOT, "src", "content", "docs", "configuration", "reference.md"); @@ -249,7 +249,7 @@ async function main(): Promise { "---", "", ":::note[Auto-generated]", - "This page is regenerated from `src/config/options.ts` on every", + "This page is regenerated from `packages/astro/src/config/options.ts` on every", "`pnpm --filter polystella-docs build`. Don't hand-edit; the", "generator overwrites it.", ":::", diff --git a/docs/src/components/Header.astro b/docs/src/components/Header.astro index 9ce2db6..503c92e 100644 --- a/docs/src/components/Header.astro +++ b/docs/src/components/Header.astro @@ -50,6 +50,7 @@ const showSections = sections.length >= 2; + {config.versions && v{config.versions.current}} {showSections && (