From 1359c037170051920adb9062d6267be730f53088 Mon Sep 17 00:00:00 2001 From: Cliff Kerr Date: Fri, 17 Apr 2026 18:04:48 -0400 Subject: [PATCH 1/3] feat: auto-populate contents if not provided --- quartodoc/_pydantic_compat.py | 3 +- quartodoc/builder/blueprint.py | 74 ++++++++++++++++++++++++++++++++++ quartodoc/layout.py | 8 +++- 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/quartodoc/_pydantic_compat.py b/quartodoc/_pydantic_compat.py index 17907ec3..c3cf259c 100644 --- a/quartodoc/_pydantic_compat.py +++ b/quartodoc/_pydantic_compat.py @@ -5,6 +5,7 @@ Extra, PrivateAttr, ValidationError, + validator, ) # noqa except ImportError: - from pydantic import BaseModel, Field, Extra, PrivateAttr, ValidationError # noqa + from pydantic import BaseModel, Field, Extra, PrivateAttr, ValidationError, validator # noqa diff --git a/quartodoc/builder/blueprint.py b/quartodoc/builder/blueprint.py index e796a894..06e10bf1 100644 --- a/quartodoc/builder/blueprint.py +++ b/quartodoc/builder/blueprint.py @@ -1,9 +1,14 @@ from __future__ import annotations +import importlib.util import logging import json import yaml +from collections import OrderedDict +from pathlib import Path +from typing import Iterable, List, Optional, Set + from .._griffe_compat import dataclasses as dc from .._griffe_compat import ( GriffeLoader, @@ -44,6 +49,47 @@ from quartodoc._pydantic_compat import BaseModel +def _identify_files_to_document( + path: Path, + file_patterns: List[str], + ignore: Optional[Iterable[str]] = None, +) -> Set[Path]: + reversed_patterns = file_patterns.copy() + reversed_patterns.reverse() + + files_to_document: dict = OrderedDict() + for pattern in reversed_patterns: + for file in path.rglob(pattern=pattern): + files_to_document[file.with_suffix("")] = file + result = set(files_to_document.values()) + + if ignore: + for pattern in ignore: + result = result.difference(set(path.glob(pattern=pattern))) + + return {p.resolve() for p in result} + + +def _auto_contents_from_package(package_name: str) -> list[Auto]: + """Return Auto entries for every .py file in *package_name*, excluding __init__ files.""" + spec = importlib.util.find_spec(package_name) + if spec is None or not spec.submodule_search_locations: + return [] + + pkg_path = Path(list(spec.submodule_search_locations)[0]) + files = _identify_files_to_document( + pkg_path, + file_patterns=["*.py"], + ignore=["**/__init__.py", "**/__pycache__/**"], + ) + + contents = [] + for file in sorted(files): + parts = list(file.relative_to(pkg_path).with_suffix("").parts) + contents.append(Auto(name=".".join(parts))) + return contents + + def _auto_package(mod: dc.Module) -> list[Section]: """Create default sections for the given package.""" @@ -246,6 +292,34 @@ def enter(self, el: Layout): return super().enter(el) + @dispatch + def enter(self, el: Section): + if el.contents: + return el + + package = self.crnt_package + label = el.title or el.subtitle or "(untitled)" + + if not package: + _log.warning( + f"Section '{label}' has no contents and no package is configured." + " Cannot auto-populate contents." + ) + return el + + _log.warning( + f"Section '{label}' has no contents. Auto-populating from package '{package}'." + ) + + contents = _auto_contents_from_package(package) + if not contents: + _log.warning(f"No Python files found in package '{package}'.") + return el + + new = el.copy() + new.contents = contents + return super().enter(new) + @dispatch def exit(self, el: Section): """Transform top-level sections, so their contents are all Pages.""" diff --git a/quartodoc/layout.py b/quartodoc/layout.py index dcd8db62..a31d1d0c 100644 --- a/quartodoc/layout.py +++ b/quartodoc/layout.py @@ -7,7 +7,7 @@ from typing_extensions import Annotated from typing import Literal, Union, Optional -from ._pydantic_compat import BaseModel, Field, Extra, PrivateAttr +from ._pydantic_compat import BaseModel, Field, Extra, PrivateAttr, validator _log = logging.getLogger(__name__) @@ -124,6 +124,12 @@ class Page(_Structural): contents: ContentList + @validator("contents") + def _contents_not_empty(cls, v): + if not v: + raise ValueError("Page contents must not be empty.") + return v + @property def obj(self): # TODO: this is for the case where pages are put as members inside From ff34434d2bf2c05e03cf6d1db00275cbd2579054 Mon Sep 17 00:00:00 2001 From: Cliff Kerr Date: Fri, 17 Apr 2026 18:10:06 -0400 Subject: [PATCH 2/3] feat: autopopulate contents documentation --- docs/get-started/overview.qmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/get-started/overview.qmd b/docs/get-started/overview.qmd index 656b0bf4..817160dc 100644 --- a/docs/get-started/overview.qmd +++ b/docs/get-started/overview.qmd @@ -161,6 +161,8 @@ quartodoc: The functions listed in `contents` are assumed to be imported from the package. +If no contents are provided, quartodoc will attempt to pull in all modules (i.e. `.py` files) from the package. + ## Learning more From 76aa6fd33ff8734a87cfe6381b15f9723ece015e Mon Sep 17 00:00:00 2001 From: Cliff Kerr Date: Tue, 9 Jun 2026 10:22:34 -0400 Subject: [PATCH 3/3] feat: add qpyd CLI --- docs/_quarto.yml | 1 + docs/get-started/basic-building.qmd | 5 + docs/get-started/qpyd-cli.qmd | 218 +++++++++++++++++ pyproject.toml | 9 +- quartodoc/qpyd/README.md | 78 ++++++ quartodoc/qpyd/__init__.py | 16 ++ quartodoc/qpyd/__main__.py | 6 + quartodoc/qpyd/clean.py | 92 +++++++ quartodoc/qpyd/cli.py | 87 +++++++ quartodoc/qpyd/config.py | 208 ++++++++++++++++ quartodoc/qpyd/convert.py | 270 +++++++++++++++++++++ quartodoc/qpyd/execute.py | 269 ++++++++++++++++++++ quartodoc/qpyd/nb.py | 88 +++++++ quartodoc/qpyd/prerender.py | 141 +++++++++++ quartodoc/qpyd/quarto_utils.py | 324 +++++++++++++++++++++++++ quartodoc/qpyd/render.py | 85 +++++++ quartodoc/qpyd/scaffold.py | 133 ++++++++++ quartodoc/qpyd/tests/__init__.py | 0 quartodoc/qpyd/tests/conftest.py | 15 ++ quartodoc/qpyd/tests/data/nb_fail.qmd | 9 + quartodoc/qpyd/tests/data/nb_ok.qmd | 11 + quartodoc/qpyd/tests/data/prose.qmd | 5 + quartodoc/qpyd/tests/test_cli.py | 65 +++++ quartodoc/qpyd/tests/test_config.py | 39 +++ quartodoc/qpyd/tests/test_convert.py | 69 ++++++ quartodoc/qpyd/tests/test_execute.py | 47 ++++ quartodoc/qpyd/tests/test_safety.py | 163 +++++++++++++ quartodoc/qpyd/tests/test_variables.py | 34 +++ quartodoc/qpyd/variables.py | 90 +++++++ 29 files changed, 2575 insertions(+), 2 deletions(-) create mode 100644 docs/get-started/qpyd-cli.qmd create mode 100644 quartodoc/qpyd/README.md create mode 100644 quartodoc/qpyd/__init__.py create mode 100644 quartodoc/qpyd/__main__.py create mode 100644 quartodoc/qpyd/clean.py create mode 100644 quartodoc/qpyd/cli.py create mode 100644 quartodoc/qpyd/config.py create mode 100644 quartodoc/qpyd/convert.py create mode 100644 quartodoc/qpyd/execute.py create mode 100644 quartodoc/qpyd/nb.py create mode 100644 quartodoc/qpyd/prerender.py create mode 100644 quartodoc/qpyd/quarto_utils.py create mode 100644 quartodoc/qpyd/render.py create mode 100644 quartodoc/qpyd/scaffold.py create mode 100644 quartodoc/qpyd/tests/__init__.py create mode 100644 quartodoc/qpyd/tests/conftest.py create mode 100644 quartodoc/qpyd/tests/data/nb_fail.qmd create mode 100644 quartodoc/qpyd/tests/data/nb_ok.qmd create mode 100644 quartodoc/qpyd/tests/data/prose.qmd create mode 100644 quartodoc/qpyd/tests/test_cli.py create mode 100644 quartodoc/qpyd/tests/test_config.py create mode 100644 quartodoc/qpyd/tests/test_convert.py create mode 100644 quartodoc/qpyd/tests/test_execute.py create mode 100644 quartodoc/qpyd/tests/test_safety.py create mode 100644 quartodoc/qpyd/tests/test_variables.py create mode 100644 quartodoc/qpyd/variables.py diff --git a/docs/_quarto.yml b/docs/_quarto.yml index 16b7a9a5..bfe9d906 100644 --- a/docs/_quarto.yml +++ b/docs/_quarto.yml @@ -58,6 +58,7 @@ website: - get-started/basic-docs.qmd - get-started/basic-content.qmd - get-started/basic-building.qmd + - get-started/qpyd-cli.qmd - get-started/crossrefs.qmd - get-started/sidebar.qmd - get-started/extending.qmd diff --git a/docs/get-started/basic-building.qmd b/docs/get-started/basic-building.qmd index 6c25a414..386a4628 100644 --- a/docs/get-started/basic-building.qmd +++ b/docs/get-started/basic-building.qmd @@ -9,6 +9,11 @@ jupyter: **tl;dr**: Once you've configured quartodoc in your `_quarto.yml` file, use the following commands to build and preview a documentation site. +:::{.callout-tip} +For a higher-level workflow that wraps these commands and also manages +notebooks, previewing, and publishing, see [the qpyd CLI](qpyd-cli.qmd). +::: + ## `quartodoc build`: Create doc files Automatically generate `.qmd` files with reference api documentation. This is written by default to the reference/ folder in your quarto project. diff --git a/docs/get-started/qpyd-cli.qmd b/docs/get-started/qpyd-cli.qmd new file mode 100644 index 00000000..cfe62ebf --- /dev/null +++ b/docs/get-started/qpyd-cli.qmd @@ -0,0 +1,218 @@ +--- +title: The qpyd CLI +--- + +**tl;dr**: `qpyd` is a higher-level command-line workflow built on top of +quartodoc. Where [`quartodoc build`](basic-building.qmd) generates your API +reference pages, `qpyd` wraps the *whole* docs lifecycle — pre-render builds, +rendering, previewing, publishing, scaffolding — and adds parallel notebook +management. + +It installs two console scripts: + +| Command | Purpose | +|---------|---------| +| `qpyd` | Build, render, preview, publish, and scaffold a docs site. | +| `qpynb` | Run, check, convert, and clean notebooks (`.qmd` / `.ipynb`). Also available as `qpyd nb …`. | + +`qpyd` builds on [`sciris`](https://sciris.org), [`jupytext`](https://jupytext.readthedocs.io), +and [`nbformat`](https://nbformat.readthedocs.io), and shells out to the +external [`quarto`](https://quarto.org) binary, so make sure Quarto is installed +and on your `PATH`. + +## Quick start + +Scaffold a docs folder, then render it: + +```bash +# Create docs/ with a starter _quarto.yml, index.qmd, and _variables.py +qpyd init docs --package your_package + +cd docs + +# Run pre-render steps, then `quarto render` +qpyd render + +# Or preview with live reload +qpyd preview +``` + +## `qpyd`: site commands + +### `qpyd prerender` + +Runs the pre-render build steps, in order: + +1. `quartodoc build` — generate the API reference pages. +2. Customize aliases — add short cross-reference aliases (e.g. `pkg.Thing` for + `pkg.submodule.Thing`) to `objects.json`. +3. `quartodoc interlinks` — build interlink inventories. +4. Build a Sphinx-compatible `objects.inv` so other projects can resolve your + references via intersphinx. + +```bash +qpyd prerender +``` + +The documented package name is read from the `quartodoc.package` key in your +`_quarto.yml`. This is the command you typically wire into your project's +`pre-render` hook (see [below](#configuring-_quartoyml)). + +### `qpyd render` + +Runs `qpyd prerender`, then `quarto render`, reporting the total build time. +Any extra arguments are passed straight through to Quarto: + +```bash +qpyd render # full build +qpyd render --to html # extra args forwarded to `quarto render` +qpyd render --no-prerender # skip the pre-render steps +``` + +### `qpyd preview` + +Like `qpyd render`, but launches `quarto preview` (a live-reloading server) +after the pre-render steps: + +```bash +qpyd preview +qpyd preview --no-prerender +``` + +### `qpyd gh-publish` + +Renders with `--cache-refresh` and publishes to the `gh-pages` branch via +`quarto publish`. + +```bash +qpyd gh-publish +``` + +:::{.callout-warning} +This pushes to a remote branch and updates your live site. It is never run as +part of any other command. +::: + +### `qpyd init` + +Scaffolds a docs folder with a starter `_quarto.yml`, `index.qmd`, and +`_variables.py`. **Existing files are never overwritten**, so it is safe to run +in an established project. + +```bash +qpyd init docs --package your_package +``` + +### `qpyd clean` + +Deletes auto-generated scratch files after a build. This is **opt-in**: it only +removes files matching the `qpyd.clean` glob patterns in your `_quarto.yml`, and +it never deletes source files (`.qmd`, `.ipynb`, `.py`, `.md`). + +```bash +qpyd clean # delete configured scratch files +qpyd clean --dry-run # show what would be deleted +``` + +## `qpynb`: notebook commands + +A "notebook" is an `.ipynb` file, or a `.qmd` file containing a `{python}` code +cell. Commands that take `paths` accept individual notebooks or folders; omit +them to operate on the whole project. + +### `qpynb run` and `qpynb check` + +Both execute notebooks **in parallel**. The difference is what they do with the +cache: + +```bash +qpynb run # execute via `quarto render`, updating the _freeze/ cache +qpynb check # execute to verify they run; touch no caches, leave no files +qpynb run tutorials # only the notebooks under tutorials/ +qpynb check --serial # one at a time (useful for debugging) +``` + +* **`run`** renders each notebook with `quarto render`, which executes it *and* + refreshes Quarto's freeze cache (`_freeze/`). Use it to pre-bake the cache in + parallel so a subsequent full-site render is fast. +* **`check`** is a pure validation pass — it executes each notebook to confirm + it runs without error, but writes no cache and leaves nothing behind. This is + ideal for CI. + +During `run`, each per-notebook render sets `QPYD_SKIP_HOOKS=1`, so a project +`pre-render: qpyd prerender` hook becomes a no-op rather than rebuilding the +whole API reference (and racing on `objects.json`) once per notebook. + +### `qpynb refresh` + +Deletes the cached copies of notebooks — Quarto's freeze cache (`_freeze/`) and +every nested jupyter cache (`.jupyter_cache/`) — so they re-execute on the next +render. + +```bash +qpynb refresh +qpynb refresh --dry-run +``` + +### `qpynb to-py` / `to-qmd` / `to-ipynb` + +Convert a notebook between formats. `.qmd` ⇄ `.ipynb` conversions go through +`quarto convert`; anything involving `.py` uses `jupytext`. The destination is +not overwritten unless you pass `--force`. + +```bash +qpynb to-py tutorials/intro.qmd # -> tutorials/intro.py +qpynb to-ipynb tutorials/intro.qmd # -> tutorials/intro.ipynb +qpynb to-qmd notebook.ipynb --force # overwrite an existing notebook.qmd +``` + +### `qpynb clear` + +Strips saved outputs and execution counts from `.ipynb` notebooks and +normalizes them. Files that are already clean are left untouched. + +```bash +qpynb clear +qpynb clear --dry-run +``` + +## `_variables.py` + +You can place an optional `_variables.py` file alongside `_quarto.yml`. Its +public, non-callable, YAML-serializable values are passed to `quarto render` as +`-M key:value` metadata by `qpyd render` and `qpyd preview`: + +```python +# _variables.py +import your_package +version = your_package.__version__ +versiondate = your_package.__versiondate__ +``` + +Values are YAML-encoded, so version strings survive intact (e.g. `"1.10"` is +passed as the string `'1.10'`, not coerced to the number `1.1`). Reference them +in documents with the [`meta` shortcode](https://quarto.org/docs/authoring/variables.html#meta): + +```markdown +Docs for version {{{< meta version >}}} ({{{< meta versiondate >}}}). +``` + +## Configuring `_quarto.yml` + +The keys `qpyd` reads or writes: + +```yaml +project: + pre-render: qpyd prerender # build API docs etc. before each render + # post-render: qpyd clean # optional; opt-in scratch cleanup + +quartodoc: + package: your_package # used by prerender for aliases / objects.inv + +qpyd: + clean: # optional glob patterns for `qpyd clean` + - '**/my-*.png' # source files are never deleted, even if matched + +execute: + freeze: auto # let `qpynb run` pre-bake the freeze cache +``` diff --git a/pyproject.toml b/pyproject.toml index 68c1c1f8..8b9e590a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] [tool.setuptools.packages.find] -include = ["quartodoc"] +include = ["quartodoc", "quartodoc.*"] [tool.pytest.ini_options] markers = [] @@ -43,7 +43,10 @@ dependencies = [ "requests", "typing-extensions >= 4.4.0", "watchdog >= 3.0.0", - "plum-dispatch > 2.0.0" + "plum-dispatch > 2.0.0", + "sciris >= 3.2.0", + "jupytext >= 1.16.0", + "nbformat >= 5.0.0" ] [project.urls] @@ -54,6 +57,8 @@ ci = "https://github.com/machow/quartodoc/actions" [project.scripts] quartodoc = "quartodoc.__main__:cli" +qpyd = "quartodoc.qpyd:cli" +qpynb = "quartodoc.qpyd:nb_cli" [dependency-groups] dev = [ diff --git a/quartodoc/qpyd/README.md b/quartodoc/qpyd/README.md new file mode 100644 index 00000000..e39ff0de --- /dev/null +++ b/quartodoc/qpyd/README.md @@ -0,0 +1,78 @@ +# qpyd + +A CLI for building [Quarto](https://quarto.org)-based Python API documentation +with quartodoc. It bundles the docs workflow (render / preview / publish / +scaffold) and parallel notebook management into two console scripts: + +- **`qpyd`** — build, render, preview, publish, and scaffold a docs site +- **`qpynb`** — manage notebooks (`.qmd` / `.ipynb`); also available as `qpyd nb …` + +It generalizes the starsim `docs/quarto_utils.py` workflow (kept verbatim as +[`quarto_utils.py`](quarto_utils.py) for reference). Built on `sciris`, +`jupytext`, `nbformat`, and the external `quarto` binary. + +## Commands + +### `qpyd` + +| Command | Description | +|---|---| +| `qpyd prerender` | Build the pre-render artifacts: API docs (`quartodoc build`), alias customization, interlinks, and a Sphinx `objects.inv`. | +| `qpyd render [quarto args]` | Run `prerender`, then `quarto render` (with timing). Extra args pass through to Quarto. | +| `qpyd preview [quarto args]` | Run `prerender`, then `quarto preview` (live reload). | +| `qpyd gh-publish` | `quarto render --cache-refresh` then `quarto publish gh-pages`. **Publishes to a remote branch.** | +| `qpyd init [path] [--package NAME]` | Scaffold a docs folder with a starter `_quarto.yml`, `index.qmd`, and `_variables.py` (never overwrites existing files). | +| `qpyd clean [--dry-run]` | Delete generated scratch files matching the `qpyd.clean` config patterns (opt-in; never deletes source files). | +| `qpyd nb …` | Alias for `qpynb …`. | + +`render` / `preview` / `gh-publish` accept `--no-prerender` to skip the build steps. + +### `qpynb` + +| Command | Description | +|---|---| +| `qpynb run [paths] [--serial]` | Execute notebooks **in parallel** via `quarto render`, updating the `_freeze/` cache. | +| `qpynb check [paths] [--serial]` | Execute notebooks **in parallel** to verify they run; touches no caches and leaves no artifacts. | +| `qpynb refresh [--dry-run]` | Delete cached copies (`_freeze/` and all nested `.jupyter_cache/`) so notebooks re-execute. | +| `qpynb to-py / to-qmd / to-ipynb PATH [--force]` | Convert between notebook formats. Won't overwrite an existing destination without `--force`. | +| `qpynb clear [paths] [--dry-run]` | Strip saved outputs from `.ipynb` notebooks and normalize them. | + +`paths` may be individual notebooks or folders; omit them to act on the whole +project. A notebook is a `.ipynb`, or a `.qmd` containing a `{python}` cell. + +## `run` vs `check` and the cache + +"Cached copies" are Quarto's freeze cache (`_freeze/`), the project-level, +commit-friendly mechanism that lets `quarto render` skip re-execution +(`freeze: auto`). `qpynb run` pre-bakes that cache in parallel so a later full +site render is fast; `qpynb check` is a pure validation pass that never writes +it. During `run`, each per-notebook render runs with `QPYD_SKIP_HOOKS=1`, so a +project `pre-render: qpyd prerender` / `post-render: qpyd clean` hook no-ops +instead of running redundantly (and racing) once per notebook. + +## `_variables.py` + +An optional file alongside `_quarto.yml`. Public, non-callable, +YAML-serializable names defined in it are passed to `quarto render` as +`-M key:value` metadata (YAML-encoded, so strings like `"1.10"` stay strings): + +```python +import starsim as ss +version = ss.__version__ +versiondate = ss.__versiondate__ +``` + +## Relevant `_quarto.yml` keys + +```yaml +project: + pre-render: qpyd prerender + # post-render: qpyd clean # optional, opt-in +quartodoc: + package: your_package # read by prerender for aliases / objects.inv +qpyd: + clean: # optional glob patterns for `qpyd clean` + - '**/my-*.png' # source files (.qmd/.ipynb/.py/.md) are never deleted +execute: + freeze: auto +``` diff --git a/quartodoc/qpyd/__init__.py b/quartodoc/qpyd/__init__.py new file mode 100644 index 00000000..bfeb8538 --- /dev/null +++ b/quartodoc/qpyd/__init__.py @@ -0,0 +1,16 @@ +""" +qpyd — a CLI for building Quarto-based Python API documentation with quartodoc. + +Two console scripts are exposed (see ``[project.scripts]`` in pyproject.toml): + +* ``qpyd`` -> :data:`cli` — build / render / preview / publish / init +* ``qpynb`` -> :data:`nb_cli` — notebook management (also available as ``qpyd nb``) + +Both are re-exported here so the entry points stay stable as internal modules +are added or moved. +""" + +from .cli import cli +from .nb import nb_cli + +__all__ = ["cli", "nb_cli"] diff --git a/quartodoc/qpyd/__main__.py b/quartodoc/qpyd/__main__.py new file mode 100644 index 00000000..9784f301 --- /dev/null +++ b/quartodoc/qpyd/__main__.py @@ -0,0 +1,6 @@ +"""Enable ``python -m quartodoc.qpyd``.""" + +from .cli import cli + +if __name__ == "__main__": + cli() diff --git a/quartodoc/qpyd/clean.py b/quartodoc/qpyd/clean.py new file mode 100644 index 00000000..4973b528 --- /dev/null +++ b/quartodoc/qpyd/clean.py @@ -0,0 +1,92 @@ +""" +Removal of auto-generated temporary files (an optional ``post-render`` step). + +This is intentionally **opt-in and conservative**. Unlike the original +starsim-specific ``clean_outputs()`` (which knew that ``tutorials/`` and +``user_guide/`` held only scratch output), a general-purpose tool cannot assume +which files are disposable. So: + +* Patterns are read from the ``qpyd.clean`` key of ``_quarto.yml`` and default + to **nothing** — running ``qpyd clean`` in an unconfigured project is a no-op. +* It refuses to run outside a real Quarto project (no cwd fallback). +* It never deletes notebook/source files (``.qmd``, ``.ipynb``, ``.py``, + ``.md``), even if a configured pattern matches them. +""" + +import os +from pathlib import Path + +import sciris as sc + +from .config import _excluded, load_quarto_config, project_root + +# File extensions that are (almost) always hand-authored source and must never +# be deleted by a glob-based cleanup, regardless of configured patterns. +PROTECTED_SUFFIXES = {".qmd", ".ipynb", ".py", ".md"} + + +def _clean_patterns(): + """Read ``qpyd.clean`` glob patterns from ``_quarto.yml`` (default: none).""" + data, _ = load_quarto_config() + qpyd_cfg = data.get("qpyd") or {} + patterns = qpyd_cfg.get("clean") or [] + if isinstance(patterns, str): + patterns = [patterns] + return list(patterns) + + +def clean_outputs(patterns=None, dry_run=False): + """ + Delete auto-generated temporary files within the docs directory. + + Args: + patterns: glob patterns (relative to the docs dir) to remove. If + omitted, patterns are read from the ``qpyd.clean`` config key, which + defaults to an empty list (so cleaning is a no-op until configured). + dry_run: if True, only print what *would* be removed. + + Returns the list of matched files. Refuses to run outside a Quarto project, + skips build/hidden directories, and never deletes protected source files + (see :data:`PROTECTED_SUFFIXES`). + """ + if os.environ.get("QPYD_SKIP_HOOKS"): + # We are running inside a per-notebook `quarto render` (see + # execute.render_notebook); skip the project post-render cleanup. + return [] + + root = project_root() + if root is None: + print("No _quarto.yml found; refusing to clean outside a Quarto project.") + return [] + + patterns = patterns if patterns is not None else _clean_patterns() + if not patterns: + print( + "No clean patterns configured. Set 'qpyd.clean' in _quarto.yml, e.g.:\n" + " qpyd:\n clean:\n - '**/my-*.png'" + ) + return [] + + matched = set() + for pattern in patterns: + for path in root.glob(pattern): + if not path.is_file(): + continue + if _excluded(path, root): + continue + if path.suffix.lower() in PROTECTED_SUFFIXES: + print(f"Skipping protected source file: {path}") + continue + matched.add(path.resolve()) + + files = sorted(matched) + if not files: + print("No temporary files to clean.") + return [] + + for path in files: + verb = "Would delete" if dry_run else "Deleting" + print(f"{verb}: {path}") + if not dry_run: + sc.rmpath(path, die=False) + return files diff --git a/quartodoc/qpyd/cli.py b/quartodoc/qpyd/cli.py new file mode 100644 index 00000000..0b5bce83 --- /dev/null +++ b/quartodoc/qpyd/cli.py @@ -0,0 +1,87 @@ +""" +The top-level ``qpyd`` command group. + +``qpyd`` orchestrates the docs build (prerender / render / preview / publish / +init / clean) and embeds the notebook commands under ``qpyd nb`` (an alias for +the standalone ``qpynb`` command). +""" + +import click + +from .nb import nb_cli +from .prerender import prerender +from .render import gh_publish, preview, render +from .scaffold import init + +_PASSTHROUGH = dict(ignore_unknown_options=True, allow_extra_args=True) + + +@click.group(name="qpyd", invoke_without_command=True) +@click.version_option(package_name="quartodoc") +@click.pass_context +def cli(ctx): + """ + qpyd — build, render, and publish Quarto-based Python API docs. + + Run without a subcommand to show this help. + """ + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + + +@cli.command("prerender") +def prerender_cmd(): + """Run pre-render build steps (API docs, aliases, interlinks, objects.inv).""" + prerender() + + +@cli.command("render", context_settings=_PASSTHROUGH) +@click.option("--no-prerender", is_flag=True, help="Skip the pre-render build steps.") +@click.argument("quarto_args", nargs=-1, type=click.UNPROCESSED) +def render_cmd(no_prerender, quarto_args): + """ + Run the pre-render steps, then 'quarto render', timing the build. + + Any extra arguments are passed through to 'quarto render'. + """ + render(extra_args=quarto_args, do_prerender=not no_prerender) + + +@cli.command("preview", context_settings=_PASSTHROUGH) +@click.option("--no-prerender", is_flag=True, help="Skip the pre-render build steps.") +@click.argument("quarto_args", nargs=-1, type=click.UNPROCESSED) +def preview_cmd(no_prerender, quarto_args): + """ + Run the pre-render steps, then 'quarto preview' (live-reloading server). + + Any extra arguments are passed through to 'quarto preview'. + """ + preview(extra_args=quarto_args, do_prerender=not no_prerender) + + +@cli.command("gh-publish") +@click.option("--no-prerender", is_flag=True, help="Skip the pre-render build steps.") +def gh_publish_cmd(no_prerender): + """Render and publish the site to GitHub Pages (the gh-pages branch).""" + gh_publish(do_prerender=not no_prerender) + + +@cli.command("init") +@click.argument("path", default="docs") +@click.option("--package", default=None, help="Package name to document.") +def init_cmd(path, package): + """Scaffold a docs/ folder with a starter _quarto.yml (existing files kept).""" + init(path=path, package=package) + + +@cli.command("clean") +@click.option("--dry-run", is_flag=True, help="Show what would be deleted, without deleting.") +def clean_cmd(dry_run): + """Remove auto-generated temporary files (my-*, example*) from the docs dir.""" + from .clean import clean_outputs + + clean_outputs(dry_run=dry_run) + + +# `qpyd nb ...` is an alias for the standalone `qpynb ...` command. +cli.add_command(nb_cli, name="nb") diff --git a/quartodoc/qpyd/config.py b/quartodoc/qpyd/config.py new file mode 100644 index 00000000..68c4cf5e --- /dev/null +++ b/quartodoc/qpyd/config.py @@ -0,0 +1,208 @@ +""" +Shared configuration, layout discovery, and helpers for the qpyd / qpynb CLIs. + +These helpers locate the Quarto project (the directory containing ``_quarto.yml``), +read the relevant settings out of it, and resolve which notebooks a command +should operate on. +""" + +import os +import shutil +from pathlib import Path + +import yaml + +# --- Display / behaviour constants --------------------------------------- + +TIMEOUT = 600 # Maximum time (s) allowed for a single notebook execution +YAY = "✓" # Success marker used in summaries +BOO = "😢" # Failure marker used in summaries + +# --- Layout constants ---------------------------------------------------- + +CONFIG_NAME = "_quarto.yml" +VARIABLES_PY = "_variables.py" +FREEZE_DIR = "_freeze" +JUPYTER_CACHE_DIR = ".jupyter_cache" + +# Notebook file types qpynb knows how to run / convert +NB_SUFFIXES = (".qmd", ".ipynb") + +# Directories that never contain runnable source notebooks. Anything whose +# path passes through one of these (or any hidden, dot-prefixed directory) is +# skipped during automatic discovery. +EXCLUDE_DIRS = { + "_site", + "_freeze", + "_extensions", + "_inv", + "api", # generated by `quartodoc build` + "assets", + "images", + "__pycache__", +} + + +def find_quarto_config(start=None): + """ + Walk up from ``start`` (default: cwd) looking for ``_quarto.yml``. + + Returns the :class:`~pathlib.Path` to the config file, or ``None`` if no + config is found in ``start`` or any of its parents. + """ + start = Path(start or os.getcwd()).resolve() + for directory in [start, *start.parents]: + candidate = directory / CONFIG_NAME + if candidate.exists(): + return candidate + return None + + +def load_quarto_config(start=None): + """ + Load and parse ``_quarto.yml``. + + Returns a ``(data, path)`` tuple, where ``data`` is the parsed dict (empty + if no config was found) and ``path`` is the config :class:`~pathlib.Path` + (or ``None``). + """ + cfg_path = find_quarto_config(start) + if cfg_path is None: + return {}, None + with open(cfg_path) as f: + data = yaml.safe_load(f) or {} + return data, cfg_path + + +def docs_dir(start=None): + """ + Return the Quarto project directory (where ``_quarto.yml`` lives). + + Falls back to ``start`` (or the cwd) if no config can be found, so callers + always get a usable directory. **Destructive** operations must use + :func:`project_root` instead, which never falls back to the cwd. + """ + cfg_path = find_quarto_config(start) + if cfg_path is not None: + return cfg_path.parent + return Path(start or os.getcwd()).resolve() + + +def project_root(start=None): + """ + Return the Quarto project directory only if a ``_quarto.yml`` exists. + + Unlike :func:`docs_dir`, this returns ``None`` when no config is found, + so destructive operations can refuse to run outside a real project rather + than acting on an arbitrary cwd. + """ + cfg_path = find_quarto_config(start) + return cfg_path.parent if cfg_path is not None else None + + +def require_tool(name, hint=None): + """ + Ensure an external command-line tool is available on ``PATH``. + + Raises a clear :class:`RuntimeError` (rather than letting a later + ``subprocess`` call surface a bare ``FileNotFoundError``) if it is missing. + """ + if shutil.which(name) is None: + msg = f"Required tool {name!r} was not found on your PATH." + if hint: + msg += f" {hint}" + raise RuntimeError(msg) + return name + + +def get_package_name(start=None, default=None): + """ + Return the documented package name from the ``quartodoc.package`` config + key, or ``default`` if it is not set. + """ + data, _ = load_quarto_config(start) + quartodoc_cfg = data.get("quartodoc") or {} + return quartodoc_cfg.get("package", default) + + +def is_python_qmd(path): + """Return True if a ``.qmd`` file contains an executable ``{python}`` cell.""" + try: + text = Path(path).read_text(encoding="utf-8", errors="ignore") + except OSError: + return False + return "```{python}" in text + + +def _excluded(path, base): + """ + True if any *directory* component of ``path`` *below* ``base`` is an + excluded build dir or hidden. + + Exclusion is judged relative to ``base`` (the directory actually being + scanned), so an out-of-project scan target whose absolute path happens to + pass through a hidden/excluded directory (e.g. ``~/.cache/proj``) is not + spuriously skipped. + """ + try: + parts = Path(path).resolve().relative_to(Path(base).resolve()).parts + except ValueError: + # Not under base; judge only on the filename's own dir components is + # impossible, so do not exclude (an explicitly targeted path wins). + return False + # parts[:-1] are the directory components (drop the filename itself) + return any( + part in EXCLUDE_DIRS or part.startswith(".") for part in parts[:-1] + ) + + +def discover_notebooks(paths=None, root=None): + """ + Resolve the list of notebooks a command should operate on. + + Args: + paths: optional iterable of file or directory paths. Files are taken + as-is (even plain ``.qmd`` without code cells); directories are + searched recursively. If omitted, the whole project is scanned. + root: project root for scanning and exclusion (default: the docs dir). + + Returns a sorted, de-duplicated list of resolved :class:`~pathlib.Path` + objects. During scanning, build/output directories and hidden directories + are skipped, and ``.qmd`` files without ``{python}`` cells are ignored + (they are prose pages, not notebooks). + """ + root = Path(root or docs_dir()).resolve() + results = [] + seen = set() + + def add(candidate, from_scan, base): + candidate = Path(candidate).resolve() + if candidate in seen: + return + if not candidate.is_file() or candidate.suffix not in NB_SUFFIXES: + return + if from_scan: + if _excluded(candidate, base): + return + if candidate.suffix == ".qmd" and not is_python_qmd(candidate): + return + seen.add(candidate) + results.append(candidate) + + if paths: + for raw in paths: + target = Path(raw) + if target.is_dir(): + # Exclusion is judged relative to the scanned dir, not root. + for suffix in NB_SUFFIXES: + for found in target.rglob(f"*{suffix}"): + add(found, from_scan=True, base=target) + else: + # Explicitly named file: include it even if it is a prose .qmd + add(target, from_scan=False, base=root) + else: + for suffix in NB_SUFFIXES: + for found in root.rglob(f"*{suffix}"): + add(found, from_scan=True, base=root) + + return sorted(results) diff --git a/quartodoc/qpyd/convert.py b/quartodoc/qpyd/convert.py new file mode 100644 index 00000000..1e01a2b6 --- /dev/null +++ b/quartodoc/qpyd/convert.py @@ -0,0 +1,270 @@ +""" +Notebook format conversions for qpynb: ``.qmd`` <-> ``.ipynb`` <-> ``.py``, +plus clearing/normalising executed ``.ipynb`` outputs. + +``.qmd`` <-> ``.ipynb`` conversions go through ``quarto convert`` (the +canonical tool); anything involving ``.py`` goes through ``jupytext``. The +``.qmd`` -> ``.py`` path uses :func:`qmd2py`, which extracts ``{python}`` +cells into a flat script (closely matching the original ``quarto_utils.py``). +""" + +import subprocess +import sys +from pathlib import Path + +import sciris as sc + +from .config import discover_notebooks + + +def qmd2py(qmd_path, py_path=None, keep_text=True): + """ + Convert a ``.qmd`` file to a ``.py`` file by extracting Python code cells. + + Each ``` ```{python} ... ``` ``` block becomes a cell, separated by + ``#%% Cell N`` headers and two blank lines between cells. Raises an + exception if blocks are ambiguous (e.g., nested or unclosed code fences). + + Lines starting with ``%`` or ``!`` (IPython magics / shell commands) are + commented out since they are not valid Python. + + Args: + qmd_path (str/Path): path to the ``.qmd`` file + py_path (str/Path): path to write the ``.py`` file (default: same name + with a ``.py`` extension) + keep_text (bool): if True, include non-code text as comments prefixed + with ``# `` + + Returns the :class:`~pathlib.Path` of the written ``.py`` file. + """ + qmd_path = sc.path(qmd_path) + if py_path is None: + py_path = qmd_path.with_suffix(".py") + else: + py_path = sc.path(py_path) + + text = sc.loadtext(qmd_path) + lines = text.splitlines() + + chunks = [] # List of (type, content) tuples; type is 'code' or 'text' + in_block = False + current_cell = [] + current_text = [] + block_start_line = None + + for i, line in enumerate(lines, start=1): + stripped = line.strip() + if stripped.startswith("```{python}"): + if in_block: + raise ValueError( + f"Nested or unclosed code block: new block at line {i}, " + f"previous block started at line {block_start_line}" + ) + if keep_text and current_text: + chunks.append(("text", current_text)) + current_text = [] + in_block = True + block_start_line = i + current_cell = [] + elif stripped == "```" and in_block: + chunks.append(("code", current_cell)) + in_block = False + current_cell = [] + block_start_line = None + elif in_block: + current_cell.append(line) + elif keep_text: + current_text.append(line) + + if in_block: + raise ValueError(f"Unclosed code block starting at line {block_start_line}") + + if keep_text and current_text: + chunks.append(("text", current_text)) + + # Build output + parts = [] + cell_num = 0 + for kind, content in chunks: + if kind == "code": + cell_num += 1 + processed = [] + for line in content: + if line.lstrip().startswith(("%", "!")): + processed.append(f"# {line} # IPython not supported in Python files") + else: + processed.append(line) + parts.append(f"#%% Cell {cell_num}\n" + "\n".join(processed)) + else: # text + commented = "\n".join( + f"# {line}" if line.strip() else "#" for line in content + ) + parts.append(commented) + + output = "\n\n\n".join(parts) + "\n" + sc.savetext(py_path, output) + return py_path + + +def _run(cmd, cwd=None): + """Run a subprocess, raising a clear error (with captured output) on failure.""" + try: + proc = subprocess.run(cmd, capture_output=True, text=True, cwd=cwd) + except FileNotFoundError as e: + tool = cmd[0] + hint = ( + "Install Quarto from https://quarto.org and ensure it is on your PATH." + if tool == "quarto" + else f"Install it (e.g. `pip install {tool}`)." + ) + raise RuntimeError(f"Required tool {tool!r} was not found on your PATH. {hint}") from e + if proc.returncode != 0: + raise RuntimeError( + f"Command failed ({' '.join(cmd)}):\n{proc.stdout}\n{proc.stderr}" + ) + return proc + + +def _guard_overwrite(out_path, src_path, force): + """Raise unless it is safe to write ``out_path`` (a different, absent, or forced dest).""" + out_path = sc.path(out_path) + if out_path.resolve() == sc.path(src_path).resolve(): + return # converting to the same file (no-op cases) is fine + if out_path.exists() and not force: + raise FileExistsError( + f"{out_path} already exists; refusing to overwrite. Pass --force to overwrite." + ) + + +def _quarto_convert(path, force=False): + """ + Convert ``.qmd`` <-> ``.ipynb`` using ``quarto convert``, returning the + path of the produced file. + + ``quarto convert`` toggles between the two formats and writes the output + alongside the input with the opposite extension, so the destination is + overwrite-guarded first. + """ + path = sc.path(path) + other = ".ipynb" if path.suffix == ".qmd" else ".qmd" + out_path = path.with_suffix(other) + _guard_overwrite(out_path, path, force) + # Run in the file's own directory: `quarto convert` will otherwise append + # quarto ignores to the .gitignore of whatever git repo the cwd sits in. + _run(["quarto", "convert", path.name], cwd=str(path.parent)) + return out_path + + +def to_py(path, force=False): + """ + Convert a notebook to a Python (``.py``) file. + + ``.qmd`` files use :func:`qmd2py`; ``.ipynb`` files use ``jupytext`` + (percent format). An existing destination is not overwritten unless + ``force`` is set. Returns the output :class:`~pathlib.Path`. + """ + path = sc.path(path) + if path.suffix == ".py": + return path + out_path = path.with_suffix(".py") + _guard_overwrite(out_path, path, force) + if path.suffix == ".qmd": + return qmd2py(path, out_path) + if path.suffix == ".ipynb": + _run(["jupytext", "--to", "py:percent", "--output", str(out_path), str(path)]) + return out_path + raise ValueError(f"Don't know how to convert {path} to .py") + + +def to_qmd(path, force=False): + """ + Convert a notebook to a Quarto (``.qmd``) file. + + ``.ipynb`` uses ``quarto convert``; ``.py`` uses ``jupytext``. An existing + destination is not overwritten unless ``force`` is set. Returns the output + :class:`~pathlib.Path`. + """ + path = sc.path(path) + if path.suffix == ".qmd": + return path + if path.suffix == ".ipynb": + return _quarto_convert(path, force=force) + if path.suffix == ".py": + out_path = path.with_suffix(".qmd") + _guard_overwrite(out_path, path, force) + _run(["jupytext", "--to", "qmd", "--output", str(out_path), str(path)]) + return out_path + raise ValueError(f"Don't know how to convert {path} to .qmd") + + +def to_ipynb(path, force=False): + """ + Convert a notebook to a Jupyter (``.ipynb``) file. + + ``.qmd`` uses ``quarto convert``; ``.py`` uses ``jupytext``. An existing + destination is not overwritten unless ``force`` is set. Returns the output + :class:`~pathlib.Path`. + """ + path = sc.path(path) + if path.suffix == ".ipynb": + return path + if path.suffix == ".qmd": + return _quarto_convert(path, force=force) + if path.suffix == ".py": + out_path = path.with_suffix(".ipynb") + _guard_overwrite(out_path, path, force) + _run(["jupytext", "--to", "ipynb", "--output", str(out_path), str(path)]) + return out_path + raise ValueError(f"Don't know how to convert {path} to .ipynb") + + +def clear_outputs(*paths, dry_run=False): + """ + Clear saved outputs from ``.ipynb`` notebooks and normalise them. + + Removes cell outputs and execution counts, then rewrites each notebook via + ``nbformat`` (which normalises structure). Operates on the ``.ipynb`` files + found in ``paths`` (or the whole project if none are given). Files that are + already clean are left untouched. With ``dry_run`` set, nothing is written. + + Returns the list of notebooks that were (or would be) modified. + """ + import nbformat + + notebooks = discover_notebooks(list(paths) or None) + ipynbs = [nb for nb in notebooks if nb.suffix == ".ipynb"] + if not ipynbs: + print("No .ipynb notebooks found to clear.") + return [] + + cleared = [] + for nb_path in ipynbs: + nb = nbformat.read(str(nb_path), as_version=4) + changed = False + for cell in nb.cells: + if cell.get("cell_type") != "code": + continue + if cell.get("outputs") or cell.get("execution_count") is not None: + changed = True + cell["outputs"] = [] + cell["execution_count"] = None + try: # normalise structure where supported; never fatal + n_norm, nb = nbformat.validator.normalize(nb) + changed = changed or bool(n_norm) + except Exception: + pass + if not changed: + print(f"Already clean: {nb_path.name}") + continue + cleared.append(nb_path) + verb = "Would clear" if dry_run else "Cleared" + print(f"{verb}: {nb_path.name}") + if not dry_run: + nbformat.write(nb, str(nb_path)) + return cleared + + +# Allow `python -m quartodoc.qpyd.convert ` for ad-hoc qmd->py conversion +if __name__ == "__main__": + for arg in sys.argv[1:]: + print(to_py(arg)) diff --git a/quartodoc/qpyd/execute.py b/quartodoc/qpyd/execute.py new file mode 100644 index 00000000..99adc21c --- /dev/null +++ b/quartodoc/qpyd/execute.py @@ -0,0 +1,269 @@ +""" +Notebook execution for qpynb. + +Two execution modes, both run **in parallel**: + +* ``check`` (:func:`execute_notebooks`) — extract each notebook to a temporary + ``.py`` script and run it, purely to verify it executes without error. It + touches no caches and leaves nothing behind. This mirrors the original + ``execute_notebooks()`` from ``quarto_utils.py``. +* ``run`` (:func:`run_notebooks`) — ``quarto render`` each notebook, which + executes it *and* updates the Quarto freeze cache (``_freeze/``) so a + subsequent full-site render can reuse the results. + +:func:`refresh_cache` deletes those cached copies so notebooks re-execute. +""" + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import sciris as sc + +from .config import ( + BOO, + FREEZE_DIR, + JUPYTER_CACHE_DIR, + TIMEOUT, + YAY, + discover_notebooks, + docs_dir, + project_root, + require_tool, +) +from .convert import qmd2py + +# Non-interactive matplotlib backend, so notebooks that plot don't try to open +# a GUI window during headless execution. +_AGG_ENV = {**os.environ, "MPLBACKEND": "agg"} + +_QUARTO_HINT = "Install Quarto from https://quarto.org and ensure it is on your PATH." +_JUPYTEXT_HINT = "Install it with `pip install jupytext`." + + +def _make_runnable_py(path, out_dir): + """Produce a runnable ``.py`` for a ``.qmd`` or ``.ipynb`` in ``out_dir``.""" + path = sc.path(path) + py_path = Path(out_dir) / (path.stem + ".py") + if path.suffix == ".qmd": + qmd2py(path, py_path) + elif path.suffix == ".ipynb": + subprocess.run( + ["jupytext", "--to", "py:percent", "--output", str(py_path), str(path)], + check=True, + capture_output=True, + ) + else: + raise ValueError(f"Cannot execute {path}: not a .qmd or .ipynb notebook") + return py_path + + +def execute_notebook(path): + """ + Execute a single notebook by extracting it to a temp ``.py`` and running it. + + Returns a one-line result string (containing :data:`YAY` or :data:`BOO` + and a ``(time: N s)`` suffix) describing success or failure. + """ + path = sc.path(path).resolve() + with sc.timer(label=sc.ansi.green(f" Execution time for {path.name}")) as T: + try: + with tempfile.TemporaryDirectory() as tmp: + py_path = _make_runnable_py(path, tmp) + print(f"Executing {path.name}...") + subprocess.run( + [sys.executable, str(py_path)], + check=True, + capture_output=True, + cwd=str(path.parent), + env=_AGG_ENV, + timeout=TIMEOUT, + ) + string = f"{YAY} {path.stem} executed successfully " + except subprocess.CalledProcessError as e: + err = (e.stderr or b"").decode(errors="ignore") + string = f"{BOO} Execution failed for {path.stem}:\n{err[-1500:]}\n" + except Exception as e: + string = f"{BOO} Error processing {path.stem}: {e}\n" + string += f"(time: {T.total:0.1f} s)" + print(string) + return string + + +def render_notebook(path, cwd=None): + """ + Render a single notebook with ``quarto render``, updating the freeze cache. + + Returns a one-line result string, in the same format as + :func:`execute_notebook`. + """ + path = sc.path(path).resolve() + cwd = str(cwd or docs_dir()) + # Suppress the project's own pre-/post-render hooks (e.g. `qpyd prerender`, + # `qpyd clean`) for these per-notebook renders: running them once per + # notebook in parallel would race on shared files (objects.json) and be + # hugely redundant. The hooks honour QPYD_SKIP_HOOKS and no-op. + env = {**_AGG_ENV, "QPYD_SKIP_HOOKS": "1"} + with sc.timer(label=sc.ansi.green(f" Render time for {path.name}")) as T: + try: + print(f"Rendering {path.name}...") + subprocess.run( + ["quarto", "render", str(path)], + check=True, + capture_output=True, + cwd=cwd, + env=env, + timeout=TIMEOUT, + ) + string = f"{YAY} {path.stem} rendered successfully " + except subprocess.CalledProcessError as e: + err = (e.stderr or b"").decode(errors="ignore") + string = f"{BOO} Render failed for {path.stem}:\n{err[-1500:]}\n" + except Exception as e: + string = f"{BOO} Error rendering {path.stem}: {e}\n" + string += f"(time: {T.total:0.1f} s)" + print(string) + return string + + +def _parallel(func, notebooks, serial=False): + """ + Run ``func(i, path)`` over ``notebooks`` in parallel. + + A small staggered delay between launches avoids thundering-herd startup + contention (the original code found the ``interval`` arg unreliable, so the + delay is applied inside the worker). + """ + + def worker(i, path, pause=1.0): + if not serial: # staggering only matters for parallel startup + sc.timedsleep(i * pause) + return func(path) + + notebook_list = list(enumerate(notebooks)) + return sc.parallelize( + worker, + notebook_list, + maxcpu=0.9, + interval=1.0, + lbkwargs=dict(verbose=False), + serial=serial, + ) + + +def _succeeded(res): + """ + Classify a result string by its *leading* marker. + + Result strings always start with YAY or BOO, but failure strings also embed + captured stderr, which can itself contain a YAY checkmark (✓). So we must + anchor on the start of the string, not test substring membership. + """ + return res.lstrip().startswith(YAY) + + +def _summarize(notebooks, results): + """Print per-notebook results and a sorted pass/fail summary.""" + table = sc.objdict() + for nb, res in zip(notebooks, results): + table[str(nb)] = res + + sc.heading("Results") + print(sc.strjoin(results, sep=f'\n\n\n{"—" * 90}\n')) + + sc.heading("Summary") + n_yay = sum(_succeeded(res) for res in results) + n_boo = len(results) - n_yay + summary = f"{n_yay} succeeded, {n_boo} failed\n" + + table.sort("values") + for nb, res in table.items(): + if "time: " in res: + timestr = res.rsplit("time: ", 1)[-1].split(")")[0].strip() + else: + timestr = "?" + suffix = f"{sc.path(nb).name:30s} ({timestr})" + if _succeeded(res): + summary += f'\n{sc.ansi.green("Succeeded")}: {suffix}' + else: + summary += f'\n{sc.ansi.red(" Failed")}: {suffix}' + print(summary) + return table + + +@sc.timer("Check notebooks") +def execute_notebooks(*paths, serial=False): + """ + Execute notebooks in parallel to verify they run (no caches updated). + + Args: + paths: notebook files or folders (default: the whole project). + serial: run one at a time (useful for debugging). + + Returns an :class:`sciris.objdict` mapping notebook path -> result string. + """ + notebooks = discover_notebooks(list(paths) or None) + if not notebooks: + print("No notebooks found.") + return sc.objdict() + if any(nb.suffix == ".ipynb" for nb in notebooks): + require_tool("jupytext", _JUPYTEXT_HINT) + sc.heading(f"Checking {len(notebooks)} notebooks (no cache update)...") + results = _parallel(execute_notebook, notebooks, serial=serial) + return _summarize(notebooks, results) + + +@sc.timer("Run notebooks") +def run_notebooks(*paths, serial=False): + """ + Render notebooks in parallel, updating the Quarto freeze cache. + + Args: + paths: notebook files or folders (default: the whole project). + serial: render one at a time (useful for debugging or to avoid + concurrent ``quarto render`` contention). + + Returns an :class:`sciris.objdict` mapping notebook path -> result string. + """ + notebooks = discover_notebooks(list(paths) or None) + if not notebooks: + print("No notebooks found.") + return sc.objdict() + require_tool("quarto", _QUARTO_HINT) + sc.heading(f"Running {len(notebooks)} notebooks (updating {FREEZE_DIR}/)...") + results = _parallel(render_notebook, notebooks, serial=serial) + return _summarize(notebooks, results) + + +def refresh_cache(dry_run=False): + """ + Delete cached copies of notebooks so they re-execute on the next render. + + Removes the Quarto freeze cache (``_freeze/``) and every jupyter-cache + (``.jupyter_cache/``) under the docs directory — with ``cache: true``, + Quarto creates the jupyter cache next to each input document, so these can + be nested in subfolders rather than only at the project root. Quarto's + internal ``.quarto/`` working dir is left alone (it regenerates itself). + + Returns the list of removed paths. With ``dry_run`` set, nothing is deleted. + Refuses to run outside a Quarto project. + """ + root = project_root() + if root is None: + print("No _quarto.yml found; refusing to refresh outside a Quarto project.") + return [] + + targets = [root / FREEZE_DIR] + targets += sorted(root.rglob(JUPYTER_CACHE_DIR)) # nested per-folder caches + existing = sorted({t for t in targets if t.exists()}) + if not existing: + print(f"No cached copies found ({FREEZE_DIR}/, {JUPYTER_CACHE_DIR}/).") + return [] + for target in existing: + verb = "Would delete" if dry_run else "Deleting" + print(f"{verb}: {target}") + if not dry_run: + sc.rmpath(target, die=False) + return existing diff --git a/quartodoc/qpyd/nb.py b/quartodoc/qpyd/nb.py new file mode 100644 index 00000000..b3b2fa53 --- /dev/null +++ b/quartodoc/qpyd/nb.py @@ -0,0 +1,88 @@ +"""The ``qpynb`` command group: notebook management for Quarto docs.""" + +import click + +from .convert import clear_outputs, to_ipynb, to_py, to_qmd +from .execute import execute_notebooks, refresh_cache, run_notebooks + +_SERIAL_HELP = "Process one notebook at a time instead of in parallel (for debugging)." + + +@click.group(name="qpynb", invoke_without_command=True) +@click.pass_context +def nb_cli(ctx): + """ + Manage Quarto notebooks (.qmd / .ipynb): run, check, convert, and clean. + + Run without a subcommand to show this help. + """ + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + + +@nb_cli.command("run") +@click.argument("paths", nargs=-1, type=click.Path()) +@click.option("--serial", is_flag=True, help=_SERIAL_HELP) +def run_cmd(paths, serial): + """ + Execute notebooks in parallel and update cached copies (Quarto _freeze/). + + With no PATHS, every notebook in the project is run. PATHS may be + individual notebooks or folders of notebooks. + """ + run_notebooks(*paths, serial=serial) + + +@nb_cli.command("check") +@click.argument("paths", nargs=-1, type=click.Path()) +@click.option("--serial", is_flag=True, help=_SERIAL_HELP) +def check_cmd(paths, serial): + """ + Execute notebooks in parallel to verify they run; do NOT update caches. + + Like 'run', but a pure validation pass that leaves no artifacts and does + not touch the freeze cache. + """ + execute_notebooks(*paths, serial=serial) + + +@nb_cli.command("refresh") +@click.option("--dry-run", is_flag=True, help="Show what would be deleted, without deleting.") +def refresh_cmd(dry_run): + """Delete cached copies of notebooks (_freeze/ and .jupyter_cache/).""" + refresh_cache(dry_run=dry_run) + + +_FORCE_HELP = "Overwrite the destination file if it already exists." + + +@nb_cli.command("to-py") +@click.argument("path", type=click.Path(exists=True)) +@click.option("--force", is_flag=True, help=_FORCE_HELP) +def to_py_cmd(path, force): + """Convert a notebook (.qmd or .ipynb) to a Python (.py) file.""" + click.echo(f"Wrote {to_py(path, force=force)}") + + +@nb_cli.command("to-qmd") +@click.argument("path", type=click.Path(exists=True)) +@click.option("--force", is_flag=True, help=_FORCE_HELP) +def to_qmd_cmd(path, force): + """Convert a notebook (.ipynb or .py) to a Quarto (.qmd) file.""" + click.echo(f"Wrote {to_qmd(path, force=force)}") + + +@nb_cli.command("to-ipynb") +@click.argument("path", type=click.Path(exists=True)) +@click.option("--force", is_flag=True, help=_FORCE_HELP) +def to_ipynb_cmd(path, force): + """Convert a notebook (.qmd or .py) to a Jupyter (.ipynb) file.""" + click.echo(f"Wrote {to_ipynb(path, force=force)}") + + +@nb_cli.command("clear") +@click.argument("paths", nargs=-1, type=click.Path()) +@click.option("--dry-run", is_flag=True, help="Show what would be cleared, without writing.") +def clear_cmd(paths, dry_run): + """Clear saved outputs from .ipynb notebooks and normalize them.""" + clear_outputs(*paths, dry_run=dry_run) diff --git a/quartodoc/qpyd/prerender.py b/quartodoc/qpyd/prerender.py new file mode 100644 index 00000000..795dfae5 --- /dev/null +++ b/quartodoc/qpyd/prerender.py @@ -0,0 +1,141 @@ +""" +Pre-render build steps (the ``pre-render`` hook). + +Generalised from the ``pre`` branch of the original ``quarto_utils.py``: + +* :func:`build_api_docs` — ``quartodoc build`` +* :func:`customize_aliases` — add short aliases to ``objects.json`` +* :func:`build_interlinks` — ``quartodoc interlinks`` +* :func:`build_objects_inv` — write a Sphinx-compatible ``objects.inv`` + +The package name is read from the ``quartodoc.package`` key in ``_quarto.yml`` +rather than hard-coded. The original starsim-specific ``update_version`` step +is intentionally dropped; version injection is now handled generically by +``_variables.py`` (see :mod:`.variables`). +""" + +import importlib +import sys +from pathlib import Path + +import sciris as sc + +from .config import get_package_name +from .render import run + + +@sc.timer("Build API docs") +def build_api_docs(): + """Generate the API reference pages via ``quartodoc build``.""" + sc.heading("Building API documentation...") + return run([sys.executable, "-m", "quartodoc", "build"]) + + +@sc.timer("Customize aliases") +def customize_aliases(mod_name=None, json_path="objects.json"): + """ + Add short aliases to the objects inventory. + + For each ``pkg.submodule.Object`` that is also importable directly as + ``pkg.Object``, register the shorter ``pkg.Object`` name so cross-references + can use it. The package name defaults to ``quartodoc.package`` from + ``_quarto.yml``. + """ + mod_name = mod_name or get_package_name() + if not mod_name: + print(" No quartodoc.package configured; skipping alias customization.") + return + if not Path(json_path).exists(): + print(f" {json_path} not found; skipping alias customization.") + return + + sc.heading("Customizing aliases ...") + try: + mod = importlib.import_module(mod_name) + except ImportError as e: + print(f" Could not import {mod_name!r} ({e}); skipping alias customization.") + return + mod_items = dir(mod) + + data = sc.loadjson(json_path) + items = data["items"] + names = [item["name"] for item in items] + print(f' Loaded {len(data["items"])} items') + + dups = [] + for item in items: + parts = item["name"].split(".") + if len(parts) < 3 or parts[0] != mod_name: + continue + objname = parts[2] # e.g. 'Analyzer' from starsim.analyzers.Analyzer + if objname in mod_items: + remainder = ".".join(parts[2:]) + alias = f"{mod_name}.{remainder}" + if alias not in names: + dup = sc.dcp(item) + dup["name"] = alias + dups.append(dup) + + items.extend(dups) + sc.savejson(json_path, data) + print(f' Saved {len(data["items"])} items') + + +@sc.timer("Build interlinks") +def build_interlinks(): + """Generate interlink inventories via ``quartodoc interlinks``.""" + sc.heading("Building docs links...") + return run([sys.executable, "-m", "quartodoc", "interlinks"]) + + +@sc.timer("Build objects.inv") +def build_objects_inv(json_path="objects.json", inv_path="objects.inv"): + """ + Convert the quartodoc JSON inventory into a Sphinx-compatible + ``objects.inv`` so other projects can resolve references via intersphinx. + """ + import sphobjinv as soi + + if not Path(json_path).exists(): + print(f" {json_path} not found; skipping objects.inv.") + return + + sc.heading("Building Sphinx objects.inv ...") + data = sc.loadjson(json_path) + inv = soi.Inventory() + inv.project = data.get("project", get_package_name(default="project")) + inv.version = str(data.get("version", "0.0.0")) + for item in data["items"]: + inv.objects.append( + soi.DataObjStr( + name=item["name"], + domain=item["domain"], + role=item["role"], + priority=str(item.get("priority", "1")), + uri=item["uri"], + dispname=item.get("dispname", "-") or "-", + ) + ) + with open(inv_path, "wb") as f: + f.write(soi.compress(inv.data_file())) + print(f" Wrote {len(inv.objects)} entries to {inv_path}") + + +@sc.timer("Pre-render") +def prerender(): + """Run all pre-render build steps in order. + + No-ops when ``QPYD_SKIP_HOOKS`` is set, which ``qpynb run`` does for its + per-notebook ``quarto render`` calls — otherwise this heavy, shared-state + build (rewriting ``objects.json``) would run once per notebook in parallel + and race with itself. + """ + import os + + if os.environ.get("QPYD_SKIP_HOOKS"): + return + sc.heading("Starting Quarto docs build", divider="★") + build_api_docs() + customize_aliases() + build_interlinks() + build_objects_inv() diff --git a/quartodoc/qpyd/quarto_utils.py b/quartodoc/qpyd/quarto_utils.py new file mode 100644 index 00000000..f46664e7 --- /dev/null +++ b/quartodoc/qpyd/quarto_utils.py @@ -0,0 +1,324 @@ +""" +Utilities for processing docs (notebooks mostly) +""" + +import os +import sys +import importlib +import subprocess +import sciris as sc +import starsim as ss + +default_folders = ['tutorials', 'user_guide'] # Folders with Jupyter notebooks +temp_patterns = ['**/my-*.*', '**/example*.*'] # Temporary files to be removed +temp_items = ['user_guide/results'] # Temporary files/folders to be removed + +timeout = 600 # Maximum time for notebook execution +yay = '✓' +boo = '😢' + + +def run(cmd): + """ Verbose version of subprocess.run """ + sc.printgreen(f'\n> {cmd}\n') + return subprocess.run(cmd, check=True, shell=True) + + +@sc.timer('Update version') +def update_version(pkg=ss): + sc.heading('Updating docs version number...') + filename = '_variables.yml' + data = dict(version=ss.__version__, versiondate=ss.__versiondate__) + orig = sc.loadyaml(filename) + if data != orig: + sc.saveyaml(filename, data) + print('Version updated to:', data) + else: + print('Version already correct:', orig) + return + + +@sc.timer('Build API docs') +def build_api_docs(): + sc.heading('Building API documentation...') + return run('python -m quartodoc build') + + +@sc.timer('Customize aliases') +def customize_aliases(mod_name='starsim', json_path='objects.json'): + """ + Manually add aliases to functions, so instead of e.g. starsim.analyzers.Analyzer, you can link via starsim.Analyzer (and therefore ss.Analyzer) + """ + sc.heading('Customizing aliases ...') + mod = importlib.import_module(mod_name) + mod_items = dir(mod) + + # Load the current objects inventory + json = sc.loadjson(json_path) + items = json['items'] + names = [item['name'] for item in items] + print(f' Loaded {len(json["items"])} items') + + # Collect duplicates + dups = [] + for item in items: + parts = item['name'].split('.') + if len(parts) < 3 or parts[0] != mod_name: + continue + objname = parts[2] # e.g. 'Analyzer' from starsim.analyzers.Analyzer + if objname in mod_items: + remainder = '.'.join(parts[2:]) + alias = f'{mod_name}.{remainder}' + if alias not in names: + dup = sc.dcp(item) + dup['name'] = alias + dups.append(dup) + + # Add them back into the JSON + items.extend(dups) + + # Save the modified version + sc.savejson(json_path, json) + print(f' Saved {len(json["items"])} items') + return + + +@sc.timer('Build interlinks') +def build_interlinks(): + sc.heading('Building docs links...') + return run('python -m quartodoc interlinks') + + +@sc.timer('Build objects.inv') +def build_objects_inv(json_path='objects.json', inv_path='objects.inv'): + """ + Convert the quartodoc JSON inventory into a Sphinx-compatible objects.inv, + so other projects can resolve Starsim references via intersphinx. + """ + import sphobjinv as soi + sc.heading('Building Sphinx objects.inv ...') + data = sc.loadjson(json_path) + inv = soi.Inventory() + inv.project = data.get('project', 'starsim') + inv.version = str(data.get('version', ss.__version__)) + for item in data['items']: + inv.objects.append(soi.DataObjStr( + name=item['name'], + domain=item['domain'], + role=item['role'], + priority=str(item.get('priority', '1')), + uri=item['uri'], + dispname=item.get('dispname', '-') or '-', + )) + with open(inv_path, 'wb') as f: + f.write(soi.compress(inv.data_file())) + print(f' Wrote {len(inv.objects)} entries to {inv_path}') + return + + +def qmd2py(qmd_path, py_path=None, keep_text=True): + """ + Convert a .qmd file to a .py file by extracting Python code cells. + + Each ```{python} ... ``` block becomes a cell, separated by "#%% Cell N" + headers and two blank lines between cells. Raises an exception if blocks + are ambiguous (e.g., nested or unclosed code fences). + + Lines starting with "%" or "!" (IPython magics/shell commands) are commented + out since they are not valid Python. + + Args: + qmd_path (str/Path): path to the .qmd file + py_path (str/Path): path to write the .py file (default: same name with .py extension) + keep_text (bool): if True, include non-code text as comments prefixed with "# " + + Returns: + Path to the written .py file + """ + qmd_path = sc.path(qmd_path) + if py_path is None: + py_path = qmd_path.with_suffix('.py') + else: + py_path = sc.path(py_path) + + text = sc.loadtext(qmd_path) + lines = text.splitlines() + + chunks = [] # List of (type, content) tuples; type is 'code' or 'text' + in_block = False + current_cell = [] + current_text = [] + block_start_line = None + + for i, line in enumerate(lines, start=1): + stripped = line.strip() + if stripped.startswith('```{python}'): + if in_block: + raise ValueError(f'Nested or unclosed code block: new block at line {i}, previous block started at line {block_start_line}') + if keep_text and current_text: + chunks.append(('text', current_text)) + current_text = [] + in_block = True + block_start_line = i + current_cell = [] + elif stripped == '```' and in_block: + chunks.append(('code', current_cell)) + in_block = False + current_cell = [] + block_start_line = None + elif in_block: + current_cell.append(line) + elif keep_text: + current_text.append(line) + + if in_block: + raise ValueError(f'Unclosed code block starting at line {block_start_line}') + + if keep_text and current_text: + chunks.append(('text', current_text)) + + # Build output + parts = [] + cell_num = 0 + for kind, content in chunks: + if kind == 'code': + cell_num += 1 + processed = [] + for line in content: + if line.lstrip().startswith(('%', '!')): + processed.append(f'# {line} # IPython not supported in Python files') + else: + processed.append(line) + parts.append(f'#%% Cell {cell_num}\n' + '\n'.join(processed)) + else: # text + commented = '\n'.join(f'# {line}' if line.strip() else '#' for line in content) + parts.append(commented) + + output = '\n\n\n'.join(parts) + '\n' + sc.savetext(py_path, output) + return py_path + + +def execute_notebook(path, tidy=True): + """ Executes a single Jupyter notebook and returns success/failure """ + qmd_path = path.name + os.chdir(path.parent) + with sc.timer(label=sc.ansi.green(f' Execution time for {qmd_path}')) as T: + base = qmd_path.removesuffix('.qmd') + py_path = base + '.py' + try: + print(f'Converting {qmd_path}...') + qmd2py(path) + print(f'Executing {py_path}...') + env = {**os.environ, 'MPLBACKEND': 'agg'} # Use non-interactive backend for matplotlib + subprocess.run(['python', py_path], check=True, capture_output=True,cwd=path.parent, env=env) # Use ipython so get_ipython() is available + string = f'{yay} {base} executed successfully ' + except subprocess.CalledProcessError as e: + string = f'{boo} Execution failed for {base}: {e}\n' + except Exception as e: + string = f'{boo} Error processing {base}: {str(e)}\n' + finally: + if tidy: + sc.rmpath(py_path) + + string += f'(time: {T.total:0.1f} s)' + print(string) + return string + + + +@sc.timer('Execute notebooks') +def execute_notebooks(*args, folders=None, tidy=True, debug=False): + """ Execute notebooks in parallel and print which succeeded / failed """ + T = sc.timer() + cwd = sc.thispath(__file__) + results = sc.objdict() + string = '' + + # Determine which notebooks to run + if args: + notebooks = [sc.path(notebook).resolve() for notebook in args] + else: + notebooks = [] + folders = sc.ifelse(folders, default_folders) + for folder in folders: + folder_path = cwd / folder + notebooks += [folder_path / f for f in sc.getfilepaths(folder_path, '*.qmd')] + + # Wrapper to chdir before execution + def execute(i, path, pause=1.0): + delay = i*pause + sc.timedsleep(delay) # For some reason the interval argument to parallelize() isn't working + return execute_notebook(path, tidy=tidy) + + sc.heading(f'Running {len(notebooks)} notebooks...') + notebook_list = list(enumerate(notebooks)) + out = sc.parallelize(execute, notebook_list, maxcpu=0.9, interval=1.0, lbkwargs=dict(verbose=False), serial=debug) + string += sc.strjoin(out, sep=f'\n\n\n{"—"*90}\n') + for nb, res in zip(notebooks, out): + results[str(nb)] = res + + sc.heading('Results') + print(string) + + sc.heading('Summary') + n_yay = string.count(yay) + n_boo = string.count(boo) + summary = f'{n_yay} succeeded, {n_boo} failed\n' + + results.sort('values') + for nb,res in results.items(): + timestr = res.split('time: ')[1][:-1] + suffix = f'{sc.path(nb).name:30s} ({timestr})' + if yay in res: + summary += f'\n{sc.ansi.green("Succeeded")}: {suffix}' + elif boo in res: + summary += f'\n{sc.ansi.red(" Failed")}: {suffix}' + else: + print('Unexpected result for {nb}, neither succeeded nor failed!') + print(summary) + + T.toc() + + return results + + +@sc.timer('Clean outputs') +def clean_outputs(folders=None, sleep=3, patterns=None): + """ Clears outputs from notebooks """ + sc.heading('Cleaning outputs ...') + if folders is None: + folders = default_folders + if patterns is None: + patterns = temp_patterns + filenames = sc.dcp(temp_items) + for pattern in patterns: + for folder in folders: + filenames += sc.getfilelist(folder=folder, pattern=pattern, recursive=True) + if len(filenames): + print(f'Deleting: {sc.newlinejoin(filenames)}\nin {sleep} seconds') + sc.timedsleep(sleep) + for filename in filenames: + sc.rmpath(filename, verbose=True, die=False) + else: + print('No files found to clean') + return + + +# Process arguments (called by _quarto.yml) +if __name__ == '__main__': + + if 'pre' in sys.argv: + sc.heading('Starting Quarto docs build', divider='★') + update_version() + build_api_docs() + customize_aliases() + build_interlinks() + build_objects_inv() + + elif 'post' in sys.argv: + clean_outputs() + + elif len(sys.argv) > 1: + errormsg = f'Argument must be "pre" or "post", not {sys.argv}' + raise ValueError(errormsg) diff --git a/quartodoc/qpyd/render.py b/quartodoc/qpyd/render.py new file mode 100644 index 00000000..31b19fbe --- /dev/null +++ b/quartodoc/qpyd/render.py @@ -0,0 +1,85 @@ +""" +Wrappers around ``quarto render`` / ``quarto preview`` / ``quarto publish``. + +Each of these optionally runs the pre-render build steps first (see +:mod:`.prerender`) and appends any ``_variables.py`` values as ``-M`` metadata +flags (see :mod:`.variables`). +""" + +import subprocess +import time + +import sciris as sc + +from .variables import variable_args + + +def run(cmd, **kwargs): + """ + Verbose ``subprocess.run`` that echoes the command first and raises on + failure. + + ``cmd`` may be a string (run via the shell) or a list of arguments. A + missing executable is reported with an actionable message rather than a + bare ``FileNotFoundError`` traceback. + """ + shell = isinstance(cmd, str) + shown = cmd if shell else sc.strjoin(cmd, sep=" ") + sc.printgreen(f"\n> {shown}\n") + try: + return subprocess.run(cmd, check=True, shell=shell, **kwargs) + except FileNotFoundError as e: + tool = cmd.split()[0] if shell else cmd[0] + hint = ( + "Install Quarto from https://quarto.org and ensure it is on your PATH." + if tool == "quarto" + else f"Ensure {tool!r} is installed and on your PATH." + ) + raise RuntimeError(f"Required tool {tool!r} was not found. {hint}") from e + + +def _maybe_prerender(do_prerender): + if do_prerender: + from .prerender import prerender + + prerender() + + +def render(extra_args=(), do_prerender=True): + """ + Run the pre-render steps, then ``quarto render``, reporting elapsed time. + + Args: + extra_args: extra arguments passed through to ``quarto render``. + do_prerender: run :func:`.prerender.prerender` first (default True). + """ + t0 = time.time() + _maybe_prerender(do_prerender) + run(["quarto", "render", *extra_args, *variable_args()]) + elapsed = time.time() - t0 + print(f"\nDone: docs built in {elapsed:0.1f} s") + + +def preview(extra_args=(), do_prerender=True): + """ + Run the pre-render steps, then ``quarto preview`` (live-reloading server). + + Args: + extra_args: extra arguments passed through to ``quarto preview``. + do_prerender: run :func:`.prerender.prerender` first (default True). + """ + _maybe_prerender(do_prerender) + run(["quarto", "preview", *extra_args, *variable_args()]) + + +def gh_publish(do_prerender=True): + """ + Render and publish the site to GitHub Pages (the ``gh-pages`` branch). + + Mirrors the original ``publish`` script: a full ``quarto render + --cache-refresh`` followed by ``quarto publish gh-pages --no-render``. This + pushes to a remote branch, so it is intentionally never run automatically. + """ + _maybe_prerender(do_prerender) + run(["quarto", "render", "--cache-refresh", *variable_args()]) + run(["quarto", "publish", "gh-pages", "--no-render", "--no-prompt", "--no-browser"]) diff --git a/quartodoc/qpyd/scaffold.py b/quartodoc/qpyd/scaffold.py new file mode 100644 index 00000000..6e72f4f9 --- /dev/null +++ b/quartodoc/qpyd/scaffold.py @@ -0,0 +1,133 @@ +""" +``qpyd init``: scaffold a ``docs/`` folder with a starter ``_quarto.yml``. + +Existing files are never overwritten, so running ``init`` in an established +project is safe and idempotent. +""" + +from pathlib import Path + +from .config import CONFIG_NAME, VARIABLES_PY + +# Token-substituted rather than ``str.format``-ed to avoid escaping the many +# literal ``{{< ... >}}`` braces a Quarto config contains. +_QUARTO_YML = """\ +project: + type: website + output-dir: _site + pre-render: qpyd prerender + # Optional cleanup of generated scratch files after rendering. Disabled by + # default; enable by uncommenting and configuring the `qpyd.clean` patterns + # below (only matching, non-source files are deleted). + # post-render: qpyd clean + +website: + title: "__TITLE__" + navbar: + left: + - href: index.qmd + text: Home + - href: api/index.qmd + text: API reference + sidebar: + - id: api + contents: + - api/index.qmd + +format: + html: + theme: cosmo + toc: true + +filters: + - interlinks + +interlinks: + sources: + python: + url: https://docs.python.org/3/ + +quartodoc: + package: __PACKAGE__ + title: API reference + dir: api + parser: google + sections: + - title: API reference + desc: API documentation for __PACKAGE__. + contents: [] + +# Optional: glob patterns for `qpyd clean` to delete after rendering. Source +# files (.qmd/.ipynb/.py/.md) are never deleted, even if matched. +# qpyd: +# clean: +# - '**/my-*.png' + +execute: + freeze: auto # Only re-execute notebooks that have changed + cache: true + error: false # Stop the build if a notebook errors +""" + +_INDEX_QMD = """\ +--- +title: "__TITLE__" +--- + +Welcome to the __PACKAGE__ documentation. + +See the [API reference](api/index.qmd) to get started. +""" + +_VARIABLES_PY = '''\ +""" +Optional variables made available to `quarto render` as `-M key:value` flags. + +Public, YAML-serialisable, non-callable names defined here are passed through +by `qpyd render` / `qpyd preview`. For example: + + import __PACKAGE__ + version = __PACKAGE__.__version__ +""" +''' + + +def _write_if_absent(path, content, created): + if path.exists(): + print(f" exists, leaving as-is: {path}") + return + path.write_text(content) + created.append(path) + print(f" created: {path}") + + +def init(path="docs", package=None): + """ + Create ``path`` (default ``docs/``) with a starter ``_quarto.yml``, + ``index.qmd``, and ``_variables.py`` if they do not already exist. + + Args: + path: directory to scaffold. + package: package name to document. If omitted, the generated files use + the literal ``your_package`` placeholder, which you should edit. + + Returns the list of files that were created. + """ + docs = Path(path) + docs.mkdir(parents=True, exist_ok=True) + + package = package or "your_package" + title = package + + def fill(template): + return template.replace("__PACKAGE__", package).replace("__TITLE__", title) + + created = [] + print(f"Initializing docs in {docs.resolve()} (package={package!r})") + _write_if_absent(docs / CONFIG_NAME, fill(_QUARTO_YML), created) + _write_if_absent(docs / "index.qmd", fill(_INDEX_QMD), created) + _write_if_absent(docs / VARIABLES_PY, fill(_VARIABLES_PY), created) + + if not created: + print("Nothing to do; all starter files already exist.") + return created diff --git a/quartodoc/qpyd/tests/__init__.py b/quartodoc/qpyd/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/quartodoc/qpyd/tests/conftest.py b/quartodoc/qpyd/tests/conftest.py new file mode 100644 index 00000000..b8a32b83 --- /dev/null +++ b/quartodoc/qpyd/tests/conftest.py @@ -0,0 +1,15 @@ +import shutil +from pathlib import Path + +import pytest + +DATA = Path(__file__).parent / "data" + + +@pytest.fixture +def docs(tmp_path): + """A temp docs dir with a minimal _quarto.yml and the sample notebooks.""" + (tmp_path / "_quarto.yml").write_text("quartodoc:\n package: quartodoc\n") + for f in DATA.glob("*.qmd"): + shutil.copy(f, tmp_path / f.name) + return tmp_path diff --git a/quartodoc/qpyd/tests/data/nb_fail.qmd b/quartodoc/qpyd/tests/data/nb_fail.qmd new file mode 100644 index 00000000..5007e046 --- /dev/null +++ b/quartodoc/qpyd/tests/data/nb_fail.qmd @@ -0,0 +1,9 @@ +--- +title: Failing notebook +--- + +A notebook that raises an error. + +```{python} +raise ValueError("intentional failure") +``` diff --git a/quartodoc/qpyd/tests/data/nb_ok.qmd b/quartodoc/qpyd/tests/data/nb_ok.qmd new file mode 100644 index 00000000..02fe0ad9 --- /dev/null +++ b/quartodoc/qpyd/tests/data/nb_ok.qmd @@ -0,0 +1,11 @@ +--- +title: OK notebook +--- + +A notebook that runs successfully. + +```{python} +x = 1 + 1 +assert x == 2 +print("ok", x) +``` diff --git a/quartodoc/qpyd/tests/data/prose.qmd b/quartodoc/qpyd/tests/data/prose.qmd new file mode 100644 index 00000000..d981102e --- /dev/null +++ b/quartodoc/qpyd/tests/data/prose.qmd @@ -0,0 +1,5 @@ +--- +title: Prose only +--- + +This page has no executable code cells, so it is not a runnable notebook. diff --git a/quartodoc/qpyd/tests/test_cli.py b/quartodoc/qpyd/tests/test_cli.py new file mode 100644 index 00000000..b48a8714 --- /dev/null +++ b/quartodoc/qpyd/tests/test_cli.py @@ -0,0 +1,65 @@ +from click.testing import CliRunner + +from quartodoc.qpyd import cli, nb_cli + + +def test_qpyd_help(): + result = CliRunner().invoke(cli, ["--help"]) + assert result.exit_code == 0 + for cmd in ["render", "preview", "prerender", "gh-publish", "init", "clean", "nb"]: + assert cmd in result.output + + +def test_qpynb_no_subcommand_shows_help(): + result = CliRunner().invoke(nb_cli, []) + assert result.exit_code == 0 + for cmd in ["run", "check", "refresh", "to-py", "to-qmd", "to-ipynb", "clear"]: + assert cmd in result.output + + +def test_nb_alias(): + result = CliRunner().invoke(cli, ["nb", "--help"]) + assert result.exit_code == 0 + assert "check" in result.output + + +def test_init_scaffolds(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(cli, ["init", "mydocs", "--package", "foo"]) + assert result.exit_code == 0, result.output + cfg = tmp_path / "mydocs" / "_quarto.yml" + assert cfg.exists() + assert (tmp_path / "mydocs" / "index.qmd").exists() + assert "package: foo" in cfg.read_text() + + +def test_init_idempotent(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + CliRunner().invoke(cli, ["init", "d", "--package", "foo"]) + cfg = tmp_path / "d" / "_quarto.yml" + cfg.write_text("custom") # tamper + CliRunner().invoke(cli, ["init", "d", "--package", "foo"]) + assert cfg.read_text() == "custom" # existing file not overwritten + + +def test_to_py_cli(tmp_path): + qmd = tmp_path / "n.qmd" + qmd.write_text("```{python}\nx = 1\n```\n") + result = CliRunner().invoke(nb_cli, ["to-py", str(qmd)]) + assert result.exit_code == 0, result.output + assert (tmp_path / "n.py").exists() + + +def test_clean_dry_run(tmp_path, monkeypatch): + # clean is opt-in via the qpyd.clean config key, and only deletes + # non-source files that match. + (tmp_path / "_quarto.yml").write_text( + "quartodoc:\n package: x\nqpyd:\n clean:\n - '**/my-*.png'\n" + ) + junk = tmp_path / "my-scratch.png" + junk.write_bytes(b"x") + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(cli, ["clean", "--dry-run"]) + assert result.exit_code == 0, result.output + assert junk.exists() # dry-run must not delete + assert "my-scratch.png" in result.output diff --git a/quartodoc/qpyd/tests/test_config.py b/quartodoc/qpyd/tests/test_config.py new file mode 100644 index 00000000..1718e16a --- /dev/null +++ b/quartodoc/qpyd/tests/test_config.py @@ -0,0 +1,39 @@ +from quartodoc.qpyd import config + + +def test_discover_skips_prose_and_finds_python(docs): + names = sorted(p.name for p in config.discover_notebooks(root=docs)) + assert "nb_ok.qmd" in names + assert "nb_fail.qmd" in names + assert "prose.qmd" not in names # no {python} cell -> not a notebook + + +def test_discover_explicit_file_includes_prose(docs): + nbs = config.discover_notebooks([str(docs / "prose.qmd")]) + assert [p.name for p in nbs] == ["prose.qmd"] + + +def test_discover_excludes_build_dirs(docs): + freeze = docs / "_freeze" + freeze.mkdir() + (freeze / "cached.qmd").write_text("```{python}\nx = 1\n```\n") + nbs = config.discover_notebooks(root=docs) + assert all("_freeze" not in str(p) for p in nbs) + + +def test_discover_finds_ipynb(docs): + (docs / "extra.ipynb").write_text("{}") + names = [p.name for p in config.discover_notebooks(root=docs)] + assert "extra.ipynb" in names + + +def test_find_config_and_package(docs, monkeypatch): + monkeypatch.chdir(docs) + assert config.find_quarto_config().name == "_quarto.yml" + assert config.get_package_name() == "quartodoc" + assert config.docs_dir().resolve() == docs.resolve() + + +def test_get_package_name_missing(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + assert config.get_package_name(default="fallback") == "fallback" diff --git a/quartodoc/qpyd/tests/test_convert.py b/quartodoc/qpyd/tests/test_convert.py new file mode 100644 index 00000000..f1dcb86b --- /dev/null +++ b/quartodoc/qpyd/tests/test_convert.py @@ -0,0 +1,69 @@ +import shutil + +import nbformat +import pytest + +from quartodoc.qpyd import convert + + +def test_qmd2py_extracts_cells(tmp_path): + qmd = tmp_path / "n.qmd" + qmd.write_text( + "---\ntitle: t\n---\n\nIntro text\n\n" + "```{python}\nx = 1\n%timeit x\n!ls\n```\n\nmore\n" + ) + text = convert.qmd2py(qmd).read_text() + assert "#%% Cell 1" in text + assert "x = 1" in text + assert "# %timeit x" in text # magic commented out + assert "# !ls" in text # shell command commented out + assert "# Intro text" in text # prose preserved as comment + + +def test_qmd2py_unclosed_raises(tmp_path): + qmd = tmp_path / "bad.qmd" + qmd.write_text("```{python}\nx = 1\n") + with pytest.raises(ValueError): + convert.qmd2py(qmd) + + +def test_to_py_on_qmd(tmp_path): + qmd = tmp_path / "n.qmd" + qmd.write_text("```{python}\nx = 1\n```\n") + out = convert.to_py(qmd) + assert out.suffix == ".py" and out.exists() + + +def test_clear_outputs(tmp_path): + nb = nbformat.v4.new_notebook() + cell = nbformat.v4.new_code_cell("print(1)") + cell.execution_count = 3 + cell.outputs = [nbformat.v4.new_output("stream", name="stdout", text="1\n")] + nb.cells = [cell] + path = tmp_path / "nb.ipynb" + nbformat.write(nb, str(path)) + + convert.clear_outputs(str(path)) + + cleared = nbformat.read(str(path), as_version=4) + assert cleared.cells[0].outputs == [] + assert cleared.cells[0].execution_count is None + + +@pytest.mark.skipif(not shutil.which("quarto"), reason="quarto not installed") +def test_qmd_to_ipynb(tmp_path): + qmd = tmp_path / "n.qmd" + qmd.write_text("---\ntitle: t\n---\n\n```{python}\nx = 1\n```\n") + out = convert.to_ipynb(qmd) + assert out.suffix == ".ipynb" and out.exists() + + +@pytest.mark.skipif(not shutil.which("jupytext"), reason="jupytext not installed") +def test_ipynb_to_py(tmp_path): + nb = nbformat.v4.new_notebook() + nb.cells = [nbformat.v4.new_code_cell("x = 1")] + path = tmp_path / "nb.ipynb" + nbformat.write(nb, str(path)) + out = convert.to_py(path) + assert out.suffix == ".py" and out.exists() + assert "x = 1" in out.read_text() diff --git a/quartodoc/qpyd/tests/test_execute.py b/quartodoc/qpyd/tests/test_execute.py new file mode 100644 index 00000000..ee01086c --- /dev/null +++ b/quartodoc/qpyd/tests/test_execute.py @@ -0,0 +1,47 @@ +from quartodoc.qpyd import execute + + +def _write(directory, name, code): + path = directory / name + path.write_text(f"```{{python}}\n{code}\n```\n") + return path + + +def test_execute_notebook_pass(tmp_path): + nb = _write(tmp_path, "ok.qmd", "x = 1 + 1\nassert x == 2\nprint('ok')") + assert execute.YAY in execute.execute_notebook(nb) + + +def test_execute_notebook_fail(tmp_path): + nb = _write(tmp_path, "bad.qmd", "raise ValueError('boom')") + assert execute.BOO in execute.execute_notebook(nb) + + +def test_execute_notebooks_serial(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text("quartodoc:\n package: x\n") + _write(tmp_path, "a.qmd", "print('a')") + _write(tmp_path, "b.qmd", "raise RuntimeError('x')") + monkeypatch.chdir(tmp_path) + + results = execute.execute_notebooks(serial=True) + + assert len(results) == 2 + combined = "".join(results.values()) + assert execute.YAY in combined and execute.BOO in combined + + +def test_refresh_dry_run_keeps_cache(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text("quartodoc:\n package: x\n") + (tmp_path / "_freeze").mkdir() + monkeypatch.chdir(tmp_path) + + removed = execute.refresh_cache(dry_run=True) + + assert any("_freeze" in str(p) for p in removed) + assert (tmp_path / "_freeze").exists() # dry-run must not delete + + +def test_refresh_empty(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text("quartodoc:\n package: x\n") + monkeypatch.chdir(tmp_path) + assert execute.refresh_cache() == [] diff --git a/quartodoc/qpyd/tests/test_safety.py b/quartodoc/qpyd/tests/test_safety.py new file mode 100644 index 00000000..6ab0db13 --- /dev/null +++ b/quartodoc/qpyd/tests/test_safety.py @@ -0,0 +1,163 @@ +"""Regression tests for the safety / correctness fixes from the review.""" + +import nbformat +import pytest + +from quartodoc.qpyd import clean, config, convert, execute, variables + + +# --- clean: opt-in, project-scoped, source-protecting (CRITICAL/HIGH) ------- + + +def test_clean_refuses_without_quarto_yml(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "example-thing.qmd").write_text("x") + (tmp_path / "my-scratch.txt").write_text("x") + removed = clean.clean_outputs() + assert removed == [] + assert (tmp_path / "example-thing.qmd").exists() # nothing deleted + + +def test_clean_no_patterns_is_noop(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text("quartodoc:\n package: x\n") + monkeypatch.chdir(tmp_path) + (tmp_path / "my-scratch.png").write_bytes(b"x") + removed = clean.clean_outputs() + assert removed == [] + assert (tmp_path / "my-scratch.png").exists() + + +def test_clean_never_deletes_source_files(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text( + "quartodoc:\n package: x\nqpyd:\n clean:\n - '**/example*.*'\n" + ) + monkeypatch.chdir(tmp_path) + (tmp_path / "example-usage.qmd").write_text("source!") + (tmp_path / "example-output.png").write_bytes(b"x") + removed = clean.clean_outputs() + assert (tmp_path / "example-usage.qmd").exists() # .qmd protected + assert (tmp_path / "example-output.png") not in [] # png is deletable + assert not (tmp_path / "example-output.png").exists() + assert all(p.suffix != ".qmd" for p in removed) + + +def test_clean_dry_run_keeps_files(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text( + "quartodoc:\n package: x\nqpyd:\n clean:\n - '**/my-*.png'\n" + ) + monkeypatch.chdir(tmp_path) + junk = tmp_path / "my-fig.png" + junk.write_bytes(b"x") + removed = clean.clean_outputs(dry_run=True) + assert junk in [p for p in removed] + assert junk.exists() + + +def test_clean_skips_under_hook_guard(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text( + "quartodoc:\n package: x\nqpyd:\n clean:\n - '**/my-*.png'\n" + ) + monkeypatch.chdir(tmp_path) + junk = tmp_path / "my-fig.png" + junk.write_bytes(b"x") + monkeypatch.setenv("QPYD_SKIP_HOOKS", "1") + assert clean.clean_outputs() == [] + assert junk.exists() + + +# --- refresh: project-scoped, finds nested caches (HIGH/LOW) ---------------- + + +def test_refresh_refuses_without_quarto_yml(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "_freeze").mkdir() + assert execute.refresh_cache() == [] + assert (tmp_path / "_freeze").exists() + + +def test_refresh_finds_nested_jupyter_cache(tmp_path, monkeypatch): + (tmp_path / "_quarto.yml").write_text("quartodoc:\n package: x\n") + (tmp_path / "_freeze").mkdir() + nested = tmp_path / "tutorials" / ".jupyter_cache" + nested.mkdir(parents=True) + monkeypatch.chdir(tmp_path) + removed = execute.refresh_cache(dry_run=True) + assert any(p.name == ".jupyter_cache" for p in removed) + assert any(p.name == "_freeze" for p in removed) + + +# --- prerender hook guard (HIGH) -------------------------------------------- + + +def test_prerender_skips_under_hook_guard(monkeypatch, capsys): + from quartodoc.qpyd import prerender + + monkeypatch.setenv("QPYD_SKIP_HOOKS", "1") + prerender.prerender() # must not invoke quartodoc build etc. + # If it tried to build, it would print the "Starting Quarto docs build" + # heading; assert it did not run. + assert "Starting Quarto docs build" not in capsys.readouterr().out + + +# --- variables -M serialization (MED) --------------------------------------- + + +def test_variables_to_args_yaml_scalars(): + args = variables.variables_to_args( + {"v": "1.2.3", "flag": True, "missing": None, "n": 5, "items": [1, 2]} + ) + flat = dict(zip(args[::2], args[1::2])) if False else args + assert "v:1.2.3" in args + assert "flag:true" in args # not Python "True" + assert "missing:null" in args # not "None" + assert "n:5" in args + assert "items:[1, 2]" in args + + +# --- conversion overwrite guard (MED) --------------------------------------- + + +def test_to_py_refuses_overwrite(tmp_path): + nb = nbformat.v4.new_notebook() + nb.cells = [nbformat.v4.new_code_cell("x = 1")] + ipynb = tmp_path / "n.ipynb" + nbformat.write(nb, str(ipynb)) + (tmp_path / "n.py").write_text("# precious hand-written file\n") + with pytest.raises(FileExistsError): + convert.to_py(ipynb) + assert "precious" in (tmp_path / "n.py").read_text() # untouched + + +def test_to_py_force_overwrites(tmp_path): + qmd = tmp_path / "n.qmd" + qmd.write_text("```{python}\nx = 1\n```\n") + (tmp_path / "n.py").write_text("old") + out = convert.to_py(qmd, force=True) + assert out.exists() + assert "old" not in out.read_text() + + +# --- _excluded base relative to scan target (LOW) --------------------------- + + +def test_discover_handles_dotted_absolute_target(tmp_path): + # A scan target whose absolute path passes through a hidden dir must still + # have its notebooks discovered. + hidden = tmp_path / ".cache" / "proj" + hidden.mkdir(parents=True) + (hidden / "nb.qmd").write_text("```{python}\nx = 1\n```\n") + found = config.discover_notebooks([str(hidden)]) + assert [p.name for p in found] == ["nb.qmd"] + + +# --- summary classification by leading marker (LOW) ------------------------- + + +def test_summary_not_fooled_by_checkmark_in_stderr(): + # A failure result that embeds a YAY (✓) in captured stderr must still be + # classified as a failure. + failing = f"{execute.BOO} Execution failed for x:\n assert ok {execute.YAY}\n(time: 0.1 s)" + passing = f"{execute.YAY} y executed successfully (time: 0.1 s)" + table = execute._summarize(["x", "y"], [failing, passing]) + assert not execute._succeeded(failing) + assert execute._succeeded(passing) diff --git a/quartodoc/qpyd/tests/test_variables.py b/quartodoc/qpyd/tests/test_variables.py new file mode 100644 index 00000000..9b983f78 --- /dev/null +++ b/quartodoc/qpyd/tests/test_variables.py @@ -0,0 +1,34 @@ +from quartodoc.qpyd import variables + + +def test_load_variables_filters(tmp_path): + vf = tmp_path / "_variables.py" + vf.write_text( + "import os\n" # a module: not YAML-serialisable -> skipped + "version = '1.2.3'\n" + "count = 5\n" + "_private = 'hidden'\n" # underscore -> skipped + "def helper():\n return 1\n" # callable -> skipped + ) + assert variables.load_variables(vf) == {"version": "1.2.3", "count": 5} + + +def test_variables_to_args(): + # Values are YAML-encoded so Quarto (which parses -M values as YAML) + # preserves them as strings rather than coercing "1.2"->float 1.2. + args = variables.variables_to_args({"version": "1.2", "date": "2026"}) + assert args == ["-M", "version:'1.2'", "-M", "date:'2026'"] + + +def test_variables_to_args_plain_string_unquoted(): + # A string that isn't number/bool/null-like needs no quoting. + assert variables.variables_to_args({"name": "starsim"}) == ["-M", "name:starsim"] + + +def test_load_missing_returns_empty(tmp_path): + assert variables.load_variables(tmp_path / "nope.py") == {} + + +def test_variable_args_roundtrip(tmp_path): + (tmp_path / "_variables.py").write_text("name = 'starsim'\n") + assert variables.variable_args(tmp_path / "_variables.py") == ["-M", "name:starsim"] diff --git a/quartodoc/qpyd/variables.py b/quartodoc/qpyd/variables.py new file mode 100644 index 00000000..0047e0f3 --- /dev/null +++ b/quartodoc/qpyd/variables.py @@ -0,0 +1,90 @@ +""" +Support for an optional ``_variables.py`` file living alongside ``_quarto.yml``. + +The file defines plain Python values (often derived from the documented +package), for example:: + + import starsim as ss + + version = ss.__version__ + versiondate = ss.__versiondate__ + +These are collected into a dict (skipping private names, callables, and +anything that is not YAML-serialisable) and turned into ``quarto render`` +metadata flags (``-M key:value``) so they can be referenced in documents. +""" + +import runpy +from pathlib import Path + +import yaml + +from .config import VARIABLES_PY, docs_dir + + +def load_variables(path=None): + """ + Execute ``_variables.py`` and return the public, serialisable values. + + Args: + path: path to the variables file (default: ``_variables.py`` in the + docs dir). If it does not exist, an empty dict is returned. + + Returns a ``{name: value}`` dict. Names beginning with ``_``, callables, + and values that cannot be safely dumped to YAML are skipped. + + Note: this executes ``_variables.py``, just like a ``conftest.py``. Only + run it on files you trust. + """ + if path is None: + path = docs_dir() / VARIABLES_PY + path = Path(path) + if not path.exists(): + return {} + + namespace = runpy.run_path(str(path)) + + variables = {} + for key, value in namespace.items(): + if key.startswith("_"): + continue + if callable(value): + continue + try: + yaml.safe_dump({key: value}) + except Exception: + continue + variables[key] = value + return variables + + +def _yaml_scalar(value): + """ + Render a Python value as a single-line YAML scalar. + + This keeps the ``-M`` payload valid YAML so Quarto parses it as the intended + type: ``True`` -> ``true``, ``None`` -> ``null``, ``"2026"`` -> ``'2026'`` + (a quoted string, not the int 2026), ``[1, 2]`` -> ``[1, 2]``. Using plain + ``str()`` would emit Python casing (``True``/``None``) that YAML mis-parses. + """ + return yaml.safe_dump(value, default_flow_style=True).splitlines()[0] + + +def variables_to_args(variables): + """ + Convert a ``{name: value}`` dict into a flat ``quarto render`` arg list. + + For example ``{"version": "1.2"}`` becomes ``["-M", "version:1.2"]``. Each + pair is a separate ``argv`` element, so no shell quoting is required even + for values containing spaces. Values are encoded as YAML scalars (see + :func:`_yaml_scalar`) so non-string types round-trip correctly. + """ + args = [] + for key, value in variables.items(): + args += ["-M", f"{key}:{_yaml_scalar(value)}"] + return args + + +def variable_args(path=None): + """Convenience wrapper: load ``_variables.py`` and return its ``-M`` args.""" + return variables_to_args(load_variables(path))