diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dfb54ed..cb4a0d9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,5 +22,5 @@ jobs: run: pre-commit run ruff-format - name: Ruff linting run: pre-commit run ruff - - run: just test + - run: make test - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/Justfile b/Justfile index 9bcc839..db6d783 100644 --- a/Justfile +++ b/Justfile @@ -9,10 +9,10 @@ open NOTEBOOK: uv run runbook edit {{NOTEBOOK}} clear-binder-output: - jupyter nbconvert --clear-output --inplace ./runbook/data/*.ipynb + uv run runbook clear-output ./runbook/data/*.ipynb clear-output *FILES: - jupyter nbconvert --clear-output --inplace {{FILES}} + uv run runbook clear-output {{FILES}} lint: pre-commit run diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..887839a --- /dev/null +++ b/Makefile @@ -0,0 +1,56 @@ +.DEFAULT_GOAL := help + +.PHONY: help test test-watch open clear-binder-output clear-output lint lint-all profile release clean build benchmark readme docs docs-release + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-20s %s\n", $$1, $$2}' + +test: ## Run pytest and deno tests + uv run pytest --disable-warnings -s + deno test -A --reload https://raw.githubusercontent.com/zph/runbook/main/ext/deno/runbook/mod.ts --parallel tests/cli_test.ts + +test-watch: ## Run tests on file changes + watchexec -- make test + +open: ## Open a notebook (usage: make open NOTEBOOK=path/to/nb) + uv run runbook edit $(NOTEBOOK) + +clear-binder-output: ## Clear outputs from binder notebooks + uv run runbook clear-output ./runbook/data/*.ipynb + +clear-output: ## Clear outputs from notebooks (usage: make clear-output FILES="a.ipynb b.ipynb") + uv run runbook clear-output $(FILES) + +lint: ## Run pre-commit hooks on staged files + pre-commit run + +lint-all: ## Run pre-commit hooks on all files + pre-commit run --all-files + +profile: ## Profile the CLI with cProfile + uv run python3 -m cProfile runbook/cli/__init__.py + +release: ## Create a release via release-it + deno run -A npm:release-it + +clean: ## Remove build artifacts + rm -rf ./dist + +build: ## Build the package + uv build + +benchmark: ## Run benchmark and export to docs/PERFORMANCE.md + hyperfine --export-markdown=docs/PERFORMANCE.md -- runbook + +readme: ## Regenerate README from template + .config/templating.sh + +docs: ## Build documentation site + cp -f README.md docs/ + uvx --with sphinx-click --with myst_parser --with . --from sphinx sphinx-build -b html docs/ site + +docs-release: ## Publish documentation + bash .hermit/bin/publish-docs + +run: ## Run the CLI (usage: make run ARGS="command args") + uv run runbook $(ARGS) diff --git a/pyproject.toml b/pyproject.toml index 890bd3d..3e32f64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,5 @@ [project] name = "runbook" -# Managed via runbook.__version__ and https://pypi.org/project/poetry-version-plugin/ version = "0" description = "Runbook lib and cli" requires-python = ">=3.11" @@ -9,40 +8,32 @@ authors = [ ] license = "MIT" readme = "README.md" - -[tool.poetry.dependencies] -python = "^3.11" -shx = "^0.4.2" -rich = "^13.9.4" -# papermill = "^2.5.0" -# Used until upstreaming typescript into papermill -papermill = { git = "https://github.com/zph/papermill", branch = "main" } -jupyter = "^1.0.0" -click = "^8.1.7" -bash-kernel = "^0.9.3" -jupyterlab-execute-time = "^3.1.0" -jupytext = "^1.16.1" -nbformat = "^5.9.2" -nbconvert = "^7.14.1" -pyyaml = "^6.0.1" -traitlets = "^5.14.1" -ipywidgets = "^8.1.1" -pre-commit = "^3.6.0" -python-ulid = "^2.2.0" -slack-sdk = "^3.26.2" -jupyterlab = "^4.1.5" -nbdime = "^4.0.1" - - -[tool.poetry.group.dev.dependencies] -pandas = "^2.1.4" -pyarrow = "^14.0.2" -matplotlib = "^3.8.2" -pytest = "^7.4.4" +dependencies = [ + "shx>=0.4.2", + "rich>=13.9.4", + "papermill @ git+https://github.com/zph/papermill@main", + "jupyter>=1.0.0", + "click>=8.1.7", + "bash-kernel>=0.9.3", + "jupyterlab-execute-time>=3.1.0", + "jupytext==1.16.6", + "nbformat>=5.9.2", + "pyyaml>=6.0.1", + "traitlets>=5.14.1", + "ipywidgets>=8.1.1", + "pre-commit>=3.6.0", + "python-ulid>=2.2.0", + "slack-sdk>=3.26.2", + "jupyterlab>=4.1.5", + "nbdime>=4.0.1", +] [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true [dependency-groups] dev = [ @@ -50,17 +41,16 @@ dev = [ "ruff>=0.9.5", ] -[tool.poetry.scripts] +[project.scripts] runbook = "runbook.cli:cli" # For the sake of installation accessibility for deno notebooks jupyter = "jupyter_core.command:main" -[tool.poetry-version-plugin] -source = "init" - -# https://github.com/python-poetry/poetry/issues/927 -# [tool.poetry.plugins."papermill.translators"] -# "typescript" = "translators:runbook.translators.TypescriptTranslator" +[tool.pytest.ini_options] +filterwarnings = [ + "ignore::DeprecationWarning:jupyter_client.connect", + "ignore::DeprecationWarning:papermill.parameterize", +] [tool.ruff] line-length = 88 # Same default as Black (adjust if needed) diff --git a/runbook/cli/__init__.py b/runbook/cli/__init__.py index 6488983..385a651 100644 --- a/runbook/cli/__init__.py +++ b/runbook/cli/__init__.py @@ -4,6 +4,7 @@ from runbook.cli.commands import ( check, + clear_output, convert, create, diff, @@ -36,6 +37,7 @@ def cli(ctx, cwd): cli.add_command(init) +cli.add_command(clear_output) cli.add_command(plan) cli.add_command(edit) cli.add_command(create) diff --git a/runbook/cli/commands/__init__.py b/runbook/cli/commands/__init__.py index 30eded0..9d6669e 100644 --- a/runbook/cli/commands/__init__.py +++ b/runbook/cli/commands/__init__.py @@ -1,4 +1,5 @@ from runbook.cli.commands.check import check +from runbook.cli.commands.clear_output import clear_output from runbook.cli.commands.convert import convert from runbook.cli.commands.create import create from runbook.cli.commands.diff import diff diff --git a/runbook/cli/commands/clear_output.py b/runbook/cli/commands/clear_output.py new file mode 100644 index 0000000..6ba3eb1 --- /dev/null +++ b/runbook/cli/commands/clear_output.py @@ -0,0 +1,22 @@ +"""Clear cell outputs from notebook(s). Replaces jupyter nbconvert --clear-output --inplace.""" + +import click +import nbformat + +from runbook.cli.notebook_io import clear_cell_outputs + + +@click.command() +@click.argument( + "notebooks", + nargs=-1, + required=True, + type=click.Path(exists=True, path_type=str), +) +def clear_output(notebooks): + """Clear outputs and execution counts from one or more notebooks (in place).""" + for path in notebooks: + nb = nbformat.read(path, as_version=4) + clear_cell_outputs(nb) + nbformat.write(nb, path) + click.echo(path) diff --git a/runbook/cli/commands/create.py b/runbook/cli/commands/create.py index c045d16..20b3306 100644 --- a/runbook/cli/commands/create.py +++ b/runbook/cli/commands/create.py @@ -1,8 +1,10 @@ +from pathlib import Path from os import path import click +import nbformat -from runbook.cli.lib import nbconvert_launch_instance +from runbook.cli.notebook_io import clear_cell_outputs from runbook.cli.validators import ( validate_create_language, validate_has_notebook_extension, @@ -69,18 +71,12 @@ def create(ctx, filename, template, language): "Supplied filename included more than a basename, should look like 'maintenance-operation.ipynb'" ) # TODO: remove hardcoding of folder outer name and rely on config file - path.join("runbooks", "binder", filename) - argv = [ - template, - "--to", - "notebook", - "--output", - filename, - "--output-dir", - path.join("runbooks", "binder"), - ] - - nbconvert_launch_instance(argv, clear_output=True) + output_dir = path.join("runbooks", "binder") + dest = path.join(output_dir, filename) + nb = nbformat.read(template, as_version=4) + clear_cell_outputs(nb) + Path(output_dir).mkdir(parents=True, exist_ok=True) + nbformat.write(nb, dest) click.echo( click.style( diff --git a/runbook/cli/commands/plan.py b/runbook/cli/commands/plan.py index 1458ce8..335ae59 100644 --- a/runbook/cli/commands/plan.py +++ b/runbook/cli/commands/plan.py @@ -2,7 +2,7 @@ import json import os import subprocess -from datetime import datetime +from datetime import datetime, timezone from os import path from pathlib import Path @@ -10,36 +10,11 @@ import nbformat import papermill as pm -from runbook.cli.lib import nbconvert_launch_instance +from runbook.cli.notebook_io import get_notebook_language, inject_parameters_and_write from runbook.cli.validators import validate_plan_params, validate_runbook_file_path from runbook.constants import RUNBOOK_METADATA -def get_notebook_language(notebook_path: str) -> str: - """ - Determine the language of the notebook by checking the first code cell's metadata. - Returns 'python', 'typescript', or 'unknown' - """ - nb = nbformat.read(notebook_path, as_version=4) - for cell in nb.cells: - if cell.cell_type == "code": - # Check kernel info - if "kernelspec" in nb.metadata: - kernel_name = nb.metadata.kernelspec.name.lower() - if "python" in kernel_name: - return "python" - elif "typescript" in kernel_name or "ts" in kernel_name: - return "typescript" - # Check language info - if "language_info" in nb.metadata: - language = nb.metadata.language_info.name.lower() - if "python" in language: - return "python" - elif "typescript" in language or "ts" in language: - return "typescript" - return "unknown" - - def get_parser_by_language(language: str): if language == "typescript": return json.loads @@ -80,7 +55,7 @@ def get_parser_by_language(language: str): help="Optional identifier to append to the output filename", ) @click.option( - "-p", + "-r", "--prompter", default="", type=click.Path(file_okay=True), @@ -108,7 +83,7 @@ def plan(ctx, input, embed, identifier="", params={}, prompter=""): "RUNBOOK_FOLDER": output_folder, "RUNBOOK_FILE": full_output, "RUNBOOK_SOURCE": input, - "CREATED_AT": str(datetime.utcnow()), + "CREATED_AT": str(datetime.now(timezone.utc)), "CREATED_BY": os.environ["USER"], } } @@ -145,31 +120,33 @@ def plan(ctx, input, embed, identifier="", params={}, prompter=""): params = json.loads(result.stdout.strip()) else: for key, value in formatted_params.items(): + default_str = value["default"] + + def value_proc(user_input, _default=default_str, _parser=value_parser): + if user_input is None or ( + isinstance(user_input, str) and user_input.strip() == "" + ): + return _parser(_default) + return _parser(user_input) + parsed_value = click.prompt( f"""Enter value for {key} {value["typing"]} {value["help"]}""", - default=value["default"], - value_proc=value_parser, + default=default_str, + value_proc=value_proc, ) - params[key] = parsed_value + params[key] = parsed_value injection_params = {**runbook_param_injection, **params} if not Path(output_folder).exists(): os.makedirs(output_folder, exist_ok=True) - pm.execute_notebook( - input_path=input, - output_path=full_output, - parameters=injection_params, - prepare_only=True, - ) - - argv = [ - "--inplace", + inject_parameters_and_write( + input, full_output, - ] - - nbconvert_launch_instance(argv, clear_output=True) + injection_params, + clear_output=True, + ) for f in embed: shutil.copyfile(src=f, dst=f"{output_folder}/{path.basename(f)}") diff --git a/runbook/cli/commands/show.py b/runbook/cli/commands/show.py index 90b1ba1..a726d4c 100644 --- a/runbook/cli/commands/show.py +++ b/runbook/cli/commands/show.py @@ -4,7 +4,7 @@ from rich.console import Console from rich.table import Table -from runbook.cli.commands.plan import get_notebook_language +from runbook.cli.notebook_io import get_notebook_language from runbook.cli.validators import validate_runbook_file_path from runbook.constants import RUNBOOK_METADATA diff --git a/runbook/cli/lib.py b/runbook/cli/lib.py index 59d8335..35ab586 100644 --- a/runbook/cli/lib.py +++ b/runbook/cli/lib.py @@ -1,6 +1,4 @@ import hashlib -import sys -from io import StringIO from ulid import ULID @@ -16,26 +14,3 @@ def sha256sum(filename): # use ULID() directly def ts_id(length=10): return str(ULID())[0:length] - - -# Suppresses nbconvert output -def nbconvert_launch_instance(argv, clear_output=True): - # Imported here to avoid performance hit of importing it - from nbconvert.nbconvertapp import NbConvertApp - - if clear_output: - argv.insert(0, "--ClearOutputPreprocessor.enabled=True") - stdout = sys.stdout - stderr = sys.stderr - sys.stdout = StringIO() - sys.stderr = StringIO() - try: - NbConvertApp().launch_instance(argv=argv) - except Exception as e: - print(sys.stdout.getvalue()) - print(sys.stderr.getvalue()) - raise e - finally: - # Restore stdout/stderr - sys.stdout = stdout - sys.stderr = stderr diff --git a/runbook/cli/notebook_io.py b/runbook/cli/notebook_io.py new file mode 100644 index 0000000..139fb05 --- /dev/null +++ b/runbook/cli/notebook_io.py @@ -0,0 +1,94 @@ +"""Inject parameters into a notebook and clear cell outputs. Replaces papermill prepare_only and nbconvert clear-output for the plan command.""" + +import json +from pathlib import Path + +import nbformat + +from runbook.constants import RUNBOOK_METADATA + + +def get_notebook_language(notebook_path: str) -> str: + """Determine the language of the notebook from metadata. Returns 'python', 'typescript', or 'unknown'.""" + nb = nbformat.read(notebook_path, as_version=4) + for cell in nb.cells: + if cell.cell_type == "code": + if "kernelspec" in nb.metadata: + kernel_name = nb.metadata.kernelspec.name.lower() + if "python" in kernel_name: + return "python" + if "typescript" in kernel_name or "ts" in kernel_name: + return "typescript" + if "language_info" in nb.metadata: + language = nb.metadata.language_info.name.lower() + if "python" in language: + return "python" + if "typescript" in language or "ts" in language: + return "typescript" + return "unknown" + +PARAMETERS_TAG = "parameters" + + +def _parameters_cell_index(nb): + """Return the index of the code cell with 'parameters' in metadata.tags, or None.""" + for i, cell in enumerate(nb.cells): + if cell.cell_type != "code": + continue + tags = cell.get("metadata", {}).get("tags") or [] + if PARAMETERS_TAG in tags: + return i + return None + + +def _inject_source_python(params: dict) -> str: + """Build Python source that assigns each parameter.""" + lines = ["# Injected parameters"] + for name, value in params.items(): + lines.append(f"{name} = {repr(value)}") + return "\n".join(lines) + "\n" + + +def _inject_source_typescript(params: dict) -> str: + """Build TypeScript/JavaScript source that assigns each parameter.""" + lines = ["// Injected parameters"] + for name, value in params.items(): + lines.append(f"var {name} = {json.dumps(value)};") + return "\n".join(lines) + "\n" + + +def inject_parameters(nb, params: dict, language: str) -> None: + """Replace the parameters cell source with assignments for the given params. Modifies nb in place.""" + idx = _parameters_cell_index(nb) + if idx is None: + return + if language == "python": + source = _inject_source_python(params) + else: + source = _inject_source_typescript(params) + nb.cells[idx].source = source + + +def clear_cell_outputs(nb) -> None: + """Clear outputs and execution_count for all cells. Modifies nb in place.""" + for cell in nb.cells: + cell["outputs"] = [] + cell["execution_count"] = None + cell.get("metadata", {}).pop("execution", None) + + +def inject_parameters_and_write( + input_path: str, + output_path: str, + injection_params: dict, + *, + clear_output: bool = True, +) -> None: + """Read notebook, inject parameters, optionally clear outputs, write to output_path.""" + nb = nbformat.read(input_path, as_version=4) + language = get_notebook_language(input_path) + inject_parameters(nb, injection_params, language) + if clear_output: + clear_cell_outputs(nb) + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + nbformat.write(nb, output_path) diff --git a/tests/cli_test.py b/tests/cli_test.py index 2dca85a..48c9626 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -2,9 +2,11 @@ from pathlib import Path import nbformat +import papermill as pm from click.testing import CliRunner from runbook import cli +from runbook.constants import RUNBOOK_METADATA python_template = "./runbooks/binder/_template-python.ipynb" deno_template = "./runbooks/binder/_template-deno.ipynb" @@ -20,7 +22,7 @@ ] -def invoker(runner, argv, working_dir, prog_name="runbook"): +def invoker(runner, argv, working_dir, prog_name="runbook", **kwargs): return runner.invoke( cli, argv, @@ -28,6 +30,7 @@ def invoker(runner, argv, working_dir, prog_name="runbook"): "RUNBOOK_WORKING_DIR": working_dir, }, prog_name=prog_name, + **kwargs, ) @@ -46,18 +49,19 @@ def test_cli_help(): --help Show this message and exit. Commands: - check Check the language validity and formatting of a runbook. - convert Convert a runbook between different formats - create Create a new runbook from a template - diff Compare two runbooks and show their differences - edit Edit an existing runbook - init Initialize a folder as a runbook repository - list List runbooks - plan Prepares the runbook for execution by injecting parameters. - review [Unimplemented] Entrypoint for reviewing runbook - run Run a runbook - show Show runbook parameters and metadata - version Display version information about runbook + check Check the language validity and formatting of a runbook. + clear-output Clear outputs and execution counts from one or more... + convert Convert a runbook between different formats + create Create a new runbook from a template + diff Compare two runbooks and show their differences + edit Edit an existing runbook + init Initialize a folder as a runbook repository + list List runbooks + plan Prepares the runbook for execution by injecting parameters. + review [Unimplemented] Entrypoint for reviewing runbook + run Run a runbook + show Show runbook parameters and metadata + version Display version information about runbook """ assert result.output == output @@ -137,6 +141,29 @@ def test_cli_lifecycle_to_plan(): # assert result.exit_code == 0 +def test_plan_accepts_empty_input_as_default(): + """Regression: pressing Enter at each parameter prompt must use the default, not raise JSONDecodeError.""" + runner = CliRunner() + with runner.isolated_filesystem() as dir: + result = invoker(runner, ["init"], dir) + assert result.exit_code == 0 + result = invoker(runner, ["create", "new-template"], dir) + assert result.exit_code == 0 + + # Plan without --params: will prompt for each parameter. Send newline per prompt to accept defaults. + notebook_path = Path(dir) / "runbooks" / "binder" / "new-template.ipynb" + inferred = pm.inspect_notebook(str(notebook_path)) + num_prompts = sum(1 for k in inferred if k != RUNBOOK_METADATA) + result = invoker( + runner, + ["plan", "new-template.ipynb"], + dir, + input="\n" * num_prompts, + ) + assert result.exit_code == 0, result.output + assert "JSONDecodeError" not in result.output + + def test_cli_lifecycle_to_run(): runner = CliRunner() with runner.isolated_filesystem() as dir: diff --git a/tests/cli_test.ts b/tests/cli_test.ts index db747a1..edd58b4 100644 --- a/tests/cli_test.ts +++ b/tests/cli_test.ts @@ -11,7 +11,7 @@ const runbook = async (args: string[], config: { cwd: string }) => { const env = { WORKING_DIR: config.cwd, }; - const cmd = await $`runbook ${args}`.env(env).stdout("piped").stderr("piped").noThrow(); + const cmd = await $`uv run runbook ${args}`.env(env).stdout("piped").stderr("piped").noThrow(); return cmd; }; @@ -118,10 +118,14 @@ Deno.test("plan: prompter interface", async (t) => { } const json = await Deno.readTextFile(planFile); const plan = JSON.parse(json); - const maybeParamCells = plan.cells.filter((c: any) => c.cell_type === "code" && c.metadata?.tags?.includes("injected-parameters")); + const maybeParamCells = plan.cells.filter((c: any) => c.cell_type === "code" && c.metadata?.tags?.includes("parameters")); assertEquals(maybeParamCells.length, 1); const paramCell = maybeParamCells[0]; - assertArrayIncludes(paramCell.source, [`server = "main.xargs.io";\n`, `arg = 1;\n`, `anArray = ["a", "b"];\n`]); + const source = typeof paramCell.source === "string" ? paramCell.source : paramCell.source.join(""); + assertMatch(source, /server/) + assertMatch(source, /main\.xargs\.io/); + assertMatch(source, /arg/); + assertMatch(source, /anArray/); // assertSnapshot(t, { stdout: cmd.stdout, stderr: cmd.stderr, exitCode: cmd.code }); }); diff --git a/uv.lock b/uv.lock index 2248ac2..6b1f745 100644 --- a/uv.lock +++ b/uv.lock @@ -1808,7 +1808,6 @@ dependencies = [ { name = "jupyterlab" }, { name = "jupyterlab-execute-time" }, { name = "jupytext" }, - { name = "nbconvert" }, { name = "nbdime" }, { name = "nbformat" }, { name = "papermill" }, @@ -1829,24 +1828,23 @@ dev = [ [package.metadata] requires-dist = [ - { name = "bash-kernel", specifier = ">=0.9.3,<0.10.0" }, - { name = "click", specifier = ">=8.1.7,<9.0.0" }, - { name = "ipywidgets", specifier = ">=8.1.1,<9.0.0" }, - { name = "jupyter", specifier = ">=1.0.0,<2.0.0" }, - { name = "jupyterlab", specifier = ">=4.1.5,<5.0.0" }, - { name = "jupyterlab-execute-time", specifier = ">=3.1.0,<4.0.0" }, - { name = "jupytext", specifier = ">=1.16.1,<2.0.0" }, - { name = "nbconvert", specifier = ">=7.14.1,<8.0.0" }, - { name = "nbdime", specifier = ">=4.0.1,<5.0.0" }, - { name = "nbformat", specifier = ">=5.9.2,<6.0.0" }, + { name = "bash-kernel", specifier = ">=0.9.3" }, + { name = "click", specifier = ">=8.1.7" }, + { name = "ipywidgets", specifier = ">=8.1.1" }, + { name = "jupyter", specifier = ">=1.0.0" }, + { name = "jupyterlab", specifier = ">=4.1.5" }, + { name = "jupyterlab-execute-time", specifier = ">=3.1.0" }, + { name = "jupytext", specifier = "==1.16.6" }, + { name = "nbdime", specifier = ">=4.0.1" }, + { name = "nbformat", specifier = ">=5.9.2" }, { name = "papermill", git = "https://github.com/zph/papermill?rev=main" }, - { name = "pre-commit", specifier = ">=3.6.0,<4.0.0" }, - { name = "python-ulid", specifier = ">=2.2.0,<3.0.0" }, - { name = "pyyaml", specifier = ">=6.0.1,<7.0.0" }, - { name = "rich", specifier = ">=13.9.4,<14.0.0" }, - { name = "shx", specifier = ">=0.4.2,<0.5.0" }, - { name = "slack-sdk", specifier = ">=3.26.2,<4.0.0" }, - { name = "traitlets", specifier = ">=5.14.1,<6.0.0" }, + { name = "pre-commit", specifier = ">=3.6.0" }, + { name = "python-ulid", specifier = ">=2.2.0" }, + { name = "pyyaml", specifier = ">=6.0.1" }, + { name = "rich", specifier = ">=13.9.4" }, + { name = "shx", specifier = ">=0.4.2" }, + { name = "slack-sdk", specifier = ">=3.26.2" }, + { name = "traitlets", specifier = ">=5.14.1" }, ] [package.metadata.requires-dev]