diff --git a/.ado/publish.yml b/.ado/publish.yml index ed4d4faa95e..f0a3d662dd3 100644 --- a/.ado/publish.yml +++ b/.ado/publish.yml @@ -469,6 +469,10 @@ extends: displayName: Build Platform-Dependent Py Packages (Other) condition: not(and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['arch'], 'aarch64'))) + - script: | + python ./build.py --no-check --no-check-prereqs --course-notebook-tests + displayName: Test course notebooks + - script: | dir target\wheels\* displayName: List Py Packages on Win diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 142fc00f601..4a5a1f836a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,6 +220,54 @@ jobs: # which requires a display to run run: xvfb-run -a python ./build.py --no-check --no-test --wasm --npm --vscode --widgets --qdk --integration-tests + course-notebook-changes: + name: Detect course notebook changes + runs-on: ubuntu-latest + outputs: + run-tests: ${{ steps.changes.outputs.run-tests }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: github.event_name != 'workflow_dispatch' + with: + fetch-depth: 0 + - name: Determine whether to run course notebook tests + id: changes + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + run: | + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + run_tests=true + elif git diff --quiet "$BASE_SHA...$HEAD_SHA" -- \ + source/vscode/resources/qdk-learning/courses/ \ + source/vscode/test/course-notebooks/; then + run_tests=false + else + run_tests=true + fi + echo "run-tests=$run_tests" >> "$GITHUB_OUTPUT" + + course-notebook-tests: + name: Course notebooks + needs: course-notebook-changes + if: needs.course-notebook-changes.outputs.run-tests == 'true' + timeout-minutes: 45 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: | + source/vscode/test/course-notebooks/test_requirements.txt + source/vscode/resources/qdk-learning/courses/*/requirements.txt + - name: Test course notebooks + run: python ./build.py --no-check --no-check-prereqs --course-notebook-tests + status-check: name: Status Check needs: @@ -233,6 +281,8 @@ jobs: unit-tests, format-qsc, integration-tests, + course-notebook-changes, + course-notebook-tests, ] runs-on: ubuntu-latest if: failure() diff --git a/.github/workflows/multiplat.yml b/.github/workflows/multiplat.yml index 24a0b2eb6c9..e0598c3f0e6 100644 --- a/.github/workflows/multiplat.yml +++ b/.github/workflows/multiplat.yml @@ -59,6 +59,8 @@ jobs: - name: Build and Test run: python ./build.py --integration-tests if: runner.os != 'Linux' + - name: Test course notebooks + run: python ./build.py --no-check --no-check-prereqs --course-notebook-tests - name: File issue on failure if: ${{ failure() && github.event_name != 'workflow_dispatch' }} id: create-issue diff --git a/build.py b/build.py index 285f026aa78..026bab13e3e 100755 --- a/build.py +++ b/build.py @@ -76,6 +76,13 @@ help="Build and run the integration tests (default is --no-integration-tests)", ) +parser.add_argument( + "--course-notebook-tests", + action=argparse.BooleanOptionalAction, + default=False, + help="Run the QDK learning course notebook tests (default is --no-course-notebook-tests)", +) + parser.add_argument( "--ci-bench", action=argparse.BooleanOptionalAction, @@ -105,18 +112,25 @@ args = parser.parse_args() -# If no specific project given then build all -build_all = ( - not args.cli - and not args.widgets - and not args.qdk - and not args.wasm - and not args.npm - and not args.play - and not args.vscode - and not args.jupyterlab - and not args.ci_bench +specific_project_requested = any( + ( + args.cli, + args.widgets, + args.qdk, + args.wasm, + args.npm, + args.play, + args.vscode, + args.jupyterlab, + ) +) + +standalone_action_requested = args.course_notebook_tests or args.ci_bench + +build_all = not specific_project_requested and ( + args.integration_tests or not standalone_action_requested ) + build_cli = build_all or args.cli build_widgets = build_all or args.widgets build_qdk = build_all or args.qdk @@ -171,6 +185,9 @@ def step_end(): raw_wheels_dir = os.path.join(root_dir, "target", "raw_wheels") vscode_src = os.path.join(qdk_src_dir, "vscode") jupyterlab_src = os.path.join(qdk_src_dir, "jupyterlab") +course_notebook_tests_dir = os.path.join( + qdk_src_dir, "vscode", "test", "course-notebooks" +) QISKIT_VERSION_MATRIX = [ { @@ -445,12 +462,12 @@ def install_python_test_requirements(cwd, interpreter, check: bool = True): subprocess.run(command_args, check=check, text=True, cwd=cwd) -def run_python_tests(cwd, interpreter, pip_env): +def run_python_tests(cwd, interpreter, pip_env, *pytest_args): test_env = pip_env.copy() if args.gpu_tests: test_env["QDK_GPU_TESTS"] = "1" - command_args = [interpreter, "-m", "pytest"] + command_args = [interpreter, "-m", "pytest", *pytest_args] subprocess.run(command_args, check=True, text=True, cwd=cwd, env=test_env) @@ -590,6 +607,19 @@ def run_ci_historic_benchmark(): step_end() +if args.course_notebook_tests: + python_bin, pip_env = use_python_env(course_notebook_tests_dir) + + step_start("Installing course notebook test requirements") + install_python_test_requirements(course_notebook_tests_dir, python_bin) + step_end() + + step_start("Testing course notebooks") + # Suppress output capturing since it's useful to have the timing information for successful runs too + run_python_tests(course_notebook_tests_dir, python_bin, pip_env, "-v", "-s") + step_end() + + if build_widgets: step_start("Building the Python widgets") diff --git a/source/vscode/resources/qdk-learning/courses/chemistry-qpe/06-iterative-phase-estimation/iterative_phase_estimation.ipynb b/source/vscode/resources/qdk-learning/courses/chemistry-qpe/06-iterative-phase-estimation/iterative_phase_estimation.ipynb index 380339c7d64..9d5c6e49fdc 100644 --- a/source/vscode/resources/qdk-learning/courses/chemistry-qpe/06-iterative-phase-estimation/iterative_phase_estimation.ipynb +++ b/source/vscode/resources/qdk-learning/courses/chemistry-qpe/06-iterative-phase-estimation/iterative_phase_estimation.ipynb @@ -322,7 +322,11 @@ "cell_type": "code", "id": "c-0a2a3fcf99d8", "execution_count": null, - "metadata": {}, + "metadata": { + "tags": [ + "skip-test" + ] + }, "outputs": [], "source": "iqpe_result = run_iqpe_workflow()\nprint_iqpe_results(iqpe_result)" }, diff --git a/source/vscode/resources/qdk-learning/courses/chemistry-qpe/_course_lib.py b/source/vscode/resources/qdk-learning/courses/chemistry-qpe/_course_lib.py index 33e7ae86d9b..36280b65d8c 100644 --- a/source/vscode/resources/qdk-learning/courses/chemistry-qpe/_course_lib.py +++ b/source/vscode/resources/qdk-learning/courses/chemistry-qpe/_course_lib.py @@ -16,7 +16,6 @@ from typing import Callable -from IPython.core.getipython import get_ipython from IPython.display import HTML, display # A learner exercise is a no-argument function whose result is checked. @@ -32,21 +31,8 @@ class ExerciseError(AssertionError): """Raised when an exercise is not yet correct.""" - -def _hide_traceback() -> None: - """Show only the failure banner for a wrong answer. - - The cell still ends in an error, which marks the exercise as incomplete, - but real errors raised by learner code keep their traceback. - """ - shell = get_ipython() - if shell is None: - return - - shell.set_custom_exc((ExerciseError,), lambda *args, **kwargs: None) - - -_hide_traceback() + def _render_traceback_(self) -> list[str]: + return [] def _register(name: str, checker: Checker) -> str: diff --git a/source/vscode/resources/qdk-learning/utils/chemistry-qpe/rst_to_notebook.py b/source/vscode/resources/qdk-learning/utils/chemistry-qpe/rst_to_notebook.py index e910e6de41b..9df3c9e5292 100644 --- a/source/vscode/resources/qdk-learning/utils/chemistry-qpe/rst_to_notebook.py +++ b/source/vscode/resources/qdk-learning/utils/chemistry-qpe/rst_to_notebook.py @@ -797,7 +797,7 @@ "section": "The complete workflow", "cells": [ ( - "code", + "skip-test", "iqpe_result = run_iqpe_workflow()\n" "print_iqpe_results(iqpe_result)\n", ), @@ -1773,7 +1773,12 @@ def splice(section, block): cells[end:end] = block owner[end:end] = [section] * len(block) - kinds = {"md": md, "code": code, "region": lambda v: code(notebook_region(v))} + kinds = { + "md": md, + "code": code, + "skip-test": lambda value: code(value, tags=["skip-test"]), + "region": lambda value: code(notebook_region(value)), + } for spec in recipe.get("inserts", []): splice(spec["section"], [kinds[k](v) for k, v in spec["cells"]]) diff --git a/source/vscode/src/learning/notebookExercises.ts b/source/vscode/src/learning/notebookExercises.ts index fd4dde50f75..c6889245fc2 100644 --- a/source/vscode/src/learning/notebookExercises.ts +++ b/source/vscode/src/learning/notebookExercises.ts @@ -42,6 +42,9 @@ const EXERCISE_TAG = "exercise"; /** Tags marking author-only cells, removed from the learner's working copy. */ const AUTHORING_TAGS = ["hint", "solution", "explanation"] as const; +/** Test-only tag removed from cells in the learner's working copy. */ +const SKIP_TEST_TAG = "skip-test"; + type AuthoringTag = (typeof AUTHORING_TAGS)[number]; /** The subset of an nbformat cell this module reads. */ @@ -190,10 +193,10 @@ export function parseNotebookActivities( * Remove the author-only cells from a notebook's JSON text, returning the * notebook the learner works in. * - * Everything else — including cell ids and the `exercise` tag — is preserved - * verbatim, so metadata parsed from the authored notebook still resolves - * against the working copy. Returns `undefined` if the text isn't a notebook, - * leaving the caller to decide on a fallback. + * Retained cells preserve their content, ids, and other tags, including the + * `exercise` tag. The test-only `skip-test` tag is removed so it doesn't leak + * into the learner's working copy. Returns `undefined` if the text isn't a + * notebook, leaving the caller to decide on a fallback. */ export function stripAuthoringCells( text: string, @@ -209,6 +212,13 @@ export function stripAuthoringCells( return !AUTHORING_TAGS.some((t) => tags.includes(t)); }); + for (const cell of notebook.cells) { + const metadata = cell.metadata; + if (metadata && Array.isArray(metadata.tags)) { + metadata.tags = metadata.tags.filter((tag) => tag !== SKIP_TEST_TAG); + } + } + // Match the ipynb serializer's formatting so the file stays diff-stable // once VS Code starts saving it: one space of indent, trailing newline. return `${JSON.stringify(notebook, undefined, 1)}\n`; diff --git a/source/vscode/test/course-notebooks/README.md b/source/vscode/test/course-notebooks/README.md new file mode 100644 index 00000000000..3e6eaf414e5 --- /dev/null +++ b/source/vscode/test/course-notebooks/README.md @@ -0,0 +1,49 @@ +# Course notebook tests + +These tests execute the source notebooks for one QDK learning course in fresh +Python kernels. The notebooks run from one shared copy of the full course +directory so relative imports work, generated files do not modify the source +tree, and later notebooks see filesystem state produced by earlier notebooks. +The suite also verifies that each notebook corresponds one-to-one with a unit +directory listed in the course's `course.json`. + +Pytest creates a `.venv` inside the temporary course copy and installs the +course requirements there. Notebook kernels use that environment, while pytest +continues to use the component's test environment. + +## Local setup + +Run the course suite from the repository root with Python 3.11 or later: + +```shell +python ./build.py --no-check --no-check-prereqs --course-notebook-tests +``` + +Like `--integration-tests`, `--course-notebook-tests` runs independently of +the regular `--test`/`--no-test` option. + +Following the other Python test suites, `build.py` uses an active Python +environment when available. Otherwise, it creates +`source/vscode/test/course-notebooks/.venv` and installs the test requirements +there. + +Run only the fast runner policy tests without creating a course environment: + +```shell +source/vscode/test/course-notebooks/.venv/bin/python -m pytest source/vscode/test/course-notebooks/test_notebook_runner.py -v +``` + +## Cell metadata + +- An `exercise` code cell must raise `ExerciseError`. Any other exception, or + successful execution, fails the test. +- A `solution` code cell must execute without an error. +- A `skip-test` code cell is skipped only by this test suite. VS Code, Jupyter, + and ordinary notebook execution do not interpret this custom tag. It also + takes precedence when combined with `exercise`. + +When a skipped setup cell supplies state to later cells, tag those dependent +cells with `skip-test` as well. + +Each cell has a 120-second timeout. Cells taking longer than 30 seconds are +reported so expensive cells can be reviewed before adding a skip. diff --git a/source/vscode/test/course-notebooks/conftest.py b/source/vscode/test/course-notebooks/conftest.py new file mode 100644 index 00000000000..4437ef1f8f4 --- /dev/null +++ b/source/vscode/test/course-notebooks/conftest.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import venv +from pathlib import Path + +import pytest + +from notebook_runner import discover_notebooks + +REPO_ROOT = Path(__file__).resolve().parents[4] +COURSES_ROOT = REPO_ROOT / "source/vscode/resources/qdk-learning/courses" + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + course_dirs = _course_dirs() + if "course_dir" in metafunc.fixturenames: + metafunc.parametrize( + "course_dir", + course_dirs, + ids=[path.name for path in course_dirs], + ) + + if "course_notebook" not in metafunc.fixturenames: + return + + notebooks = [ + notebook + for course_dir in course_dirs + for notebook in discover_notebooks(course_dir) + ] + if not notebooks: + raise pytest.UsageError(f"no source notebooks found under {COURSES_ROOT}") + metafunc.parametrize( + "course_notebook", + notebooks, + ids=[str(path.relative_to(COURSES_ROOT)) for path in notebooks], + ) + + +@pytest.fixture(scope="session") +def copied_course_dirs( + tmp_path_factory: pytest.TempPathFactory, +) -> dict[Path, Path]: + copies_root = tmp_path_factory.mktemp("course-notebooks") + copied_course_dirs = {} + for course_dir in _course_dirs(): + copied_course_dir = copies_root / course_dir.name + shutil.copytree(course_dir, copied_course_dir) + _create_notebook_environment(copied_course_dir) + copied_course_dirs[course_dir] = copied_course_dir + return copied_course_dirs + + +@pytest.fixture +def copied_course_notebook( + course_notebook: Path, + copied_course_dirs: dict[Path, Path], +) -> tuple[Path, Path, Path]: + course_name = course_notebook.relative_to(COURSES_ROOT).parts[0] + course_dir = COURSES_ROOT / course_name + copied_course_dir = copied_course_dirs[course_dir] + copied_notebook = copied_course_dir / course_notebook.relative_to(course_dir) + kernel_specs_dir = copied_course_dir / ".venv" / "share" / "jupyter" / "kernels" + return ( + copied_notebook, + course_notebook.relative_to(REPO_ROOT), + kernel_specs_dir, + ) + + +def _create_notebook_environment(course_dir: Path) -> None: + venv_dir = course_dir / ".venv" + venv.create(venv_dir, with_pip=True) + python = venv_dir / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + subprocess.run( + [ + python, + "-m", + "pip", + "install", + "--quiet", + "-r", + course_dir / "requirements.txt", + ], + check=True, + ) + + kernel_dir = venv_dir / "share/jupyter/kernels/python3" + kernel_dir.mkdir(parents=True, exist_ok=True) + kernel_spec = { + "argv": [str(python), "-m", "ipykernel_launcher", "-f", "{connection_file}"], + "display_name": "Course notebook tests", + "language": "python", + } + (kernel_dir / "kernel.json").write_text( + json.dumps(kernel_spec, indent=2), + encoding="utf-8", + ) + + +def _course_dirs() -> list[Path]: + return sorted(path for path in COURSES_ROOT.iterdir() if path.is_dir()) diff --git a/source/vscode/test/course-notebooks/notebook_runner.py b/source/vscode/test/course-notebooks/notebook_runner.py new file mode 100644 index 00000000000..b72c228a907 --- /dev/null +++ b/source/vscode/test/course-notebooks/notebook_runner.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from time import perf_counter + +import nbformat +from jupyter_client.manager import AsyncKernelManager +from jupyter_client.kernelspec import KernelSpecManager +from nbclient import NotebookClient +from nbformat import NotebookNode + +CELL_TIMEOUT_SECONDS = 120 +SLOW_CELL_SECONDS = 30 +SKIP_TEST_TAG = "skip-test" +EXERCISE_TAG = "exercise" + + +@dataclass(frozen=True) +class CellFailure: + cell_number: int + source_line: str + message: str + + +@dataclass(frozen=True) +class SlowCell: + cell_number: int + source_line: str + duration_seconds: float + + +@dataclass(frozen=True) +class NotebookRunReport: + notebook_path: Path + elapsed_seconds: float + executed_cells: int + skipped_cells: tuple[int, ...] + slow_cells: tuple[SlowCell, ...] + failures: tuple[CellFailure, ...] + + def format_failures(self) -> str: + return "\n".join( + f"{self.notebook_path}: cell {failure.cell_number} " + f"({failure.source_line}): {failure.message}" + for failure in self.failures + ) + + +def discover_notebooks(course_dir: Path) -> list[Path]: + return sorted( + path + for path in course_dir.rglob("*.ipynb") + if not path.name.endswith(".workbook.ipynb") + ) + + +def clear_notebook_outputs(notebook: NotebookNode) -> None: + for cell in notebook.cells: + if cell.cell_type != "code": + continue + cell.outputs = [] + cell.execution_count = None + cell.metadata.pop("execution", None) + + +def collect_cell_failures(notebook: NotebookNode) -> list[CellFailure]: + failures: list[CellFailure] = [] + for cell_number, cell in enumerate(notebook.cells, start=1): + if cell.cell_type != "code": + continue + + tags = set(cell.metadata.get("tags", [])) + source_line = _first_source_line(cell.source) + if SKIP_TEST_TAG in tags: + continue + + errors = [ + output + for output in cell.get("outputs", []) + if output.get("output_type") == "error" + ] + if EXERCISE_TAG in tags: + if not errors: + failures.append( + CellFailure( + cell_number, + source_line, + "exercise cell did not raise ExerciseError", + ) + ) + elif errors[0].get("ename") != "ExerciseError": + failures.append( + CellFailure( + cell_number, + source_line, + "exercise cell raised " + _format_error(errors[0]), + ) + ) + continue + + failures.extend( + CellFailure( + cell_number, + source_line, + "unexpected error: " + _format_error(error), + ) + for error in errors + ) + return failures + + +def run_notebook( + notebook_path: Path, + display_path: Path, + kernel_specs_dir: Path, +) -> NotebookRunReport: + notebook = nbformat.read(notebook_path, as_version=4) + clear_notebook_outputs(notebook) + + started = perf_counter() + kernel_manager = AsyncKernelManager( + kernel_name="python3", + kernel_spec_manager=KernelSpecManager( + kernel_dirs=[str(kernel_specs_dir)], + ), + ) + NotebookClient( + notebook, + km=kernel_manager, + timeout=CELL_TIMEOUT_SECONDS, + allow_errors=True, + kernel_name="python3", + resources={"metadata": {"path": str(notebook_path.parent)}}, + skip_cells_with_tag=SKIP_TEST_TAG, + store_widget_state=False, + ).execute(cleanup_kc=True) + elapsed_seconds = perf_counter() - started + + skipped_cells = tuple( + cell_number + for cell_number, cell in enumerate(notebook.cells, start=1) + if cell.cell_type == "code" and SKIP_TEST_TAG in cell.metadata.get("tags", []) + ) + # Cells that are slow enough to consider skipping, but not slow enough to fail with a timeout. + # Reported as a convenience for test authors. + slow_cells = tuple( + slow_cell + for cell_number, cell in enumerate(notebook.cells, start=1) + if (slow_cell := _slow_cell(cell_number, cell)) is not None + ) + executed_cells = sum( + 1 + for cell in notebook.cells + if cell.cell_type == "code" + and SKIP_TEST_TAG not in cell.metadata.get("tags", []) + ) + report = NotebookRunReport( + display_path, + elapsed_seconds, + executed_cells, + skipped_cells, + slow_cells, + tuple(collect_cell_failures(notebook)), + ) + print_notebook_report(report) + return report + + +def print_notebook_report(report: NotebookRunReport) -> None: + print( + f"{report.notebook_path}: {report.elapsed_seconds:.1f}s, " + f"{report.executed_cells} executed, {len(report.skipped_cells)} skipped" + ) + for cell_number in report.skipped_cells: + print(f" skipped cell {cell_number} ({SKIP_TEST_TAG})") + for cell in report.slow_cells: + print( + f" slow cell {cell.cell_number}: {cell.duration_seconds:.1f}s " + f"({cell.source_line})" + ) + + +def _first_source_line(source: str) -> str: + for line in source.splitlines(): + if stripped := line.strip(): + return stripped + return "" + + +def _format_error(error: NotebookNode) -> str: + name = error.get("ename", "Error") + value = error.get("evalue", "") + return f"{name}: {value}" if value else name + + +def _slow_cell(cell_number: int, cell: NotebookNode) -> SlowCell | None: + if cell.cell_type != "code": + return None + execution = cell.metadata.get("execution", {}) + started = execution.get("iopub.status.busy") + finished = execution.get("iopub.status.idle") + if not started or not finished: + return None + duration = _parse_timestamp(finished) - _parse_timestamp(started) + duration_seconds = duration.total_seconds() + if duration_seconds <= SLOW_CELL_SECONDS: + return None + return SlowCell( + cell_number, + _first_source_line(cell.source), + duration_seconds, + ) + + +def _parse_timestamp(value: str) -> datetime: + return datetime.fromisoformat(value) diff --git a/source/vscode/test/course-notebooks/test_course_notebooks.py b/source/vscode/test/course-notebooks/test_course_notebooks.py new file mode 100644 index 00000000000..46ac9d23a78 --- /dev/null +++ b/source/vscode/test/course-notebooks/test_course_notebooks.py @@ -0,0 +1,38 @@ +import json +from collections import Counter +from pathlib import Path + +from notebook_runner import discover_notebooks, run_notebook + + +def test_discovered_notebooks_match_course_manifest(course_dir: Path) -> None: + manifest = json.loads((course_dir / "course.json").read_text(encoding="utf-8")) + manifest_unit_dirs = [Path(unit["dir"]) for unit in manifest["units"]] + discovered_notebooks = discover_notebooks(course_dir) + discovered_unit_dirs = [ + notebook.relative_to(course_dir).parent for notebook in discovered_notebooks + ] + + manifest_counts = Counter(manifest_unit_dirs) + discovered_counts = Counter(discovered_unit_dirs) + missing = list((manifest_counts - discovered_counts).elements()) + unmatched = [ + notebook.relative_to(course_dir) + for notebook in discovered_notebooks + if discovered_counts[notebook.relative_to(course_dir).parent] + > manifest_counts[notebook.relative_to(course_dir).parent] + ] + + assert manifest_counts == discovered_counts, ( + f"{course_dir / 'course.json'} does not match discovered notebooks; " + f"missing notebooks for units: {missing}; " + f"unmatched notebooks: {unmatched}" + ) + + +def test_course_notebook(copied_course_notebook: tuple[Path, Path, Path]) -> None: + notebook_path, display_path, kernel_specs_dir = copied_course_notebook + + report = run_notebook(notebook_path, display_path, kernel_specs_dir) + + assert not report.failures, report.format_failures() diff --git a/source/vscode/test/course-notebooks/test_notebook_runner.py b/source/vscode/test/course-notebooks/test_notebook_runner.py new file mode 100644 index 00000000000..9745acea72d --- /dev/null +++ b/source/vscode/test/course-notebooks/test_notebook_runner.py @@ -0,0 +1,80 @@ +from collections.abc import Iterable + +import nbformat +from nbformat import NotebookNode + +from notebook_runner import collect_cell_failures + + +def _notebook(*cells: NotebookNode) -> NotebookNode: + return nbformat.v4.new_notebook(cells=list(cells)) + + +def _code_cell( + *, + tags: Iterable[str] = (), + error: tuple[str, str] | None = None, +) -> NotebookNode: + cell = nbformat.v4.new_code_cell("answer = 42", metadata={"tags": list(tags)}) + if error is not None: + name, value = error + cell.outputs = [ + nbformat.v4.new_output( + "error", + ename=name, + evalue=value, + traceback=[], + ) + ] + return cell + + +def test_exercise_requires_exercise_error() -> None: + notebook = _notebook( + _code_cell(tags=["exercise"], error=("ExerciseError", "try again")) + ) + + assert collect_cell_failures(notebook) == [] + + +def test_exercise_that_succeeds_fails_policy() -> None: + notebook = _notebook(_code_cell(tags=["exercise"])) + + failures = collect_cell_failures(notebook) + + assert len(failures) == 1 + assert failures[0].message == "exercise cell did not raise ExerciseError" + + +def test_exercise_with_wrong_error_fails_policy() -> None: + notebook = _notebook( + _code_cell(tags=["exercise"], error=("ValueError", "bad value")) + ) + + failures = collect_cell_failures(notebook) + + assert len(failures) == 1 + assert failures[0].message == "exercise cell raised ValueError: bad value" + + +def test_ordinary_cell_error_fails_policy() -> None: + notebook = _notebook(_code_cell(error=("RuntimeError", "broken"))) + + failures = collect_cell_failures(notebook) + + assert len(failures) == 1 + assert failures[0].message == "unexpected error: RuntimeError: broken" + + +def test_skip_test_cell_is_not_evaluated() -> None: + notebook = _notebook( + _code_cell(tags=["skip-test"], error=("RuntimeError", "ignored")) + ) + + assert collect_cell_failures(notebook) == [] + + +def test_skipped_exercise_is_not_evaluated() -> None: + notebook = _notebook(_code_cell(tags=["exercise", "skip-test"])) + + assert collect_cell_failures(notebook) == [] diff --git a/source/vscode/test/course-notebooks/test_requirements.txt b/source/vscode/test/course-notebooks/test_requirements.txt new file mode 100644 index 00000000000..4386aee0934 --- /dev/null +++ b/source/vscode/test/course-notebooks/test_requirements.txt @@ -0,0 +1,3 @@ +nbclient +nbformat +pytest