Test notebook courses

.ado/publish.yml

@@ -469,6 +469,10 @@ extends:
                     displayName: Build Platform-Dependent Py Packages (Other)
                     condition: not(and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['arch'], 'aarch64')))
 
+                  - script: |
+                      python ./build.py --no-check --no-check-prereqs --course-notebook-tests
+                    displayName: Test course notebooks
+
                   - script: |
                       dir target\wheels\*
                     displayName: List Py Packages on Win

.github/workflows/ci.yml

@@ -220,6 +220,54 @@ jobs:
           # which requires a display to run
         run: xvfb-run -a python ./build.py --no-check --no-test --wasm --npm --vscode --widgets --qdk --integration-tests
 
+  course-notebook-changes:
+    name: Detect course notebook changes
+    runs-on: ubuntu-latest
+    outputs:
+      run-tests: ${{ steps.changes.outputs.run-tests }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        if: github.event_name != 'workflow_dispatch'
+        with:
+          fetch-depth: 0
+      - name: Determine whether to run course notebook tests
+        id: changes
+        shell: bash
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+        run: |
+          if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
+            run_tests=true
+          elif git diff --quiet "$BASE_SHA...$HEAD_SHA" -- \
+            source/vscode/resources/qdk-learning/courses/ \
+            source/vscode/test/course-notebooks/; then
+            run_tests=false
+          else
+            run_tests=true
+          fi
+          echo "run-tests=$run_tests" >> "$GITHUB_OUTPUT"
+
+  course-notebook-tests:
+    name: Course notebooks
+    needs: course-notebook-changes
+    if: needs.course-notebook-changes.outputs.run-tests == 'true'
+    timeout-minutes: 45
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: |
+            source/vscode/test/course-notebooks/test_requirements.txt
+            source/vscode/resources/qdk-learning/courses/*/requirements.txt
+      - name: Test course notebooks
+        run: python ./build.py --no-check --no-check-prereqs --course-notebook-tests
+
   status-check:
     name: Status Check
     needs:
@@ -233,6 +281,8 @@ jobs:
         unit-tests,
         format-qsc,
         integration-tests,
+        course-notebook-changes,
+        course-notebook-tests,
       ]
     runs-on: ubuntu-latest
     if: failure()

.github/workflows/multiplat.yml

@@ -59,6 +59,8 @@ jobs:
       - name: Build and Test
         run: python ./build.py --integration-tests
         if: runner.os != 'Linux'
+      - name: Test course notebooks
+        run: python ./build.py --no-check --no-check-prereqs --course-notebook-tests
       - name: File issue on failure
         if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
         id: create-issue

build.py

@@ -76,6 +76,13 @@
     help="Build and run the integration tests (default is --no-integration-tests)",
 )
 
+parser.add_argument(
+    "--course-notebook-tests",
+    action=argparse.BooleanOptionalAction,
+    default=False,
+    help="Run the QDK learning course notebook tests (default is --no-course-notebook-tests)",
+)
+
 parser.add_argument(
     "--ci-bench",
     action=argparse.BooleanOptionalAction,
@@ -105,18 +112,25 @@
 
 args = parser.parse_args()
 
-# If no specific project given then build all
-build_all = (
-    not args.cli
-    and not args.widgets
-    and not args.qdk
-    and not args.wasm
-    and not args.npm
-    and not args.play
-    and not args.vscode
-    and not args.jupyterlab
-    and not args.ci_bench
+specific_project_requested = any(
+    (
+        args.cli,
+        args.widgets,
+        args.qdk,
+        args.wasm,
+        args.npm,
+        args.play,
+        args.vscode,
+        args.jupyterlab,
+    )
+)
+
+standalone_action_requested = args.course_notebook_tests or args.ci_bench
+
+build_all = not specific_project_requested and (
+    args.integration_tests or not standalone_action_requested
 )
+
 build_cli = build_all or args.cli
 build_widgets = build_all or args.widgets
 build_qdk = build_all or args.qdk
@@ -171,6 +185,9 @@ def step_end():
 raw_wheels_dir = os.path.join(root_dir, "target", "raw_wheels")
 vscode_src = os.path.join(qdk_src_dir, "vscode")
 jupyterlab_src = os.path.join(qdk_src_dir, "jupyterlab")
+course_notebook_tests_dir = os.path.join(
+    qdk_src_dir, "vscode", "test", "course-notebooks"
+)
 
 QISKIT_VERSION_MATRIX = [
     {
@@ -445,12 +462,12 @@ def install_python_test_requirements(cwd, interpreter, check: bool = True):
         subprocess.run(command_args, check=check, text=True, cwd=cwd)
 
 
-def run_python_tests(cwd, interpreter, pip_env):
+def run_python_tests(cwd, interpreter, pip_env, *pytest_args):
     test_env = pip_env.copy()
     if args.gpu_tests:
         test_env["QDK_GPU_TESTS"] = "1"
 
-    command_args = [interpreter, "-m", "pytest"]
+    command_args = [interpreter, "-m", "pytest", *pytest_args]
     subprocess.run(command_args, check=True, text=True, cwd=cwd, env=test_env)
 
 
@@ -590,6 +607,19 @@ def run_ci_historic_benchmark():
             step_end()
 
 
+if args.course_notebook_tests:
+    python_bin, pip_env = use_python_env(course_notebook_tests_dir)
+
+    step_start("Installing course notebook test requirements")
+    install_python_test_requirements(course_notebook_tests_dir, python_bin)
+    step_end()
+
+    step_start("Testing course notebooks")
+    # Suppress output capturing since it's useful to have the timing information for successful runs too
+    run_python_tests(course_notebook_tests_dir, python_bin, pip_env, "-v", "-s")
+    step_end()
+
+
 if build_widgets:
     step_start("Building the Python widgets")
 

source/vscode/resources/qdk-learning/courses/chemistry-qpe/06-iterative-phase-estimation/iterative_phase_estimation.ipynb

@@ -322,7 +322,11 @@
    "cell_type": "code",
    "id": "c-0a2a3fcf99d8",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-test"
+    ]
+   },
    "outputs": [],
    "source": "iqpe_result = run_iqpe_workflow()\nprint_iqpe_results(iqpe_result)"
   },

source/vscode/resources/qdk-learning/courses/chemistry-qpe/_course_lib.py

@@ -16,7 +16,6 @@
 
 from typing import Callable
 
-from IPython.core.getipython import get_ipython
 from IPython.display import HTML, display
 
 # A learner exercise is a no-argument function whose result is checked.
@@ -32,21 +31,8 @@
 class ExerciseError(AssertionError):
     """Raised when an exercise is not yet correct."""
 
-
-def _hide_traceback() -> None:
-    """Show only the failure banner for a wrong answer.
-
-    The cell still ends in an error, which marks the exercise as incomplete,
-    but real errors raised by learner code keep their traceback.
-    """
-    shell = get_ipython()
-    if shell is None:
-        return
-
-    shell.set_custom_exc((ExerciseError,), lambda *args, **kwargs: None)
-
-
-_hide_traceback()
+    def _render_traceback_(self) -> list[str]:
+        return []
 
 
 def _register(name: str, checker: Checker) -> str:

source/vscode/resources/qdk-learning/utils/chemistry-qpe/rst_to_notebook.py

@@ -797,7 +797,7 @@
                 "section": "The complete workflow",
                 "cells": [
                     (
-                        "code",
+                        "skip-test",
                         "iqpe_result = run_iqpe_workflow()\n"
                         "print_iqpe_results(iqpe_result)\n",
                     ),
@@ -1773,7 +1773,12 @@ def splice(section, block):
         cells[end:end] = block
         owner[end:end] = [section] * len(block)
 
-    kinds = {"md": md, "code": code, "region": lambda v: code(notebook_region(v))}
+    kinds = {
+        "md": md,
+        "code": code,
+        "skip-test": lambda value: code(value, tags=["skip-test"]),
+        "region": lambda value: code(notebook_region(value)),
+    }
     for spec in recipe.get("inserts", []):
         splice(spec["section"], [kinds[k](v) for k, v in spec["cells"]])
 

source/vscode/src/learning/notebookExercises.ts

@@ -42,6 +42,9 @@ const EXERCISE_TAG = "exercise";
 /** Tags marking author-only cells, removed from the learner's working copy. */
 const AUTHORING_TAGS = ["hint", "solution", "explanation"] as const;
 
+/** Test-only tag removed from cells in the learner's working copy. */
+const SKIP_TEST_TAG = "skip-test";
+
 type AuthoringTag = (typeof AUTHORING_TAGS)[number];
 
 /** The subset of an nbformat cell this module reads. */
@@ -190,10 +193,10 @@ export function parseNotebookActivities(
  * Remove the author-only cells from a notebook's JSON text, returning the
  * notebook the learner works in.
  *
- * Everything else — including cell ids and the `exercise` tag — is preserved
- * verbatim, so metadata parsed from the authored notebook still resolves
- * against the working copy. Returns `undefined` if the text isn't a notebook,
- * leaving the caller to decide on a fallback.
+ * Retained cells preserve their content, ids, and other tags, including the
+ * `exercise` tag. The test-only `skip-test` tag is removed so it doesn't leak
+ * into the learner's working copy. Returns `undefined` if the text isn't a
+ * notebook, leaving the caller to decide on a fallback.
  */
 export function stripAuthoringCells(
   text: string,
@@ -209,6 +212,13 @@ export function stripAuthoringCells(
     return !AUTHORING_TAGS.some((t) => tags.includes(t));
   });
 
+  for (const cell of notebook.cells) {
+    const metadata = cell.metadata;
+    if (metadata && Array.isArray(metadata.tags)) {
+      metadata.tags = metadata.tags.filter((tag) => tag !== SKIP_TEST_TAG);
+    }
+  }
+
   // Match the ipynb serializer's formatting so the file stays diff-stable
   // once VS Code starts saving it: one space of indent, trailing newline.
   return `${JSON.stringify(notebook, undefined, 1)}\n`;

source/vscode/test/course-notebooks/README.md

@@ -0,0 +1,49 @@
+# Course notebook tests
+
+These tests execute the source notebooks for one QDK learning course in fresh
+Python kernels. The notebooks run from one shared copy of the full course
+directory so relative imports work, generated files do not modify the source
+tree, and later notebooks see filesystem state produced by earlier notebooks.
+The suite also verifies that each notebook corresponds one-to-one with a unit
+directory listed in the course's `course.json`.
+
+Pytest creates a `.venv` inside the temporary course copy and installs the
+course requirements there. Notebook kernels use that environment, while pytest
+continues to use the component's test environment.
+
+## Local setup
+
+Run the course suite from the repository root with Python 3.11 or later:
+
+```shell
+python ./build.py --no-check --no-check-prereqs --course-notebook-tests
+```
+
+Like `--integration-tests`, `--course-notebook-tests` runs independently of
+the regular `--test`/`--no-test` option.
+
+Following the other Python test suites, `build.py` uses an active Python
+environment when available. Otherwise, it creates
+`source/vscode/test/course-notebooks/.venv` and installs the test requirements
+there.
+
+Run only the fast runner policy tests without creating a course environment:
+
+```shell
+source/vscode/test/course-notebooks/.venv/bin/python -m pytest source/vscode/test/course-notebooks/test_notebook_runner.py -v
+```
+
+## Cell metadata
+
+- An `exercise` code cell must raise `ExerciseError`. Any other exception, or
+  successful execution, fails the test.
+- A `solution` code cell must execute without an error.
+- A `skip-test` code cell is skipped only by this test suite. VS Code, Jupyter,
+  and ordinary notebook execution do not interpret this custom tag. It also
+  takes precedence when combined with `exercise`.
+
+When a skipped setup cell supplies state to later cells, tag those dependent
+cells with `skip-test` as well.
+
+Each cell has a 120-second timeout. Cells taking longer than 30 seconds are
+reported so expensive cells can be reviewed before adding a skip.