Add integration tests

.github/workflows/integration.yml

@@ -0,0 +1,103 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Filesystem integration
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - pull-request/[0-9]+
+
+permissions: {}
+
+concurrency:
+  group: filesystem-integration
+  cancel-in-progress: false
+
+jobs:
+  integration:
+    name: Setup and test all (${{ matrix.architecture }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - architecture: amd64
+            runner: linux-amd64-cpu4
+          - architecture: arm64
+            runner: linux-arm64-cpu4
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 90
+    permissions:
+      contents: read
+    steps:
+      - name: Check out source
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+        with:
+          python-version: "3.12"
+      - name: Configure the package proxy
+        uses: nv-gha-runners/setup-proxy-cache@eadc57871cde0c92a223d0f486ab669a2360f978 # main
+        with:
+          enable-apt: "true"
+          enable-pip: "true"
+          enable-conda: "false"
+          enable-pypi-anaconda: "false"
+      - name: Verify privileged runner prerequisites
+        run: |
+          sudo -n true
+          docker info >/dev/null
+      - name: Set up the integration environment
+        run: |
+          "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup
+      - name: Verify repeated setup
+        run: |
+          "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup
+      - name: Stop the integration environment
+        run: |
+          "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs stop
+      - name: Restart the integration environment
+        run: |
+          "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs start
+      - name: Verify root lifecycle execution is rejected
+        run: |
+          for action in setup test; do
+            if sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs "$action"; then
+              echo "integration $action action unexpectedly accepted root" >&2
+              exit 1
+            fi
+          done
+      - name: Run all integration tests
+        run: |
+          "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs test
+      - name: Tear down the integration environment
+        if: ${{ always() }}
+        run: |
+          "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown
+          "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown
+
+  integration-status:
+    name: Filesystem integration status
+    if: ${{ always() }}
+    needs: integration
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require every architecture to pass
+        env:
+          INTEGRATION_RESULT: ${{ needs.integration.result }}
+        run: test "$INTEGRATION_RESULT" = success

AGENTS.md

@@ -63,6 +63,9 @@ See the README "Getting Started" section for the full quickstart and
 ./utils/run_ci_checks.sh
 ```
 
+After changing `.github/`, parse every `.yml` and `.yaml` file beneath it with
+a YAML parser before committing.
+
 If any required check tool is missing from the sandbox, install it into the
 repo's local environment and rerun the check. Do not skip required tooling just
 because it is not preinstalled.
@@ -72,6 +75,8 @@ The script creates and reuses `.venv-ci` with the pinned tools. Pass
 `pylint`, or `pytest` to run one check. Set `CI_CHECK_JOBS=1` in a constrained
 sandbox. In a pre-provisioned, network-restricted sandbox, set `CI_BOOTSTRAP=0`
 and use `CI_PYTHON` or `CI_SHELLCHECK` to select installed tools.
+Run tests and lint through this script or an environment populated from both
+requirements files; never treat ambient Python tooling as authoritative.
 
 `black` must be 25.9.0+; `pylint` must score 10.00/10. Details and rationale:
 [docs/CODING_STANDARDS.md](docs/CODING_STANDARDS.md).
@@ -93,6 +98,8 @@ and use `CI_PYTHON` or `CI_SHELLCHECK` to select installed tools.
 
 ## Pull requests
 
+Wrap commit-message lines at about 72 characters.
+
 This project is currently not accepting external contributions. For maintainer
 changes: keep PRs focused, ensure the checks above pass, and update
 `README.md`/`docs/` when behavior or configuration changes.

docs/CONTEXT.md

@@ -56,6 +56,37 @@ and are responsible for every binary they place in it.
 
 Slurm is the default execution substrate. Setting `SSH_HOST_LIST` selects
 passwordless SSH instead. Kubernetes execution is not implemented.
+The separate `integration-tests/` fixture provisions a three-node kind cluster,
+a shared RWX backend, two passwordless-SSH workers, and Slinky Slurm. Its `nfs`
+backend uses loop-backed NFSv4 and NFS CSI; Docker SBX instead shares a
+repository path across kind nodes through static RWX volumes. Both validate the
+same in-scope storage contract, and the selected backend persists until
+teardown. The SBX profile pairs kind/Kubernetes 1.34 with kubectl 1.34.
+
+Lifecycle actions run as an ordinary user and keep state below
+`tmp/integration-state`; only narrow NFS host operations use `sudo`. Dedicated
+state and export leaves require exact ownership markers, and setup locks its
+namespace and export path until teardown. Cleanup removes only harness-owned
+resources, preserves pre-existing NFS services and exports, tolerates partial
+bootstrap, and verifies temporary and final unmounts before deleting paths.
+
+The test CLI selects substrates and named scenarios independently. Its planner
+uses substrate-aware preflight and batches the shared-home SSH scenario behind
+a crash-recoverable transition, with separate homes as the canonical state.
+All pod-side work uses the fixed `tester` UID/GID 2000 identity, matching the
+all-squashed NFS export and Slurm account.
+
+The harness builds the ordinary deployment archive from an immutable tracked
+source snapshot, caches it by source, architecture, and seeded Elbencho/runtime
+identity, and extracts isolated scenario workspaces. The real catalog covers
+baseline and default I/O, failure/resume, retained datasets, live capture,
+Cartesian sweeps, single-file and weighted-root behavior, shared SSH homes,
+and Slurm scheduling. Fast tests cover parsing, precedence, workload and path
+safety, sizing, scheduler boundaries, failure contracts, and reporting. CI
+runs the complete NFS-backed catalog concurrently on amd64 and arm64; SBX is a
+supported local backend. Kubernetes remains fixture infrastructure rather than
+a benchmark execution substrate.
+
 GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks
 alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python
 3.14 unit tests run weekly and on manual request.
@@ -79,6 +110,7 @@ alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python
 | `utils/build_tarball.sh` | User-local deployment-tarball builder |
 | `utils/build/` | Helpers for building Warp and the in-tree s3test program |
 | `tests/` | Python and shell-behavior regression tests collected by `pytest` |
+| `integration-tests/` | Single-host kind, RWX storage, SSH, and Slinky fixture |
 
 The checked-in benchmark entry points are:
 
@@ -119,6 +151,9 @@ Important configuration relationships:
   inclusive ranges, and `+step` increments. Entry points validate the complete
   expanded list before dispatch.
 - `TEST_DIRS` is an associative array of filesystem test roots and weights.
+  `validate_env.sh` compares each path's `stat -c %d` device number with `/`
+  through both dispatch interfaces; this establishes distinct backing storage,
+  not that the configured path is itself the exact mountpoint.
   Object tests use a dedicated `OBJ_BUCKET`, endpoint settings, and credentials
   sourced from `OBJ_AUTH_FILE`.
 

docs/DESIGN.md

@@ -210,7 +210,7 @@ edit-validate loop:
    | Slurm connectivity | `sinfo`, `sbatch` a test job, wait for completion |
    | SSH connectivity | `ssh` command execution + scriptlet execution on each host |
    | Binary architecture match | `file` on binary vs. `uname -m` on remote |
-   | Filesystem paths are mountpoints | `mountpoint` on compute nodes (via Slurm/SSH) |
+   | Filesystem paths use storage distinct from `/` | Compare `stat -c %d` device IDs on compute nodes (via Slurm/SSH) |
    | Filesystem paths are writable | Touch test on compute nodes |
    | S3 credentials and bucket access | `s3test` binary |
    | S3 bucket emptiness | Object count check (warning if non-empty) |
@@ -692,7 +692,10 @@ All analysis scripts support filtering to reduce clutter:
 
 Intermediate data can be cached (`--to-csv`, `--to-json`) and reloaded
 (`--from-csv`, `--from-json`) to avoid re-parsing raw files, enabling fast
-iterative report refinement.
+iterative report refinement. For Elbencho, `--from-csv` is an alternative input
+source and cannot be combined with raw result directories. Malformed filters
+and filters that match neither aggregate nor live metrics fail instead of
+silently producing an unfiltered or empty report.
 
 ### 9.5 Plot Design Principles
 

docs/REQUIREMENTS.md

@@ -219,7 +219,7 @@ These requirements address the infrastructure and runtime constraints the tool m
 | CV-2.3 | Validation shall verify that benchmark binaries are present, executable, and compiled for the correct architecture. | Yes |
 | CV-2.4 | Validation shall verify Slurm connectivity (if Slurm mode is enabled): partition access, account, reservation, sbatch and srun functionality. | Yes |
 | CV-2.5 | Validation shall verify SSH connectivity (if SSH mode is enabled): ability to run commands and scriptlets on remote hosts. | Yes |
-| CV-2.6 | Validation shall verify that filesystem test paths are mountpoints and writable on compute nodes. | Yes |
+| CV-2.6 | Validation shall verify that filesystem test paths reside on a filesystem device distinct from `/` and are writable on compute nodes. | Yes |
 | CV-2.7 | Validation shall verify S3 object storage credentials and bucket accessibility (if object tests are enabled). | Yes |
 | CV-2.8 | Validation shall warn if the target S3 bucket contains existing objects (warp deletes all objects). | Yes |
 | CV-2.9 | Validation shall validate elbencho configuration parameters (thread list contains integers, IO sizes are valid, duration is valid). | Yes |