From 7959b88fb8aed4299e14b6ebed6767db9420dc48 Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Fri, 18 Sep 2026 17:12:17 +0000 Subject: [PATCH 01/10] Add single-host integration harness Add an idempotent Python driver for the kind, NFS CSI, SSH, MariaDB, and Slinky fixture. Setup and start reconcile a complete running environment. Stop deletes the disposable cluster while preserving reusable host state and external NFS data. Capture the sanitized single-host feasibility study and implementation handoff. Add a bounded test action that builds and validates the deployment tarball, then runs one-node and two-node filesystem sweeps through SSH and Slurm. Run the SSH entry point on the host, and transfer the same archive to the Slinky login node for Slurm execution. Install the standard file utility in the test images, validate generated environments, retain diagnostic logs, and assert results and cleanup. Fix two SSH dispatch variable-shadowing bugs exposed by the regression cases. --- docs/CONTEXT.md | 14 + .../single-host-integration-feasibility.md | 295 +++ integration-tests/README.md | 94 + integration-tests/bin/integration-test.py | 1926 +++++++++++++++++ .../lib/filesystem_integration.py | 1059 +++++++++ integration-tests/manifests/kind.yaml.tmpl | 28 + .../manifests/mariadb-accounting.yaml.tmpl | 120 + .../manifests/nfs-csi-values.yaml | 28 + .../manifests/nfs-storage.yaml.tmpl | 60 + .../manifests/slinky-operator-values.yaml | 42 + .../manifests/slinky-slurm-values.yaml | 142 ++ .../manifests/ssh-workers.yaml.tmpl | 116 + .../slinky-login-image.Dockerfile | 23 + integration-tests/ssh-image.Dockerfile | 46 + lib/env_functions.sh | 11 +- utils/build_tarball.sh | 59 +- 16 files changed, 4046 insertions(+), 17 deletions(-) create mode 100644 docs/research/single-host-integration-feasibility.md create mode 100644 integration-tests/README.md create mode 100755 integration-tests/bin/integration-test.py create mode 100644 integration-tests/lib/filesystem_integration.py create mode 100644 integration-tests/manifests/kind.yaml.tmpl create mode 100644 integration-tests/manifests/mariadb-accounting.yaml.tmpl create mode 100644 integration-tests/manifests/nfs-csi-values.yaml create mode 100644 integration-tests/manifests/nfs-storage.yaml.tmpl create mode 100644 integration-tests/manifests/slinky-operator-values.yaml create mode 100644 integration-tests/manifests/slinky-slurm-values.yaml create mode 100644 integration-tests/manifests/ssh-workers.yaml.tmpl create mode 100644 integration-tests/slinky-login-image.Dockerfile create mode 100644 integration-tests/ssh-image.Dockerfile diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index ea9d032..5676ca7 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -56,6 +56,14 @@ and are responsible for every binary they place in it. Slurm is the default execution substrate. Setting `SSH_HOST_LIST` selects passwordless SSH instead. Kubernetes execution is not implemented. +The separate `integration-tests/` fixture provisions a three-node kind cluster, +NFS CSI storage, two passwordless-SSH workers, and a Slinky Slurm environment +on one Linux host. Its test action builds and validates a deployment tarball, +derives environments from the packaged `env.sh.template`, and runs bounded +one-node and two-node filesystem sweeps through SSH and Slurm. The SSH entry +point runs on the host; the Slurm entry point runs from the archive extracted by +the LoginSet on shared storage. It does not add Kubernetes dispatch to the +benchmark entry points. GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. @@ -79,6 +87,8 @@ alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python | `utils/build_tarball.sh` | User-local deployment-tarball builder | | `utils/build/` | Helpers for building Warp and the in-tree s3test program | | `tests/` | Python and shell-behavior regression tests collected by `pytest` | +| `integration-tests/` | Single-host kind, NFS CSI, SSH, and Slinky fixture provisioner and manifests | +| `docs/research/` | Feasibility studies and implementation handoffs for future integration work | The checked-in benchmark entry points are: @@ -407,6 +417,10 @@ into a benchmark environment. It: - includes existing Warp binaries but does not download or automatically build them, warning when an architecture is missing. +By default the helper retains that full behavior. `--arch` can select only the +native elbencho architecture, and `--skip-object-tools` omits Warp checks and +s3test builds when creating a filesystem-only deployment archive. + `utils/build/build_s3test_from_source.sh` tries suitable local compilers, Docker, and Docker Buildx. Failure to produce one architecture warns and permits tarball creation, but object testing on that architecture will be unavailable. diff --git a/docs/research/single-host-integration-feasibility.md b/docs/research/single-host-integration-feasibility.md new file mode 100644 index 0000000..8d23ae6 --- /dev/null +++ b/docs/research/single-host-integration-feasibility.md @@ -0,0 +1,295 @@ + + +# Single-host integration environment feasibility and handoff + +## Outcome + +A functional Kubernetes, passwordless-SSH, shared-storage, and Slurm fixture is +feasible on one Linux server. The implemented harness uses rootful Docker and +kind to create three logical Kubernetes nodes: + +| Node | Label | Fixture roles | +|---|---|---| +| Control plane | `storage-scale-test/login=true` | Kubernetes control plane, negative scheduling control, Slinky LoginSet, MariaDB, and `slurmdbd` | +| Worker 1 | `storage-scale-test/target=true` | SSH worker and one `slurmd` | +| Worker 2 | `storage-scale-test/target=true` | SSH worker and one `slurmd` | + +The control-plane taint is removed so its lack of the target label is the +negative-selection test. A fourth node is unnecessary: the lightweight Slinky +LoginSet is schedulable on the control plane and is not a compute node. + +This fixture validates orchestration and substrate behavior. Logical nodes on +one server share physical resources, so it is not suitable for performance, +scaling, failure-domain, or network-isolation measurements. + +## Implemented lifecycle + +The executable driver is `integration-tests/bin/integration-test.py` and +supports four actions: + +- `setup` installs missing host dependencies, creates or reconciles the + environment, and validates every substrate. +- `start` is an exact synonym for `setup`. +- `stop` is disposable: it deletes only the marker-owned kind cluster and then + stops the NFS service only when the harness started it and no unrelated + exports exist. +- `test` requires an already-running setup and runs selected bounded filesystem + regression cases without reconciling the environment. + +Stop preserves installed packages and tools, downloaded charts, Docker image +caches, generated SSH and database credentials, rendered state, and external +NFS data. It intentionally discards Kubernetes objects, kind containers, and +MariaDB's node-local storage. A later start creates and validates a fresh +cluster, so it takes longer than setup against an already-running cluster. +The disposable stop and subsequent fresh start were exercised end to end, as +was a repeated stop with the cluster already absent. + +Deleting the cluster avoids relying on resumed kind container addresses. +Kubernetes Service DNS stabilizes application endpoints inside the cluster but +cannot make the node-container underlay stable, and the host cannot normally +route cluster-local DNS names. A fresh cluster is consequently simpler and more +reliable than restoring persisted CNI, kube-proxy, and host-network state. + +Lifecycle operations are serialized by a state-directory lock. The driver +uses a private kubeconfig on every kind, kubectl, and Helm command. A persistent +ownership marker prevents adoption or deletion of an unrelated same-named +cluster or export directory. + +## Pinned software + +The initial harness pins: + +- kind 0.33.0; +- Kubernetes node image and kubectl 1.37.0; +- Helm 3.22.0; +- NFS CSI chart 4.13.4 and its sidecar versions; +- Slinky charts 1.2.0; and +- a digest-pinned MariaDB 11.4 image. + +Client binaries are downloaded with their published checksums. The NFS CSI +chart is extracted from a versioned source archive after verifying both the +source archive and embedded chart SHA-256 values. CSI images use the production +Kubernetes registry with the official staging registry as a bounded fallback. + +The setup path currently targets Linux on x86-64 or ARM64, Python 3.12 or newer, +an accessible rootful Docker daemon, at least two CPUs, 8 GiB total memory, +6 GiB available memory, and 20 GiB free space. These are admission limits for +the fixture, not a benchmark sizing recommendation. + +## Shared storage + +The host runs a narrowly configured NFSv4.1 server with two server threads. +Its export is backed by a persistent 128 MiB sparse ext4 image. This gives the +project's mount validation a filesystem distinct from the host root while +keeping disk consumption bounded and retaining data across disposable stops. +Provisioning discovers the kind Docker network's IPv4 subnet and gateway; it +does not assume a fixed bridge address. The export: + +- is restricted to the discovered kind subnet; +- uses `all_squash` and maps requests to numeric UID/GID 2000; +- is installed through dedicated files in `/etc/exports.d` and + `/etc/nfs.conf.d` without replacing unrelated configuration; and +- opens TCP 2049 only for the kind subnet when UFW is already active. + +Numeric ownership is applied as `+2000:+2000` to avoid accidental name-service +resolution of numeric-looking account names. + +The upstream NFS CSI driver dynamically provisions two independent +`ReadWriteMany` claims: + +- `storage-test-rwx` is mounted at `/mnt/storage-test` by the SSH workers, + Slinky LoginSet, and Slurm workers. +- `ssh-home-rwx` optionally supplies the shared `/home/tester` mode. + +Both claims use deterministic namespace/PVC subdirectories and retain their +external NFS data when the disposable Kubernetes cluster is deleted. Requested +PVC sizes are metadata rather than NFS server quotas. MariaDB uses kind's local +`ReadWriteOnce` provisioner and is deliberately disposable. + +## SSH fixture + +The SSH substrate is a two-replica StatefulSet with required pod anti-affinity +and a target-node selector. Each pod uses host networking and listens on its +kind worker's bridge address, making both workers reachable from the Linux host +without a NodePort or LAN-facing host port. + +The image provides a dedicated `tester` user with UID/GID 2000. Setup generates +one Ed25519 fixture key, stores it in protected state, and creates a Kubernetes +Secret only when absent. An init container copies key material into the home +directory with strict ownership and modes. Password and root authentication are +disabled, while `/root` remains on local overlay storage. + +Two home modes are supported: + +- `separate` mounts a per-pod `emptyDir` at `/home/tester`. +- `shared` mounts the dedicated NFS RWX home claim at `/home/tester` in both + workers. + +The host discovers both live worker addresses, writes a strict known-hosts file +and an `ssh_hosts` list into protected state, and proves public-key access to +both. Validation also proves root rejection, bidirectional worker SSH, distinct +placement, the selected home visibility semantics, shared RWX visibility, and +local `/root` storage. + +## Slurm fixture + +Slinky installs CRDs, the operator, and the Slurm custom resources in that +order. The conservative profile disables cert-manager, monitoring, accelerator +support, container plug-ins, high availability, and external load balancers. It +uses: + +- one LoginSet on the login-labeled control plane; +- one DaemonSet-mode NodeSet selecting the two target workers; +- one partition containing that NodeSet; +- controller persistence disabled for the disposable cluster; and +- one directly managed MariaDB instance plus `slurmdbd` for accounting. + +The `slurmd` containers have a small CPU request but no CPU limit. A limit can +cause effective CPU discovery to reach zero on constrained hosts. Other +components retain modest requests and limits. SSH workers are temporarily +scaled to zero during the heaviest Slinky reconciliation and restored before +setup succeeds. + +Slinky 1.2.0 renders a new self-signed webhook CA during a same-values operator +upgrade when cert-manager is disabled. The driver therefore skips releases that +are already deployed at the pinned chart version. A real operator upgrade +restarts and waits for the webhook before applying Slurm custom resources. + +Slurm validation discovers pods by their actual workload container names rather +than relying on generic labels that the operator does not publish. It verifies: + +- the LoginSet runs on the control plane; +- exactly two `slurmd` pods run on the two target nodes; +- the LoginSet sees the NFS mount; +- a two-node `srun` reaches two distinct Slurm nodes; and +- a two-node `sbatch --wait` is reported by `sacct` as `COMPLETED|0:0`. + +Commands are submitted from the LoginSet, matching the intended login/submit +execution point. + +## Filesystem regression cases + +The `test` action accepts `all`, `filesystem`, `ssh`, and `slurm` selectors. +`all` and `filesystem` run both substrate cases; `ssh` and `slurm` can run +individually or together. Test execution deliberately refuses to run without a +matching saved setup and a healthy live three-node fixture. + +The harness first copies only tracked working-tree files to an isolated staging +tree and invokes `utils/build_tarball.sh` there. It validates the resulting +archive's paths, types, size, required files, and packaged benchmark executable +before extraction. Each case renders `env.sh` from the packaged user-facing +`env.sh.template`, injects only bounded fixture overrides, and executes +`validate_env.sh` before the benchmark. + +The SSH case runs both validation and `nv-elbencho-sweep.sh` on the test host; +the checked-in SSH implementation copies and launches its worker payloads in +the two pods. The Slurm case streams the same deployment archive to the +LoginSet, extracts it in the shared NFS filesystem, and runs both commands from +that extracted tree. The sweep uses one 4 KiB file, one thread, queue depth one, +buffered I/O, and one-node and two-node dimensions. Substrates run sequentially. +This covers deployment packaging, environment validation, sweep reification, +SSH and Slurm dispatch, service startup, write/read/delete phases, result +recording, and cleanup while writing only a few KiB per execution. + +The pinned elbencho release archive is architecture-selected, capped during +download, checksum-verified, and cached in protected state. Its verified native +binary is supplied to the isolated tree as the deployment builder's documented +local cache, so the integration test does not vendor a benchmark binary. A +minimal derived Slinky login image installs the standard `file` package required +by `validate_env.sh`; the test does not replace that prerequisite with a +test-specific implementation. + +Every run retains host-side logs beneath `test-runs/`. Success requires two +successful execution records, zero exit codes, workload manifests, environment +snapshots, nonempty CSV and text output, and no remaining generated benchmark +directory at the NFS root. Performance values are not asserted. + +## Idempotency and failure behavior + +Setup reuses generated credentials, existing claims, cached downloads, and +already-current Slinky releases. It uses declarative Kubernetes apply and Helm +upgrade/install for resources that require reconciliation. Re-running setup on +a healthy cluster repeats validation without rotating credentials. + +If setup finds marker-owned stopped or incomplete kind containers, it deletes +that disposable partial cluster and creates a fresh one. An unowned same-named +cluster or nonempty export directory is a hard error with remediation text. + +Every external command and readiness gate has a timeout. Downloads use bounded +retries and checksum verification. Secret-bearing commands are redacted from +logs. On setup failure, diagnostics include bounded host capacity, filesystem, +Docker, kind, NFS service/export, Kubernetes node/pod, and sorted event output; +Kubernetes Secrets are never dumped. + +Stop is idempotent. Repeated stop calls succeed when the cluster and owned NFS +service are already absent or inactive. It never stops Docker globally and +leaves a pre-existing or shared NFS service running. + +## Provisioning sequence + +An implementation or future refactor should preserve this ordering: + +1. Acquire the lifecycle lock and initialize protected logging. +2. Validate the platform and conservative capacity floor. +3. Install the narrow host package set and verify rootful Docker. +4. Install checksum-verified kind, kubectl, and Helm clients as needed. +5. Validate cluster ownership, replacing only an owned partial cluster. +6. Create and validate the three-node kind topology and labels. +7. Create or mount the bounded sparse ext4 backing image, then configure and + probe the subnet-scoped NFSv4.1 export. +8. Preload pinned CSI images, install NFS CSI, and bind both RWX claims. +9. Build, preload, deploy, and validate the two SSH workers. +10. Scale SSH down, reconcile MariaDB and Slinky, and validate Slurm. +11. Restore and revalidate the SSH workers. +12. Persist a non-secret state summary and report success. + +For disposable stop: + +1. Acquire the lifecycle lock. +2. Discover the exact kind cluster and containers. +3. Require the matching ownership marker before deletion. +4. Delete the named kind cluster and verify its containers are gone. +5. Stop NFS only when it is harness-owned and has no unrelated exports. +6. Preserve all host packages, caches, keys, rendered state, and NFS data. + +## Checked-in implementation artifacts + +| Path | Purpose | +|---|---| +| `integration-tests/bin/integration-test.py` | Setup/start/stop driver, validation, logging, and diagnostics | +| `integration-tests/lib/filesystem_integration.py` | Filesystem test selection, staging, execution, and result assertions | +| `integration-tests/manifests/kind.yaml.tmpl` | Three-node kind topology and neutral role labels | +| `integration-tests/manifests/nfs-csi-values.yaml` | Low-footprint NFS CSI deployment values | +| `integration-tests/manifests/nfs-storage.yaml.tmpl` | StorageClass and two RWX claims | +| `integration-tests/ssh-image.Dockerfile` | Ubuntu/OpenSSH worker image with non-root fixture user | +| `integration-tests/slinky-login-image.Dockerfile` | Slinky login image with the standard `file` prerequisite | +| `integration-tests/manifests/ssh-workers.yaml.tmpl` | Two host-networked SSH workers and selectable home volume | +| `integration-tests/manifests/mariadb-accounting.yaml.tmpl` | Disposable local accounting database | +| `integration-tests/manifests/slinky-operator-values.yaml` | Low-footprint operator and webhook configuration | +| `integration-tests/manifests/slinky-slurm-values.yaml` | LoginSet, NodeSet, shared storage, partition, and accounting | + +The checked-in files are the authoritative associated artifact contents. They +should be changed together with this handoff whenever topology, versions, +resource policy, or lifecycle semantics change. + +## Remaining work + +The implemented first pass covers the basic filesystem sweep through SSH and +Slurm. Future cases can add metadata, write-only/read-from/resume, failure +recovery, shared-home mode, and Kubernetes dispatch coverage while preserving +the same bounded-data and no-performance-assertion policy. diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 0000000..807441a --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,94 @@ + + +# Single-host integration environment + +`bin/integration-test.py` provisions the lightweight, three-node kind fixture +used to exercise Kubernetes, SSH, and Slurm storage substrates on one Linux +server. The control-plane node is also the Slinky login node and negative +control for the storage-worker label. The two worker nodes run storage clients. + +The current setup target is Ubuntu 24.04 on x86-64 or ARM64 with at least two +CPUs, 8 GiB RAM, and 20 GiB free disk. Python 3.12 and an accessible rootful +Docker daemon are prerequisites. The driver installs its other host packages +and pinned client tools when needed. It also builds a small derived Slinky +login image containing the standard `file` package required by +`validate_env.sh`. + +Run setup (or its exact synonym, start) with: + +```bash +sudo -v +integration-tests/bin/integration-test.py setup +``` + +The default SSH homes are separate `emptyDir` volumes. To mount the dedicated +RWX NFS claim at `/home/tester` in both workers instead, use: + +```bash +integration-tests/bin/integration-test.py --ssh-home-mode shared setup +``` + +The generated host SSH key, strict known-hosts file, and two worker addresses +are kept under `/var/lib/storage-scale-test-integration/`. Re-running setup +reconciles and validates the environment without replacing those credentials +or retained NFS data. + +After setup succeeds, run the bounded filesystem regression cases with: + +```bash +integration-tests/bin/integration-test.py test all +``` + +`all` and `filesystem` select both substrates. `ssh` and `slurm` may be used +individually, or together as two arguments. The `test` action never installs +or reconciles the fixture: it requires the saved setup state, verifies that +the live topology is healthy, generates each test environment from the packaged +`env.sh.template`, and runs `validate_env.sh` before the sweep. + +Each substrate runs one 4 KiB buffered execution on one node and one on two +nodes through the real filesystem sweep entry point. The harness builds a real +deployment archive from a tracked-files-only snapshot with +`utils/build_tarball.sh`, validates its contents, and runs from the extracted +archive. The SSH case launches `validate_env.sh` and `nv-elbencho-sweep.sh` on +the host and reaches the two worker pods over SSH. The Slurm case streams the +same archive to the LoginSet, extracts it in the shared NFS filesystem, and +launches both commands there. + +The pinned benchmark archive is size-limited, checksum-verified, and cached +outside the repository before the binary is included in the user-built +deployment archive. Timestamped build and step logs are retained below the +state directory's `test-runs/` directory. The test also requires successful +execution records, nonempty benchmark output, environment snapshots, and +cleanup of its generated data directories. + +Delete the disposable kind cluster and, when owned exclusively by the harness, +stop NFS with: + +```bash +integration-tests/bin/integration-test.py stop +``` + +Stop preserves packages, downloaded charts, Docker images, generated keys and +passwords, rendered state, and external NFS data. Kind containers, Kubernetes +objects, and MariaDB's node-local volume are disposable and are deleted. A +subsequent start therefore creates and validates a fresh cluster and takes +longer than an idempotent setup against an already-running cluster. Timestamped +logs and rendered manifests are retained in the state directory. Add +`--verbose` for command-level logging. The failure path captures host, Docker, +NFS, Kubernetes node, pod, and event diagnostics without printing Kubernetes +Secrets. diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py new file mode 100755 index 0000000..f794e77 --- /dev/null +++ b/integration-tests/bin/integration-test.py @@ -0,0 +1,1926 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provision the single-host storage-scale integration environment.""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +import logging +import os +import platform +import secrets +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import IO + +INTEGRATION_LIB = Path(__file__).resolve().parents[1] / "lib" +sys.path.insert(0, str(INTEGRATION_LIB)) + +from filesystem_integration import ( # pylint: disable=wrong-import-position + IntegrationTestError, + TEST_SELECTORS, + run_filesystem_tests, +) + +KIND_VERSION = "v0.33.0" +KUBECTL_VERSION = "v1.37.0" +HELM_VERSION = "v3.22.0" +KIND_NODE_IMAGE = ( + "kindest/node:v1.37.0@" + "sha256:a1ed56cfb0e7b93589bdf97c8cd566405a265939e3620fc4f5de89adff580ae5" +) +NFS_CSI_VERSION = "4.13.4" +NFS_CSI_SOURCE_SHA256 = ( + "ded6ffba8b1600d4c723ce1ecb1fd91721ef48e732ce7ca30c0efeeecbb0b900" +) +NFS_CSI_CHART_SHA256 = ( + "815ac441a2dd0e48c82fa92d043e96caac4dd8ac422fbba91ed76892ed32da54" +) +SLINKY_VERSION = "1.2.0" +SLINKY_LOGIN_BASE_IMAGE = "ghcr.io/slinkyproject/login:26.05-ubuntu26.04" +SLINKY_LOGIN_IMAGE = "storage-scale-integration-login:slinky-26.05-file" +SSH_IMAGE = "storage-scale-integration-ssh:ubuntu-24.04" +STATE_SCHEMA = 1 +TARGET_LABEL = "storage-scale-test/target=true" +LOGIN_LABEL = "storage-scale-test/login=true" +NFS_UID = 2000 +NFS_GID = 2000 +NFS_IMAGE_BYTES = 128 * 1024 * 1024 +GIB = 1024**3 +DEFAULT_STATE_DIR = Path("/var/lib/storage-scale-test-integration") +DEFAULT_EXPORT_DIR = Path("/srv/storage-scale-test-integration") +LOG = logging.getLogger("storage-scale-integration") + +CSI_IMAGES = ( + ("csi-node-driver-registrar", "v2.17.0"), + ("csi-provisioner", "v6.3.0"), + ("csi-resizer", "v2.2.0"), + ("livenessprobe", "v2.19.0"), + ("nfsplugin", "v4.13.4"), +) + + +class ProvisionError(RuntimeError): + """An actionable provisioning failure.""" + + +@dataclass(frozen=True) +class Config: + """Resolved integration environment configuration.""" + + cluster_name: str + namespace: str + state_dir: Path + export_dir: Path + ssh_home_mode: str + verbose: bool + + @property + def kubeconfig(self) -> Path: + """Return the private kubeconfig path.""" + return self.state_dir / "kubeconfig" + + @property + def manifests_dir(self) -> Path: + """Return the rendered-manifest state directory.""" + return self.state_dir / "manifests" + + @property + def keys_dir(self) -> Path: + """Return the persistent test-key directory.""" + return self.state_dir / "keys" + + @property + def nfs_image(self) -> Path: + """Return the sparse backing image for the dedicated NFS export.""" + return self.state_dir / "nfs-export.ext4" + + +class Runner: + """Run commands with bounded execution and consistent diagnostics.""" + + def run( + self, + args: Sequence[str | Path], + *, + timeout: int = 300, + check: bool = True, + sensitive: bool = False, + stdin: IO[bytes] | None = None, + cwd: Path | None = None, + ) -> subprocess.CompletedProcess[str]: + """Run *args* and return its completed process.""" + command = [str(item) for item in args] + display = ( + command[0] + " [redacted arguments]" if sensitive else shlex.join(command) + ) + LOG.debug("Running: %s", display) + result = subprocess.run( + command, + check=False, + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=stdin is None, + timeout=timeout, + cwd=cwd, + ) + stdout = _output_text(result.stdout) + stderr = _output_text(result.stderr) + if stdout and not sensitive: + LOG.debug("stdout from %s:\n%s", command[0], stdout.rstrip()) + if stderr and not sensitive: + LOG.debug("stderr from %s:\n%s", command[0], stderr.rstrip()) + if check and result.returncode: + detail = _failure_detail(stdout, stderr, sensitive) + raise ProvisionError( + f"command failed ({result.returncode}): {display}{detail}" + ) + return result + + +def _output_text(output: str | bytes | None) -> str: + """Return subprocess output as text.""" + if output is None: + return "" + if isinstance(output, bytes): + return output.decode(errors="replace") + return output + + +def _failure_detail(stdout: str, stderr: str, sensitive: bool) -> str: + """Return bounded non-secret command failure output.""" + if sensitive: + return " (output redacted)" + detail = stderr.strip() or stdout.strip() + if not detail: + return "" + return f"\n{detail[-8000:]}" + + +def _repository_root() -> Path: + """Return the repository root from this script location.""" + return Path(__file__).resolve().parents[2] + + +def _resource_path(name: str) -> Path: + """Return a checked-in integration resource path.""" + return _repository_root() / "integration-tests" / name + + +def _sudo_prefix() -> list[str]: + """Return a sudo prefix when the caller is not root.""" + return [] if os.geteuid() == 0 else ["sudo"] + + +def _bootstrap_state_dir(config: Config) -> None: + """Create the operator-owned state directory before file logging.""" + command = [ + *_sudo_prefix(), + "install", + "-d", + "-m", + "0750", + "-o", + f"+{os.getuid()}", + "-g", + f"+{os.getgid()}", + config.state_dir, + config.manifests_dir, + config.keys_dir, + config.state_dir / "logs", + ] + result = subprocess.run(command, check=False, capture_output=True, text=True) + if result.returncode: + raise ProvisionError( + f"cannot create state directory {config.state_dir}: {result.stderr.strip()}" + ) + + +def _configure_logging(config: Config, action: str) -> Path: + """Configure console and timestamped file logging.""" + timestamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + log_path = config.state_dir / "logs" / f"{action}-{timestamp}.log" + formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") + console = logging.StreamHandler() + console.setLevel(logging.DEBUG if config.verbose else logging.INFO) + console.setFormatter(formatter) + file_handler = logging.FileHandler(log_path, encoding="utf-8") + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(formatter) + LOG.handlers.clear() + LOG.setLevel(logging.DEBUG) + LOG.addHandler(console) + LOG.addHandler(file_handler) + return log_path + + +def _write_text(path: Path, text: str, mode: int = 0o640) -> None: + """Atomically write *text* with an explicit file mode.""" + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, delete=False + ) as handle: + handle.write(text) + temporary = Path(handle.name) + temporary.chmod(mode) + temporary.replace(path) + + +def _write_bytes(path: Path, content: bytes, mode: int = 0o640) -> None: + """Atomically write binary *content* with an explicit file mode.""" + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as handle: + handle.write(content) + temporary = Path(handle.name) + temporary.chmod(mode) + temporary.replace(path) + + +def _render_resource(config: Config, name: str, replacements: dict[str, str]) -> Path: + """Render a checked-in template into protected state.""" + source = _resource_path(name) + text = source.read_text(encoding="utf-8") + for token, value in replacements.items(): + text = text.replace(f"@@{token}@@", value) + unresolved = [word for word in text.split() if "@@" in word] + if unresolved: + raise ProvisionError(f"unresolved template token in {source}: {unresolved[0]}") + destination = config.manifests_dir / source.name.removesuffix(".tmpl") + _write_text(destination, text) + return destination + + +def _acquire_lock(config: Config) -> IO[str]: + """Acquire the exclusive lifecycle lock.""" + lock_path = config.state_dir / "lifecycle.lock" + handle = lock_path.open("w", encoding="utf-8") + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + handle.close() + raise ProvisionError( + f"another integration lifecycle command holds {lock_path}" + ) from error + handle.write(f"pid={os.getpid()}\n") + handle.flush() + return handle + + +def _require_python() -> None: + """Require the repository's supported Python baseline.""" + if sys.version_info < (3, 12): + raise ProvisionError("integration-test.py requires Python 3.12 or newer") + + +def _check_host_capacity() -> None: + """Fail before provisioning an undersized host.""" + cpu_count = os.cpu_count() or 0 + memory = _meminfo() + disk = shutil.disk_usage("/") + failures: list[str] = [] + if cpu_count < 2: + failures.append(f"need at least 2 CPUs; found {cpu_count}") + if memory.get("MemTotal", 0) < 8 * GIB: + failures.append("need at least 8 GiB total memory") + if memory.get("MemAvailable", 0) < 6 * GIB: + failures.append("need at least 6 GiB available memory") + if disk.free < 20 * GIB: + failures.append("need at least 20 GiB free on /") + if failures: + raise ProvisionError("host capacity check failed: " + "; ".join(failures)) + LOG.info( + "Host capacity accepted: %s CPUs, %.1f GiB available RAM, %.1f GiB free disk", + cpu_count, + memory["MemAvailable"] / GIB, + disk.free / GIB, + ) + + +def _meminfo() -> dict[str, int]: + """Read selected Linux memory counters in bytes.""" + result: dict[str, int] = {} + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + name, value = line.split(":", maxsplit=1) + result[name] = int(value.strip().split()[0]) * 1024 + return result + + +def _check_platform() -> str: + """Validate Linux and return the download architecture.""" + if sys.platform != "linux": + raise ProvisionError( + "the integration environment currently supports Linux only" + ) + architectures = {"x86_64": "amd64", "aarch64": "arm64"} + try: + return architectures[platform.machine()] + except KeyError as error: + raise ProvisionError( + f"unsupported architecture: {platform.machine()}" + ) from error + + +def _ensure_apt_packages(runner: Runner) -> None: + """Install the narrow tested Ubuntu/Debian package set when absent.""" + packages = ( + "ca-certificates", + "curl", + "e2fsprogs", + "file", + "jq", + "nfs-common", + "nfs-kernel-server", + "openssh-client", + "openssl", + ) + missing = [ + package + for package in packages + if runner.run( + ["dpkg-query", "-W", "-f=${db:Status-Abbrev}", package], check=False + ).stdout.strip() + != "ii" + ] + if not missing: + LOG.info("Required operating-system packages are already installed") + return + if not shutil.which("apt-get"): + raise ProvisionError(f"missing packages and apt-get is unavailable: {missing}") + LOG.info("Installing required packages: %s", ", ".join(missing)) + runner.run([*_sudo_prefix(), "apt-get", "update"], timeout=600) + runner.run( + [ + *_sudo_prefix(), + "env", + "DEBIAN_FRONTEND=noninteractive", + "apt-get", + "install", + "-y", + "--no-install-recommends", + *missing, + ], + timeout=600, + ) + + +def _ensure_docker(runner: Runner) -> None: + """Require a working rootful Docker daemon.""" + if not shutil.which("docker"): + raise ProvisionError( + "Docker is required but absent; install a supported rootful Docker Engine first" + ) + result = runner.run( + ["docker", "info", "--format", "{{json .SecurityOptions}}"], timeout=60 + ) + if "rootless" in result.stdout.lower(): + raise ProvisionError( + "rootless Docker is not supported by this integration fixture" + ) + LOG.info("Rootful Docker is available") + + +def _command_version(runner: Runner, command: str) -> str: + """Return normalized version output, or an empty string if unavailable.""" + if not shutil.which(command): + return "" + arguments = { + "kind": ["kind", "version"], + "kubectl": ["kubectl", "version", "--client"], + "helm": ["helm", "version", "--short"], + }[command] + return runner.run(arguments, check=False, timeout=30).stdout + + +def _ensure_client_tools(runner: Runner, architecture: str) -> None: + """Install checksum-verified kind, kubectl, and Helm when versions differ.""" + expected = { + "kind": KIND_VERSION, + "kubectl": KUBECTL_VERSION, + "helm": HELM_VERSION, + } + for command, version in expected.items(): + if version in _command_version(runner, command): + LOG.info("Using %s %s", command, version) + continue + _install_client_tool(runner, command, version, architecture) + + +def _install_client_tool( + runner: Runner, command: str, version: str, architecture: str +) -> None: + """Download, verify, and install one client binary.""" + LOG.info("Installing %s %s", command, version) + with tempfile.TemporaryDirectory(prefix="storage-scale-tool-") as directory: + target = Path(directory) + if command == "kind": + binary = _download_kind(runner, target, version, architecture) + elif command == "kubectl": + binary = _download_kubectl(runner, target, version, architecture) + else: + binary = _download_helm(runner, target, version, architecture) + runner.run( + [*_sudo_prefix(), "install", "-m", "0755", binary, "/usr/local/bin/"] + ) + + +def _curl(runner: Runner, url: str, destination: Path) -> None: + """Download one URL with bounded retries.""" + runner.run( + [ + "curl", + "--fail", + "--location", + "--retry", + "3", + "--retry-all-errors", + "--output", + destination, + url, + ], + timeout=300, + ) + + +def _verify_sha256(path: Path, expected: str) -> None: + """Verify one downloaded file against an expected SHA-256.""" + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected.lower(): + raise ProvisionError( + f"SHA-256 mismatch for {path.name}: {actual} != {expected}" + ) + + +def _download_kind( + runner: Runner, directory: Path, version: str, architecture: str +) -> Path: + """Download and verify kind.""" + name = f"kind-linux-{architecture}" + base = f"https://github.com/kubernetes-sigs/kind/releases/download/{version}" + binary = directory / "kind" + checksum = directory / "kind.sha256sum" + _curl(runner, f"{base}/{name}", binary) + _curl(runner, f"{base}/{name}.sha256sum", checksum) + _verify_sha256(binary, checksum.read_text(encoding="utf-8").split()[0]) + return binary + + +def _download_kubectl( + runner: Runner, directory: Path, version: str, architecture: str +) -> Path: + """Download and verify kubectl.""" + base = f"https://dl.k8s.io/release/{version}/bin/linux/{architecture}/kubectl" + binary = directory / "kubectl" + checksum = directory / "kubectl.sha256" + _curl(runner, base, binary) + _curl(runner, f"{base}.sha256", checksum) + _verify_sha256(binary, checksum.read_text(encoding="utf-8").strip()) + return binary + + +def _download_helm( + runner: Runner, directory: Path, version: str, architecture: str +) -> Path: + """Download and verify Helm 3.""" + archive_name = f"helm-{version}-linux-{architecture}.tar.gz" + base = f"https://get.helm.sh/{archive_name}" + archive = directory / archive_name + checksum = directory / f"{archive_name}.sha256sum" + _curl(runner, base, archive) + _curl(runner, f"{base}.sha256sum", checksum) + _verify_sha256(archive, checksum.read_text(encoding="utf-8").split()[0]) + with tarfile.open(archive, "r:gz") as tar: + member = tar.getmember(f"linux-{architecture}/helm") + member.name = "helm" + tar.extract(member, directory, filter="data") + return directory / "helm" + + +def _kubectl(config: Config, *arguments: str | Path) -> list[str | Path]: + """Build a kubectl command using the private kubeconfig.""" + return ["kubectl", "--kubeconfig", config.kubeconfig, *arguments] + + +def _kind_clusters(runner: Runner) -> set[str]: + """Return running kind cluster names.""" + result = runner.run(["kind", "get", "clusters"], check=False, timeout=30) + return { + line.strip() + for line in result.stdout.splitlines() + if line.strip() and "No kind clusters" not in line + } + + +def _kind_containers(runner: Runner, config: Config, running_only: bool) -> list[str]: + """Return kind node container IDs owned by this cluster.""" + arguments = ["docker", "ps"] + if not running_only: + arguments.append("--all") + arguments.extend( + [ + "--filter", + f"label=io.x-k8s.kind.cluster={config.cluster_name}", + "--format", + "{{.ID}}", + ] + ) + output = runner.run(arguments, timeout=30).stdout + return [line for line in output.splitlines() if line] + + +def _render_kind_config(config: Config) -> Path: + """Render the immutable three-node topology.""" + return _render_resource( + config, + "manifests/kind.yaml.tmpl", + {"CLUSTER_NAME": config.cluster_name}, + ) + + +def _create_cluster(runner: Runner, config: Config) -> None: + """Create a new owned kind cluster.""" + manifest = _render_kind_config(config) + LOG.info("Creating three-node kind cluster %s", config.cluster_name) + runner.run( + [ + "kind", + "create", + "cluster", + "--name", + config.cluster_name, + "--image", + KIND_NODE_IMAGE, + "--config", + manifest, + "--kubeconfig", + config.kubeconfig, + "--wait", + "180s", + ], + timeout=600, + ) + + +def _ensure_cluster_ownership(config: Config, cluster_exists: bool) -> None: + """Claim a new cluster name or validate its persistent ownership marker.""" + marker = config.state_dir / "cluster-owner.json" + expected = {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name} + if marker.exists(): + if json.loads(marker.read_text(encoding="utf-8")) != expected: + raise ProvisionError(f"cluster ownership marker does not match: {marker}") + return + if cluster_exists: + raise ProvisionError( + f"refusing to adopt existing unowned kind cluster {config.cluster_name}; " + f"use another --cluster-name or restore {marker}" + ) + _write_text(marker, json.dumps(expected, sort_keys=True) + "\n") + + +def _export_kubeconfig(runner: Runner, config: Config) -> None: + """Refresh the private kubeconfig for an existing cluster.""" + runner.run( + [ + "kind", + "export", + "kubeconfig", + "--name", + config.cluster_name, + "--kubeconfig", + config.kubeconfig, + ], + timeout=60, + ) + + +def _delete_cluster(runner: Runner, config: Config) -> None: + """Delete the exact marker-owned disposable kind cluster.""" + LOG.info("Deleting disposable kind cluster %s", config.cluster_name) + runner.run( + ["kind", "delete", "cluster", "--name", config.cluster_name], timeout=300 + ) + leftovers = _kind_containers(runner, config, running_only=False) + if leftovers: + raise ProvisionError( + "kind reported successful deletion, but cluster containers remain: " + + ", ".join(leftovers) + ) + + +def _wait_for_cluster(runner: Runner, config: Config) -> None: + """Wait for exactly three Ready nodes and enforce fixture labels.""" + _wait_for_kube_api(runner, config) + runner.run( + _kubectl( + config, + "wait", + "--for=condition=Ready", + "nodes", + "--all", + "--timeout=180s", + ), + timeout=210, + ) + runner.run( + _kubectl( + config, + "taint", + "nodes", + f"{config.cluster_name}-control-plane", + "node-role.kubernetes.io/control-plane:NoSchedule-", + ), + check=False, + ) + nodes = json.loads( + runner.run(_kubectl(config, "get", "nodes", "-o", "json")).stdout + ) + if len(nodes["items"]) != 3: + raise ProvisionError( + f"expected exactly 3 Kubernetes nodes; found {len(nodes['items'])}" + ) + _validate_node_labels(nodes) + + +def _wait_for_kube_api(runner: Runner, config: Config) -> None: + """Wait for the Kubernetes API to become usable.""" + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + probe = runner.run( + _kubectl(config, "get", "nodes", "--request-timeout=5s"), check=False + ) + if probe.returncode == 0: + return + LOG.info("Waiting for the Kubernetes API") + time.sleep(3) + raise ProvisionError("Kubernetes API did not become usable within 90 seconds") + + +def _validate_node_labels(nodes: dict[str, object]) -> None: + """Validate two targets and one target-negative login node.""" + target_count = 0 + login_count = 0 + for node in nodes["items"]: # type: ignore[index] + labels = node["metadata"]["labels"] # type: ignore[index] + target_count += labels.get(TARGET_LABEL.split("=")[0]) == "true" + login_count += labels.get(LOGIN_LABEL.split("=")[0]) == "true" + if target_count != 2 or login_count != 1: + raise ProvisionError( + f"node label invariant failed: target nodes={target_count}, login nodes={login_count}" + ) + + +def _kind_ipv4_network(runner: Runner) -> tuple[str, str]: + """Return the kind Docker network's IPv4 subnet and gateway.""" + data = json.loads(runner.run(["docker", "network", "inspect", "kind"]).stdout) + for entry in data[0]["IPAM"]["Config"]: + subnet = entry.get("Subnet", "") + if "." in subnet: + return subnet, entry["Gateway"] + raise ProvisionError("kind Docker network has no IPv4 IPAM entry") + + +def _ensure_export_marker(runner: Runner, config: Config) -> None: + """Create or validate ownership of the dedicated host export.""" + marker_path = config.export_dir / ".storage-scale-test-integration.json" + expected = json.dumps( + {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name}, sort_keys=True + ) + existing = runner.run( + [*_sudo_prefix(), "cat", marker_path], check=False, timeout=30 + ) + if existing.returncode == 0 and existing.stdout.strip() != expected: + raise ProvisionError( + f"refusing export with mismatched ownership marker: {marker_path}" + ) + if existing.returncode != 0: + probe = runner.run( + [ + *_sudo_prefix(), + "find", + config.export_dir, + "-mindepth", + "1", + "-maxdepth", + "1", + ], + check=False, + ) + if probe.returncode == 0 and probe.stdout.strip(): + raise ProvisionError( + f"refusing nonempty unowned export directory: {config.export_dir}" + ) + runner.run([*_sudo_prefix(), "install", "-d", "-m", "0770", config.export_dir]) + runner.run([*_sudo_prefix(), "chown", f"+{NFS_UID}:+{NFS_GID}", config.export_dir]) + marker_source = config.state_dir / "export-marker.json" + _write_text(marker_source, expected + "\n") + runner.run( + [ + *_sudo_prefix(), + "install", + "-m", + "0644", + marker_source, + marker_path, + ] + ) + + +def _export_mount_type(runner: Runner, config: Config) -> str: + """Return the export mount filesystem type, or an empty string.""" + result = runner.run( + [ + "findmnt", + "--noheadings", + "--output", + "FSTYPE", + "--mountpoint", + config.export_dir, + ], + check=False, + ) + return result.stdout.strip() if result.returncode == 0 else "" + + +def _ensure_export_filesystem(runner: Runner, config: Config) -> None: + """Mount a small persistent filesystem for realistic mount validation.""" + runner.run([*_sudo_prefix(), "install", "-d", "-m", "0770", config.export_dir]) + mounted_type = _export_mount_type(runner, config) + if mounted_type: + if mounted_type != "ext4": + raise ProvisionError( + f"refusing non-ext4 mount at dedicated export {config.export_dir}: " + f"{mounted_type}" + ) + loops = runner.run( + [*_sudo_prefix(), "losetup", "--associated", config.nfs_image], + check=False, + ).stdout + if not loops.strip(): + raise ProvisionError( + f"mounted export {config.export_dir} is not backed by " + f"{config.nfs_image}" + ) + return + + _ensure_export_marker(runner, config) + if not config.nfs_image.exists(): + LOG.info("Creating sparse %s-byte NFS backing filesystem", NFS_IMAGE_BYTES) + temporary_image = config.nfs_image.with_suffix(".ext4.new") + temporary_image.unlink(missing_ok=True) + runner.run(["truncate", "--size", str(NFS_IMAGE_BYTES), temporary_image]) + runner.run(["/usr/sbin/mkfs.ext4", "-F", "-q", "-m", "0", temporary_image]) + temporary_image.replace(config.nfs_image) + with tempfile.TemporaryDirectory(dir=config.state_dir) as directory: + migration_mount = Path(directory) + runner.run( + [ + *_sudo_prefix(), + "mount", + "-o", + "loop", + config.nfs_image, + migration_mount, + ] + ) + try: + runner.run( + [ + *_sudo_prefix(), + "cp", + "-a", + f"{config.export_dir}/.", + f"{migration_mount}/", + ] + ) + finally: + runner.run([*_sudo_prefix(), "umount", migration_mount], check=False) + runner.run([*_sudo_prefix(), "exportfs", "-u", config.export_dir], check=False) + runner.run( + [*_sudo_prefix(), "mount", "-o", "loop", config.nfs_image, config.export_dir] + ) + _ensure_export_marker(runner, config) + + +def _configure_nfs(runner: Runner, config: Config, subnet: str, gateway: str) -> None: + """Reconcile the narrow NFSv4 export and firewall rule.""" + LOG.info("Configuring NFSv4 export for kind subnet %s", subnet) + _record_nfs_service_state(runner, config) + _ensure_export_filesystem(runner, config) + _ensure_export_marker(runner, config) + export_line = ( + f"{config.export_dir} {subnet}(rw,sync,no_subtree_check,fsid=0," + f"all_squash,anonuid={NFS_UID},anongid={NFS_GID})\n" + ) + export_source = config.manifests_dir / "storage-scale-test.exports" + nfs_source = config.manifests_dir / "storage-scale-test-nfs.conf" + _write_text(export_source, export_line) + _write_text(nfs_source, "[nfsd]\nvers3 = n\nvers4 = y\nthreads = 2\n") + runner.run([*_sudo_prefix(), "install", "-d", "/etc/exports.d", "/etc/nfs.conf.d"]) + runner.run( + [ + *_sudo_prefix(), + "install", + "-m", + "0644", + export_source, + "/etc/exports.d/storage-scale-test-integration.exports", + ] + ) + runner.run( + [ + *_sudo_prefix(), + "install", + "-m", + "0644", + nfs_source, + "/etc/nfs.conf.d/storage-scale-test-integration.conf", + ] + ) + _ensure_nfs_firewall(runner, subnet) + runner.run([*_sudo_prefix(), "exportfs", "-rav"]) + runner.run([*_sudo_prefix(), "systemctl", "enable", "--now", "nfs-server"]) + state = {"subnet": subnet, "gateway": gateway} + _write_text(config.state_dir / "network.json", json.dumps(state, indent=2) + "\n") + + +def _record_nfs_service_state(runner: Runner, config: Config) -> None: + """Remember whether this harness was responsible for starting NFS.""" + path = config.state_dir / "nfs-service.json" + if path.exists(): + return + active = ( + runner.run( + [*_sudo_prefix(), "systemctl", "is-active", "nfs-server"], check=False + ).returncode + == 0 + ) + _write_text(path, json.dumps({"started_by_harness": not active}) + "\n") + + +def _ensure_nfs_firewall(runner: Runner, subnet: str) -> None: + """Allow NFS only from kind when UFW is active.""" + if not shutil.which("ufw"): + LOG.warning("ufw is absent; verify an equivalent TCP-2049 restriction") + return + status = runner.run([*_sudo_prefix(), "ufw", "status"], check=False) + if not status.stdout.startswith("Status: active"): + LOG.info("ufw is inactive; exportfs remains restricted to %s", subnet) + return + runner.run( + [ + *_sudo_prefix(), + "ufw", + "allow", + "from", + subnet, + "to", + "any", + "port", + "2049", + "proto", + "tcp", + "comment", + "storage-scale-test integration NFSv4", + ] + ) + + +def _probe_nfs(runner: Runner, config: Config, gateway: str) -> None: + """Prove a kind node can mount and write the export.""" + node = f"{config.cluster_name}-control-plane" + script = ( + "set -eu; mkdir -p /tmp/storage-scale-nfs-probe; " + f"mount -t nfs4 -o vers=4.1 {gateway}:/ /tmp/storage-scale-nfs-probe; " + "touch /tmp/storage-scale-nfs-probe/.provisioner-probe; " + "rm /tmp/storage-scale-nfs-probe/.provisioner-probe; " + "umount /tmp/storage-scale-nfs-probe" + ) + runner.run(["docker", "exec", node, "sh", "-c", script], timeout=90) + + +def _image_exists(runner: Runner, image: str) -> bool: + """Return whether Docker has *image*.""" + return ( + runner.run(["docker", "image", "inspect", image], check=False).returncode == 0 + ) + + +def _prepare_csi_images(runner: Runner, config: Config) -> None: + """Pull CSI images with an official staging fallback and load all nodes.""" + LOG.info("Preparing pinned NFS CSI images") + destinations: list[str] = [] + for name, tag in CSI_IMAGES: + destination = f"registry.k8s.io/sig-storage/{name}:{tag}" + destinations.append(destination) + if not _image_exists(runner, destination): + pull = runner.run(["docker", "pull", destination], check=False, timeout=180) + if pull.returncode: + source = f"gcr.io/k8s-staging-sig-storage/{name}:{tag}" + LOG.warning( + "Production registry failed for %s; using official staging", name + ) + runner.run(["docker", "pull", source], timeout=300) + runner.run(["docker", "tag", source, destination]) + nodes = _kind_containers(runner, config, running_only=True) + if len(nodes) != 3: + raise ProvisionError( + f"cannot preload CSI images: expected 3 nodes, found {len(nodes)}" + ) + for image in destinations: + _load_image_into_nodes(runner, image, nodes) + + +def _load_image_into_nodes(runner: Runner, image: str, nodes: list[str]) -> None: + """Import one amd64/arm64 Docker image into each kind containerd store.""" + with tempfile.TemporaryDirectory(prefix="storage-scale-image-") as directory: + archive_path = Path(directory) / "image.tar" + runner.run(["docker", "save", "--output", archive_path, image], timeout=300) + for node in nodes: + with archive_path.open("rb") as archive: + runner.run( + [ + "docker", + "exec", + "-i", + node, + "ctr", + "-n", + "k8s.io", + "images", + "import", + "--snapshotter=overlayfs", + "-", + ], + stdin=archive, + timeout=300, + ) + + +def _install_nfs_csi(runner: Runner, config: Config, gateway: str) -> None: + """Install NFS CSI and bind the two RWX claims.""" + _prepare_csi_images(runner, config) + chart = _ensure_nfs_csi_chart(runner, config) + values = _resource_path("manifests/nfs-csi-values.yaml") + runner.run( + [ + "helm", + "upgrade", + "--install", + "csi-driver-nfs", + chart, + "--namespace", + "kube-system", + "--values", + values, + "--wait", + "--timeout", + "5m", + "--kubeconfig", + config.kubeconfig, + ], + timeout=420, + ) + _ensure_namespace(runner, config) + storage = _render_resource( + config, + "manifests/nfs-storage.yaml.tmpl", + {"NAMESPACE": config.namespace, "NFS_SERVER": gateway}, + ) + runner.run(_kubectl(config, "apply", "-f", storage)) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "wait", + "--for=jsonpath={.status.phase}=Bound", + "pvc/storage-test-rwx", + "pvc/ssh-home-rwx", + "--timeout=180s", + ), + timeout=210, + ) + + +def _ensure_nfs_csi_chart(runner: Runner, config: Config) -> Path: + """Cache the pinned NFS CSI chart from a checksum-verified source archive.""" + chart = config.state_dir / "charts" / f"csi-driver-nfs-{NFS_CSI_VERSION}.tgz" + if chart.exists(): + _verify_sha256(chart, NFS_CSI_CHART_SHA256) + return chart + url = ( + "https://codeload.github.com/kubernetes-csi/csi-driver-nfs/tar.gz/" + f"refs/tags/v{NFS_CSI_VERSION}" + ) + with tempfile.TemporaryDirectory(prefix="storage-scale-nfs-chart-") as directory: + source = Path(directory) / "source.tar.gz" + _curl(runner, url, source) + _verify_sha256(source, NFS_CSI_SOURCE_SHA256) + member_name = ( + f"csi-driver-nfs-{NFS_CSI_VERSION}/charts/latest/" + f"csi-driver-nfs-{NFS_CSI_VERSION}.tgz" + ) + with tarfile.open(source, "r:gz") as archive: + member = archive.extractfile(member_name) + if member is None: + raise ProvisionError(f"NFS CSI chart is absent from {source.name}") + content = member.read() + if hashlib.sha256(content).hexdigest() != NFS_CSI_CHART_SHA256: + raise ProvisionError("SHA-256 mismatch for embedded NFS CSI chart") + _write_bytes(chart, content) + return chart + + +def _ensure_namespace(runner: Runner, config: Config) -> None: + """Create the integration namespace when absent.""" + probe = runner.run( + _kubectl(config, "get", "namespace", config.namespace), check=False + ) + if probe.returncode: + runner.run(_kubectl(config, "create", "namespace", config.namespace)) + + +def _ensure_ssh_key(runner: Runner, config: Config) -> tuple[Path, Path]: + """Create or reuse the dedicated integration-test SSH key.""" + private_key = config.keys_dir / "id_ed25519" + public_key = config.keys_dir / "id_ed25519.pub" + if private_key.exists() and public_key.exists(): + return private_key, public_key + if private_key.exists() or public_key.exists(): + raise ProvisionError(f"incomplete SSH identity in {config.keys_dir}") + runner.run( + [ + "ssh-keygen", + "-q", + "-t", + "ed25519", + "-N", + "", + "-C", + "storage-scale-integration", + "-f", + private_key, + ], + sensitive=True, + ) + private_key.chmod(0o600) + public_key.chmod(0o644) + return private_key, public_key + + +def _ensure_file_secret( + runner: Runner, + config: Config, + name: str, + files: dict[str, Path], +) -> None: + """Create an immutable-input file Secret when absent.""" + probe = runner.run( + _kubectl(config, "-n", config.namespace, "get", "secret", name), check=False + ) + if probe.returncode == 0: + return + arguments: list[str | Path] = [ + *_kubectl(config, "-n", config.namespace, "create", "secret", "generic", name) + ] + arguments.extend(f"--from-file={key}={value}" for key, value in files.items()) + runner.run(arguments, sensitive=True) + + +def _install_ssh_workers(runner: Runner, config: Config) -> None: + """Build, deploy, and validate the two SSH workers.""" + private_key, public_key = _ensure_ssh_key(runner, config) + runner.run( + [ + "docker", + "build", + "--tag", + SSH_IMAGE, + "--file", + _resource_path("ssh-image.Dockerfile"), + _resource_path("."), + ], + timeout=600, + ) + nodes = _kind_containers(runner, config, running_only=True) + _load_image_into_nodes(runner, SSH_IMAGE, nodes) + _ensure_file_secret( + runner, + config, + "storage-ssh-identity", + {"id_ed25519": private_key, "authorized_keys": public_key}, + ) + home_volume = ( + "persistentVolumeClaim:\n claimName: ssh-home-rwx" + if config.ssh_home_mode == "shared" + else "emptyDir:\n sizeLimit: 64Mi" + ) + manifest = _render_resource( + config, + "manifests/ssh-workers.yaml.tmpl", + {"NAMESPACE": config.namespace, "SSH_HOME_VOLUME": home_volume}, + ) + runner.run( + _kubectl( + config, + "apply", + "--server-side", + "--force-conflicts", + "--field-manager=storage-scale-integration", + "-f", + manifest, + ) + ) + _wait_for_ssh(runner, config) + _validate_ssh_workers(runner, config, private_key) + + +def _ssh_pods(runner: Runner, config: Config) -> list[dict[str, object]]: + """Return SSH pod objects in ordinal order.""" + result = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "pods", + "-l", + "app.kubernetes.io/name=storage-ssh-worker", + "-o", + "json", + ) + ) + return sorted( + json.loads(result.stdout)["items"], key=lambda item: item["metadata"]["name"] + ) + + +def _validate_ssh_workers(runner: Runner, config: Config, private_key: Path) -> None: + """Validate placement, host SSH, home semantics, and RWX visibility.""" + pods = _ssh_pods(runner, config) + if len(pods) != 2: + raise ProvisionError(f"expected 2 SSH pods; found {len(pods)}") + nodes = {pod["spec"]["nodeName"] for pod in pods} + if len(nodes) != 2 or f"{config.cluster_name}-control-plane" in nodes: + raise ProvisionError(f"SSH placement invariant failed: {sorted(nodes)}") + known_hosts = config.state_dir / "ssh_known_hosts" + scan_lines: list[str] = [] + addresses: list[str] = [] + for pod in pods: + address = str(pod["status"]["podIP"]) + addresses.append(address) + scan = runner.run(["ssh-keyscan", "-T", "10", str(address)], timeout=30) + scan_lines.extend(scan.stdout.splitlines()) + _write_text(known_hosts, "\n".join(scan_lines) + "\n", mode=0o600) + _write_text(config.state_dir / "ssh_hosts", "\n".join(addresses) + "\n") + for address in addresses: + runner.run( + [ + "ssh", + "-i", + private_key, + "-o", + "BatchMode=yes", + "-o", + f"UserKnownHostsFile={known_hosts}", + f"tester@{address}", + "test $(id -u) -eq 2000 && test $(stat -f -c %T /root) = overlayfs", + ], + timeout=30, + ) + root_probe = runner.run( + [ + "ssh", + "-i", + private_key, + "-o", + "BatchMode=yes", + "-o", + f"UserKnownHostsFile={known_hosts}", + f"root@{addresses[0]}", + "true", + ], + check=False, + timeout=30, + ) + if root_probe.returncode == 0: + raise ProvisionError("SSH root-login rejection validation failed") + for pod in pods: + _pod_exec( + runner, + config, + str(pod["metadata"]["name"]), + "rm -f /home/tester/.ssh/known_hosts", + ) + for source, destination in ((0, 1), (1, 0)): + _pod_exec( + runner, + config, + str(pods[source]["metadata"]["name"]), + "runuser -u tester -- ssh -o BatchMode=yes " + "-o StrictHostKeyChecking=accept-new " + f"tester@{addresses[destination]} true", + ) + _validate_ssh_storage(runner, config, pods) + + +def _pod_exec( + runner: Runner, config: Config, pod: str, script: str, *, check: bool = True +) -> subprocess.CompletedProcess[str]: + """Run a bounded shell command in an SSH worker pod.""" + return runner.run( + _kubectl( + config, + "-n", + config.namespace, + "exec", + pod, + "--", + "bash", + "-c", + script, + ), + check=check, + timeout=60, + ) + + +def _validate_ssh_storage( + runner: Runner, config: Config, pods: list[dict[str, object]] +) -> None: + """Validate the selected home mode and shared storage claim.""" + names = [str(pod["metadata"]["name"]) for pod in pods] + _pod_exec( + runner, + config, + names[0], + "touch /home/tester/.integration-home-probe /mnt/storage-test/.integration-rwx-probe", + ) + home_probe = _pod_exec( + runner, + config, + names[1], + "test -e /home/tester/.integration-home-probe", + check=False, + ) + rwx_probe = _pod_exec( + runner, + config, + names[1], + "test -e /mnt/storage-test/.integration-rwx-probe && " + "case $(stat -f -c %T /mnt/storage-test) in nfs|nfs4) true;; *) false;; esac", + check=False, + ) + expected_home_rc = 0 if config.ssh_home_mode == "shared" else 1 + if home_probe.returncode != expected_home_rc or rwx_probe.returncode: + raise ProvisionError("SSH home or RWX visibility validation failed") + cleanup = ( + "rm -f /home/tester/.integration-home-probe " + "/mnt/storage-test/.integration-rwx-probe" + ) + _pod_exec(runner, config, names[0], cleanup) + + +def _load_or_create_db_credentials(config: Config) -> dict[str, str]: + """Return stable protected MariaDB credentials.""" + path = config.keys_dir / "mariadb.json" + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + credentials = { + "password": secrets.token_hex(24), + "root_password": secrets.token_hex(24), + } + _write_text(path, json.dumps(credentials) + "\n", mode=0o600) + return credentials + + +def _ensure_mariadb_secret(runner: Runner, config: Config) -> None: + """Create the stable MariaDB Secret when absent.""" + name = "mariadb-password" + probe = runner.run( + _kubectl(config, "-n", config.namespace, "get", "secret", name), check=False + ) + if probe.returncode == 0: + return + credentials = _load_or_create_db_credentials(config) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "create", + "secret", + "generic", + name, + f"--from-literal=password={credentials['password']}", + f"--from-literal=root-password={credentials['root_password']}", + ), + sensitive=True, + ) + + +def _install_slurm(runner: Runner, config: Config) -> None: + """Install MariaDB, Slinky, and the two-node Slurm fixture.""" + _prepare_slinky_login_image(runner, config) + _ensure_mariadb_secret(runner, config) + mariadb = _render_resource( + config, + "manifests/mariadb-accounting.yaml.tmpl", + {"NAMESPACE": config.namespace}, + ) + runner.run(_kubectl(config, "apply", "-f", mariadb)) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "statefulset/mariadb-accounting", + "--timeout=240s", + ), + timeout=270, + ) + _helm_slinky(runner, config) + _wait_for_slurm(runner, config) + _validate_slurm(runner, config) + + +def _prepare_slinky_login_image(runner: Runner, config: Config) -> None: + """Build and preload the login image with declared test prerequisites.""" + runner.run( + [ + "docker", + "build", + "--tag", + SLINKY_LOGIN_IMAGE, + "--build-arg", + f"BASE_IMAGE={SLINKY_LOGIN_BASE_IMAGE}", + "--file", + _resource_path("slinky-login-image.Dockerfile"), + _resource_path("."), + ], + timeout=600, + ) + _load_image_into_nodes( + runner, + SLINKY_LOGIN_IMAGE, + [f"{config.cluster_name}-control-plane"], + ) + + +def _slinky_login_image_current(runner: Runner, config: Config) -> bool: + """Return whether the live LoginSet selects the prepared image.""" + result = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "loginsets", + "-o", + "json", + ), + check=False, + timeout=30, + ) + if result.returncode: + return False + return any( + item.get("spec", {}).get("login", {}).get("image") == SLINKY_LOGIN_IMAGE + for item in json.loads(result.stdout).get("items", []) + ) + + +def _helm_slinky(runner: Runner, config: Config) -> None: + """Reconcile the three pinned Slinky releases.""" + releases = ( + ( + "slurm-operator-crds", + "slurm-operator-crds", + None, + ), + ( + "slurm-operator", + "slurm-operator", + _resource_path("manifests/slinky-operator-values.yaml"), + ), + ( + "slurm", + "slurm", + _resource_path("manifests/slinky-slurm-values.yaml"), + ), + ) + for release, chart, values in releases: + current = _helm_release_current(runner, config, release, chart) + if current and ( + release != "slurm" or _slinky_login_image_current(runner, config) + ): + LOG.info("Slinky release %s is already at %s", release, SLINKY_VERSION) + continue + arguments: list[str | Path] = [ + "helm", + "upgrade", + "--install", + release, + f"oci://ghcr.io/slinkyproject/charts/{chart}", + "--version", + SLINKY_VERSION, + "--namespace", + config.namespace, + "--kubeconfig", + config.kubeconfig, + "--wait", + "--timeout", + "8m", + ] + if values: + arguments.extend(["--values", values]) + runner.run(arguments, timeout=600) + if release == "slurm-operator": + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "restart", + "deployment/slurm-operator-webhook", + ) + ) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "deployment/slurm-operator-webhook", + "--timeout=180s", + ), + timeout=210, + ) + + +def _helm_release_current( + runner: Runner, config: Config, release: str, chart: str +) -> bool: + """Return whether a healthy release already has the pinned chart version.""" + result = runner.run( + [ + "helm", + "list", + "--namespace", + config.namespace, + "--all", + "--output", + "json", + "--kubeconfig", + config.kubeconfig, + ], + check=False, + timeout=60, + ) + if result.returncode: + return False + expected_chart = f"{chart}-{SLINKY_VERSION}" + return any( + item.get("name") == release + and item.get("chart") == expected_chart + and item.get("status") == "deployed" + for item in json.loads(result.stdout) + ) + + +def _wait_for_slurm(runner: Runner, config: Config) -> None: + """Wait for Slinky child resources not covered by Helm's wait.""" + expected = {"slurmdbd": 1, "slurmctld": 1, "slurmrestd": 1, "login": 1, "slurmd": 2} + deadline = time.monotonic() + 300 + while time.monotonic() < deadline: + pods = _namespace_pods(runner, config) + ready = { + container: sum( + _pod_is_ready(pod) for pod in _pods_with_container(pods, container) + ) + for container in expected + } + if ready == expected: + return + LOG.info("Waiting for Slinky child pods: ready=%s expected=%s", ready, expected) + time.sleep(5) + raise ProvisionError(f"Slinky child pods did not become ready: {ready}") + + +def _namespace_pods(runner: Runner, config: Config) -> list[dict[str, object]]: + """Return all pod objects in the integration namespace.""" + result = runner.run( + _kubectl(config, "-n", config.namespace, "get", "pods", "-o", "json") + ) + return json.loads(result.stdout)["items"] + + +def _pods_with_container( + pods: list[dict[str, object]], container: str +) -> list[dict[str, object]]: + """Select pods containing a named Slinky workload container.""" + return [ + pod + for pod in pods + if container + in {entry["name"] for entry in pod["spec"]["containers"]} # type: ignore[index] + ] + + +def _pod_is_ready(pod: dict[str, object]) -> bool: + """Return whether a pod is running with all containers ready.""" + status = pod["status"] # type: ignore[index] + containers = status.get("containerStatuses", []) + return ( + status.get("phase") == "Running" + and bool(containers) + and all(container.get("ready", False) for container in containers) + ) + + +def _login_pod(runner: Runner, config: Config) -> str: + """Return the single Slinky LoginSet pod name.""" + pods = _pods_with_container(_namespace_pods(runner, config), "login") + if len(pods) != 1: + raise ProvisionError(f"expected one LoginSet pod; found {len(pods)}") + return str(pods[0]["metadata"]["name"]) # type: ignore[index] + + +def _validate_slurm(runner: Runner, config: Config) -> None: + """Validate LoginSet placement, NFS, two-node fan-out, and accounting.""" + login = _login_pod(runner, config) + login_node = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "pod", + login, + "-o", + "jsonpath={.spec.nodeName}", + ) + ).stdout.strip() + expected_login_node = f"{config.cluster_name}-control-plane" + if login_node != expected_login_node: + raise ProvisionError( + f"LoginSet placement failed: {login} is on {login_node}, " + f"expected {expected_login_node}" + ) + pods = _namespace_pods(runner, config) + workers = _pods_with_container(pods, "slurmd") + worker_nodes = {str(pod["spec"]["nodeName"]) for pod in workers} # type: ignore[index] + target_result = runner.run( + _kubectl( + config, + "get", + "nodes", + "-l", + TARGET_LABEL, + "-o", + "jsonpath={.items[*].metadata.name}", + ) + ) + target_nodes = set(target_result.stdout.split()) + if worker_nodes != target_nodes or len(worker_nodes) != 2: + raise ProvisionError( + f"Slurm worker placement failed: pods={sorted(worker_nodes)}, " + f"targets={sorted(target_nodes)}" + ) + prefix = _kubectl(config, "-n", config.namespace, "exec", login, "--") + runner.run( + [ + *prefix, + "bash", + "-c", + "case $(stat -f -c %T /mnt/storage-test) in " + "nfs|nfs4) true;; *) false;; esac", + ] + ) + fanout = runner.run( + [ + *prefix, + "srun", + "-p", + "all", + "-N2", + "-n2", + "--ntasks-per-node=1", + "hostname", + ], + timeout=120, + ).stdout.splitlines() + if len(set(fanout)) != 2: + raise ProvisionError( + f"Slurm fan-out did not reach two distinct nodes: {fanout}" + ) + job = runner.run( + [ + *prefix, + "sbatch", + "--wait", + "--parsable", + "-p", + "all", + "-N2", + "-n2", + "--ntasks-per-node=1", + "--output=/mnt/storage-test/integration-accounting-%j.out", + "--wrap=srun hostname", + ], + timeout=180, + ).stdout.strip() + accounting = runner.run( + [ + *prefix, + "sacct", + "-X", + "-j", + job, + "--format=State,ExitCode", + "-n", + "-P", + ] + ).stdout + if "COMPLETED|0:0" not in accounting: + raise ProvisionError( + f"Slurm accounting validation failed for job {job}: {accounting}" + ) + + +def _write_state_summary(config: Config, subnet: str, gateway: str) -> None: + """Persist non-secret desired state for later diagnostics.""" + state = { + "schema": STATE_SCHEMA, + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "export_dir": str(config.export_dir), + "ssh_home_mode": config.ssh_home_mode, + "kind_version": KIND_VERSION, + "kubernetes_version": KUBECTL_VERSION, + "nfs_csi_version": NFS_CSI_VERSION, + "slinky_version": SLINKY_VERSION, + "kind_subnet": subnet, + "kind_gateway": gateway, + } + _write_text(config.state_dir / "state.json", json.dumps(state, indent=2) + "\n") + + +def setup_environment(runner: Runner, config: Config) -> None: + """Idempotently provision and validate the complete fixture.""" + architecture = _check_platform() + _check_host_capacity() + _ensure_apt_packages(runner) + _ensure_docker(runner) + _ensure_client_tools(runner, architecture) + running_clusters = _kind_clusters(runner) + containers = _kind_containers(runner, config, running_only=False) + running_containers = _kind_containers(runner, config, running_only=True) + cluster_exists = config.cluster_name in running_clusters or bool(containers) + _ensure_cluster_ownership(config, cluster_exists) + if cluster_exists and not _export_mount_type(runner, config): + LOG.warning( + "Replacing the disposable cluster before initializing the NFS " + "backing filesystem" + ) + _delete_cluster(runner, config) + running_clusters = set() + running_containers = set() + cluster_exists = False + _ensure_export_filesystem(runner, config) + if config.cluster_name in running_clusters and len(running_containers) == 3: + _export_kubeconfig(runner, config) + elif cluster_exists: + LOG.warning("Replacing incomplete or stopped disposable kind cluster") + _delete_cluster(runner, config) + _create_cluster(runner, config) + else: + _create_cluster(runner, config) + _wait_for_cluster(runner, config) + subnet, gateway = _kind_ipv4_network(runner) + _configure_nfs(runner, config, subnet, gateway) + _probe_nfs(runner, config, gateway) + _install_nfs_csi(runner, config, gateway) + _install_ssh_workers(runner, config) + _scale_ssh(runner, config, replicas=0) + _install_slurm(runner, config) + _scale_ssh(runner, config, replicas=2) + _wait_for_ssh(runner, config) + private_key = config.keys_dir / "id_ed25519" + _validate_ssh_workers(runner, config, private_key) + _write_state_summary(config, subnet, gateway) + LOG.info("Integration environment is provisioned and running") + + +def _scale_ssh(runner: Runner, config: Config, replicas: int) -> None: + """Scale SSH workers to conserve host capacity between checks.""" + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "scale", + "statefulset/ssh-worker", + f"--replicas={replicas}", + ) + ) + + +def _wait_for_ssh(runner: Runner, config: Config) -> None: + """Wait for both SSH workers after restoring the running fixture.""" + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "statefulset/ssh-worker", + "--timeout=180s", + ), + timeout=210, + ) + + +def stop_environment(runner: Runner, config: Config) -> None: + """Delete the disposable cluster and stop owned host services.""" + clusters = _kind_clusters(runner) + containers = _kind_containers(runner, config, running_only=False) + cluster_exists = config.cluster_name in clusters or bool(containers) + if cluster_exists: + _ensure_cluster_ownership(config, cluster_exists=True) + _delete_cluster(runner, config) + else: + LOG.info("Disposable kind cluster %s is already absent", config.cluster_name) + _stop_owned_nfs(runner, config) + LOG.info( + "Integration environment stopped; host packages, caches, keys, and NFS data " + "were preserved" + ) + + +def _stop_owned_nfs(runner: Runner, config: Config) -> None: + """Stop NFS only when this harness started the otherwise-dedicated service.""" + state_path = config.state_dir / "nfs-service.json" + if not state_path.exists(): + LOG.info("NFS ownership state is absent; leaving nfs-server unchanged") + return + state = json.loads(state_path.read_text(encoding="utf-8")) + if not state.get("started_by_harness", False): + LOG.info("nfs-server predated this fixture; leaving it running") + return + exports = runner.run([*_sudo_prefix(), "exportfs", "-v"], check=False).stdout + export_paths = [ + line.split()[0] for line in exports.splitlines() if line.startswith("/") + ] + unrelated = [path for path in export_paths if path != str(config.export_dir)] + if unrelated: + LOG.warning( + "Leaving nfs-server running because unrelated exports exist: %s", unrelated + ) + return + service = runner.run( + [*_sudo_prefix(), "systemctl", "is-active", "nfs-server"], check=False + ) + if service.returncode == 0: + runner.run([*_sudo_prefix(), "systemctl", "stop", "nfs-server"]) + LOG.info("Stopped harness-owned nfs-server") + else: + LOG.info("nfs-server is already stopped") + + +def _collect_diagnostics(runner: Runner, config: Config) -> None: + """Collect bounded troubleshooting state after a setup failure.""" + LOG.error("Collecting troubleshooting diagnostics") + commands: tuple[Sequence[str | Path], ...] = ( + ("free", "-h"), + ("df", "-h", "/"), + ("docker", "ps", "--all"), + ("kind", "get", "clusters"), + (*_sudo_prefix(), "systemctl", "status", "nfs-server", "--no-pager"), + (*_sudo_prefix(), "exportfs", "-v"), + ) + for command in commands: + result = runner.run(command, check=False, timeout=30) + output = (result.stdout + result.stderr).strip() + if output: + LOG.error("diagnostic %s:\n%s", command[0], output[-12000:]) + if config.kubeconfig.exists(): + for arguments in ( + ("get", "nodes", "-o", "wide"), + ("get", "pods", "--all-namespaces", "-o", "wide"), + ("get", "events", "--all-namespaces", "--sort-by=.lastTimestamp"), + ): + result = runner.run(_kubectl(config, *arguments), check=False, timeout=30) + output = (result.stdout + result.stderr).strip() + if output: + LOG.error("kubectl diagnostic:\n%s", output[-16000:]) + + +def _parser() -> argparse.ArgumentParser: + """Build the command-line parser.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cluster-name", default="storage-scale-integration") + parser.add_argument("--namespace", default="storage-scale-integration") + parser.add_argument("--state-dir", type=Path, default=DEFAULT_STATE_DIR) + parser.add_argument("--export-dir", type=Path, default=DEFAULT_EXPORT_DIR) + parser.add_argument( + "--ssh-home-mode", choices=("separate", "shared"), default="separate" + ) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("action", choices=("setup", "start", "stop", "test")) + parser.add_argument( + "tests", + nargs="*", + metavar="TEST", + help="test selectors for the test action: " + ", ".join(TEST_SELECTORS), + ) + return parser + + +def _config(arguments: argparse.Namespace) -> Config: + """Convert parsed arguments into immutable configuration.""" + return Config( + cluster_name=arguments.cluster_name, + namespace=arguments.namespace, + state_dir=arguments.state_dir.resolve(), + export_dir=arguments.export_dir.resolve(), + ssh_home_mode=arguments.ssh_home_mode, + verbose=arguments.verbose, + ) + + +def main() -> int: + """Run one integration environment lifecycle action.""" + _require_python() + arguments = _parser().parse_args() + config = _config(arguments) + try: + if arguments.action != "test" and arguments.tests: + raise ProvisionError("test selectors are valid only with the test action") + if arguments.action == "test" and not config.state_dir.is_dir(): + raise ProvisionError( + f"setup state directory is absent at {config.state_dir}; run setup first" + ) + _bootstrap_state_dir(config) + log_path = _configure_logging(config, arguments.action) + LOG.info("Detailed log: %s", log_path) + with _acquire_lock(config): + runner = Runner() + if arguments.action == "stop": + stop_environment(runner, config) + elif arguments.action == "test": + run_filesystem_tests( + runner, config, _repository_root(), arguments.tests + ) + else: + setup_environment(runner, config) + return 0 + except ( + ProvisionError, + IntegrationTestError, + OSError, + subprocess.TimeoutExpired, + json.JSONDecodeError, + ) as error: + LOG.error("%s", error) + if "runner" in locals() and arguments.action != "stop": + _collect_diagnostics(runner, config) + return 1 + except KeyboardInterrupt: + LOG.error("Interrupted") + return 130 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py new file mode 100644 index 0000000..f458e32 --- /dev/null +++ b/integration-tests/lib/filesystem_integration.py @@ -0,0 +1,1059 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run bounded filesystem integration tests against the provisioned fixture.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import platform +import shlex +import shutil +import tarfile +import tempfile +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +LOG = logging.getLogger("storage-scale-integration") + +ELBENCHO_VERSION = "v3.1-11" +ELBENCHO_RELEASE_API = ( + "https://api.github.com/repos/breuner/elbencho/releases/tags/" + ELBENCHO_VERSION +) +MAX_ARCHIVE_BYTES = 32 * 1024 * 1024 +MAX_DEPLOYMENT_ARCHIVE_BYTES = 128 * 1024 * 1024 +MAX_DEPLOYMENT_FILES = 10_000 +MAX_DEPLOYMENT_CONTENT_BYTES = 512 * 1024 * 1024 +ELBENCHO_ARCHIVES = { + "x86_64": ( + "elbencho-static-x86_64.tar.gz", + "8d7cf885481dbd8f39908b7f4ff588d9e80cbc0fd26eeaba0b764c77587884d2", + "elbencho", + ), + "aarch64": ( + "elbencho-static-aarch64.tar.gz", + "a744c82ab4e15d8cf4023f7f5053c2008e148f77349e2ba53d2352c4f7508683", + "elbencho.aarch64", + ), +} +REMOTE_BASE = "/mnt/storage-test/integration-regression" +VALIDATION_SUCCESS = "All validation checks passed successfully" +TEST_SELECTORS = ("all", "filesystem", "ssh", "slurm") + + +class IntegrationTestError(RuntimeError): + """An actionable filesystem integration test failure.""" + + +@dataclass(frozen=True) +class Fixture: + """Live fixture details discovered from Kubernetes and Slurm.""" + + login_pod: str + login_container: str + ssh_addresses: tuple[str, str] + slurm_nodes: tuple[str, str] + architecture: str + ssh_home_mode: str + + +def _kubectl(config: Any, *arguments: str | Path) -> list[str | Path]: + """Build a kubectl command using the fixture's private kubeconfig.""" + return ["kubectl", "--kubeconfig", config.kubeconfig, *arguments] + + +def _pod_command( + config: Any, + pod: str, + container: str, + command: str, + *, + as_user: str | None = None, + timeout: int = 600, +) -> list[str | Path]: + """Build a remotely bounded command for one fixture pod.""" + prefix: list[str | Path] = _kubectl( + config, + "-n", + config.namespace, + "exec", + pod, + "-c", + container, + "--", + ) + if as_user: + prefix.extend(("runuser", "-u", as_user, "--")) + prefix.extend( + ( + "timeout", + "--foreground", + "--kill-after=10s", + f"{timeout}s", + "bash", + "-lc", + command, + ) + ) + return prefix + + +def _ready(pod: dict[str, Any]) -> bool: + """Return whether all containers in a running pod are ready.""" + status = pod.get("status", {}) + containers = status.get("containerStatuses", []) + return ( + status.get("phase") == "Running" + and bool(containers) + and all(item.get("ready", False) for item in containers) + ) + + +def _pods_with_container( + pods: list[dict[str, Any]], container: str +) -> list[dict[str, Any]]: + """Return ready pods that contain *container*.""" + return [ + pod + for pod in pods + if _ready(pod) + and container + in {item["name"] for item in pod.get("spec", {}).get("containers", [])} + ] + + +def _load_state(config: Any) -> dict[str, Any]: + """Load and validate the successful-setup marker.""" + path = config.state_dir / "state.json" + if not path.is_file(): + raise IntegrationTestError( + f"setup state is absent at {path}; run integration-test.py setup first" + ) + state = json.loads(path.read_text(encoding="utf-8")) + expected = { + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "export_dir": str(config.export_dir), + } + mismatches = [ + f"{name}={state.get(name)!r} (expected {value!r})" + for name, value in expected.items() + if state.get(name) != value + ] + if mismatches: + raise IntegrationTestError( + "setup state does not match the requested fixture: " + ", ".join(mismatches) + ) + mode = state.get("ssh_home_mode") + if mode not in {"separate", "shared"}: + raise IntegrationTestError(f"invalid ssh_home_mode in {path}: {mode!r}") + return state + + +def _require_nodes(runner: Any, config: Any) -> None: + """Require the exact ready three-node topology and labels.""" + clusters = runner.run(["kind", "get", "clusters"], timeout=30).stdout.split() + if config.cluster_name not in clusters: + raise IntegrationTestError( + f"kind cluster {config.cluster_name!r} is not running; run setup first" + ) + result = runner.run(_kubectl(config, "get", "nodes", "-o", "json"), timeout=30) + nodes = json.loads(result.stdout)["items"] + if len(nodes) != 3: + raise IntegrationTestError(f"expected 3 Kubernetes nodes; found {len(nodes)}") + if not all( + any( + condition.get("type") == "Ready" and condition.get("status") == "True" + for condition in node.get("status", {}).get("conditions", []) + ) + for node in nodes + ): + raise IntegrationTestError("all three Kubernetes nodes must be Ready") + target = sum( + node["metadata"].get("labels", {}).get("storage-scale-test/target") == "true" + for node in nodes + ) + login = sum( + node["metadata"].get("labels", {}).get("storage-scale-test/login") == "true" + for node in nodes + ) + if (target, login) != (2, 1): + raise IntegrationTestError( + f"node label invariant failed: target={target}, login={login}" + ) + + +def _require_storage(runner: Any, config: Any) -> None: + """Require both integration PVCs to be bound.""" + result = runner.run( + _kubectl(config, "-n", config.namespace, "get", "pvc", "-o", "json"), + timeout=30, + ) + claims = { + item["metadata"]["name"]: item.get("status", {}).get("phase") + for item in json.loads(result.stdout)["items"] + } + expected = {"storage-test-rwx", "ssh-home-rwx"} + if any(claims.get(name) != "Bound" for name in expected): + raise IntegrationTestError(f"integration PVCs are not Bound: {claims}") + + +def _pod_inventory(runner: Any, config: Any) -> list[dict[str, Any]]: + """Return the namespace pod inventory.""" + result = runner.run( + _kubectl(config, "-n", config.namespace, "get", "pods", "-o", "json"), + timeout=30, + ) + return json.loads(result.stdout)["items"] + + +def _require_pods( + pods: list[dict[str, Any]], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Require one LoginSet and two SSH worker pods.""" + login = _pods_with_container(pods, "login") + ssh = sorted( + _pods_with_container(pods, "sshd"), key=lambda item: item["metadata"]["name"] + ) + slurmd = _pods_with_container(pods, "slurmd") + if len(login) != 1 or len(ssh) != 2 or len(slurmd) != 2: + raise IntegrationTestError( + "fixture workloads are incomplete: " + f"login={len(login)}, ssh={len(ssh)}, slurmd={len(slurmd)}; run setup" + ) + return login[0], ssh + + +def _probe_pod( + runner: Any, + config: Any, + pod: str, + container: str, + command: str, + *, + as_user: str | None = None, +) -> str: + """Run a short fixture probe and return stripped stdout.""" + result = runner.run( + _pod_command( + config, + pod, + container, + command, + as_user=as_user, + timeout=30, + ), + timeout=45, + ) + return result.stdout.strip() + + +def _require_fixture(runner: Any, config: Any) -> Fixture: + """Validate setup without reconciling or installing anything.""" + state = _load_state(config) + required_host_tools = ( + "bash", + "file", + "find", + "git", + "ssh-add", + "ssh-agent", + "tar", + "timeout", + ) + missing_host_tools = [ + tool for tool in required_host_tools if shutil.which(tool) is None + ] + if missing_host_tools: + raise IntegrationTestError( + "required host tools are absent: " + ", ".join(missing_host_tools) + ) + if not config.kubeconfig.is_file(): + raise IntegrationTestError( + f"private kubeconfig is absent at {config.kubeconfig}; run setup first" + ) + _require_nodes(runner, config) + _require_storage(runner, config) + login, ssh = _require_pods(_pod_inventory(runner, config)) + login_name = str(login["metadata"]["name"]) + ssh_name = str(ssh[0]["metadata"]["name"]) + addresses = tuple(str(item["status"]["podIP"]) for item in ssh) + if len(set(addresses)) != 2: + raise IntegrationTestError(f"SSH workers lack distinct addresses: {addresses}") + tools = ( + "for tool in bash file find tar timeout; do " + 'command -v "$tool" >/dev/null || exit 1; done' + ) + _probe_pod(runner, config, login_name, "login", tools) + _probe_pod(runner, config, ssh_name, "sshd", tools, as_user="tester") + mount_probe = ( + "case $(stat -f -c %T /mnt/storage-test) in nfs|nfs4) true;; *) false;; esac" + ) + _probe_pod(runner, config, login_name, "login", mount_probe) + _probe_pod(runner, config, ssh_name, "sshd", mount_probe, as_user="tester") + login_arch = _probe_pod(runner, config, login_name, "login", "uname -m") + ssh_arch = _probe_pod( + runner, config, ssh_name, "sshd", "uname -m", as_user="tester" + ) + host_arch = platform.machine() + if login_arch != ssh_arch or login_arch != host_arch: + raise IntegrationTestError( + "fixture architecture mismatch: " + f"host={host_arch}, login={login_arch}, ssh={ssh_arch}" + ) + if host_arch not in ELBENCHO_ARCHIVES: + raise IntegrationTestError(f"unsupported fixture architecture: {host_arch}") + slurm_output = _probe_pod( + runner, + config, + login_name, + "login", + "sinfo -N -h -o %N | sort -u", + ) + slurm_nodes = tuple(line for line in slurm_output.splitlines() if line) + if len(slurm_nodes) != 2: + raise IntegrationTestError( + f"expected two Slurm compute nodes; found {slurm_nodes}" + ) + return Fixture( + login_pod=login_name, + login_container="login", + ssh_addresses=(addresses[0], addresses[1]), + slurm_nodes=(slurm_nodes[0], slurm_nodes[1]), + architecture=host_arch, + ssh_home_mode=str(state["ssh_home_mode"]), + ) + + +def _sha256(path: Path) -> str: + """Return the SHA-256 digest of *path*.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _request(url: str, accept: str = "application/vnd.github+json") -> Any: + """Open a bounded upstream request with retries.""" + request = urllib.request.Request( + url, + headers={ + "Accept": accept, + "User-Agent": "storage-scale-test-integration", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + last_error: Exception | None = None + for attempt in range(1, 4): + try: + return urllib.request.urlopen(request, timeout=60) + except (OSError, urllib.error.URLError) as error: + last_error = error + if attempt < 3: + time.sleep(attempt * 2) + raise IntegrationTestError(f"download failed after 3 attempts: {url}: {last_error}") + + +def _asset_url(name: str) -> str: + """Resolve a pinned release asset through the public GitHub API.""" + with _request(ELBENCHO_RELEASE_API) as response: + release = json.load(response) + matches = [ + asset for asset in release.get("assets", []) if asset.get("name") == name + ] + if len(matches) != 1: + raise IntegrationTestError( + f"expected one {name!r} asset in {ELBENCHO_VERSION}; found {len(matches)}" + ) + return str(matches[0]["url"]) + + +def _download_archive(destination: Path, name: str, expected: str) -> None: + """Download and verify one pinned elbencho archive atomically.""" + destination.parent.mkdir(parents=True, exist_ok=True) + url = _asset_url(name) + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as handle: + temporary = Path(handle.name) + try: + with _request(url, "application/octet-stream") as response: + content_length = response.headers.get("Content-Length") + if content_length and int(content_length) > MAX_ARCHIVE_BYTES: + raise IntegrationTestError( + f"refusing oversized archive {name}: {content_length} bytes" + ) + copied = 0 + while chunk := response.read(1024 * 1024): + copied += len(chunk) + if copied > MAX_ARCHIVE_BYTES: + raise IntegrationTestError( + f"refusing archive {name} larger than " + f"{MAX_ARCHIVE_BYTES} bytes" + ) + handle.write(chunk) + except Exception: + temporary.unlink(missing_ok=True) + raise + actual = _sha256(temporary) + if actual != expected: + temporary.unlink(missing_ok=True) + raise IntegrationTestError( + f"checksum mismatch for {name}: expected {expected}, got {actual}" + ) + temporary.chmod(0o640) + temporary.replace(destination) + + +def _ensure_elbencho(config: Any, architecture: str) -> tuple[Path, str]: + """Return a verified, extracted pinned elbencho binary and staged name.""" + archive_name, expected, binary_name = ELBENCHO_ARCHIVES[architecture] + cache = config.state_dir / "test-cache" + archive = cache / f"{ELBENCHO_VERSION}-{archive_name}" + if not archive.is_file() or _sha256(archive) != expected: + LOG.info( + "Downloading pinned elbencho %s for %s", ELBENCHO_VERSION, architecture + ) + _download_archive(archive, archive_name, expected) + else: + LOG.info("Using cached pinned elbencho archive for %s", architecture) + binary = cache / f"{ELBENCHO_VERSION}-{binary_name}" + with tarfile.open(archive, "r:gz") as tar: + members = [ + member + for member in tar.getmembers() + if member.isfile() and Path(member.name).name == "elbencho" + ] + if len(members) != 1: + raise IntegrationTestError( + f"expected one elbencho binary in {archive}; found {len(members)}" + ) + source = tar.extractfile(members[0]) + if source is None: + raise IntegrationTestError(f"cannot extract elbencho from {archive}") + with tempfile.NamedTemporaryFile(dir=cache, delete=False) as handle: + temporary = Path(handle.name) + shutil.copyfileobj(source, handle) + temporary.chmod(0o755) + temporary.replace(binary) + return binary, binary_name + + +def _shell(value: str | Path) -> str: + """Quote one value for the generated Bash environment.""" + return shlex.quote(str(value)) + + +def _override_block( + selector: str, remote_root: str, fixture: Fixture +) -> tuple[str, dict[str, str]]: + """Return template overrides and small support-file contents.""" + data = "/mnt/storage-test" + lines = [ + "# Bounded integration regression overrides.", + f"export RESULTS_DIR={_shell(remote_root + '/results')}", + f"export LOGS_DIR={_shell(remote_root + '/logs')}", + "ORDER_NODES=1", + 'client_type="cpu"', + f"client_arch={_shell(fixture.architecture)}", + "unset TEST_DIRS", + f"declare -A TEST_DIRS=([{_shell(data)}]=1)", + "export FS_MAX_AGG_THROUGHPUT=1", + "export FS_MAX_NODE_THROUGHPUT_GBPS=1", + "export FS_MAX_NODE_IOPS=100", + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + "export ELBENCHO_FILE_SIZE_MULTIPLIER=1", + 'export ELBENCHO_FILE_LAYOUT="shared-directory"', + "export ELBENCHO_FILES_PER_NODE=1", + 'export ELBENCHO_FILE_SIZE="4K"', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_READ_AFTER_WRITE_PAUSE=0", + "export ELBENCHO_LIVE_CSV_EXTENDED=0", + "export ELBENCHO_SINGLE_BIG_FILE=0", + 'export OBJ_BUCKET=""', + ] + support: dict[str, str] = {} + if selector == "ssh": + host_file = f"{remote_root}/ssh-hosts" + lines.extend( + ( + f"export SSH_HOST_LIST={_shell(host_file)}", + 'export SSH_USER="tester"', + "unset SLURM_NODE_INCLUDES SLURM_NODE_IGNORES", + ) + ) + if fixture.ssh_home_mode == "shared": + lines.append("export SSH_HOMEDIR_SHARED=1") + else: + lines.append("unset SSH_HOMEDIR_SHARED") + support["ssh-hosts"] = "\n".join(fixture.ssh_addresses) + "\n" + else: + include_file = f"{remote_root}/slurm-nodes" + ignore_file = f"{remote_root}/slurm-ignore" + lines.extend( + ( + "unset SSH_HOST_LIST SSH_USER SSH_HOMEDIR_SHARED", + 'account=""', + 'reservation=""', + 'partition="all"', + 'run_time="00:05:00"', + "SLURM_EXCLUSIVE_USER=0", + f"export SLURM_NODE_INCLUDES={_shell(include_file)}", + f"export SLURM_NODE_IGNORES={_shell(ignore_file)}", + ) + ) + support["slurm-nodes"] = "\n".join(fixture.slurm_nodes) + "\n" + support["slurm-ignore"] = "" + return "\n".join(lines) + "\n", support + + +def _render_env( + template: Path, selector: str, remote_root: str, fixture: Fixture +) -> tuple[str, dict[str, str]]: + """Render one runtime env from the repository's real user template.""" + text = template.read_text(encoding="utf-8") + anchor = 'source "${SCALE_TEST_BASE}/lib/env_base.sh"' + if text.count(anchor) != 1: + raise IntegrationTestError( + f"expected exactly one env_base source anchor in {template}" + ) + overrides, support = _override_block(selector, remote_root, fixture) + rendered = text.replace(anchor, overrides + "\n" + anchor) + return rendered, support + + +def _copy_tracked_snapshot(runner: Any, repo_root: Path, destination: Path) -> None: + """Copy only tracked working-tree files into an isolated packaging tree.""" + required = (repo_root / "utils" / "build_tarball.sh", repo_root / "NOTICE") + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise IntegrationTestError( + "deployment tarball sources are absent: " + ", ".join(missing) + ) + result = runner.run( + ["git", "ls-files", "-z", "--cached"], cwd=repo_root, timeout=30 + ) + destination.mkdir(parents=True) + for name in result.stdout.split("\0"): + if not name: + continue + relative = PurePosixPath(name) + if relative.is_absolute() or ".." in relative.parts: + raise IntegrationTestError(f"unsafe tracked path from git: {name!r}") + source = repo_root / Path(*relative.parts) + target = destination / Path(*relative.parts) + if not source.is_file() or source.is_symlink(): + raise IntegrationTestError( + f"deployment snapshot requires a regular tracked file: {source}" + ) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def _validate_deployment_archive( + archive: Path, + destination: Path, + binary_name: str, + binary_digest: str, + architecture: str, + runner: Any, +) -> Path: + """Validate and safely extract the deployment tarball.""" + if not archive.is_file() or archive.stat().st_size > MAX_DEPLOYMENT_ARCHIVE_BYTES: + raise IntegrationTestError( + f"deployment archive is absent or oversized: {archive}" + ) + required = { + "storage-scale-test/NOTICE", + "storage-scale-test/env.sh.template", + "storage-scale-test/validate_env.sh", + "storage-scale-test/lib/env_base.sh", + "storage-scale-test/storage-tests/fs/nv-elbencho-sweep.sh", + f"storage-scale-test/utils/{binary_name}", + } + names: set[str] = set() + content_bytes = 0 + with tarfile.open(archive, "r:gz") as tar: + members = tar.getmembers() + if len(members) > MAX_DEPLOYMENT_FILES: + raise IntegrationTestError( + f"deployment archive has too many members: {len(members)}" + ) + for member in members: + path = PurePosixPath(member.name) + if ( + path.is_absolute() + or not path.parts + or path.parts[0] != "storage-scale-test" + or ".." in path.parts + or member.name in names + or member.issym() + or member.islnk() + or member.isdev() + or member.isfifo() + or not (member.isfile() or member.isdir()) + ): + raise IntegrationTestError( + f"unsafe deployment archive member: {member.name!r}" + ) + if "env.sh" == path.name or ".obj_auth" in path.parts: + raise IntegrationTestError( + f"unexpected private deployment member: {member.name!r}" + ) + names.add(member.name) + content_bytes += member.size + if content_bytes > MAX_DEPLOYMENT_CONTENT_BYTES: + raise IntegrationTestError("deployment archive content is oversized") + absent = sorted(required - names) + if absent: + raise IntegrationTestError( + "deployment archive lacks required files: " + ", ".join(absent) + ) + destination.mkdir(parents=True) + tar.extractall(destination, filter="data") + root = destination / "storage-scale-test" + packaged_binary = root / "utils" / binary_name + if ( + not packaged_binary.is_file() + or packaged_binary.is_symlink() + or not os.access(packaged_binary, os.X_OK) + or _sha256(packaged_binary) != binary_digest + ): + raise IntegrationTestError( + f"packaged elbencho failed identity checks: {packaged_binary}" + ) + description = runner.run(["file", packaged_binary], timeout=30).stdout + expected_arch = "x86-64" if architecture == "x86_64" else "aarch64" + if expected_arch not in description: + raise IntegrationTestError( + f"packaged elbencho architecture mismatch: {description.strip()}" + ) + runner.run([packaged_binary, "--help"], timeout=30) + return root + + +def _build_deployment_archive( + runner: Any, + repo_root: Path, + build_root: Path, + binary: Path, + binary_name: str, + architecture: str, +) -> tuple[Path, Path]: + """Build and inspect one filesystem-only deployment tarball.""" + snapshot = build_root / "source" + _copy_tracked_snapshot(runner, repo_root, snapshot) + seeded_binary = snapshot / "utils" / binary_name + shutil.copy2(binary, seeded_binary) + seeded_binary.chmod(0o755) + LOG.info("Building deployment tarball from a tracked-files-only snapshot") + result = runner.run( + [ + snapshot / "utils" / "build_tarball.sh", + "--arch", + architecture, + "--skip-object-tools", + ], + cwd=snapshot, + timeout=600, + ) + (build_root / "build-tarball.log").write_text( + result.stdout + result.stderr, encoding="utf-8" + ) + archive = build_root / "storage-scale-test.tar.gz" + extracted = _validate_deployment_archive( + archive, + build_root / "extracted", + binary_name, + _sha256(binary), + architecture, + runner, + ) + return archive, extracted + + +def _write_runtime_files( + workspace: Path, + selector: str, + runtime_root: str, + fixture: Fixture, + *, + template: Path | None = None, +) -> None: + """Add the generated environment and support files to a deployment.""" + rendered, support = _render_env( + template or workspace / "env.sh.template", selector, runtime_root, fixture + ) + (workspace / "env.sh").write_text(rendered, encoding="utf-8") + (workspace / "env.sh").chmod(0o640) + for name, content in support.items(): + (workspace / name).write_text(content, encoding="utf-8") + (workspace / name).chmod(0o640) + + +def _stream_to_login( + runner: Any, + config: Any, + fixture: Fixture, + source: Path, + command: list[str | Path], + timeout: int = 180, +) -> None: + """Stream one local archive to a command in the Slinky LoginSet.""" + with source.open("rb") as stream: + runner.run( + [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + "-i", + fixture.login_pod, + "-c", + fixture.login_container, + "--", + ), + *command, + ], + stdin=stream, + timeout=timeout, + ) + + +def _stage_workspace( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + archive: Path, + extracted: Path, + build_root: Path, +) -> str: + """Stage a packaged deployment for host SSH or LoginSet Slurm execution.""" + if selector == "ssh": + workspace = build_root / "ssh" / "storage-scale-test" + shutil.copytree(extracted, workspace) + _write_runtime_files(workspace, selector, str(workspace), fixture) + return str(workspace) + + remote_base = f"{REMOTE_BASE}/{selector}" + remote_root = f"{remote_base}/storage-scale-test" + reset = f"rm -rf -- {_shell(remote_base)} && mkdir -p -- {_shell(remote_base)}" + runner.run( + _pod_command( + config, + fixture.login_pod, + fixture.login_container, + reset, + timeout=60, + ), + timeout=75, + ) + extract_command = [ + "tar", + "--no-same-owner", + "--no-same-permissions", + "-xzf", + "-", + "-C", + remote_base, + ] + _stream_to_login(runner, config, fixture, archive, extract_command) + support_stage = build_root / "slurm-runtime" + support_stage.mkdir() + _write_runtime_files( + support_stage, + selector, + remote_root, + fixture, + template=extracted / "env.sh.template", + ) + support_archive = build_root / "slurm-runtime.tar" + with tarfile.open(support_archive, "w") as tar: + for child in sorted(support_stage.iterdir()): + tar.add(child, arcname=child.name) + support_command = [ + "tar", + "--no-same-owner", + "--no-same-permissions", + "-xf", + "-", + "-C", + remote_root, + ] + _stream_to_login(runner, config, fixture, support_archive, support_command) + return remote_root + + +def _test_command( + config: Any, fixture: Fixture, selector: str, command: str, timeout: int +) -> list[str | Path]: + """Build one substrate test command.""" + if selector == "ssh": + private_key = config.keys_dir / "id_ed25519" + host_command = "\n".join( + ( + "set -euo pipefail", + 'eval "$(ssh-agent -s)" >/dev/null', + "trap 'ssh-agent -k >/dev/null 2>&1 || true' EXIT", + f"ssh-add -- {_shell(private_key)} >/dev/null 2>&1", + command, + ) + ) + return [ + "timeout", + "--foreground", + "--kill-after=10s", + f"{timeout}s", + "bash", + "-lc", + host_command, + ] + return _pod_command( + config, + fixture.login_pod, + fixture.login_container, + command, + timeout=timeout, + ) + + +def _run_step( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + name: str, + command: str, + log_dir: Path, + timeout: int, +) -> str: + """Run one bounded substrate step and preserve diagnostic output.""" + LOG.info("Running %s filesystem step: %s", selector, name) + result = runner.run( + _test_command(config, fixture, selector, command, timeout), + check=False, + timeout=timeout + 30, + ) + output = result.stdout + result.stderr + log_path = log_dir / f"{selector}-{name}.log" + log_path.write_text(output, encoding="utf-8") + log_path.chmod(0o640) + if result.returncode: + detail = output.strip()[-8000:] + raise IntegrationTestError( + f"{selector} {name} failed with exit code {result.returncode}; " + f"full output: {log_path}\n{detail}" + ) + LOG.info("Passed %s filesystem step: %s", selector, name) + return output + + +def _assert_results( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + remote_root: str, + log_dir: Path, +) -> None: + """Assert the two execution records and bounded-data cleanup.""" + results = f"{remote_root}/results" + discover = ( + f"find {_shell(results)} -mindepth 1 -maxdepth 1 -type d " + "-name 'elbencho-*' -printf '%p\\n'" + ) + output = _run_step( + runner, + config, + fixture, + selector, + "discover-results", + discover, + log_dir, + 30, + ) + directories = [line for line in output.splitlines() if line.strip()] + if len(directories) != 1: + raise IntegrationTestError( + f"expected one {selector} results directory; found {directories}" + ) + result_dir = directories[0] + if selector == "ssh": + quoted_addresses = " ".join( + _shell(address) for address in fixture.ssh_addresses + ) + remote_cleanup = ( + 'test -z "$(find /mnt/storage-test -mindepth 1 -maxdepth 1 ' + "-type d -name 'elbencho-sweep-target-*' -print -quit)\"" + ) + cleanup_probe = f""" +for host in {quoted_addresses}; do + ssh -T -o BatchMode=yes -o ConnectTimeout=15 \\ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \\ + -o PreferredAuthentications=publickey -o LogLevel=ERROR \\ + "tester@$host" {_shell(remote_cleanup)} +done +""".strip() + else: + cleanup_probe = ( + 'test -z "$(find /mnt/storage-test -mindepth 1 -maxdepth 1 ' + "-type d -name 'elbencho-sweep-target-*' -print -quit)\"" + ) + assertion = f""" +set -euo pipefail +result={_shell(result_dir)} +test -s "$result/env_used.yaml" +test -s "$result/env_used.sh" +test "$(find "$result/executions" -maxdepth 1 -name '*.status' | wc -l)" -eq 2 +for id in 0001 0002; do + grep -qx SUCCESS "$result/executions/$id.status" + grep -qx 0 "$result/executions/$id.exitcode" + test -s "$result/executions/$id.log" + test -s "$result/executions/$id.workload.tsv" +done +test "$(find "$result" -type f -name '*.csv' -size +0c | wc -l)" -ge 2 +test "$(find "$result" -type f -name '*.out' -size +0c | wc -l)" -ge 2 +{cleanup_probe} +""".strip() + _run_step( + runner, + config, + fixture, + selector, + "assert-results", + assertion, + log_dir, + 60, + ) + + +def _run_substrate( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + archive: Path, + extracted: Path, + build_root: Path, + log_dir: Path, +) -> None: + """Stage, validate, run, and inspect one filesystem substrate.""" + remote_root = _stage_workspace( + runner, + config, + fixture, + selector, + archive, + extracted, + build_root, + ) + prefix = f"cd -- {_shell(remote_root)} && " + validation = _run_step( + runner, + config, + fixture, + selector, + "validate-env", + prefix + "./validate_env.sh", + log_dir, + 180, + ) + if VALIDATION_SUCCESS not in validation: + raise IntegrationTestError( + f"{selector} validate_env.sh omitted its success marker" + ) + sweep = prefix + "./storage-tests/fs/nv-elbencho-sweep.sh -b --nodes 1,2" + _run_step( + runner, + config, + fixture, + selector, + "filesystem-sweep", + sweep, + log_dir, + 600, + ) + _assert_results(runner, config, fixture, selector, remote_root, log_dir) + + +def _selected_tests(selectors: list[str]) -> tuple[str, ...]: + """Normalize public selectors to ordered substrate names.""" + requested = selectors or ["all"] + unknown = sorted(set(requested) - set(TEST_SELECTORS)) + if unknown: + raise IntegrationTestError( + f"unknown test selector(s): {', '.join(unknown)}; " + f"choose from {', '.join(TEST_SELECTORS)}" + ) + duplicates = sorted(name for name in set(requested) if requested.count(name) > 1) + if duplicates: + raise IntegrationTestError( + f"duplicate test selector(s): {', '.join(duplicates)}" + ) + if "all" in requested and len(requested) > 1: + raise IntegrationTestError("test selector 'all' cannot be combined") + if "filesystem" in requested and len(requested) > 1: + raise IntegrationTestError("test selector 'filesystem' cannot be combined") + if requested in (["all"], ["filesystem"]): + return ("ssh", "slurm") + return tuple(name for name in ("ssh", "slurm") if name in requested) + + +def run_filesystem_tests( + runner: Any, config: Any, repo_root: Path, selectors: list[str] +) -> None: + """Run selected filesystem regression cases against an existing setup.""" + selected = _selected_tests(selectors) + LOG.info("Requiring an already-running integration setup") + fixture = _require_fixture(runner, config) + binary, binary_name = _ensure_elbencho(config, fixture.architecture) + run_id = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + f"-{os.getpid()}" + log_dir = config.state_dir / "test-runs" / run_id + log_dir.mkdir(parents=True, exist_ok=False) + LOG.info("Filesystem integration artifacts: %s", log_dir) + scratch_parent = config.state_dir / "test-runs" + with tempfile.TemporaryDirectory(dir=scratch_parent) as temporary: + build_root = Path(temporary) + archive, extracted = _build_deployment_archive( + runner, + repo_root, + build_root, + binary, + binary_name, + fixture.architecture, + ) + shutil.copy2(build_root / "build-tarball.log", log_dir / "build-tarball.log") + for selector in selected: + _run_substrate( + runner, + config, + fixture, + selector, + archive, + extracted, + build_root, + log_dir, + ) + LOG.info("Filesystem integration tests passed: %s", ", ".join(selected)) diff --git a/integration-tests/manifests/kind.yaml.tmpl b/integration-tests/manifests/kind.yaml.tmpl new file mode 100644 index 0000000..6f73886 --- /dev/null +++ b/integration-tests/manifests/kind.yaml.tmpl @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: @@CLUSTER_NAME@@ +nodes: + - role: control-plane + labels: + storage-scale-test/login: "true" + - role: worker + labels: + storage-scale-test/target: "true" + - role: worker + labels: + storage-scale-test/target: "true" diff --git a/integration-tests/manifests/mariadb-accounting.yaml.tmpl b/integration-tests/manifests/mariadb-accounting.yaml.tmpl new file mode 100644 index 0000000..7298e85 --- /dev/null +++ b/integration-tests/manifests/mariadb-accounting.yaml.tmpl @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-accounting-config + namespace: @@NAMESPACE@@ +data: + integration.cnf: | + [mariadb] + bind-address=0.0.0.0 + default_storage_engine=InnoDB + binlog_format=row + innodb_autoinc_lock_mode=2 + innodb_buffer_pool_size=128M + innodb_lock_wait_timeout=120 + max_allowed_packet=64M +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb + namespace: @@NAMESPACE@@ +spec: + clusterIP: None + selector: + app.kubernetes.io/name: mariadb-accounting + ports: + - name: mysql + port: 3306 + targetPort: mysql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mariadb-accounting + namespace: @@NAMESPACE@@ +spec: + serviceName: mariadb + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mariadb-accounting + template: + metadata: + labels: + app.kubernetes.io/name: mariadb-accounting + spec: + nodeSelector: + storage-scale-test/login: "true" + containers: + - name: mariadb + image: mariadb:11.4@sha256:70cc072b29b4a89ae07abb2d4da2c64678a7f2dfe092751bb51c87d67dc1338b + ports: + - name: mysql + containerPort: 3306 + env: + - name: MARIADB_DATABASE + value: slurm_acct_db + - name: MARIADB_USER + value: slurm + - name: MARIADB_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-password + key: password + - name: MARIADB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-password + key: root-password + readinessProbe: + exec: + command: ["healthcheck.sh", "--connect", "--innodb_initialized"] + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + exec: + command: ["healthcheck.sh", "--connect", "--innodb_initialized"] + initialDelaySeconds: 30 + periodSeconds: 10 + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 500m + memory: 768Mi + volumeMounts: + - name: data + mountPath: /var/lib/mysql + - name: config + mountPath: /etc/mysql/conf.d/integration.cnf + subPath: integration.cnf + volumes: + - name: config + configMap: + name: mariadb-accounting-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: standard + resources: + requests: + storage: 1Gi diff --git a/integration-tests/manifests/nfs-csi-values.yaml b/integration-tests/manifests/nfs-csi-values.yaml new file mode 100644 index 0000000..cbed0e4 --- /dev/null +++ b/integration-tests/manifests/nfs-csi-values.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +controller: + replicas: 1 + runOnControlPlane: true + enableSnapshotter: false + +externalSnapshotter: + enabled: false + +feature: + enableFSGroupPolicy: true + +node: + maxUnavailable: 1 diff --git a/integration-tests/manifests/nfs-storage.yaml.tmpl b/integration-tests/manifests/nfs-storage.yaml.tmpl new file mode 100644 index 0000000..dc443e1 --- /dev/null +++ b/integration-tests/manifests/nfs-storage.yaml.tmpl @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: storage-scale-nfs-csi +provisioner: nfs.csi.k8s.io +parameters: + server: @@NFS_SERVER@@ + share: / + subDir: ${pvc.metadata.namespace}/${pvc.metadata.name} + mountPermissions: "0770" + onDelete: retain +reclaimPolicy: Retain +volumeBindingMode: Immediate +allowVolumeExpansion: true +mountOptions: + - nfsvers=4.1 + - hard + - timeo=600 + - retrans=2 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-test-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: storage-scale-nfs-csi + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssh-home-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: storage-scale-nfs-csi + resources: + requests: + storage: 64Mi diff --git a/integration-tests/manifests/slinky-operator-values.yaml b/integration-tests/manifests/slinky-operator-values.yaml new file mode 100644 index 0000000..843f55e --- /dev/null +++ b/integration-tests/manifests/slinky-operator-values.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +certManager: + enabled: false + +operator: + accountingWorkers: 1 + controllerWorkers: 1 + loginsetWorkers: 1 + nodesetWorkers: 1 + restapiWorkers: 1 + tokenWorkers: 1 + slurmclientWorkers: 1 + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + cpu: 300m + memory: 256Mi + +webhook: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi diff --git a/integration-tests/manifests/slinky-slurm-values.yaml b/integration-tests/manifests/slinky-slurm-values.yaml new file mode 100644 index 0000000..16f81c5 --- /dev/null +++ b/integration-tests/manifests/slinky-slurm-values.yaml @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +controller: + persistence: + enabled: false + slurmctld: + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + reconfigure: + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + logfile: + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 64Mi + +restapi: + slurmrestd: + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + +accounting: + enabled: true + slurmdbd: + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 300m + memory: 512Mi + podSpec: + nodeSelector: + storage-scale-test/login: "true" + +loginsets: + test: + replicas: 1 + login: + image: + repository: storage-scale-integration-login + tag: slinky-26.05-file + digest: null + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: shared-storage + mountPath: /mnt/storage-test + initconf: + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 64Mi + podSpec: + nodeSelector: + storage-scale-test/login: "true" + volumes: + - name: shared-storage + persistentVolumeClaim: + claimName: storage-test-rwx + service: + spec: + type: ClusterIP + +nodesets: + storage: + scalingMode: DaemonSet + workloadDisruptionProtection: false + slurmd: + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + memory: 768Mi + volumeMounts: + - name: shared-storage + mountPath: /mnt/storage-test + logfile: + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 64Mi + podSpec: + nodeSelector: + storage-scale-test/target: "true" + volumes: + - name: shared-storage + persistentVolumeClaim: + claimName: storage-test-rwx + +partitions: + all: + enabled: true + nodesets: + - storage + configMap: + Default: "YES" + MaxTime: UNLIMITED diff --git a/integration-tests/manifests/ssh-workers.yaml.tmpl b/integration-tests/manifests/ssh-workers.yaml.tmpl new file mode 100644 index 0000000..541341f --- /dev/null +++ b/integration-tests/manifests/ssh-workers.yaml.tmpl @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: ssh-workers + namespace: @@NAMESPACE@@ +spec: + clusterIP: None + selector: + app.kubernetes.io/name: storage-ssh-worker + ports: + - name: ssh + port: 22 + targetPort: ssh +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ssh-worker + namespace: @@NAMESPACE@@ +spec: + serviceName: ssh-workers + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: storage-ssh-worker + template: + metadata: + labels: + app.kubernetes.io/name: storage-ssh-worker + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + storage-scale-test/target: "true" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: storage-ssh-worker + topologyKey: kubernetes.io/hostname + initContainers: + - name: install-ssh-identity + image: storage-scale-integration-ssh:ubuntu-24.04 + imagePullPolicy: Never + command: + - sh + - -ec + - | + chown 2000:2000 /home/tester + chmod 0700 /home/tester + install -d -o 2000 -g 2000 -m 0700 /home/tester/.ssh + install -o 2000 -g 2000 -m 0600 /ssh-secret/id_ed25519 /home/tester/.ssh/id_ed25519 + install -o 2000 -g 2000 -m 0600 /ssh-secret/authorized_keys /home/tester/.ssh/authorized_keys + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + volumeMounts: + - name: ssh-secret + mountPath: /ssh-secret + readOnly: true + - name: ssh-home + mountPath: /home/tester + containers: + - name: sshd + image: storage-scale-integration-ssh:ubuntu-24.04 + imagePullPolicy: Never + ports: + - name: ssh + containerPort: 22 + readinessProbe: + tcpSocket: + port: ssh + initialDelaySeconds: 1 + periodSeconds: 2 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: ssh-home + mountPath: /home/tester + - name: shared-storage + mountPath: /mnt/storage-test + volumes: + - name: ssh-secret + secret: + secretName: storage-ssh-identity + defaultMode: 0600 + - name: ssh-home + @@SSH_HOME_VOLUME@@ + - name: shared-storage + persistentVolumeClaim: + claimName: storage-test-rwx diff --git a/integration-tests/slinky-login-image.Dockerfile b/integration-tests/slinky-login-image.Dockerfile new file mode 100644 index 0000000..24236f2 --- /dev/null +++ b/integration-tests/slinky-login-image.Dockerfile @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + file \ + && rm -rf /var/lib/apt/lists/* diff --git a/integration-tests/ssh-image.Dockerfile b/integration-tests/ssh-image.Dockerfile new file mode 100644 index 0000000..38efcee --- /dev/null +++ b/integration-tests/ssh-image.Dockerfile @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:24.04 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + coreutils \ + file \ + findutils \ + gawk \ + gzip \ + iproute2 \ + openssh-client \ + openssh-server \ + procps \ + psmisc \ + tar \ + util-linux \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --gid 2000 storage-test \ + && useradd --uid 2000 --gid 2000 --create-home --shell /bin/bash tester \ + && install -d -m 0755 /run/sshd \ + && printf '%s\n' \ + 'PasswordAuthentication no' \ + 'PermitRootLogin no' \ + 'PubkeyAuthentication yes' \ + 'AllowUsers tester' \ + >> /etc/ssh/sshd_config \ + && rm -f /etc/ssh/ssh_host_* + +EXPOSE 22 +CMD ["bash", "-c", "ssh-keygen -A && exec /usr/sbin/sshd -D -e"] diff --git a/lib/env_functions.sh b/lib/env_functions.sh index 0f8b4df..d9b6cf8 100644 --- a/lib/env_functions.sh +++ b/lib/env_functions.sh @@ -2223,7 +2223,7 @@ dispatch_ssh_executions() { # The nameref lets the entry point stop services before releasing the lock. _dispatch_ssh_executions_owned() { local output_dir="$1" - local -n ssh_services_started="$2" + local -n ssh_services_started_ref="$2" local executions_dir="${output_dir}/executions" _elbencho_sweep_running_to_pending "$executions_dir" || return 1 @@ -2248,7 +2248,8 @@ _dispatch_ssh_executions_owned() { echo "Error: failed to start elbencho services on any reachable SSH host" >&2 return 1 fi - ssh_services_started=1 + # shellcheck disable=SC2034 # Nameref assignment updates the caller's flag. + ssh_services_started_ref=1 local max_nodes if ! max_nodes=$(max_nodes_remaining_executions "$executions_dir"); then @@ -2323,7 +2324,7 @@ _ssh_fan_out_to_each_host() { } local rc=0 - local -a successful_hosts=() + local -a completed_hosts=() local spawn_output if ! spawn_output=$(spawn_N_ssh "$status_dir" true "$scriptlet"); then rc=1 @@ -2341,7 +2342,7 @@ _ssh_fan_out_to_each_host() { echo "Warning: SSH command failed on $hostname (rc=$ssh_rc)" >&2 rc=1 else - successful_hosts+=("$hostname") + completed_hosts+=("$hostname") fi done fi @@ -2354,7 +2355,7 @@ _ssh_fan_out_to_each_host() { if [[ -n "$successful_hosts_array_name" ]]; then local -n successful_hosts_ref="$successful_hosts_array_name" # shellcheck disable=SC2034 # Assignment intentionally updates the caller's named array - successful_hosts_ref=("${successful_hosts[@]}") + successful_hosts_ref=("${completed_hosts[@]}") fi return "$rc" } diff --git a/utils/build_tarball.sh b/utils/build_tarball.sh index 7e6446c..934bc1f 100755 --- a/utils/build_tarball.sh +++ b/utils/build_tarball.sh @@ -288,15 +288,21 @@ check_s3test_binaries() { # Command-line argument processing usage() { cat <&2 + usage >&2 + exit 1 + fi + case "$2" in + x86_64|aarch64|all) + selected_arch="$2" + ;; + *) + echo "Unsupported architecture: $2" >&2 + usage >&2 + exit 1 + ;; + esac + shift 2 + ;; + --skip-object-tools) + skip_object_tools=true + shift + ;; -h|--help) usage exit 0 @@ -320,24 +348,31 @@ echo "Creating deployment tarball..." # Build list of elbencho binaries to download/extract. Format per entry: # arch|output_path|display_name -declare -a binaries_to_download=( - "x86_64|${UTILS_DIR}/elbencho|elbencho amd64" - "aarch64|${UTILS_DIR}/elbencho.aarch64|elbencho arm64" -) +declare -a binaries_to_download=() +if [[ "${selected_arch}" == x86_64 || "${selected_arch}" == all ]]; then + binaries_to_download+=("x86_64|${UTILS_DIR}/elbencho|elbencho amd64") +fi +if [[ "${selected_arch}" == aarch64 || "${selected_arch}" == all ]]; then + binaries_to_download+=("aarch64|${UTILS_DIR}/elbencho.aarch64|elbencho arm64") +fi # Refuse to produce a deployment tarball until the pinned upstream checksums have # been filled in. This prevents placeholder values from degrading into the # historical warn-and-continue download behavior below. validate_elbencho_checksum_configuration || exit 1 -# Warp binaries are user-built from OSS source when object storage testing is needed. -# Warn if either architecture is missing, but keep building the tarball. -check_warp_binaries +if [[ "${skip_object_tools}" == false ]]; then + # Warp binaries are user-built from OSS source when object storage testing is needed. + # Warn if either architecture is missing, but keep building the tarball. + check_warp_binaries -# Build s3test from in-tree source when the binaries are missing or stale. -# If one architecture cannot be built, keep creating the tarball but warn that -# object storage testing will not work for that architecture. -check_s3test_binaries ||: + # Build s3test from in-tree source when the binaries are missing or stale. + # If one architecture cannot be built, keep creating the tarball but warn that + # object storage testing will not work for that architecture. + check_s3test_binaries ||: +else + echo "Skipping object-storage tool checks for filesystem-only packaging." +fi # Download all selected files concurrently. echo "Downloading required files..." From e5ccbada00952747e470df533411c9f9dfc96247 Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Fri, 18 Sep 2026 16:29:57 -0700 Subject: [PATCH 02/10] Run integration coverage in CI Add full integration environment teardown Add an idempotent teardown lifecycle action that removes marker-owned NFS configuration and data while preserving installed tools. Track ownership for host state, loop devices, firewall rules, and locally built image tags. Use full teardown for CI cleanup and document its distinction from disposable stop. Run fixture lifecycle and sweep tests concurrently on amd64 and arm64, with a final job requiring both matrix legs to pass. Exercise idempotent setup and teardown, stop/start recovery, and enforce that sweeps run as the provisioned pre-sudo user instead of root. Validate exact one- and two-node workload metadata and ordered worker selection for SSH and Slurm. Log Slurm's per-cell host subset so the harness can prove prefix ordering. Copy completed results to the host, run extract-elbencho.sh, and require reports to expose both node counts. Document strengthened coverage and resource prerequisites, improve missing-command failure diagnostics, and retain bounded artifacts for troubleshooting. --- .github/workflows/integration.yml | 97 +++ AGENTS.md | 3 + docs/CONTEXT.md | 11 +- .../single-host-integration-feasibility.md | 33 +- integration-tests/README.md | 66 +- integration-tests/bin/integration-test.py | 588 +++++++++++++++++- .../lib/filesystem_integration.py | 143 ++++- lib/env_functions.sh | 2 +- 8 files changed, 906 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..1657b4e --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Filesystem integration + +on: + workflow_dispatch: + push: + branches: + - pull-request/[0-9]+ + +permissions: {} + +concurrency: + group: filesystem-integration + cancel-in-progress: false + +jobs: + integration: + name: Setup and test all (${{ matrix.architecture }}) + strategy: + fail-fast: false + matrix: + include: + - architecture: amd64 + runner: linux-amd64-cpu4 + - architecture: arm64 + runner: linux-arm64-cpu4 + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + permissions: + contents: read + steps: + - name: Check out source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + - name: Configure the package proxy + uses: nv-gha-runners/setup-proxy-cache@eadc57871cde0c92a223d0f486ab669a2360f978 # main + with: + enable-apt: "true" + enable-pip: "true" + enable-conda: "false" + enable-pypi-anaconda: "false" + - name: Verify privileged runner prerequisites + run: | + sudo -n true + docker info >/dev/null + - name: Set up the integration environment + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py setup + - name: Verify repeated setup + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py setup + - name: Stop the integration environment + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py stop + - name: Restart the integration environment + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py start + - name: Verify root test execution is rejected + run: | + if sudo "$(command -v python)" integration-tests/bin/integration-test.py test filesystem; then + echo "integration test action unexpectedly accepted root" >&2 + exit 1 + fi + - name: Run all integration tests + run: | + "$(command -v python)" integration-tests/bin/integration-test.py test all + - name: Tear down the integration environment + if: ${{ always() }} + run: | + sudo "$(command -v python)" integration-tests/bin/integration-test.py teardown + sudo "$(command -v python)" integration-tests/bin/integration-test.py teardown + + integration-status: + name: Filesystem integration status + if: ${{ always() }} + needs: integration + runs-on: ubuntu-latest + steps: + - name: Require every architecture to pass + env: + INTEGRATION_RESULT: ${{ needs.integration.result }} + run: test "$INTEGRATION_RESULT" = success diff --git a/AGENTS.md b/AGENTS.md index a434577..fa0df59 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,6 +63,9 @@ See the README "Getting Started" section for the full quickstart and ./utils/run_ci_checks.sh ``` +After changing `.github/`, parse every `.yml` and `.yaml` file beneath it with +a YAML parser before committing. + If any required check tool is missing from the sandbox, install it into the repo's local environment and rerun the check. Do not skip required tooling just because it is not preinstalled. diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 5676ca7..f69e6bd 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -60,10 +60,13 @@ The separate `integration-tests/` fixture provisions a three-node kind cluster, NFS CSI storage, two passwordless-SSH workers, and a Slinky Slurm environment on one Linux host. Its test action builds and validates a deployment tarball, derives environments from the packaged `env.sh.template`, and runs bounded -one-node and two-node filesystem sweeps through SSH and Slurm. The SSH entry -point runs on the host; the Slurm entry point runs from the archive extracted by -the LoginSet on shared storage. It does not add Kubernetes dispatch to the -benchmark entry points. +one-node and two-node filesystem sweeps through SSH and Slurm as the non-root +account recorded by setup. It verifies ordered worker selection, exact workload +totals, cleanup, and report extraction for both node counts. The SSH entry point +runs on the host; the Slurm entry point runs from the archive extracted by the +LoginSet on shared storage. The on-demand integration workflow runs the full +lifecycle concurrently on amd64 and arm64. It does not add Kubernetes dispatch +to the benchmark entry points. GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. diff --git a/docs/research/single-host-integration-feasibility.md b/docs/research/single-host-integration-feasibility.md index 8d23ae6..48ee557 100644 --- a/docs/research/single-host-integration-feasibility.md +++ b/docs/research/single-host-integration-feasibility.md @@ -40,7 +40,7 @@ scaling, failure-domain, or network-isolation measurements. ## Implemented lifecycle The executable driver is `integration-tests/bin/integration-test.py` and -supports four actions: +supports five actions: - `setup` installs missing host dependencies, creates or reconciles the environment, and validates every substrate. @@ -48,6 +48,8 @@ supports four actions: - `stop` is disposable: it deletes only the marker-owned kind cluster and then stops the NFS service only when the harness started it and no unrelated exports exist. +- `teardown` performs stop and then removes all marker-owned fixture data and + host configuration while leaving installed packages and client tools. - `test` requires an already-running setup and runs selected bounded filesystem regression cases without reconciling the environment. @@ -57,7 +59,10 @@ NFS data. It intentionally discards Kubernetes objects, kind containers, and MariaDB's node-local storage. A later start creates and validates a fresh cluster, so it takes longer than setup against an already-running cluster. The disposable stop and subsequent fresh start were exercised end to end, as -was a repeated stop with the cluster already absent. +was a repeated stop with the cluster already absent. Full teardown was also +exercised after both filesystem substrate tests, its absence/disabled-state +postconditions were verified, and a second teardown succeeded with all fixture +state already absent. Deleting the cluster avoids relying on resumed kind container addresses. Kubernetes Service DNS stabilizes application endpoints inside the cluster but @@ -240,6 +245,16 @@ Stop is idempotent. Repeated stop calls succeed when the cluster and owned NFS service are already absent or inactive. It never stops Docker globally and leaves a pre-existing or shared NFS service running. +Teardown is also idempotent and is intended for CI workers and other disposable +fixtures. It first performs stop, then removes only marker-owned NFS data and +host configuration, stops and disables the dedicated NFS service, removes an +owned UFW rule, unmounts the verified loop device, and deletes generated state +and locally built image tags. It refuses cleanup if ownership, configuration, +mount-backing, or unrelated-export checks fail. Installed packages, client +tools, and reusable upstream image layers remain available. Image IDs are +recorded before and after local builds so teardown can remove only owned tags +and restore any prior tag target. + ## Provisioning sequence An implementation or future refactor should preserve this ordering: @@ -267,11 +282,23 @@ For disposable stop: 5. Stop NFS only when it is harness-owned and has no unrelated exports. 6. Preserve all host packages, caches, keys, rendered state, and NFS data. +For full teardown: + +1. Validate all ownership markers, installed configuration content, the ext4 + loop backing file, and the absence of unrelated NFS exports. +2. Perform the idempotent disposable stop. +3. Unexport the dedicated path and remove the two dedicated host config files. +4. Stop and disable NFS and remove only a UFW rule recorded as harness-created. +5. Unmount the export, detach its verified loop device, and remove the two + locally built fixture image tags. +6. Delete the dedicated export and state directories while retaining installed + packages, client tools, and reusable upstream container layers. + ## Checked-in implementation artifacts | Path | Purpose | |---|---| -| `integration-tests/bin/integration-test.py` | Setup/start/stop driver, validation, logging, and diagnostics | +| `integration-tests/bin/integration-test.py` | Setup/start/stop/teardown driver, validation, logging, and diagnostics | | `integration-tests/lib/filesystem_integration.py` | Filesystem test selection, staging, execution, and result assertions | | `integration-tests/manifests/kind.yaml.tmpl` | Three-node kind topology and neutral role labels | | `integration-tests/manifests/nfs-csi-values.yaml` | Low-footprint NFS CSI deployment values | diff --git a/integration-tests/README.md b/integration-tests/README.md index 807441a..35e7411 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -23,19 +23,24 @@ server. The control-plane node is also the Slinky login node and negative control for the storage-worker label. The two worker nodes run storage clients. The current setup target is Ubuntu 24.04 on x86-64 or ARM64 with at least two -CPUs, 8 GiB RAM, and 20 GiB free disk. Python 3.12 and an accessible rootful -Docker daemon are prerequisites. The driver installs its other host packages -and pinned client tools when needed. It also builds a small derived Slinky -login image containing the standard `file` package required by -`validate_env.sh`. +CPUs, 8 GiB total RAM, 6 GiB available RAM, and 20 GiB free on `/`. Python +3.12 and an accessible rootful Docker daemon are prerequisites. The driver +installs its other host packages and pinned client tools when needed. It also +builds a small derived Slinky login image containing the standard `file` +package required by `validate_env.sh`. Run setup (or its exact synonym, start) with: ```bash sudo -v -integration-tests/bin/integration-test.py setup +sudo integration-tests/bin/integration-test.py setup ``` +Setup records the invoking pre-sudo account, gives that non-root account +access to the private kubeconfig, key, and test-run workspace, and verifies it +can use Docker, kind, and kubectl. A root shell without `SUDO_USER` must name +the account explicitly with `--test-user USER`. + The default SSH homes are separate `emptyDir` volumes. To mount the dedicated RWX NFS claim at `/home/tester` in both workers instead, use: @@ -56,8 +61,9 @@ integration-tests/bin/integration-test.py test all `all` and `filesystem` select both substrates. `ssh` and `slurm` may be used individually, or together as two arguments. The `test` action never installs -or reconciles the fixture: it requires the saved setup state, verifies that -the live topology is healthy, generates each test environment from the packaged +or reconciles the fixture, and it refuses to run as root. It requires the saved +setup state to match the current non-root account, verifies that the live +topology is healthy, generates each test environment from the packaged `env.sh.template`, and runs `validate_env.sh` before the sweep. Each substrate runs one 4 KiB buffered execution on one node and one on two @@ -73,8 +79,32 @@ The pinned benchmark archive is size-limited, checksum-verified, and cached outside the repository before the binary is included in the user-built deployment archive. Timestamped build and step logs are retained below the state directory's `test-runs/` directory. The test also requires successful -execution records, nonempty benchmark output, environment snapshots, and -cleanup of its generated data directories. +execution records, exact one- and two-node workload totals, ordered worker +selection, nonempty benchmark output, environment snapshots, and cleanup of +its generated data directories. It then runs `utils/extract-elbencho.sh` on a +host-side copy of each result and requires the report to contain both node +counts. + +## On-demand CI + +The `Filesystem integration` GitHub Actions workflow runs independent amd64 +and arm64 jobs concurrently. Each job runs setup twice, stops and restarts the +fixture, proves that root test execution is rejected, runs `test all` as the +ordinary runner account, and tears down twice. A final status job requires both +architectures to pass. The workflow is deliberately absent from ordinary +pull-request and default-branch events. + +For a pull request, use the repository's existing PR authorization control—the +same control used to start the regular PR checks. Authorization copies the +reviewed PR commit to the trusted `pull-request/` branch. A push to +that narrowly matched branch starts the integration workflow. Updating a PR +requires authorizing its new head before a new integration run can start. An +existing run can instead be repeated with **Re-run jobs** in GitHub Actions. + +Before this workflow file is present on the default branch, that authorized PR +branch is the way to run it. After the workflow is merged, a maintainer can also +open **Actions**, choose **Filesystem integration**, select **Run workflow**, +and choose an authorized branch or the default branch. Delete the disposable kind cluster and, when owned exclusively by the harness, stop NFS with: @@ -92,3 +122,19 @@ logs and rendered manifests are retained in the state directory. Add `--verbose` for command-level logging. The failure path captures host, Docker, NFS, Kubernetes node, pod, and event diagnostics without printing Kubernetes Secrets. + +For CI workers or any host where retained fixture data is not wanted, run: + +```bash +integration-tests/bin/integration-test.py teardown +``` + +Teardown is idempotent. It performs the disposable stop, removes the dedicated +NFS export and configuration, disables and stops `nfs-server`, removes any UFW +rule that the harness added, unmounts the verified loop-backed filesystem, and +deletes the fixture's generated data, keys, logs, and locally built image tags. +It refuses destructive cleanup when ownership markers, rendered host +configuration, mount backing, or unrelated NFS exports do not match the +fixture. Operating-system packages, kind, kubectl, Helm, and reusable upstream +Docker image layers are not uninstalled. If a locally built tag existed before +setup, teardown restores that exact prior image ID instead of deleting it. diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py index f794e77..f6e1c24 100755 --- a/integration-tests/bin/integration-test.py +++ b/integration-tests/bin/integration-test.py @@ -26,6 +26,7 @@ import logging import os import platform +import pwd import secrets import shlex import shutil @@ -75,6 +76,11 @@ GIB = 1024**3 DEFAULT_STATE_DIR = Path("/var/lib/storage-scale-test-integration") DEFAULT_EXPORT_DIR = Path("/srv/storage-scale-test-integration") +NFS_EXPORT_CONFIG = Path("/etc/exports.d/storage-scale-test-integration.exports") +NFS_DAEMON_CONFIG = Path("/etc/nfs.conf.d/storage-scale-test-integration.conf") +EXPORT_MARKER = ".storage-scale-test-integration.json" +STATE_MARKER = "state-owner.json" +UFW_COMMENT = "storage-scale-test integration NFSv4" LOG = logging.getLogger("storage-scale-integration") CSI_IMAGES = ( @@ -99,6 +105,9 @@ class Config: state_dir: Path export_dir: Path ssh_home_mode: str + test_user: str + test_uid: int + test_gid: int verbose: bool @property @@ -201,6 +210,7 @@ def _sudo_prefix() -> list[str]: def _bootstrap_state_dir(config: Config) -> None: """Create the operator-owned state directory before file logging.""" + create_marker = not config.state_dir.exists() command = [ *_sudo_prefix(), "install", @@ -208,9 +218,9 @@ def _bootstrap_state_dir(config: Config) -> None: "-m", "0750", "-o", - f"+{os.getuid()}", + f"+{config.test_uid}", "-g", - f"+{os.getgid()}", + f"+{config.test_gid}", config.state_dir, config.manifests_dir, config.keys_dir, @@ -221,6 +231,15 @@ def _bootstrap_state_dir(config: Config) -> None: raise ProvisionError( f"cannot create state directory {config.state_dir}: {result.stderr.strip()}" ) + if create_marker: + _write_text( + config.state_dir / STATE_MARKER, + json.dumps( + {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name}, + sort_keys=True, + ) + + "\n", + ) def _configure_logging(config: Config, action: str) -> Path: @@ -707,7 +726,7 @@ def _kind_ipv4_network(runner: Runner) -> tuple[str, str]: def _ensure_export_marker(runner: Runner, config: Config) -> None: """Create or validate ownership of the dedicated host export.""" - marker_path = config.export_dir / ".storage-scale-test-integration.json" + marker_path = config.export_dir / EXPORT_MARKER expected = json.dumps( {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name}, sort_keys=True ) @@ -849,7 +868,7 @@ def _configure_nfs(runner: Runner, config: Config, subnet: str, gateway: str) -> "-m", "0644", export_source, - "/etc/exports.d/storage-scale-test-integration.exports", + NFS_EXPORT_CONFIG, ] ) runner.run( @@ -859,10 +878,10 @@ def _configure_nfs(runner: Runner, config: Config, subnet: str, gateway: str) -> "-m", "0644", nfs_source, - "/etc/nfs.conf.d/storage-scale-test-integration.conf", + NFS_DAEMON_CONFIG, ] ) - _ensure_nfs_firewall(runner, subnet) + _ensure_nfs_firewall(runner, config, subnet) runner.run([*_sudo_prefix(), "exportfs", "-rav"]) runner.run([*_sudo_prefix(), "systemctl", "enable", "--now", "nfs-server"]) state = {"subnet": subnet, "gateway": gateway} @@ -883,19 +902,68 @@ def _record_nfs_service_state(runner: Runner, config: Config) -> None: _write_text(path, json.dumps({"started_by_harness": not active}) + "\n") -def _ensure_nfs_firewall(runner: Runner, subnet: str) -> None: +def _ensure_nfs_firewall(runner: Runner, config: Config, subnet: str) -> None: """Allow NFS only from kind when UFW is active.""" + state_path = config.state_dir / "ufw-rule.json" + previous: dict[str, object] = {} + if state_path.exists(): + previous = json.loads(state_path.read_text(encoding="utf-8")) if not shutil.which("ufw"): LOG.warning("ufw is absent; verify an equivalent TCP-2049 restriction") return status = runner.run([*_sudo_prefix(), "ufw", "status"], check=False) if not status.stdout.startswith("Status: active"): LOG.info("ufw is inactive; exportfs remains restricted to %s", subnet) + if not state_path.exists(): + _write_text( + state_path, + json.dumps({"added_by_harness": False, "subnet": subnet}) + "\n", + ) return + if previous.get("added_by_harness") and previous.get("subnet") != subnet: + _delete_nfs_firewall_rule(runner, str(previous["subnet"])) + previous = {} + status = runner.run([*_sudo_prefix(), "ufw", "status"], check=False) + rule_exists = any( + subnet in line and UFW_COMMENT in line for line in status.stdout.splitlines() + ) + added_by_harness = bool(previous.get("added_by_harness")) + if rule_exists: + LOG.info("The dedicated UFW NFS rule is already present") + else: + runner.run( + [ + *_sudo_prefix(), + "ufw", + "allow", + "from", + subnet, + "to", + "any", + "port", + "2049", + "proto", + "tcp", + "comment", + UFW_COMMENT, + ] + ) + added_by_harness = True + _write_text( + state_path, + json.dumps({"added_by_harness": added_by_harness, "subnet": subnet}) + "\n", + ) + + +def _delete_nfs_firewall_rule( + runner: Runner, subnet: str, *, check: bool = False +) -> None: + """Delete the exact UFW rule installed by this harness.""" runner.run( [ *_sudo_prefix(), "ufw", + "delete", "allow", "from", subnet, @@ -906,8 +974,9 @@ def _ensure_nfs_firewall(runner: Runner, subnet: str) -> None: "proto", "tcp", "comment", - "storage-scale-test integration NFSv4", - ] + UFW_COMMENT, + ], + check=check, ) @@ -931,6 +1000,36 @@ def _image_exists(runner: Runner, image: str) -> bool: ) +def _image_id(runner: Runner, image: str) -> str | None: + """Return a local Docker image ID, or None when its tag is absent.""" + result = runner.run( + ["docker", "image", "inspect", "--format", "{{.Id}}", image], check=False + ) + return result.stdout.strip() if result.returncode == 0 else None + + +def _record_image_build_start(runner: Runner, config: Config, image: str) -> None: + """Remember a fixed tag's original owner before the first local build.""" + state_path = config.state_dir / "built-images.json" + state: dict[str, dict[str, str | None]] = {} + if state_path.exists(): + state = json.loads(state_path.read_text(encoding="utf-8")) + if image not in state: + state[image] = {"previous_id": _image_id(runner, image), "built_id": None} + _write_text(state_path, json.dumps(state, indent=2, sort_keys=True) + "\n") + + +def _record_image_build_complete(runner: Runner, config: Config, image: str) -> None: + """Record the exact image ID produced by a successful local build.""" + state_path = config.state_dir / "built-images.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + built_id = _image_id(runner, image) + if built_id is None: + raise ProvisionError(f"Docker build did not produce expected image tag {image}") + state[image]["built_id"] = built_id + _write_text(state_path, json.dumps(state, indent=2, sort_keys=True) + "\n") + + def _prepare_csi_images(runner: Runner, config: Config) -> None: """Pull CSI images with an official staging fallback and load all nodes.""" LOG.info("Preparing pinned NFS CSI images") @@ -1116,6 +1215,7 @@ def _ensure_file_secret( def _install_ssh_workers(runner: Runner, config: Config) -> None: """Build, deploy, and validate the two SSH workers.""" private_key, public_key = _ensure_ssh_key(runner, config) + _record_image_build_start(runner, config, SSH_IMAGE) runner.run( [ "docker", @@ -1128,6 +1228,7 @@ def _install_ssh_workers(runner: Runner, config: Config) -> None: ], timeout=600, ) + _record_image_build_complete(runner, config, SSH_IMAGE) nodes = _kind_containers(runner, config, running_only=True) _load_image_into_nodes(runner, SSH_IMAGE, nodes) _ensure_file_secret( @@ -1374,6 +1475,7 @@ def _install_slurm(runner: Runner, config: Config) -> None: def _prepare_slinky_login_image(runner: Runner, config: Config) -> None: """Build and preload the login image with declared test prerequisites.""" + _record_image_build_start(runner, config, SLINKY_LOGIN_IMAGE) runner.run( [ "docker", @@ -1388,6 +1490,7 @@ def _prepare_slinky_login_image(runner: Runner, config: Config) -> None: ], timeout=600, ) + _record_image_build_complete(runner, config, SLINKY_LOGIN_IMAGE) _load_image_into_nodes( runner, SLINKY_LOGIN_IMAGE, @@ -1686,6 +1789,9 @@ def _write_state_summary(config: Config, subnet: str, gateway: str) -> None: "namespace": config.namespace, "export_dir": str(config.export_dir), "ssh_home_mode": config.ssh_home_mode, + "test_user": config.test_user, + "test_uid": config.test_uid, + "test_gid": config.test_gid, "kind_version": KIND_VERSION, "kubernetes_version": KUBECTL_VERSION, "nfs_csi_version": NFS_CSI_VERSION, @@ -1742,6 +1848,55 @@ def setup_environment(runner: Runner, config: Config) -> None: LOG.info("Integration environment is provisioned and running") +def _grant_test_user_access(runner: Runner, config: Config) -> None: + """Give the non-root test identity access to its private setup state.""" + marker = config.state_dir / STATE_MARKER + if not marker.is_file() or json.loads(marker.read_text(encoding="utf-8")) != ( + _owner_document(config) + ): + raise ProvisionError( + f"refusing to change ownership of unverified setup state: {config.state_dir}" + ) + runner.run( + [ + *_sudo_prefix(), + "chown", + "-R", + f"{config.test_uid}:{config.test_gid}", + config.state_dir, + ] + ) + + +def _test_user_command(config: Config, *command: str | Path) -> list[str | Path]: + """Build a command that executes as the configured non-root test user.""" + if os.geteuid() != 0: + return list(command) + return ["runuser", "-u", config.test_user, "--", *command] + + +def _verify_test_user_access(runner: Runner, config: Config) -> None: + """Verify the test identity can use the provisioned cluster and state.""" + probe = config.state_dir / "test-runs" / ".access-probe" + runner.run(_test_user_command(config, "mkdir", "-p", probe.parent)) + runner.run(_test_user_command(config, "touch", probe)) + runner.run(_test_user_command(config, "rm", "--", probe)) + runner.run(_test_user_command(config, "docker", "info"), timeout=60) + runner.run(_test_user_command(config, "kind", "get", "clusters"), timeout=30) + runner.run( + _test_user_command( + config, + "kubectl", + "--kubeconfig", + config.kubeconfig, + "get", + "nodes", + ), + timeout=30, + ) + LOG.info("Verified integration test access for non-root user %s", config.test_user) + + def _scale_ssh(runner: Runner, config: Config, replicas: int) -> None: """Scale SSH workers to conserve host capacity between checks.""" runner.run( @@ -1789,6 +1944,351 @@ def stop_environment(runner: Runner, config: Config) -> None: ) +def teardown_environment(runner: Runner, config: Config) -> None: + """Stop the fixture and remove all harness-owned data and host config.""" + state_owned, setup_owned, export_owned, nfs_configured = ( + _validate_teardown_ownership(runner, config) + ) + stop_environment(runner, config) + if nfs_configured: + _remove_nfs_configuration(runner, config) + if export_owned: + _unmount_export_filesystem(runner, config) + if setup_owned: + _remove_harness_images(runner, config) + _remove_owned_directories( + runner, config, remove_state=state_owned, remove_export=export_owned + ) + LOG.info( + "Integration environment torn down; installed host packages and client " + "tools were preserved" + ) + + +def _owner_document(config: Config) -> dict[str, object]: + """Return the exact ownership document used by persistent markers.""" + return {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name} + + +def _read_system_file(runner: Runner, path: Path) -> str | None: + """Read a root-owned file, returning None when it is absent.""" + result = runner.run([*_sudo_prefix(), "cat", path], check=False, timeout=30) + return result.stdout if result.returncode == 0 else None + + +def _validate_teardown_ownership( + runner: Runner, config: Config +) -> tuple[bool, bool, bool, bool]: + """Validate every persistent artifact before destructive cleanup.""" + _validate_cleanup_paths(config) + expected_owner = _owner_document(config) + state_marker = config.state_dir / STATE_MARKER + state_owned = False + if state_marker.exists(): + if json.loads(state_marker.read_text(encoding="utf-8")) != expected_owner: + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {state_marker}" + ) + state_owned = True + cluster_marker = config.state_dir / "cluster-owner.json" + cluster_owned = False + if cluster_marker.exists(): + if json.loads(cluster_marker.read_text(encoding="utf-8")) != expected_owner: + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {cluster_marker}" + ) + cluster_owned = True + state_owned = True + elif config.state_dir.exists() and not state_owned: + raise ProvisionError( + f"refusing to remove unowned setup state: {config.state_dir}" + ) + + export_marker = config.export_dir / EXPORT_MARKER + marker_text = _read_system_file(runner, export_marker) + export_owned = False + if marker_text is not None: + try: + marker = json.loads(marker_text) + except json.JSONDecodeError as error: + raise ProvisionError( + f"refusing teardown with invalid export marker: {export_marker}" + ) from error + if marker != expected_owner: + raise ProvisionError( + f"refusing teardown with mismatched export marker: {export_marker}" + ) + export_owned = True + elif ( + runner.run( + [*_sudo_prefix(), "test", "-d", config.export_dir], check=False + ).returncode + == 0 + ): + contents = runner.run( + [ + *_sudo_prefix(), + "find", + config.export_dir, + "-mindepth", + "1", + "-maxdepth", + "1", + "-print", + ] + ).stdout.strip() + if contents: + raise ProvisionError( + f"refusing to remove nonempty unowned export: {config.export_dir}" + ) + + export_config = _read_system_file(runner, NFS_EXPORT_CONFIG) + daemon_config = _read_system_file(runner, NFS_DAEMON_CONFIG) + _validate_installed_config( + export_config, + config.manifests_dir / "storage-scale-test.exports", + NFS_EXPORT_CONFIG, + ) + _validate_installed_config( + daemon_config, + config.manifests_dir / "storage-scale-test-nfs.conf", + NFS_DAEMON_CONFIG, + ) + service_state = config.state_dir / "nfs-service.json" + nfs_configured = ( + export_config is not None or daemon_config is not None or service_state.exists() + ) + if (export_owned or nfs_configured) and not cluster_owned: + raise ProvisionError( + "refusing to remove NFS artifacts without the matching setup state marker" + ) + if (export_config is not None or daemon_config is not None) and not export_owned: + raise ProvisionError( + "refusing to remove NFS configuration without the matching export marker" + ) + if _export_mount_type(runner, config): + if not export_owned: + raise ProvisionError( + f"refusing to unmount unowned export directory: {config.export_dir}" + ) + _verified_export_loop(runner, config) + _validate_loop_associations(runner, config) + if cluster_owned: + _validate_image_ownership(runner, config) + + if nfs_configured: + unrelated = [ + path for path in _export_paths(runner) if path != str(config.export_dir) + ] + if unrelated: + raise ProvisionError( + "refusing to stop and disable nfs-server while unrelated exports " + "exist: " + ", ".join(unrelated) + ) + return state_owned, cluster_owned, export_owned, nfs_configured + + +def _validate_cleanup_paths(config: Config) -> None: + """Reject broad or overlapping destructive cleanup targets.""" + forbidden = {Path("/"), Path("/var"), Path("/srv"), Path("/etc")} + if config.state_dir in forbidden or config.export_dir in forbidden: + raise ProvisionError("refusing teardown with a broad state or export path") + if config.state_dir == config.export_dir: + raise ProvisionError("state and export directories must be different") + if config.state_dir in config.export_dir.parents: + raise ProvisionError("export directory must not be inside the state directory") + if config.export_dir in config.state_dir.parents: + raise ProvisionError("state directory must not be inside the export directory") + + +def _validate_installed_config( + installed: str | None, source: Path, destination: Path +) -> None: + """Require a host config file to match its harness-rendered source.""" + if installed is None: + return + if not source.exists() or installed != source.read_text(encoding="utf-8"): + raise ProvisionError( + f"refusing to remove modified or unowned host configuration: {destination}" + ) + + +def _export_paths(runner: Runner) -> list[str]: + """Return currently exported local paths.""" + exports = runner.run([*_sudo_prefix(), "exportfs", "-v"], check=False).stdout + return [line.split()[0] for line in exports.splitlines() if line.startswith("/")] + + +def _verified_export_loop(runner: Runner, config: Config) -> str: + """Return the export loop device after verifying its exact backing image.""" + mounted = runner.run( + [ + "findmnt", + "--noheadings", + "--output", + "SOURCE,FSTYPE", + "--mountpoint", + config.export_dir, + ] + ).stdout.split() + if ( + len(mounted) != 2 + or mounted[1] != "ext4" + or not mounted[0].startswith("/dev/loop") + ): + raise ProvisionError( + f"refusing unexpected mount at dedicated export {config.export_dir}" + ) + backing = runner.run( + [ + *_sudo_prefix(), + "losetup", + "--noheadings", + "--output", + "BACK-FILE", + mounted[0], + ] + ).stdout.strip() + if Path(backing).resolve() != config.nfs_image.resolve(): + raise ProvisionError( + f"refusing loop device {mounted[0]} backed by unexpected file {backing}" + ) + return mounted[0] + + +def _associated_loop_devices(runner: Runner, config: Config) -> list[str]: + """Return loop devices associated with the exact NFS backing image.""" + if not config.nfs_image.exists(): + return [] + result = runner.run( + [*_sudo_prefix(), "losetup", "--associated", config.nfs_image], check=False + ) + return [line.split(":", maxsplit=1)[0] for line in result.stdout.splitlines()] + + +def _validate_loop_associations(runner: Runner, config: Config) -> None: + """Reject an owned loop device mounted anywhere except the export path.""" + for device in _associated_loop_devices(runner, config): + mounts = runner.run( + ["findmnt", "--noheadings", "--output", "TARGET", "--source", device], + check=False, + ).stdout.splitlines() + unexpected = [ + target for target in mounts if Path(target).resolve() != config.export_dir + ] + if unexpected: + raise ProvisionError( + f"refusing loop device {device} mounted outside the fixture: " + + ", ".join(unexpected) + ) + + +def _validate_image_ownership(runner: Runner, config: Config) -> None: + """Reject fixture tags that no longer identify images built by setup.""" + state_path = config.state_dir / "built-images.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text(encoding="utf-8")) + for image, ownership in state.items(): + previous_id = ownership.get("previous_id") + if previous_id and _image_id(runner, previous_id) != previous_id: + raise ProvisionError( + f"cannot restore prior Docker image for fixture tag {image}: " + f"{previous_id} is absent" + ) + current = _image_id(runner, image) + allowed = {previous_id, ownership.get("built_id"), None} + if current not in allowed: + raise ProvisionError( + f"refusing to alter Docker tag changed outside the fixture: {image}" + ) + + +def _remove_nfs_configuration(runner: Runner, config: Config) -> None: + """Unexport storage, disable NFS, and remove exact host configuration.""" + LOG.info("Removing the dedicated NFS export and host configuration") + export_source = config.manifests_dir / "storage-scale-test.exports" + if export_source.exists(): + client = export_source.read_text(encoding="utf-8").split()[1].split("(", 1)[0] + runner.run( + [ + *_sudo_prefix(), + "exportfs", + "-u", + f"{client}:{config.export_dir}", + ], + check=False, + ) + for path in (NFS_EXPORT_CONFIG, NFS_DAEMON_CONFIG): + runner.run([*_sudo_prefix(), "rm", "--force", "--", path]) + runner.run([*_sudo_prefix(), "exportfs", "-ra"]) + if str(config.export_dir) in _export_paths(runner): + raise ProvisionError(f"NFS export is still active: {config.export_dir}") + runner.run([*_sudo_prefix(), "systemctl", "disable", "--now", "nfs-server"]) + firewall_state = config.state_dir / "ufw-rule.json" + if firewall_state.exists(): + state = json.loads(firewall_state.read_text(encoding="utf-8")) + if state.get("added_by_harness"): + _delete_nfs_firewall_rule(runner, str(state["subnet"]), check=True) + + +def _unmount_export_filesystem(runner: Runner, config: Config) -> None: + """Unmount and detach only the verified fixture backing filesystem.""" + if _export_mount_type(runner, config): + _verified_export_loop(runner, config) + runner.run([*_sudo_prefix(), "umount", config.export_dir]) + _validate_loop_associations(runner, config) + for device in _associated_loop_devices(runner, config): + runner.run([*_sudo_prefix(), "losetup", "--detach", device]) + if _export_mount_type(runner, config): + raise ProvisionError(f"export remains mounted: {config.export_dir}") + if _associated_loop_devices(runner, config): + raise ProvisionError(f"loop devices remain attached to {config.nfs_image}") + + +def _remove_harness_images(runner: Runner, config: Config) -> None: + """Remove owned image tags or restore the tags that setup replaced.""" + state_path = config.state_dir / "built-images.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text(encoding="utf-8")) + for image, ownership in state.items(): + built_id = ownership.get("built_id") + previous_id = ownership.get("previous_id") + if built_id is None or _image_id(runner, image) != built_id: + continue + if previous_id: + runner.run(["docker", "image", "tag", previous_id, image]) + else: + runner.run(["docker", "image", "rm", image]) + + +def _remove_owned_directories( + runner: Runner, + config: Config, + *, + remove_state: bool, + remove_export: bool, +) -> None: + """Remove the validated export and state directories without crossing mounts.""" + paths = [config.state_dir] if remove_state else [] + if remove_export: + paths.insert(0, config.export_dir) + for path in paths: + probe = runner.run([*_sudo_prefix(), "test", "-e", path], check=False) + if probe.returncode: + continue + runner.run( + [*_sudo_prefix(), "find", path, "-xdev", "-depth", "-delete"], + timeout=120, + ) + if ( + runner.run([*_sudo_prefix(), "test", "-e", path], check=False).returncode + == 0 + ): + raise ProvisionError(f"cleanup did not remove {path}") + + def _stop_owned_nfs(runner: Runner, config: Config) -> None: """Stop NFS only when this harness started the otherwise-dedicated service.""" state_path = config.state_dir / "nfs-service.json" @@ -1799,10 +2299,7 @@ def _stop_owned_nfs(runner: Runner, config: Config) -> None: if not state.get("started_by_harness", False): LOG.info("nfs-server predated this fixture; leaving it running") return - exports = runner.run([*_sudo_prefix(), "exportfs", "-v"], check=False).stdout - export_paths = [ - line.split()[0] for line in exports.splitlines() if line.startswith("/") - ] + export_paths = _export_paths(runner) unrelated = [path for path in export_paths if path != str(config.export_dir)] if unrelated: LOG.warning( @@ -1831,6 +2328,9 @@ def _collect_diagnostics(runner: Runner, config: Config) -> None: (*_sudo_prefix(), "exportfs", "-v"), ) for command in commands: + if not shutil.which(str(command[0])): + LOG.error("diagnostic command is unavailable: %s", command[0]) + continue result = runner.run(command, check=False, timeout=30) output = (result.stdout + result.stderr).strip() if output: @@ -1854,11 +2354,20 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--namespace", default="storage-scale-integration") parser.add_argument("--state-dir", type=Path, default=DEFAULT_STATE_DIR) parser.add_argument("--export-dir", type=Path, default=DEFAULT_EXPORT_DIR) + parser.add_argument( + "--test-user", + help=( + "non-root account that runs tests; required for root setup unless " + "SUDO_USER identifies it" + ), + ) parser.add_argument( "--ssh-home-mode", choices=("separate", "shared"), default="separate" ) parser.add_argument("--verbose", action="store_true") - parser.add_argument("action", choices=("setup", "start", "stop", "test")) + parser.add_argument( + "action", choices=("setup", "start", "stop", "teardown", "test") + ) parser.add_argument( "tests", nargs="*", @@ -1870,22 +2379,61 @@ def _parser() -> argparse.ArgumentParser: def _config(arguments: argparse.Namespace) -> Config: """Convert parsed arguments into immutable configuration.""" + account = _test_account(arguments.test_user) return Config( cluster_name=arguments.cluster_name, namespace=arguments.namespace, state_dir=arguments.state_dir.resolve(), export_dir=arguments.export_dir.resolve(), ssh_home_mode=arguments.ssh_home_mode, + test_user=account.pw_name, + test_uid=account.pw_uid, + test_gid=account.pw_gid, verbose=arguments.verbose, ) +def _test_account(explicit_user: str | None) -> pwd.struct_passwd: + """Resolve the non-root account that owns and runs integration tests.""" + requested = explicit_user + if os.geteuid() == 0 and requested is None: + requested = os.environ.get("SUDO_USER") + if requested is None: + try: + requested = pwd.getpwuid(os.getuid()).pw_name + except KeyError as error: + raise ProvisionError( + f"current uid has no password-database entry: {os.getuid()}" + ) from error + try: + account = pwd.getpwnam(requested) + except KeyError as error: + raise ProvisionError( + f"integration test user does not exist: {requested}" + ) from error + if account.pw_uid == 0: + raise ProvisionError( + "integration tests require a non-root account; use sudo from that " + "account or pass --test-user" + ) + if os.geteuid() != 0 and account.pw_uid != os.getuid(): + raise ProvisionError( + f"non-root caller cannot provision tests for another user: {requested}" + ) + return account + + def main() -> int: """Run one integration environment lifecycle action.""" _require_python() arguments = _parser().parse_args() - config = _config(arguments) try: + if arguments.action == "test" and os.geteuid() == 0: + raise ProvisionError( + "refusing to run integration sweeps as root; rerun the test " + "action as the account provisioned by setup" + ) + config = _config(arguments) if arguments.action != "test" and arguments.tests: raise ProvisionError("test selectors are valid only with the test action") if arguments.action == "test" and not config.state_dir.is_dir(): @@ -1899,12 +2447,16 @@ def main() -> int: runner = Runner() if arguments.action == "stop": stop_environment(runner, config) + elif arguments.action == "teardown": + teardown_environment(runner, config) elif arguments.action == "test": run_filesystem_tests( runner, config, _repository_root(), arguments.tests ) else: setup_environment(runner, config) + _grant_test_user_access(runner, config) + _verify_test_user_access(runner, config) return 0 except ( ProvisionError, @@ -1914,7 +2466,11 @@ def main() -> int: json.JSONDecodeError, ) as error: LOG.error("%s", error) - if "runner" in locals() and arguments.action != "stop": + if ( + "runner" in locals() + and "config" in locals() + and arguments.action not in ("stop", "teardown") + ): _collect_diagnostics(runner, config) return 1 except KeyboardInterrupt: diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py index f458e32..7d0889c 100644 --- a/integration-tests/lib/filesystem_integration.py +++ b/integration-tests/lib/filesystem_integration.py @@ -22,6 +22,7 @@ import logging import os import platform +import re import shlex import shutil import tarfile @@ -72,6 +73,7 @@ class Fixture: login_container: str ssh_addresses: tuple[str, str] slurm_nodes: tuple[str, str] + slurm_addresses: tuple[str, str] architecture: str ssh_home_mode: str @@ -153,6 +155,9 @@ def _load_state(config: Any) -> dict[str, Any]: "cluster_name": config.cluster_name, "namespace": config.namespace, "export_dir": str(config.export_dir), + "test_user": config.test_user, + "test_uid": config.test_uid, + "test_gid": config.test_gid, } mismatches = [ f"{name}={state.get(name)!r} (expected {value!r})" @@ -334,11 +339,26 @@ def _require_fixture(runner: Any, config: Any) -> Fixture: raise IntegrationTestError( f"expected two Slurm compute nodes; found {slurm_nodes}" ) + slurm_address_output = _probe_pod( + runner, + config, + login_name, + "login", + "for node in " + + " ".join(shlex.quote(node) for node in slurm_nodes) + + "; do getent ahostsv4 \"$node\" | awk 'NR == 1 {print $1}'; done", + ) + slurm_addresses = tuple(line for line in slurm_address_output.splitlines() if line) + if len(slurm_addresses) != 2 or len(set(slurm_addresses)) != 2: + raise IntegrationTestError( + f"Slurm workers lack distinct IPv4 addresses: {slurm_addresses}" + ) return Fixture( login_pod=login_name, login_container="login", ssh_addresses=(addresses[0], addresses[1]), slurm_nodes=(slurm_nodes[0], slurm_nodes[1]), + slurm_addresses=(slurm_addresses[0], slurm_addresses[1]), architecture=host_arch, ssh_home_mode=str(state["ssh_home_mode"]), ) @@ -877,7 +897,7 @@ def _assert_results( selector: str, remote_root: str, log_dir: Path, -) -> None: +) -> str: """Assert the two execution records and bounded-data cleanup.""" results = f"{remote_root}/results" discover = ( @@ -933,6 +953,14 @@ def _assert_results( test -s "$result/executions/$id.log" test -s "$result/executions/$id.workload.tsv" done +grep -qx $'nodes\t1' "$result/executions/0001.workload.tsv" +grep -qx $'nodes\t2' "$result/executions/0002.workload.tsv" +grep -qx $'dataset_files_total\t1' "$result/executions/0001.workload.tsv" +grep -qx $'dataset_files_total\t2' "$result/executions/0002.workload.tsv" +grep -qx $'dataset_bytes_total\t4096' "$result/executions/0001.workload.tsv" +grep -qx $'dataset_bytes_total\t8192' "$result/executions/0002.workload.tsv" +grep -qx $'completion_state\tcompleted' "$result/executions/0001.workload.tsv" +grep -qx $'completion_state\tcompleted' "$result/executions/0002.workload.tsv" test "$(find "$result" -type f -name '*.csv' -size +0c | wc -l)" -ge 2 test "$(find "$result" -type f -name '*.out' -size +0c | wc -l)" -ge 2 {cleanup_probe} @@ -947,6 +975,92 @@ def _assert_results( log_dir, 60, ) + return result_dir + + +def _assert_ordered_workers(fixture: Fixture, selector: str, sweep_output: str) -> None: + """Prove increasing cells use the configured workers in prefix order.""" + workers = fixture.ssh_addresses if selector == "ssh" else fixture.slurm_addresses + expected = { + "1": workers[0], + "2": ",".join(workers), + } + selected: dict[str, str] = {} + for line in sweep_output.splitlines(): + match = re.search(r"starting execution \d+:.*nodes=(\d+).*hosts=([^ ]+)", line) + if match: + selected[match.group(1)] = match.group(2) + if selected != expected: + raise IntegrationTestError( + f"{selector} ordered worker selection was {selected!r}; " + f"expected {expected!r}" + ) + + +def _copy_result_for_reporting( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + result_dir: str, + destination: Path, +) -> Path: + """Bring one completed result tree to the host for report validation.""" + destination.mkdir(parents=True) + if selector == "ssh": + shutil.copytree(result_dir, destination / Path(result_dir).name) + else: + runner.run( + [ + *_kubectl(config, "-n", config.namespace, "cp"), + "-c", + fixture.login_container, + f"{fixture.login_pod}:{result_dir}", + destination / Path(result_dir).name, + ], + timeout=120, + ) + return destination / Path(result_dir).name + + +def _assert_report( + runner: Any, + report_workspace: Path, + result_dir: Path, + selector: str, + log_dir: Path, +) -> None: + """Run the supported report wrapper and verify both sweep sizes appear.""" + result = runner.run( + [ + report_workspace / "utils" / "extract-elbencho.sh", + "--markdown", + result_dir, + ], + cwd=report_workspace, + timeout=600, + check=False, + ) + output = result.stdout + result.stderr + report_log = log_dir / f"{selector}-extract-elbencho.log" + report_log.write_text(output, encoding="utf-8") + report_log.chmod(0o640) + if result.returncode: + raise IntegrationTestError( + f"{selector} result reporting failed with exit code {result.returncode}; " + f"full output: {report_log}\n{output.strip()[-8000:]}" + ) + missing = [] + if "| Nodes" not in output: + missing.append("Nodes header") + for node_count in (1, 2): + if not re.search(rf"^\|\s*{node_count}\s*\|", output, re.MULTILINE): + missing.append(f"{node_count}-node row") + if missing: + raise IntegrationTestError( + f"{selector} report omitted expected node-count rows {missing}; " + f"full output: {report_log}" + ) def _run_substrate( @@ -957,6 +1071,7 @@ def _run_substrate( archive: Path, extracted: Path, build_root: Path, + report_workspace: Path, log_dir: Path, ) -> None: """Stage, validate, run, and inspect one filesystem substrate.""" @@ -985,7 +1100,7 @@ def _run_substrate( f"{selector} validate_env.sh omitted its success marker" ) sweep = prefix + "./storage-tests/fs/nv-elbencho-sweep.sh -b --nodes 1,2" - _run_step( + sweep_output = _run_step( runner, config, fixture, @@ -995,7 +1110,19 @@ def _run_substrate( log_dir, 600, ) - _assert_results(runner, config, fixture, selector, remote_root, log_dir) + _assert_ordered_workers(fixture, selector, sweep_output) + result_dir = _assert_results( + runner, config, fixture, selector, remote_root, log_dir + ) + local_result = _copy_result_for_reporting( + runner, + config, + fixture, + selector, + result_dir, + build_root / f"{selector}-report-input", + ) + _assert_report(runner, report_workspace, local_result, selector, log_dir) def _selected_tests(selectors: list[str]) -> tuple[str, ...]: @@ -1045,6 +1172,15 @@ def run_filesystem_tests( fixture.architecture, ) shutil.copy2(build_root / "build-tarball.log", log_dir / "build-tarball.log") + report_workspace = build_root / "report-workspace" + shutil.copytree(extracted, report_workspace) + _write_runtime_files( + report_workspace, + "ssh", + str(report_workspace), + fixture, + template=extracted / "env.sh.template", + ) for selector in selected: _run_substrate( runner, @@ -1054,6 +1190,7 @@ def run_filesystem_tests( archive, extracted, build_root, + report_workspace, log_dir, ) LOG.info("Filesystem integration tests passed: %s", ", ".join(selected)) diff --git a/lib/env_functions.sh b/lib/env_functions.sh index d9b6cf8..36608a5 100644 --- a/lib/env_functions.sh +++ b/lib/env_functions.sh @@ -1825,7 +1825,7 @@ coordinator_run_one_execution() { test_dirs_csv=$(_compute_test_dirs_csv_for_execution) || exit 1 # shellcheck disable=SC2154 # nodes, io_size, thread_count, io_depth come from sourcing NNNN.sh - _echo_ts "[coordinator] starting execution ${id}: nodes=${nodes} io_size=${io_size} threads=${thread_count} iodepth=${io_depth}" + _echo_ts "[coordinator] starting execution ${id}: nodes=${nodes} hosts=${first_n_hosts_csv} io_size=${io_size} threads=${thread_count} iodepth=${io_depth}" run_elbencho_io_sweep_iteration ) 2>&1 | tee "$log_file" execution_pipe_status=("${PIPESTATUS[@]}") From 84dd77fc879ef092061a811a88f58881b16172aa Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sat, 19 Sep 2026 13:15:19 -0700 Subject: [PATCH 03/10] Support and harden filesystem integration in Docker SBX Add an explicit sbx-shared storage backend alongside the existing NFS backend. Persist backend selection, limit automatic selection to recognized host capability gaps, and refuse backend changes until teardown. Run the SBX profile with the tested kind and Kubernetes versions, conditionally map the missing kmsg device, propagate the Docker SBX proxy CA, and expose a repository-backed directory through static RWX volumes. Prove cross-node and host visibility while retaining the same SSH, Slurm, tarball, sweep, result, and reporting coverage as the NFS profile. Use the digest-pinned upstream Elbencho image when release assets are unavailable in SBX. Package its dynamic runtime only in generated test artifacts, stage it for SSH workers, and keep third-party binaries out of the repository. Accept Elbencho's adjacent streamed JSON records and terminate loader-launched services by their listening port. Harden first-boot Slinky behavior by retrying only the recognized webhook startup race and refreshing the configless login client after accounting is ready. Add the reporting Python prerequisite and prevent shell startup noise from contaminating structured result discovery. Document backend selection, lifecycle and cleanup guarantees, environmental fidelity, and the Docker-SBX-specific compatibility behavior. Document the remaining filesystem sweep coverage gaps and rank future work across environment, command-line, and reporting behavior. Make the Docker SBX profile use a Kubernetes-compatible kubectl and reconcile Slinky values on every setup. Ignore terminating LoginSet pods during rollout and require ownership markers before mutating retained state or export directories. Package Elbencho with its discovered dynamic loader, use explicit SSH identities, and increase the bounded workload enough to produce measurable throughput. Handle valid all-zero scaling reports and add focused reporting and lifecycle regression tests. Restore configured Slurm worker order after allocation canonicalization so ORDER_NODES cell selection remains deterministic. --- docs/CONTEXT.md | 19 +- ...esystem-sweep-integration-coverage-gaps.md | 442 +++++++++++ integration-tests/README.md | 98 ++- integration-tests/bin/integration-test.py | 699 ++++++++++++++---- .../lib/filesystem_integration.py | 214 +++++- .../manifests/kind-sbx-shared.yaml.tmpl | 40 + .../manifests/sbx-storage.yaml.tmpl | 85 +++ lib/_elbencho_functions.sh | 40 +- .../fs/sbatch/_nv-elbencho-coordinator.sh | 30 +- tests/test_elbencho_shared_directory_shell.py | 2 + .../test_extract_elbencho_scale_efficiency.py | 33 + tests/test_integration_driver_safety.py | 255 +++++++ utils/extract-elbencho.py | 16 +- 13 files changed, 1769 insertions(+), 204 deletions(-) create mode 100644 docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md create mode 100644 integration-tests/manifests/kind-sbx-shared.yaml.tmpl create mode 100644 integration-tests/manifests/sbx-storage.yaml.tmpl create mode 100644 tests/test_extract_elbencho_scale_efficiency.py create mode 100644 tests/test_integration_driver_safety.py diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index f69e6bd..4ee3448 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -57,8 +57,21 @@ and are responsible for every binary they place in it. Slurm is the default execution substrate. Setting `SSH_HOST_LIST` selects passwordless SSH instead. Kubernetes execution is not implemented. The separate `integration-tests/` fixture provisions a three-node kind cluster, -NFS CSI storage, two passwordless-SSH workers, and a Slinky Slurm environment -on one Linux host. Its test action builds and validates a deployment tarball, +a shared RWX storage backend, two passwordless-SSH workers, and a Slinky Slurm +environment on one Linux host. The `nfs` backend uses a loop-backed NFSv4 +export and NFS CSI. The Docker-SBX-specific `sbx-shared` backend mounts a +repository-backed directory into every kind node and uses static RWX volumes; +both backends validate the repository's same shared-storage contract. Backend +selection is explicit or capability-based and persists until teardown. The +harness's state and export directories are dedicated leaves: an existing path +must carry the exact harness ownership marker before setup changes it or teardown +removes it. A completed setup also locks its namespace and, for NFS, export +directory until teardown. The SBX compatibility profile pairs kind/Kubernetes +1.34 with kubectl 1.34 and rebuilds cached container-derived Elbencho bundles +when their pinned image or bundle recipe changes. Slurm coordinators restore +configured `ORDER_NODES` include-list order after Slurm canonicalizes an +allocation's node list. +The test action builds and validates a deployment tarball, derives environments from the packaged `env.sh.template`, and runs bounded one-node and two-node filesystem sweeps through SSH and Slurm as the non-root account recorded by setup. It verifies ordered worker selection, exact workload @@ -90,7 +103,7 @@ alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python | `utils/build_tarball.sh` | User-local deployment-tarball builder | | `utils/build/` | Helpers for building Warp and the in-tree s3test program | | `tests/` | Python and shell-behavior regression tests collected by `pytest` | -| `integration-tests/` | Single-host kind, NFS CSI, SSH, and Slinky fixture provisioner and manifests | +| `integration-tests/` | Single-host kind, RWX storage, SSH, and Slinky fixture provisioner and manifests | | `docs/research/` | Feasibility studies and implementation handoffs for future integration work | The checked-in benchmark entry points are: diff --git a/docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md b/docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md new file mode 100644 index 0000000..c3a7d75 --- /dev/null +++ b/docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md @@ -0,0 +1,442 @@ + + +# Elbencho filesystem sweep integration coverage gaps + +## Scope and method + +This note compares the integration harness as of 2026-09-19 with the complete +filesystem sweep and reporting interfaces implemented by: + +- `env.sh.template` and `lib/env_base.sh`; +- `storage-tests/fs/nv-elbencho-sweep.sh`; +- `lib/env_functions.sh` and `lib/_elbencho_functions.sh`; and +- `utils/extract-elbencho.sh` and `utils/extract-elbencho.py`. + +The focus is functional integration coverage. Unit and shell tests reduce risk, +but tests with a mocked Elbencho or dispatcher are not counted as end-to-end SSH +or Slurm coverage. + +The configuration space is not a simple Cartesian product. Several variables +select a branch and make other values invalid, ignored, or operationally inert. +This note therefore distinguishes: + +- **covered**: the value changes behavior that the integration test executes + and meaningfully asserts; +- **executed but weakly asserted**: the code runs, but weak postconditions allow + a semantically wrong result to pass; +- **set but inert**: the harness sets a value that the selected workload does + not consume; and +- **not covered**: no integration case exercises the behavior. + +## Executive conclusion + +The harness covers deployment packaging, environment validation, +service/coordinator startup, one- and two-node worker selection, result +retrieval, and basic reporting over SSH and Slurm on amd64 and arm64. It does +**not** broadly cover sweep workloads. Every integration sweep is one buffered, +sequential, generated `shared-directory` workload with one 16 MiB file per node, +one thread, one I/O depth, and the default write/read/delete lifecycle. The +default `worker-directories` and direct-I/O path is absent, as are +failure/resume, retained or staged datasets, single-big-file mode, random and +split I/O, live CSV capture, weighted targets, and nearly all reporting options. + +The current report assertion is especially permissive: a successful command +with a `Nodes` header and rows beginning with `1` and `2` passes. It does not +prove that WRITE and READ were both reported, metrics are correct, workload +metadata survived extraction, or expected plots were created. + +## What the integration suite tests now + +The environment generator in +`integration-tests/lib/filesystem_integration.py:577-638` fixes one bounded +configuration. `_run_substrate()` invokes only: + +```text +./storage-tests/fs/nv-elbencho-sweep.sh -b --nodes 1,2 +``` + +The resulting effective matrix is: + +| Axis | Covered value | +|---|---| +| Execution substrate | Passwordless SSH and Slurm | +| CI architecture | amd64 and arm64 | +| Node count | Literal list `1,2` | +| Node selection | `ORDER_NODES=1`; first node, then both nodes | +| Access mode | Buffered I/O (`-b`) | +| I/O pattern | Sequential | +| Lifecycle | Generated mkdir, write, read, distributed delete | +| Layout | `ELBENCHO_FILE_LAYOUT=shared-directory` | +| Targets | One `TEST_DIRS` root with weight 1 | +| Dataset | One 16 MiB file per node | +| Sweep dimensions | One I/O size (`4K`), thread count (`1`), and depth (`1`) | +| Live capture | Disabled | +| Single-big-file mode | Disabled | +| SSH identity | Explicit user `tester` | +| Slurm selection | Two explicit include nodes, empty ignore list | +| Slurm allocation | CPU client, `partition=all`, bare `--exclusive` | +| Reporting | One raw result directory at a time, `--markdown` | + +The harness meaningfully verifies: + +- the deployment tarball and `env.sh.template` can be used from an extracted + deployment; +- `validate_env.sh` succeeds in both environments; +- the sweep dispatches through SSH and through a real Slurm coordinator; +- the one-node cell selects the first configured worker and the two-node cell + selects both workers; +- exactly two execution status files exist and both end in `SUCCESS` with exit + code zero; +- each execution has a log and workload record; +- dataset totals are one file/16777216 bytes and two files/33554432 bytes; +- at least two nonempty aggregate CSV and human-readable output files exist; +- generated target directories are removed after the default lifecycle; and +- the report wrapper exits successfully and emits a node header plus apparent + one- and two-node rows. + +The workflow also checks setup idempotence, stop/start, rejection of root test +execution, teardown idempotence, and both architecture jobs. These checks do not +expand the sweep matrix. CI uses separate SSH homes; shared homes are optional +in the harness but not a CI axis. + +## Configuration precedence and inactive-value semantics + +These relationships are important when selecting future cases. Merely changing +a value does not necessarily exercise it. + +1. A nonempty `SSH_HOST_LIST` selects SSH and disables Slurm. Slurm account, + reservation, partition, runtime, module, include/ignore, exclusivity, and + extra-argument settings cannot affect that run. This selection happens in + `lib/env_base.sh:90-105`. +2. `TEST_DIR` is a compatibility fallback only when `TEST_DIRS` is unset or + empty. A populated `TEST_DIRS` always wins (`lib/env_base.sh:58-67`). +3. `ORDER_NODES=1`, `yes`, or `true` selects deterministic prefixes. Other + values select the normal behavior. For Slurm, deterministic prefixing only + has an ordered list to use when `SLURM_NODE_INCLUDES` is nonempty + (`lib/env_base.sh:81-88,221-230`). +4. `ELBENCHO_FILE_SIZE` overrides size derivation from the write block size and + `ELBENCHO_FILE_SIZE_MULTIPLIER`. The integration harness sets both, so the + multiplier is inert (`lib/_elbencho_functions.sh:737-763`). +5. Generated `shared-directory` work is exact-completion work. Its duration is + reported as not applicable and it does not use `--timelimit` or `--infloop`. + The integration's one-second duration is therefore inert for its measured + phases (`lib/_elbencho_functions.sh:2901-2947`). +6. `-s/--single` is active only for a generated legacy + `worker-directories` workload. It is intentionally inactive for + `shared-directory`, `--read-from`, and single-big-file runs. +7. Multiple generated target paths, whether from multiple `TEST_DIRS` roots or + a weight above one, take the same computed fixed-file-count branch as + `--single`. That is where the `FS_MAX_*` estimates matter + (`lib/_elbencho_functions.sh:3161-3194`). +8. `--read-from` selects a staged-dataset reader before either generated + many-file layout. Current layout, files-per-node, generated file size, and + multiplier do not define the staged dataset. The exact treefile defines its + file and byte totals (`lib/_elbencho_functions.sh:2952-2968,2610-2643`). +9. `ELBENCHO_SINGLE_BIG_FILE=1` takes precedence over staged-directory and + generated-layout dispatch. It requires one root and sequential I/O and is + incompatible with `shared-directory`. With `--read-from `, the file + supplies the extent and `ELBENCHO_SINGLE_BIG_FILE_SIZE` is optional and + unused for that read (`lib/env_functions.sh:3360-3387`). +10. `ELBENCHO_ALL_NODES_ACCESS_ALL_DATA` is valid only in single-big-file mode; + value `1` adds Elbencho `--nosvcshare`. It is invalid elsewhere + (`lib/_elbencho_functions.sh:785-797,1100-1148`). +11. `-r/--rand` makes every applicable phase random. An `r` prefix inside one + `ELBENCHO_SCALE_IO_SIZES` component can independently force only WRITE or + READ random. There is no per-size marker that cancels a global `-r`. +12. Buffered timed reads omit `--infloop`; direct timed reads add `--direct` and + `--infloop` (`lib/_elbencho_functions.sh:3140-3159`). Upstream defines + `--infloop` as restarting completed worker workloads and `--direct` as + avoiding buffering/caching; see the upstream [changelog](https://github.com/breuner/elbencho/blob/master/CHANGELOG.md) + and [generated large-file help](https://github.com/breuner/elbencho/blob/master/docs/usage/help-large.md). +13. `--write-only`, `--write-no-read`, `--read-from`, and `--delete-only` are + mutually exclusive. A positive read-after-write pause is irrelevant to + write-only, write-no-read, and read-from operations. +14. `--resume` is exclusive with every other CLI flag. It restores the original + environment and CLI choices from `env_used.sh`; the caller's current sweep + values are not supposed to redefine the remaining cells. + +## Environment-variable coverage gaps + +### Dispatch, placement, and deployment variables + +| Variable or group | Current coverage | Gap | +|---|---|---| +| `RESULTS_DIR`, `LOGS_DIR` | Nondefault absolute fixture paths work. | Relative, unusual, and unwritable paths are not tested. | +| `SSH_HOST_LIST` | A simple newline-delimited two-IP file selects SSH. | Comma-separated and whitespace-separated entries, comments, blank lines, hostnames, duplicates, an empty file, unavailable hosts, and setting SSH alongside nonempty Slurm values are not integrated. | +| `SSH_USER` | Explicit `tester`. | Default/SSH-config identity and a wrong user are not tested. | +| `SSH_HOMEDIR_SHARED` | The harness can provision either mode, but CI uses separate homes. | No required two-mode matrix proves the different copy/staging behavior and a complete sweep in both modes. | +| `ORDER_NODES` | Only numeric true with Slurm includes and an SSH host list. | Disabled/random SSH selection, true/false aliases, Slurm without includes, changing subsets, and include ordering are absent. | +| `account`, `reservation`, `partition`, `run_time` | Empty account/reservation, `partition=all`, five-minute limit in Slurm only. | Explicit account/reservation, alternate or empty partition, time-limit propagation, and proof that all are ignored in SSH mode are absent. | +| `MODULES` | No custom module is required; defaults are attempted only if present. | Loading a real configured module, missing-module behavior, and SSH-mode irrelevance are not tested. | +| `SLURM_NODE_INCLUDES` | Simple two-line node list. | Unset/empty behavior, compressed hostlists, multiple entries per line, malformed names, ordering disabled, and capacity shorter than the requested node count are absent. | +| `SLURM_NODE_IGNORES` | Present but empty. | Real exclusions, compressed hostlists, overlap/conflict with includes, and resulting capacity checks are absent. | +| `SLURM_EXTRA_ARGS` | Empty array. | Multiple elements, an element containing spaces, duplicate/overriding scheduler options, invalid options, and consistent `sbatch`/`srun` propagation are absent. | +| `SLURM_EXCLUSIVE_USER` | `0`, producing bare `--exclusive`. | `1`/`yes`/`true`, target CPU discovery, `--cpus-per-task`, discovery failure, and execution inside an existing allocation are absent. | +| `SLURM_JOB_NAME_PREFIX` | Empty. | Nonempty naming and scheduler-safe unusual characters are absent. | +| `client_type` | `cpu`. | `gpu`, its four-GPU request, and GPU-only partition behavior are absent. | +| `client_arch` | amd64 and arm64 in separate CI jobs. | Configured/probed mismatch and wrong-binary validation are not integration cases. | + +### Filesystem target and workload variables + +| Variable or group | Current coverage | Gap and consequence | +|---|---|---| +| `TEST_DIRS` | One root, weight 1. | Multiple roots, weights above one, mixed weights, empty/invalid weights, and special-mode rejection are absent. These cases change targets and select computed file counts. | +| Legacy `TEST_DIR` | Not used. | Fallback when `TEST_DIRS` is empty and non-override when it is populated are not integrated. | +| `FS_MAX_AGG_THROUGHPUT` | Set to 1 but inert. | Aggregate-bandwidth limiting of computed file counts is absent. The compatibility fallback from `IOR_FS_MAX_AGG_THROUGHPUT` is also absent. | +| `FS_MAX_NODE_THROUGHPUT_GBPS` | Set to 1 but inert. | Per-node bandwidth limiting, scale with node count, and rounding are absent. | +| `FS_MAX_NODE_IOPS` | Set to 100 but inert. | IOPS-limited computed counts and the choice of IOPS versus bandwidth limit are absent. | +| `ELBENCHO_SCALE_THREAD_LIST` | One value, `1`. | Multiple values, Cartesian ordering, high counts, invalid/zero values, shared-file divisibility, and files-per-worker behavior are absent. | +| `ELBENCHO_SCALE_IO_SIZES` | One sequential `4K`. | Multiple values, `rSIZE`, split `WRITE,READ`, independently random components, malformed values, exact block divisibility, and Cartesian ordering are absent. | +| `ELBENCHO_IODEPTH_LIST` | One value, `1`. | Multiple depths, depth greater than one, invalid/zero values, and interaction with thread count and shared files are absent. | +| `ELBENCHO_SCALE_READ_WRITE_DURATION` | Set to 1 but inert in generated shared-directory mode. | Time-bounded worker-directory write/read, staged reads, single-file reads, validation, and actual timeout behavior are absent. | +| `ELBENCHO_READ_AFTER_WRITE_PAUSE` | Zero. | Positive pause, ordering around the pause, and interruption during the pause are absent. | +| `ELBENCHO_FILE_LAYOUT` | Only `shared-directory`. | The default `worker-directories` implementation, invalid values, and the layout's special interactions with read-from and single-file mode are absent. | +| `ELBENCHO_FILES_PER_NODE` | `1`. | Unset default, nontrivial counts, counts smaller than threads, indivisible counts, large boundary values, and rejection outside shared-directory are absent. | +| `ELBENCHO_FILE_SIZE` | Explicit `16M`. | Unset/derived size, alternate exact sizes, direct/random block divisibility, and invalid values are absent. | +| `ELBENCHO_FILE_SIZE_MULTIPLIER` | Set to 1 but shadowed by explicit file size. | Derived sizing and multiplier effects are entirely absent. | +| `ELBENCHO_LIVE_CSV_EXTENDED` | `0`. | Native live CSV production, retrieval, aggregate/service rows, large-artifact behavior, and report discovery are absent. | +| `ELBENCHO_LIVEINT` | Default value is parsed but live capture is off. | A nondefault interval, the below-250 ms warning, invalid values, and cadence in real output are absent. | +| `ELBENCHO_SINGLE_BIG_FILE` | `0`. | The whole generated and staged single-file branch is absent. | +| `ELBENCHO_SINGLE_BIG_FILE_BASENAME` | Default but inert. | Custom basename, path construction, cleanup, and unsafe/unusual names are absent. | +| `ELBENCHO_SINGLE_BIG_FILE_SIZE` | Empty and inert. | Required generated extent, exact size behavior, and optional/ignored read-from extent are absent. | +| `ELBENCHO_ALL_NODES_ACCESS_ALL_DATA` | `0` and inert outside single-file mode. | Both values and `--nosvcshare` propagation are absent. | + +Existing shell tests cover many validators, exact counters, cleanup traps, +treefile helpers, and single-file helpers with mocked calls. The integration +gap is entrypoint-to-substrate composition with the real Elbencho binary. + +## Sweep command-line coverage gaps + +The parser is defined in `storage-tests/fs/nv-elbencho-sweep.sh:53-205`. Only +`-b` and the literal `--nodes 1,2` form have integration coverage. + +| CLI interface | Semantics | Missing integration coverage | +|---|---|---| +| No `-b` (default DIO) | Adds `--direct` and, for timed reads, `--infloop`. | The default access path, alignment/exactness, direct-I/O failures, and report labeling. | +| `-b`, `--bio` | Adds `--norandalign` and omits read `--infloop`. | Short form is used, but native argv and report label are not asserted; the long alias is not tested. | +| `-r`, `--rand` | Makes applicable WRITE and READ phases random. | Both aliases, phase argv, interaction with per-size `r`, BIO/DIO, and rejection in single-big-file mode. | +| `-s`, `--single` | Forces computed file counts for one generated legacy target. | Active worker-directory behavior, `FS_MAX_*` sizing, cleanup, and documented no-op branches. | +| `--nodes X` | One node count. | Zero, negative, and nonnumeric rejection is absent. | +| `--nodes X-Y` | Inclusive ascending range. | Expansion, execution ordering, allocation size, and bad descending range. | +| `--nodes X-Y+Z` | Stepped range that always includes the stop. | Non-dividing steps, steps larger than the range, zero step, and ordering. | +| Comma node list | Preserves specified order, including a descending list. | Only `1,2`; mixed ranges, descending lists, duplicates, empty elements, and maximum-capacity rejection are absent. | +| `--write-only` | Retains one unique dataset per cell and emits its path. | Retention, emitted path, absence of READ/delete, safe later reuse/deletion, special one-root/weight rule, and both substrates. | +| `--write-no-read` | Writes then deletes without READ. | Missing-READ artifact/report semantics, distributed RMFILES, cleanup evidence, and both substrates. | +| `--read-from ` | Reads an operator dataset via scan or cached treefile. | Real cache miss, atomic publication, cache hit, stale-cache operator contract, mount-root fallback, exact totals, no mutation/deletion, random reads, and both substrates. | +| `--read-from ` with single-file mode | Reads file extent from metadata. | Optional size, no treescan, sequential enforcement, and both substrates. | +| `--delete-only ` | Deletes one strict descendant on one compute node; `--nodes` is irrelevant. | Successful SSH/Slurm deletion, preservation of root/siblings, root and outside-root rejection, symlink/realpath safety, and accepted-but-irrelevant `-b`/`-r`/`-s`/`--nodes`. | +| `--resume ` | Restores saved settings, resets stale RUNNING cells, skips SUCCESS, and retries remaining cells in order. | A real interrupted or failed sweep, mixed statuses, snapshot fidelity, lock ownership, Slurm live-job/accounting checks, SSH host reselection, repeated resume, malformed snapshots, and successful reporting of retry artifacts. | +| `-h`, `--help` | Prints usage without environment work. | Both aliases and stable documented interface. | +| Invalid CLI | Missing values, unknown options, positional arguments, missing `--nodes`, conflicting path modes, or resume plus another flag must fail. | No integration/entrypoint-level contract matrix. Many are cheap tests that do not require a live cluster. | +| Invocation inside Slurm | Direct invocation with `SLURM_JOB_ID` and no SSH is rejected. | Rejection behavior and the SSH-enabled exception. | + +The current two-cell run also does not exercise the full reification product. +Multiple node counts, I/O sizes, threads, and depths should demonstrate stable +execution numbering, unique test suffixes, read-host rotation, and result +association across more than one varying axis. + +## Failure, recovery, and lifecycle gaps + +Successful cells cover only the easiest state transition: +`PENDING -> RUNNING -> SUCCESS`. No real integration scenario covers: + +- a benchmark phase returning nonzero; +- incomplete exact counters despite process exit zero; +- stop-on-first-failure with later cells left pending; +- cleanup after mkdir, WRITE, READ, or RMFILES failure; +- a worker service dying between phases and Slurm service restart; +- an SSH worker failing startup and being pruned from the usable pool; +- signal handling while a shared-directory target is active; +- stale `RUNNING` recovery; +- dispatch-lock contention or stale lock recovery; +- Slurm coordinator disappearance versus a still-live allocation; +- result retrieval failing after remote benchmark success; or +- a successful resume that preserves earlier successes and removes duplicate + or stale artifacts from the retried cell. + +Several of these have focused shell tests in +`tests/test_elbencho_dispatch_shell.py` and +`tests/test_elbencho_shared_directory_signals.py`. Those tests use stubs and +synthetic sentinels. They do not establish the end-to-end contract among the +entrypoint, service processes, remote files, scheduler state, and reporter. + +## Reporting CLI coverage gaps + +The reporting parser is at `utils/extract-elbencho.py:4410-4507`. The only +wrapper-level integration call is `--markdown `. + +| Argument or input mode | Current status | Gap | +|---|---|---| +| Positional `input_dirs` | One directory per invocation. | Multiple directories, duplicate coordinates/datestamps, relative paths, empty/missing/unwritable directories, and first-directory output selection. | +| `--markdown` | Command succeeds; header and node-like rows are checked. | WRITE/READ sections, operation/config labels, metrics, workload metadata, representative command, image references, and exact row cardinality are not asserted. | +| Default terminal mode | Not invoked end to end. | Terminal table content, stdout mirroring to `report.txt`, and consistency with Markdown. | +| `--to-csv` | Not invoked through the CLI. | Creation of `elbencho-metrics.csv`, complete schema, histogram serialization, errors, and round-trip fidelity. | +| `--from-csv FILE` | Not invoked through the CLI. | CSV-only reports/plots, legacy optional columns, malformed required fields, filters, Markdown, and output-directory choice. | +| `--from-csv FILE` plus input dirs | Not covered and semantically unclear. | Help says “instead of” raw parsing, but implementation currently combines CSV metrics with parsed directories, risking duplicate rows. The intended contract needs a test or a validation error. | +| `--only-threads` | Not invoked. | Exact values, comma lists, inclusive ranges, malformed tokens, and empty matches. | +| `--only-nodes` | Not invoked. | Same, including proof that aggregate and live inputs are filtered consistently. | +| `--only-sizes` | Parser helper has unit tests; CLI is not invoked. | Repeated flags, semicolon lists, compound sizes such as `1M,r64K`, malformed input, and exact matching. | +| `--only-iodepths` | Not invoked. | Exact/list/range/error and empty-match behavior. | +| `--test-parse FILE` | Not invoked. | Base, `.csv`, and `.out` paths; missing partner; malformed files; histogram output; and intentional precedence over otherwise supplied report options. | +| `--no-dual-y-axis` | Default false path may generate plots, but no files are asserted. | Expected single-axis versus dual-axis filenames and graph content. | +| `--per-client-plots` | No live input exists. | CLI discovery, second-pass client selection, summary CSV/text, time series, heatmaps, missing clients, counter resets, and failover. | +| `--client-outlier-threshold` | Only the default is parsed; effect is inactive. | Nondefault threshold and zero/negative rejection through the CLI. | +| `--client-min-underperform-segments` | Only the default is parsed; effect is inactive. | Nondefault selection and zero/negative rejection. | +| `--client-max-timeseries-lines` | Only the default is parsed; effect is inactive. | Truncation/selection and zero/negative rejection. | +| `--client-max-heatmap-rows` | Only the default is parsed; effect is inactive. | Row limiting and zero/negative rejection. | + +The report also lacks integration inputs for every sweep branch absent above: +worker directories, direct/random or split I/O, multiple dimensions, +write-only, no-read, staged directory and file reads, single-big-file, +all-nodes-all-data, and resumed attempts. Those modes affect grouping, labels, +duration interpretation, deduplication, and workload metadata. + +Existing unit tests substantially reduce parser risk: + +- `tests/test_extract_elbencho_resume.py` covers workload joins, resumed CSV + deduplication, and last output sections; +- `tests/test_extract_elbencho_terminal_subgroups.py` covers subgroup keys and + some mode-sensitive display rules; +- `tests/test_extract_elbencho_treescan_scan.py` covers treescan artifact + matching; +- `tests/test_extract_elbencho_live_csv.py` covers the live-analysis library; + and +- `tests/test_parse_only_sizes.py` covers the size-filter parser. + +They mostly call Python functions directly. They do not cover `main()` argument +composition, shell-wrapper virtual-environment setup, real result discovery, +or end-to-end output artifacts for those options. + +## False-pass weaknesses in the current integration assertions + +The present assertions are useful smoke checks, but several regressions could +pass: + +1. Two successful statuses and plausible dataset totals do not prove the + intended native Elbencho argv. A regression could drop `--norandalign`, use + the wrong block size/depth/thread count, or omit host rotation. +2. The workload checks validate only total files, total bytes, and overall + completion. They do not assert WRITE, READ, and delete completion states and + counters, cleanup timing, files per worker, or layout/source fields. +3. `find ... | wc -l` requires at least two `.csv` and `.out` files, not the + exact expected set associated with execution IDs 0001 and 0002. +4. `env_used.yaml` and `env_used.sh` need only be nonempty. Snapshot omissions, + wrong CLI flags, wrong arrays, wrong `TEST_DIRS`, or values leaking from the + current `env.sh` would not be detected before a resume is attempted. +5. Ordered-worker checking parses “starting execution” log lines. It does not + independently inspect the actual SSH or `srun` command, and later duplicate + lines for a node count overwrite earlier entries in its dictionary. +6. Cleanup checks only top-level names matching + `elbencho-sweep-target-*`. A wrongly named leaked dataset or a retained + single-file artifact would escape the check. +7. The reporter needs only emit a `Nodes` header and any rows that begin with 1 + and 2. It could omit an operation, report wrong values, duplicate stale + metrics, lose workload metadata, or fail to create plots and still pass. +8. No deliberate bad artifact is injected to verify that result and report + assertions fail rather than accepting partial output. +9. The reporter logs a bad benchmark pair and continues. One valid pair can + therefore hide incomplete parsing. +10. Invalid integer filter tokens are skipped, and an empty aggregate filter + set is treated as no filter. Filtering all aggregate metrics can also exit + successfully with “No metrics to display.” +11. `--from-csv` plus input directories combines both sources despite help text + that says CSV is used “instead,” allowing unnoticed duplicate metrics. +12. Missing optional metadata can reduce report detail without violating the + current node-row assertion. + +## Stack-ranked gaps to close + +The ranking below considers expected frequency, consequence of silent error, +amount of code unique to the branch, existing lower-level coverage, and the +cost of adding a case. It is a value ranking, not a recommendation to create a +full cross product. + +1. **Default `worker-directories` plus direct I/O on SSH and Slurm.** This is + the shipped default and executes a large branch that the integration suite + currently bypasses: timed mkdir/write/read, derived file size, tree scan, + host rotation, and recursive cleanup. A small DIO case should assert exact + argv/metadata and both WRITE and READ report rows. +2. **Strengthen the existing result and report oracle.** Before multiplying + cases, require the current case to prove exact artifact names, snapshot + values, phase counters/states, expected metric count and values, BIO label, + both operations, workload metadata, and plot artifacts. This closes the + largest “tests run but can pass incorrectly” risk at modest runtime cost. +3. **Real failure followed by `--resume` on both substrates.** Long sweeps are + expensive, making partial-run recovery operationally critical. Exercise + stop-on-first-failure, preserved SUCCESS, stale or FAILED retry, lock + behavior, artifact replacement/deduplication, and final reporting. +4. **`--write-only` -> `--read-from` -> `--delete-only` lifecycle.** One chained + scenario can validate retained path publication, a real cache miss and hit, + read-only preservation, safe explicit deletion, and report annotations. + Root/outside-root rejection should be a mandatory negative check because + delete-only is destructive. +5. **`--write-no-read` and failure cleanup.** This verifies absence of READ, + distributed RMFILES evidence, no leaked data, and correct reporting of a + write-only metric without conflating the mode with retained write-only data. +6. **Single-big-file mode at one and two nodes.** Cover cooperative slicing, + `ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1`, custom basename/size, generated + cleanup, staged file read with inferred extent, and random/shared-layout + rejection. Bugs here can silently change how much of a shared file each + service accesses. +7. **A small multi-dimensional random/split-I/O matrix.** Vary at least two I/O + sizes (including independently random WRITE/READ), two threads, and two I/O + depths in one bounded run. Assert Cartesian numbering, phase argv, host + rotation, result association, report grouping, and filters. +8. **Weighted/multiple roots and active `-s` sizing.** Exercise the default + compatibility model where `FS_MAX_*` selects file counts, including one + bandwidth-limited and one IOPS-limited case. This is materially different + from exact shared-directory completion. +9. **Extended live CSV through reporting.** Generate real live files on SSH and + Slurm, then run aggregate live reporting and `--per-client-plots` with + nondefault limits. Live capture can become very large and is an important + diagnostic path, but its analysis library already has substantial unit + coverage. +10. **Slurm option variants with scheduling consequences.** Test + `SLURM_EXCLUSIVE_USER=1` and CPU derivation first, then an extra argument + containing spaces, include/exclude interaction, and a nonempty job prefix. + GPU GRES should follow when a suitable runner exists. +11. **Reporting persistence and output modes.** Add CLI round-trip coverage for + `--to-csv`/`--from-csv`, default terminal/report.txt output, + `--no-dual-y-axis`, exact plot sets, and multiple result directories. +12. **SSH parsing, selection, and shared homes.** Make separate and shared home + modes explicit CI cases or a sequential subcase, test host-file syntax, and + cover `ORDER_NODES=0`. This matters, but core SSH orchestration already has + a real two-worker happy path. +13. **CLI and validation error matrix.** Help, missing arguments, bad ranges, + conflicting modes, random single-file rejection, and invalid filters are + cheap and should be exhaustive. Most do not need the expensive kind + fixture and belong in entrypoint-level tests rather than the full workflow. +14. **Less common deployment variants.** Explicit account/reservation/module + settings, architecture mismatch, GPU client type, unusual paths, and + scheduler capacity failures are valuable environment-compatibility checks + but are less portable and lower value than the workload and recovery gaps + above. + +## Suggested coverage strategy + +A full Cartesian product would be expensive and redundant: + +1. Run one short real case for each workload/lifecycle branch on both + substrates, plus one multi-dimensional case for reification and grouping. +2. Put parsing, invalid combinations, ranges, and snapshot-schema contracts in + fast tests that stub only final dispatch. +3. Isolate process, service, retrieval, and scheduler failures in a focused + fault-injection suite. +4. Reuse real workload artifacts across reporting modes. Each case should + assert its branch, native command, phases, artifacts, snapshot, cleanup or + retention contract, and semantic report content. diff --git a/integration-tests/README.md b/integration-tests/README.md index 35e7411..09016fa 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -23,11 +23,11 @@ server. The control-plane node is also the Slinky login node and negative control for the storage-worker label. The two worker nodes run storage clients. The current setup target is Ubuntu 24.04 on x86-64 or ARM64 with at least two -CPUs, 8 GiB total RAM, 6 GiB available RAM, and 20 GiB free on `/`. Python -3.12 and an accessible rootful Docker daemon are prerequisites. The driver -installs its other host packages and pinned client tools when needed. It also -builds a small derived Slinky login image containing the standard `file` -package required by `validate_env.sh`. +CPUs, 8 GiB total RAM, 6 GiB available RAM, and 20 GiB free on the selected +backend's filesystem. Python 3.12 and an accessible rootful Docker daemon are +prerequisites. The driver installs its other host packages and pinned client +tools when needed. It also builds a small derived Slinky login image containing +the standard `file` package required by `validate_env.sh`. Run setup (or its exact synonym, start) with: @@ -36,13 +36,46 @@ sudo -v sudo integration-tests/bin/integration-test.py setup ``` +`--storage-backend auto` is the default. It selects `nfs` when the host has the +required loop, mount, systemd, and kernel NFS facilities. It selects +`sbx-shared` only when a recognized capability needed by that profile is +absent. An explicit backend never falls back, and setup records the selection; +changing it requires teardown first. Arbitrary download, image, Kubernetes, +manifest, storage-visibility, SSH, and Slurm failures remain fatal. + +The `nfs` backend uses a loop-backed NFSv4 export and NFS CSI. The +`sbx-shared` backend is specifically for Docker SBX: it mounts one +repository-backed directory into every kind node and binds static RWX claims to +separate test-data and shared-home subdirectories. Both implement the same +in-scope integration contract: cross-node and host read/write visibility, +shared-home behavior, and successful SSH and Slurm filesystem sweeps. NFS and +CSI provisioning themselves are infrastructure details outside this +repository's test scope; the backend difference is environmental fidelity, not +repository feature coverage. + +Select Docker SBX explicitly with: + +```bash +sudo integration-tests/bin/integration-test.py \ + --storage-backend sbx-shared setup +``` + +The SBX profile requires Docker's private engine to bind-mount the checked-out +repository path. It uses the tested kind v0.30.0/Kubernetes v1.34.0 profile and +maps `/dev/null` to `/dev/kmsg` in kind nodes only when the SBX environment +lacks that device. When Docker SBX exposes its proxy CA, setup installs that CA +in the disposable kind nodes so containerd can pull the fixture images. The +default shared root is `tmp/integration-sbx-shared`; an alternate path may be +set with `--sbx-shared-root`, but must remain below the repository's `tmp/` +directory. + Setup records the invoking pre-sudo account, gives that non-root account access to the private kubeconfig, key, and test-run workspace, and verifies it can use Docker, kind, and kubectl. A root shell without `SUDO_USER` must name the account explicitly with `--test-user USER`. The default SSH homes are separate `emptyDir` volumes. To mount the dedicated -RWX NFS claim at `/home/tester` in both workers instead, use: +RWX shared-home claim at `/home/tester` in both workers instead, use: ```bash integration-tests/bin/integration-test.py --ssh-home-mode shared setup @@ -51,7 +84,7 @@ integration-tests/bin/integration-test.py --ssh-home-mode shared setup The generated host SSH key, strict known-hosts file, and two worker addresses are kept under `/var/lib/storage-scale-test-integration/`. Re-running setup reconciles and validates the environment without replacing those credentials -or retained NFS data. +or retained backend data. After setup succeeds, run the bounded filesystem regression cases with: @@ -72,18 +105,19 @@ deployment archive from a tracked-files-only snapshot with `utils/build_tarball.sh`, validates its contents, and runs from the extracted archive. The SSH case launches `validate_env.sh` and `nv-elbencho-sweep.sh` on the host and reaches the two worker pods over SSH. The Slurm case streams the -same archive to the LoginSet, extracts it in the shared NFS filesystem, and +same archive to the LoginSet, extracts it in the shared storage filesystem, and launches both commands there. -The pinned benchmark archive is size-limited, checksum-verified, and cached -outside the repository before the binary is included in the user-built -deployment archive. Timestamped build and step logs are retained below the -state directory's `test-runs/` directory. The test also requires successful -execution records, exact one- and two-node workload totals, ordered worker -selection, nonempty benchmark output, environment snapshots, and cleanup of -its generated data directories. It then runs `utils/extract-elbencho.sh` on a -host-side copy of each result and requires the report to contain both node -counts. +The NFS profile uses a size-limited, checksum-verified upstream benchmark +archive. Docker SBX, where GitHub release assets may be unavailable, extracts +the binary and runtime libraries from the digest-pinned upstream +`breuner/elbencho:v3.1-11` image and includes them only in the generated test +deployment. Timestamped build and step logs are retained below the state +directory's `test-runs/` directory. The test also requires successful execution +records, exact one- and two-node workload totals, ordered worker selection, +nonempty benchmark output, environment snapshots, and cleanup of its generated +data directories. It then runs `utils/extract-elbencho.sh` on a host-side copy +of each result and requires the report to contain both node counts. ## On-demand CI @@ -106,22 +140,22 @@ branch is the way to run it. After the workflow is merged, a maintainer can also open **Actions**, choose **Filesystem integration**, select **Run workflow**, and choose an authorized branch or the default branch. -Delete the disposable kind cluster and, when owned exclusively by the harness, -stop NFS with: +Delete the disposable kind cluster and, for the NFS backend when owned +exclusively by the harness, stop NFS with: ```bash integration-tests/bin/integration-test.py stop ``` Stop preserves packages, downloaded charts, Docker images, generated keys and -passwords, rendered state, and external NFS data. Kind containers, Kubernetes +passwords, rendered state, and backend data. Kind containers, Kubernetes objects, and MariaDB's node-local volume are disposable and are deleted. A subsequent start therefore creates and validates a fresh cluster and takes longer than an idempotent setup against an already-running cluster. Timestamped logs and rendered manifests are retained in the state directory. Add `--verbose` for command-level logging. The failure path captures host, Docker, -NFS, Kubernetes node, pod, and event diagnostics without printing Kubernetes -Secrets. +backend, Kubernetes node, pod, and event diagnostics without printing +Kubernetes Secrets. For CI workers or any host where retained fixture data is not wanted, run: @@ -129,12 +163,14 @@ For CI workers or any host where retained fixture data is not wanted, run: integration-tests/bin/integration-test.py teardown ``` -Teardown is idempotent. It performs the disposable stop, removes the dedicated -NFS export and configuration, disables and stops `nfs-server`, removes any UFW -rule that the harness added, unmounts the verified loop-backed filesystem, and -deletes the fixture's generated data, keys, logs, and locally built image tags. -It refuses destructive cleanup when ownership markers, rendered host -configuration, mount backing, or unrelated NFS exports do not match the -fixture. Operating-system packages, kind, kubectl, Helm, and reusable upstream -Docker image layers are not uninstalled. If a locally built tag existed before -setup, teardown restores that exact prior image ID instead of deleting it. +Teardown is idempotent. It performs the disposable stop and deletes the +fixture's generated data, keys, logs, and locally built image tags. For NFS it +also removes the dedicated export and configuration, disables and stops +`nfs-server`, removes any harness-owned UFW rule, and unmounts the verified +loop-backed filesystem. For Docker SBX it removes only the marker-owned shared +root and never invokes NFS, systemd, firewall, loop, or mount operations. It +refuses destructive cleanup when the applicable ownership and path checks do +not match the fixture. Operating-system packages, kind, kubectl, Helm, and +reusable upstream Docker image layers are not uninstalled. If a locally built +tag existed before setup, teardown restores that exact prior image ID instead +of deleting it. diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py index f6e1c24..f5f6aae 100755 --- a/integration-tests/bin/integration-test.py +++ b/integration-tests/bin/integration-test.py @@ -50,12 +50,18 @@ ) KIND_VERSION = "v0.33.0" +SBX_KIND_VERSION = "v0.30.0" KUBECTL_VERSION = "v1.37.0" +SBX_KUBECTL_VERSION = "v1.34.0" HELM_VERSION = "v3.22.0" KIND_NODE_IMAGE = ( "kindest/node:v1.37.0@" "sha256:a1ed56cfb0e7b93589bdf97c8cd566405a265939e3620fc4f5de89adff580ae5" ) +SBX_KIND_NODE_IMAGE = ( + "kindest/node:v1.34.0@" + "sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a" +) NFS_CSI_VERSION = "4.13.4" NFS_CSI_SOURCE_SHA256 = ( "ded6ffba8b1600d4c723ce1ecb1fd91721ef48e732ce7ca30c0efeeecbb0b900" @@ -76,6 +82,9 @@ GIB = 1024**3 DEFAULT_STATE_DIR = Path("/var/lib/storage-scale-test-integration") DEFAULT_EXPORT_DIR = Path("/srv/storage-scale-test-integration") +DEFAULT_SBX_SHARED_ROOT = ( + Path(__file__).resolve().parents[2] / "tmp" / "integration-sbx-shared" +) NFS_EXPORT_CONFIG = Path("/etc/exports.d/storage-scale-test-integration.exports") NFS_DAEMON_CONFIG = Path("/etc/nfs.conf.d/storage-scale-test-integration.conf") EXPORT_MARKER = ".storage-scale-test-integration.json" @@ -104,6 +113,8 @@ class Config: namespace: str state_dir: Path export_dir: Path + storage_backend: str + sbx_shared_root: Path ssh_home_mode: str test_user: str test_uid: int @@ -318,11 +329,11 @@ def _require_python() -> None: raise ProvisionError("integration-test.py requires Python 3.12 or newer") -def _check_host_capacity() -> None: +def _check_host_capacity(disk_path: Path) -> None: """Fail before provisioning an undersized host.""" cpu_count = os.cpu_count() or 0 memory = _meminfo() - disk = shutil.disk_usage("/") + disk = shutil.disk_usage(disk_path) failures: list[str] = [] if cpu_count < 2: failures.append(f"need at least 2 CPUs; found {cpu_count}") @@ -331,15 +342,108 @@ def _check_host_capacity() -> None: if memory.get("MemAvailable", 0) < 6 * GIB: failures.append("need at least 6 GiB available memory") if disk.free < 20 * GIB: - failures.append("need at least 20 GiB free on /") + failures.append(f"need at least 20 GiB free on {disk_path}") if failures: raise ProvisionError("host capacity check failed: " + "; ".join(failures)) LOG.info( - "Host capacity accepted: %s CPUs, %.1f GiB available RAM, %.1f GiB free disk", + "Host capacity accepted: %s CPUs, %.1f GiB available RAM, %.1f GiB " + "free on %s", cpu_count, memory["MemAvailable"] / GIB, disk.free / GIB, + disk_path, + ) + + +def _nfs_capability_failures() -> list[str]: + """Return recognized reasons that the full NFS backend cannot run.""" + failures = [] + for path in (Path("/dev/kmsg"), Path("/dev/loop-control")): + if not path.exists(): + failures.append(f"{path} is absent") + # exportfs is supplied by nfs-kernel-server, which setup installs after + # selection; its pre-setup absence is not a host capability failure. + for command in ("losetup", "mount", "systemctl"): + if not shutil.which(command): + failures.append(f"{command} is unavailable") + return failures + + +def _storage_backend_document(config: Config, backend: str) -> dict[str, object]: + """Return the persistent storage-backend selection document.""" + document: dict[str, object] = { + "schema": STATE_SCHEMA, + "cluster_name": config.cluster_name, + "backend": backend, + } + if backend == "sbx-shared": + document["shared_root"] = str(config.sbx_shared_root) + return document + + +def _validate_retained_state_summary(config: Config) -> None: + """Reject immutable option changes after a setup has completed.""" + path = config.state_dir / "state.json" + if not path.exists(): + return + state = json.loads(path.read_text(encoding="utf-8")) + backend = state.get("storage_backend") + if backend not in {"nfs", "sbx-shared"}: + raise ProvisionError(f"invalid retained setup state: {path}") + expected: dict[str, object] = { + "schema": STATE_SCHEMA, + "cluster_name": config.cluster_name, + "namespace": config.namespace, + } + if backend == "nfs": + expected["export_dir"] = str(config.export_dir) + mismatches = [name for name, value in expected.items() if state.get(name) != value] + backend_changed = config.storage_backend not in {"auto", backend} + if mismatches or backend_changed: + changed = ", ".join(mismatches or ["storage_backend"]) + raise ProvisionError( + f"retained setup differs in immutable fields ({changed}); " + "use the original options to teardown first" + ) + + +def _select_storage_backend(config: Config) -> str: + """Select once, persist, and validate the integration storage backend.""" + _validate_retained_state_summary(config) + path = config.state_dir / "storage-backend.json" + if path.exists(): + document = json.loads(path.read_text(encoding="utf-8")) + backend = str(document.get("backend", "")) + expected = _storage_backend_document(config, backend) + if backend not in {"nfs", "sbx-shared"} or document != expected: + raise ProvisionError(f"invalid retained storage backend state: {path}") + if config.storage_backend not in {"auto", backend}: + raise ProvisionError( + f"retained setup uses storage backend {backend}; teardown is " + f"required before selecting {config.storage_backend}" + ) + return backend + + failures = _nfs_capability_failures() + backend = config.storage_backend + if backend == "auto": + backend = "sbx-shared" if failures else "nfs" + if backend == "nfs" and failures: + raise ProvisionError( + "NFS storage backend requirements are unavailable: " + "; ".join(failures) + ) + _write_text( + path, + json.dumps(_storage_backend_document(config, backend), sort_keys=True) + "\n", ) + if backend == "sbx-shared": + LOG.info( + "Using the Docker SBX shared-path backend for the required RWX " + "storage contract" + ) + else: + LOG.info("Using full-fidelity NFS CSI storage backend") + return backend def _meminfo() -> dict[str, int]: @@ -366,19 +470,21 @@ def _check_platform() -> str: ) from error -def _ensure_apt_packages(runner: Runner) -> None: +def _ensure_apt_packages(runner: Runner, backend: str) -> None: """Install the narrow tested Ubuntu/Debian package set when absent.""" - packages = ( + packages = [ "ca-certificates", "curl", - "e2fsprogs", "file", "jq", - "nfs-common", - "nfs-kernel-server", "openssh-client", "openssl", - ) + "python3-venv", + ] + if backend == "nfs": + packages.extend(("e2fsprogs", "nfs-common", "nfs-kernel-server")) + elif not shutil.which("kind"): + packages.append("kind") missing = [ package for package in packages @@ -437,14 +543,29 @@ def _command_version(runner: Runner, command: str) -> str: return runner.run(arguments, check=False, timeout=30).stdout -def _ensure_client_tools(runner: Runner, architecture: str) -> None: +def _ensure_client_tools( + runner: Runner, architecture: str, storage_backend: str +) -> None: """Install checksum-verified kind, kubectl, and Helm when versions differ.""" expected = { "kind": KIND_VERSION, - "kubectl": KUBECTL_VERSION, + "kubectl": ( + SBX_KUBECTL_VERSION if storage_backend == "sbx-shared" else KUBECTL_VERSION + ), "helm": HELM_VERSION, } for command, version in expected.items(): + if command == "kind" and storage_backend == "sbx-shared": + installed = _command_version(runner, command) + if SBX_KIND_VERSION not in installed: + raise ProvisionError( + "sbx-shared compatibility profile requires kind " + f"{SBX_KIND_VERSION}; found {installed.strip() or 'nothing'}" + ) + LOG.info( + "Using Docker SBX compatibility profile with %s", installed.strip() + ) + continue if version in _command_version(runner, command): LOG.info("Using %s %s", command, version) continue @@ -573,8 +694,23 @@ def _kind_containers(runner: Runner, config: Config, running_only: bool) -> list return [line for line in output.splitlines() if line] -def _render_kind_config(config: Config) -> Path: +def _render_kind_config(config: Config, backend: str) -> Path: """Render the immutable three-node topology.""" + if backend == "sbx-shared": + kmsg_mount = "" + if not Path("/dev/kmsg").exists(): + kmsg_mount = "\n".join( + (" - hostPath: /dev/null", " containerPath: /dev/kmsg") + ) + return _render_resource( + config, + "manifests/kind-sbx-shared.yaml.tmpl", + { + "CLUSTER_NAME": config.cluster_name, + "SHARED_ROOT": str(config.sbx_shared_root), + "KMSG_MOUNT": kmsg_mount, + }, + ) return _render_resource( config, "manifests/kind.yaml.tmpl", @@ -582,9 +718,10 @@ def _render_kind_config(config: Config) -> Path: ) -def _create_cluster(runner: Runner, config: Config) -> None: +def _create_cluster(runner: Runner, config: Config, backend: str) -> None: """Create a new owned kind cluster.""" - manifest = _render_kind_config(config) + manifest = _render_kind_config(config, backend) + node_image = SBX_KIND_NODE_IMAGE if backend == "sbx-shared" else KIND_NODE_IMAGE LOG.info("Creating three-node kind cluster %s", config.cluster_name) runner.run( [ @@ -594,7 +731,7 @@ def _create_cluster(runner: Runner, config: Config) -> None: "--name", config.cluster_name, "--image", - KIND_NODE_IMAGE, + node_image, "--config", manifest, "--kubeconfig", @@ -606,6 +743,100 @@ def _create_cluster(runner: Runner, config: Config) -> None: ) +def _validate_sbx_shared_root(config: Config) -> None: + """Require the Docker SBX root to be a narrow path inside repository tmp.""" + allowed_parent = (_repository_root() / "tmp").resolve() + root = config.sbx_shared_root + if root == allowed_parent or allowed_parent not in root.parents: + raise ProvisionError( + f"Docker SBX shared root must be below {allowed_parent}: {root}" + ) + + +def _sbx_shared_marker(config: Config) -> dict[str, object]: + """Return the exact marker for the repository-backed shared root.""" + return { + **_owner_document(config), + "backend": "sbx-shared", + "shared_root": str(config.sbx_shared_root), + } + + +def _prepare_sbx_shared(runner: Runner, config: Config) -> None: + """Create and prove the repository-backed Docker-shared directory.""" + _validate_sbx_shared_root(config) + root = config.sbx_shared_root + marker = root / EXPORT_MARKER + if root.exists() and not marker.exists() and any(root.iterdir()): + raise ProvisionError(f"refusing nonempty unowned SBX shared root: {root}") + root.mkdir(parents=True, exist_ok=True) + if marker.exists(): + document = json.loads(marker.read_text(encoding="utf-8")) + if document != _sbx_shared_marker(config): + raise ProvisionError(f"SBX shared marker does not match: {marker}") + else: + _write_text( + marker, + json.dumps(_sbx_shared_marker(config), sort_keys=True) + "\n", + ) + for directory in (root / "storage-test", root / "ssh-home"): + directory.mkdir(exist_ok=True) + directory.chmod(0o777) + token = secrets.token_hex(16) + source = root / "agent-probe" + source.write_text(token + "\n", encoding="utf-8") + try: + runner.run( + [ + "docker", + "run", + "--rm", + "--entrypoint", + "sh", + "--mount", + f"type=bind,src={root},dst=/probe", + SBX_KIND_NODE_IMAGE, + "-ec", + 'test "$(cat /probe/agent-probe)" = "$1"; ' + 'printf "%s\\n" "$1" >/probe/engine-probe', + "sbx-shared-probe", + token, + ], + timeout=300, + ) + if (root / "engine-probe").read_text(encoding="utf-8").strip() != token: + raise ProvisionError("Docker shared-path probe returned the wrong token") + finally: + source.unlink(missing_ok=True) + (root / "engine-probe").unlink(missing_ok=True) + + +def _configure_sbx_node_trust(runner: Runner, config: Config) -> None: + """Install Docker SBX's proxy CA in kind nodes when it is present.""" + source = Path("/usr/local/share/ca-certificates/proxy-ca.crt") + if not source.is_file(): + LOG.info("Docker SBX proxy CA is absent; leaving kind trust unchanged") + return + expected = hashlib.sha256(source.read_bytes()).hexdigest() + destination = "/usr/local/share/ca-certificates/docker-sbx-proxy-ca.crt" + nodes = _kind_containers(runner, config, running_only=True) + if len(nodes) != 3: + raise ProvisionError( + f"cannot configure Docker SBX trust: expected 3 nodes, found {len(nodes)}" + ) + for node in nodes: + current = runner.run( + ["docker", "exec", node, "sha256sum", destination], check=False + ) + if current.returncode == 0 and current.stdout.split()[0] == expected: + continue + runner.run(["docker", "cp", source, f"{node}:{destination}"]) + runner.run(["docker", "exec", node, "update-ca-certificates"]) + runner.run(["docker", "exec", node, "systemctl", "restart", "containerd"]) + runner.run(["docker", "exec", node, "systemctl", "restart", "kubelet"]) + LOG.info("Configured kind nodes to trust the Docker SBX proxy CA") + + def _ensure_cluster_ownership(config: Config, cluster_exists: bool) -> None: """Claim a new cluster name or validate its persistent ownership marker.""" marker = config.state_dir / "cluster-owner.json" @@ -730,30 +961,36 @@ def _ensure_export_marker(runner: Runner, config: Config) -> None: expected = json.dumps( {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name}, sort_keys=True ) + export_exists = ( + runner.run( + [*_sudo_prefix(), "test", "-e", config.export_dir], + check=False, + timeout=30, + ).returncode + == 0 + ) existing = runner.run( [*_sudo_prefix(), "cat", marker_path], check=False, timeout=30 ) - if existing.returncode == 0 and existing.stdout.strip() != expected: + if export_exists and existing.returncode != 0: raise ProvisionError( - f"refusing export with mismatched ownership marker: {marker_path}" + f"refusing to modify unowned export directory: {config.export_dir}" ) - if existing.returncode != 0: - probe = runner.run( - [ - *_sudo_prefix(), - "find", - config.export_dir, - "-mindepth", - "1", - "-maxdepth", - "1", - ], + if not export_exists: + parent_exists = runner.run( + [*_sudo_prefix(), "test", "-d", config.export_dir.parent], check=False, + timeout=30, ) - if probe.returncode == 0 and probe.stdout.strip(): + if parent_exists.returncode: raise ProvisionError( - f"refusing nonempty unowned export directory: {config.export_dir}" + "export directory must be a leaf below an existing directory: " + f"{config.export_dir}" ) + if existing.returncode == 0 and existing.stdout.strip() != expected: + raise ProvisionError( + f"refusing export with mismatched ownership marker: {marker_path}" + ) runner.run([*_sudo_prefix(), "install", "-d", "-m", "0770", config.export_dir]) runner.run([*_sudo_prefix(), "chown", f"+{NFS_UID}:+{NFS_GID}", config.export_dir]) marker_source = config.state_dir / "export-marker.json" @@ -788,7 +1025,7 @@ def _export_mount_type(runner: Runner, config: Config) -> str: def _ensure_export_filesystem(runner: Runner, config: Config) -> None: """Mount a small persistent filesystem for realistic mount validation.""" - runner.run([*_sudo_prefix(), "install", "-d", "-m", "0770", config.export_dir]) + _ensure_export_marker(runner, config) mounted_type = _export_mount_type(runner, config) if mounted_type: if mounted_type != "ext4": @@ -807,7 +1044,6 @@ def _ensure_export_filesystem(runner: Runner, config: Config) -> None: ) return - _ensure_export_marker(runner, config) if not config.nfs_image.exists(): LOG.info("Creating sparse %s-byte NFS backing filesystem", NFS_IMAGE_BYTES) temporary_image = config.nfs_image.with_suffix(".ext4.new") @@ -1127,6 +1363,30 @@ def _install_nfs_csi(runner: Runner, config: Config, gateway: str) -> None: ) +def _install_sbx_shared_storage(runner: Runner, config: Config) -> None: + """Bind the shared kind-node paths to the fixture's stable RWX claims.""" + _ensure_namespace(runner, config) + storage = _render_resource( + config, + "manifests/sbx-storage.yaml.tmpl", + {"NAMESPACE": config.namespace}, + ) + runner.run(_kubectl(config, "apply", "-f", storage)) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "wait", + "--for=jsonpath={.status.phase}=Bound", + "pvc/storage-test-rwx", + "pvc/ssh-home-rwx", + "--timeout=60s", + ), + timeout=90, + ) + + def _ensure_nfs_csi_chart(runner: Runner, config: Config) -> Path: """Cache the pinned NFS CSI chart from a checksum-verified source archive.""" chart = config.state_dir / "charts" / f"csi-driver-nfs-{NFS_CSI_VERSION}.tgz" @@ -1309,6 +1569,8 @@ def _validate_ssh_workers(runner: Runner, config: Config, private_key: Path) -> "-o", "BatchMode=yes", "-o", + "IdentitiesOnly=yes", + "-o", f"UserKnownHostsFile={known_hosts}", f"tester@{address}", "test $(id -u) -eq 2000 && test $(stat -f -c %T /root) = overlayfs", @@ -1323,6 +1585,8 @@ def _validate_ssh_workers(runner: Runner, config: Config, private_key: Path) -> "-o", "BatchMode=yes", "-o", + "IdentitiesOnly=yes", + "-o", f"UserKnownHostsFile={known_hosts}", f"root@{addresses[0]}", "true", @@ -1377,11 +1641,14 @@ def _validate_ssh_storage( ) -> None: """Validate the selected home mode and shared storage claim.""" names = [str(pod["metadata"]["name"]) for pod in pods] + token = secrets.token_hex(16) _pod_exec( runner, config, names[0], - "touch /home/tester/.integration-home-probe /mnt/storage-test/.integration-rwx-probe", + "touch /home/tester/.integration-home-probe; " + f"printf '%s\\n' {shlex.quote(token)} " + ">/mnt/storage-test/.integration-rwx-probe", ) home_probe = _pod_exec( runner, @@ -1390,14 +1657,29 @@ def _validate_ssh_storage( "test -e /home/tester/.integration-home-probe", check=False, ) + storage_check = ( + f'test "$(cat /mnt/storage-test/.integration-rwx-probe)" = ' + f"{shlex.quote(token)}" + ) + if _select_storage_backend(config) == "nfs": + storage_check += ( + " && case $(stat -f -c %T /mnt/storage-test) in " + "nfs|nfs4) true;; *) false;; esac" + ) rwx_probe = _pod_exec( runner, config, names[1], - "test -e /mnt/storage-test/.integration-rwx-probe && " - "case $(stat -f -c %T /mnt/storage-test) in nfs|nfs4) true;; *) false;; esac", + storage_check, check=False, ) + if _select_storage_backend(config) == "sbx-shared": + host_probe = config.sbx_shared_root / "storage-test" / ".integration-rwx-probe" + if ( + not host_probe.is_file() + or host_probe.read_text(encoding="utf-8").strip() != token + ): + raise ProvisionError("SBX shared data is not visible from the agent") expected_home_rc = 0 if config.ssh_home_mode == "shared" else 1 if home_probe.returncode != expected_home_rc or rwx_probe.returncode: raise ProvisionError("SSH home or RWX visibility validation failed") @@ -1470,6 +1752,7 @@ def _install_slurm(runner: Runner, config: Config) -> None: ) _helm_slinky(runner, config) _wait_for_slurm(runner, config) + _restart_slinky_login(runner, config) _validate_slurm(runner, config) @@ -1498,29 +1781,6 @@ def _prepare_slinky_login_image(runner: Runner, config: Config) -> None: ) -def _slinky_login_image_current(runner: Runner, config: Config) -> bool: - """Return whether the live LoginSet selects the prepared image.""" - result = runner.run( - _kubectl( - config, - "-n", - config.namespace, - "get", - "loginsets", - "-o", - "json", - ), - check=False, - timeout=30, - ) - if result.returncode: - return False - return any( - item.get("spec", {}).get("login", {}).get("image") == SLINKY_LOGIN_IMAGE - for item in json.loads(result.stdout).get("items", []) - ) - - def _helm_slinky(runner: Runner, config: Config) -> None: """Reconcile the three pinned Slinky releases.""" releases = ( @@ -1541,12 +1801,6 @@ def _helm_slinky(runner: Runner, config: Config) -> None: ), ) for release, chart, values in releases: - current = _helm_release_current(runner, config, release, chart) - if current and ( - release != "slurm" or _slinky_login_image_current(runner, config) - ): - LOG.info("Slinky release %s is already at %s", release, SLINKY_VERSION) - continue arguments: list[str | Path] = [ "helm", "upgrade", @@ -1565,7 +1819,7 @@ def _helm_slinky(runner: Runner, config: Config) -> None: ] if values: arguments.extend(["--values", values]) - runner.run(arguments, timeout=600) + _install_slinky_release(runner, arguments, release) if release == "slurm-operator": runner.run( _kubectl( @@ -1591,34 +1845,28 @@ def _helm_slinky(runner: Runner, config: Config) -> None: ) -def _helm_release_current( - runner: Runner, config: Config, release: str, chart: str -) -> bool: - """Return whether a healthy release already has the pinned chart version.""" - result = runner.run( - [ - "helm", - "list", - "--namespace", - config.namespace, - "--all", - "--output", - "json", - "--kubeconfig", - config.kubeconfig, - ], - check=False, - timeout=60, - ) - if result.returncode: - return False - expected_chart = f"{chart}-{SLINKY_VERSION}" - return any( - item.get("name") == release - and item.get("chart") == expected_chart - and item.get("status") == "deployed" - for item in json.loads(result.stdout) - ) +def _install_slinky_release( + runner: Runner, arguments: list[str | Path], release: str +) -> None: + """Install a release, retrying the operator webhook startup race once.""" + for attempt in range(2): + result = runner.run(arguments, check=False, timeout=600) + if result.returncode == 0: + return + detail = result.stdout + result.stderr + webhook_race = "failed calling webhook" in detail + if attempt == 0 and webhook_race: + LOG.warning( + "Slinky release %s reached its webhook before it was responsive; " + "retrying once", + release, + ) + time.sleep(10) + continue + raise ProvisionError( + f"Helm failed to install Slinky release {release!r}" + f"{_failure_detail(result.stdout, result.stderr, False)}" + ) def _wait_for_slurm(runner: Runner, config: Config) -> None: @@ -1672,15 +1920,54 @@ def _pod_is_ready(pod: dict[str, object]) -> bool: def _login_pod(runner: Runner, config: Config) -> str: - """Return the single Slinky LoginSet pod name.""" - pods = _pods_with_container(_namespace_pods(runner, config), "login") - if len(pods) != 1: - raise ProvisionError(f"expected one LoginSet pod; found {len(pods)}") - return str(pods[0]["metadata"]["name"]) # type: ignore[index] + """Wait for and return the single ready, nonterminating LoginSet pod.""" + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + pods = [ + pod + for pod in _pods_with_container(_namespace_pods(runner, config), "login") + if not pod["metadata"].get("deletionTimestamp") # type: ignore[index] + and _pod_is_ready(pod) + ] + if len(pods) == 1: + return str(pods[0]["metadata"]["name"]) # type: ignore[index] + LOG.info( + "Waiting for one ready, nonterminating LoginSet pod; found %s", len(pods) + ) + time.sleep(3) + raise ProvisionError( + "LoginSet did not converge to one ready pod within 180 seconds" + ) + + +def _restart_slinky_login(runner: Runner, config: Config) -> None: + """Restart the configless login client after accounting is available.""" + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "restart", + "deployment/slurm-login-test", + ) + ) + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "deployment/slurm-login-test", + "--timeout=180s", + ), + timeout=210, + ) def _validate_slurm(runner: Runner, config: Config) -> None: - """Validate LoginSet placement, NFS, two-node fan-out, and accounting.""" + """Validate LoginSet placement, storage, two-node fan-out, and accounting.""" login = _login_pod(runner, config) login_node = runner.run( _kubectl( @@ -1721,15 +2008,25 @@ def _validate_slurm(runner: Runner, config: Config) -> None: f"targets={sorted(target_nodes)}" ) prefix = _kubectl(config, "-n", config.namespace, "exec", login, "--") - runner.run( - [ - *prefix, - "bash", - "-c", - "case $(stat -f -c %T /mnt/storage-test) in " - "nfs|nfs4) true;; *) false;; esac", - ] - ) + backend = _select_storage_backend(config) + token = secrets.token_hex(16) + storage_probe = ( + f"printf '%s\\n' {shlex.quote(token)} >/mnt/storage-test/.login-probe" + ) + if backend == "nfs": + storage_probe += ( + "; case $(stat -f -c %T /mnt/storage-test) in " + "nfs|nfs4) true;; *) false;; esac" + ) + runner.run([*prefix, "bash", "-c", storage_probe]) + if backend == "sbx-shared": + host_probe = config.sbx_shared_root / "storage-test" / ".login-probe" + if ( + not host_probe.is_file() + or host_probe.read_text(encoding="utf-8").strip() != token + ): + raise ProvisionError("LoginSet SBX data is not visible from the agent") + runner.run([*prefix, "rm", "-f", "/mnt/storage-test/.login-probe"]) fanout = runner.run( [ *prefix, @@ -1781,7 +2078,9 @@ def _validate_slurm(runner: Runner, config: Config) -> None: ) -def _write_state_summary(config: Config, subnet: str, gateway: str) -> None: +def _write_state_summary( + config: Config, backend: str, subnet: str = "", gateway: str = "" +) -> None: """Persist non-secret desired state for later diagnostics.""" state = { "schema": STATE_SCHEMA, @@ -1789,12 +2088,18 @@ def _write_state_summary(config: Config, subnet: str, gateway: str) -> None: "namespace": config.namespace, "export_dir": str(config.export_dir), "ssh_home_mode": config.ssh_home_mode, + "storage_backend": backend, "test_user": config.test_user, "test_uid": config.test_uid, "test_gid": config.test_gid, - "kind_version": KIND_VERSION, - "kubernetes_version": KUBECTL_VERSION, - "nfs_csi_version": NFS_CSI_VERSION, + "kind_version": (SBX_KIND_VERSION if backend == "sbx-shared" else KIND_VERSION), + "kubectl_version": ( + SBX_KUBECTL_VERSION if backend == "sbx-shared" else KUBECTL_VERSION + ), + "kubernetes_version": ( + SBX_KUBECTL_VERSION if backend == "sbx-shared" else KUBECTL_VERSION + ), + "nfs_csi_version": NFS_CSI_VERSION if backend == "nfs" else None, "slinky_version": SLINKY_VERSION, "kind_subnet": subnet, "kind_gateway": gateway, @@ -1805,16 +2110,18 @@ def _write_state_summary(config: Config, subnet: str, gateway: str) -> None: def setup_environment(runner: Runner, config: Config) -> None: """Idempotently provision and validate the complete fixture.""" architecture = _check_platform() - _check_host_capacity() - _ensure_apt_packages(runner) + backend = _select_storage_backend(config) + capacity_path = _repository_root() if backend == "sbx-shared" else Path("/") + _check_host_capacity(capacity_path) + _ensure_apt_packages(runner, backend) _ensure_docker(runner) - _ensure_client_tools(runner, architecture) + _ensure_client_tools(runner, architecture, backend) running_clusters = _kind_clusters(runner) containers = _kind_containers(runner, config, running_only=False) running_containers = _kind_containers(runner, config, running_only=True) cluster_exists = config.cluster_name in running_clusters or bool(containers) _ensure_cluster_ownership(config, cluster_exists) - if cluster_exists and not _export_mount_type(runner, config): + if backend == "nfs" and cluster_exists and not _export_mount_type(runner, config): LOG.warning( "Replacing the disposable cluster before initializing the NFS " "backing filesystem" @@ -1823,20 +2130,30 @@ def setup_environment(runner: Runner, config: Config) -> None: running_clusters = set() running_containers = set() cluster_exists = False - _ensure_export_filesystem(runner, config) + if backend == "nfs": + _ensure_export_filesystem(runner, config) + else: + _prepare_sbx_shared(runner, config) if config.cluster_name in running_clusters and len(running_containers) == 3: _export_kubeconfig(runner, config) elif cluster_exists: LOG.warning("Replacing incomplete or stopped disposable kind cluster") _delete_cluster(runner, config) - _create_cluster(runner, config) + _create_cluster(runner, config, backend) else: - _create_cluster(runner, config) + _create_cluster(runner, config, backend) + if backend == "sbx-shared": + _configure_sbx_node_trust(runner, config) _wait_for_cluster(runner, config) - subnet, gateway = _kind_ipv4_network(runner) - _configure_nfs(runner, config, subnet, gateway) - _probe_nfs(runner, config, gateway) - _install_nfs_csi(runner, config, gateway) + subnet = "" + gateway = "" + if backend == "nfs": + subnet, gateway = _kind_ipv4_network(runner) + _configure_nfs(runner, config, subnet, gateway) + _probe_nfs(runner, config, gateway) + _install_nfs_csi(runner, config, gateway) + else: + _install_sbx_shared_storage(runner, config) _install_ssh_workers(runner, config) _scale_ssh(runner, config, replicas=0) _install_slurm(runner, config) @@ -1844,7 +2161,7 @@ def setup_environment(runner: Runner, config: Config) -> None: _wait_for_ssh(runner, config) private_key = config.keys_dir / "id_ed25519" _validate_ssh_workers(runner, config, private_key) - _write_state_summary(config, subnet, gateway) + _write_state_summary(config, backend, subnet, gateway) LOG.info("Integration environment is provisioned and running") @@ -1937,15 +2254,35 @@ def stop_environment(runner: Runner, config: Config) -> None: _delete_cluster(runner, config) else: LOG.info("Disposable kind cluster %s is already absent", config.cluster_name) - _stop_owned_nfs(runner, config) + backend = _select_storage_backend(config) + if backend == "nfs": + _stop_owned_nfs(runner, config) LOG.info( - "Integration environment stopped; host packages, caches, keys, and NFS data " - "were preserved" + "Integration environment stopped; host packages, caches, keys, and %s " + "data were preserved", + "NFS" if backend == "nfs" else "SBX shared", ) def teardown_environment(runner: Runner, config: Config) -> None: """Stop the fixture and remove all harness-owned data and host config.""" + if _select_storage_backend(config) == "sbx-shared": + state_owned, setup_owned, shared_owned = _validate_sbx_teardown_ownership( + runner, config + ) + stop_environment(runner, config) + if setup_owned: + _remove_harness_images(runner, config) + if shared_owned: + _remove_sbx_shared(runner, config) + _remove_owned_directories( + runner, config, remove_state=state_owned, remove_export=False + ) + LOG.info( + "Docker SBX integration environment torn down; installed host " + "packages and client tools were preserved" + ) + return state_owned, setup_owned, export_owned, nfs_configured = ( _validate_teardown_ownership(runner, config) ) @@ -1965,6 +2302,68 @@ def teardown_environment(runner: Runner, config: Config) -> None: ) +def _validate_sbx_teardown_ownership( + runner: Runner, config: Config +) -> tuple[bool, bool, bool]: + """Validate owned state and shared paths before Docker SBX cleanup.""" + _validate_cleanup_paths(config) + _validate_sbx_shared_root(config) + expected_owner = _owner_document(config) + state_marker = config.state_dir / STATE_MARKER + state_owned = state_marker.is_file() + if ( + state_owned + and json.loads(state_marker.read_text(encoding="utf-8")) != expected_owner + ): + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {state_marker}" + ) + if config.state_dir.exists() and not state_owned: + raise ProvisionError( + f"refusing to remove unowned setup state: {config.state_dir}" + ) + + cluster_marker = config.state_dir / "cluster-owner.json" + setup_owned = cluster_marker.is_file() + if ( + setup_owned + and json.loads(cluster_marker.read_text(encoding="utf-8")) != expected_owner + ): + raise ProvisionError( + f"refusing teardown with mismatched ownership marker: {cluster_marker}" + ) + + root = config.sbx_shared_root + shared_owned = root.exists() + if shared_owned: + marker = root / EXPORT_MARKER + if not marker.is_file() or json.loads(marker.read_text(encoding="utf-8")) != ( + _sbx_shared_marker(config) + ): + raise ProvisionError( + f"refusing teardown of unowned SBX shared root: {root}" + ) + allowed = {EXPORT_MARKER, "storage-test", "ssh-home"} + unexpected = sorted( + path.name for path in root.iterdir() if path.name not in allowed + ) + if unexpected: + raise ProvisionError( + f"refusing unexpected entries in SBX shared root: {unexpected}" + ) + if setup_owned: + _validate_image_ownership(runner, config) + return state_owned, setup_owned, shared_owned + + +def _remove_sbx_shared(runner: Runner, config: Config) -> None: + """Remove only the validated marker-owned Docker SBX shared root.""" + root = config.sbx_shared_root + runner.run(["find", root, "-xdev", "-depth", "-delete"], timeout=120) + if root.exists(): + raise ProvisionError(f"cleanup did not remove SBX shared root: {root}") + + def _owner_document(config: Config) -> dict[str, object]: """Return the exact ownership document used by persistent markers.""" return {"schema": STATE_SCHEMA, "cluster_name": config.cluster_name} @@ -2089,10 +2488,12 @@ def _validate_teardown_ownership( def _validate_cleanup_paths(config: Config) -> None: - """Reject broad or overlapping destructive cleanup targets.""" + """Reject broad or overlapping lifecycle paths.""" forbidden = {Path("/"), Path("/var"), Path("/srv"), Path("/etc")} if config.state_dir in forbidden or config.export_dir in forbidden: - raise ProvisionError("refusing teardown with a broad state or export path") + raise ProvisionError( + "refusing lifecycle action with a broad state or export path" + ) if config.state_dir == config.export_dir: raise ProvisionError("state and export directories must be different") if config.state_dir in config.export_dir.parents: @@ -2101,6 +2502,31 @@ def _validate_cleanup_paths(config: Config) -> None: raise ProvisionError("state directory must not be inside the export directory") +def _validate_lifecycle_paths(config: Config) -> None: + """Validate state ownership before bootstrap can mutate its path.""" + _validate_cleanup_paths(config) + state_dir = config.state_dir + if not state_dir.exists(): + if not state_dir.parent.is_dir(): + raise ProvisionError( + f"state directory must be a leaf below an existing directory: {state_dir}" + ) + return + if not state_dir.is_dir(): + raise ProvisionError(f"state path is not a directory: {state_dir}") + marker = state_dir / STATE_MARKER + if not marker.is_file(): + raise ProvisionError(f"refusing to modify unowned setup state: {state_dir}") + try: + owner = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ProvisionError( + f"invalid setup state ownership marker: {marker}" + ) from error + if owner != _owner_document(config): + raise ProvisionError(f"setup state ownership marker does not match: {marker}") + + def _validate_installed_config( installed: str | None, source: Path, destination: Path ) -> None: @@ -2354,6 +2780,16 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--namespace", default="storage-scale-integration") parser.add_argument("--state-dir", type=Path, default=DEFAULT_STATE_DIR) parser.add_argument("--export-dir", type=Path, default=DEFAULT_EXPORT_DIR) + parser.add_argument( + "--storage-backend", + choices=("auto", "nfs", "sbx-shared"), + default="auto", + ) + parser.add_argument( + "--sbx-shared-root", + type=Path, + default=DEFAULT_SBX_SHARED_ROOT, + ) parser.add_argument( "--test-user", help=( @@ -2385,6 +2821,8 @@ def _config(arguments: argparse.Namespace) -> Config: namespace=arguments.namespace, state_dir=arguments.state_dir.resolve(), export_dir=arguments.export_dir.resolve(), + storage_backend=arguments.storage_backend, + sbx_shared_root=arguments.sbx_shared_root.resolve(), ssh_home_mode=arguments.ssh_home_mode, test_user=account.pw_name, test_uid=account.pw_uid, @@ -2434,6 +2872,7 @@ def main() -> int: "action as the account provisioned by setup" ) config = _config(arguments) + _validate_lifecycle_paths(config) if arguments.action != "test" and arguments.tests: raise ProvisionError("test selectors are valid only with the test action") if arguments.action == "test" and not config.state_dir.is_dir(): diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py index 7d0889c..6d062e6 100644 --- a/integration-tests/lib/filesystem_integration.py +++ b/integration-tests/lib/filesystem_integration.py @@ -40,6 +40,11 @@ ELBENCHO_RELEASE_API = ( "https://api.github.com/repos/breuner/elbencho/releases/tags/" + ELBENCHO_VERSION ) +ELBENCHO_CONTAINER = ( + "breuner/elbencho:v3.1-11@" + "sha256:719fba92cab57c773ddf7a2776414b358aeb8126a15fbc8e3c52469ce3a5b8b2" +) +SBX_ELBENCHO_BUNDLE_RECIPE = 1 MAX_ARCHIVE_BYTES = 32 * 1024 * 1024 MAX_DEPLOYMENT_ARCHIVE_BYTES = 128 * 1024 * 1024 MAX_DEPLOYMENT_FILES = 10_000 @@ -76,6 +81,7 @@ class Fixture: slurm_addresses: tuple[str, str] architecture: str ssh_home_mode: str + storage_backend: str def _kubectl(config: Any, *arguments: str | Path) -> list[str | Path]: @@ -138,6 +144,7 @@ def _pods_with_container( pod for pod in pods if _ready(pod) + and not pod.get("metadata", {}).get("deletionTimestamp") and container in {item["name"] for item in pod.get("spec", {}).get("containers", [])} ] @@ -171,6 +178,9 @@ def _load_state(config: Any) -> dict[str, Any]: mode = state.get("ssh_home_mode") if mode not in {"separate", "shared"}: raise IntegrationTestError(f"invalid ssh_home_mode in {path}: {mode!r}") + backend = state.get("storage_backend") + if backend not in {"nfs", "sbx-shared"}: + raise IntegrationTestError(f"invalid storage_backend in {path}: {backend!r}") return state @@ -310,9 +320,12 @@ def _require_fixture(runner: Any, config: Any) -> Fixture: ) _probe_pod(runner, config, login_name, "login", tools) _probe_pod(runner, config, ssh_name, "sshd", tools, as_user="tester") - mount_probe = ( - "case $(stat -f -c %T /mnt/storage-test) in nfs|nfs4) true;; *) false;; esac" - ) + mount_probe = "test -w /mnt/storage-test" + if state["storage_backend"] == "nfs": + mount_probe += ( + " && case $(stat -f -c %T /mnt/storage-test) in " + "nfs|nfs4) true;; *) false;; esac" + ) _probe_pod(runner, config, login_name, "login", mount_probe) _probe_pod(runner, config, ssh_name, "sshd", mount_probe, as_user="tester") login_arch = _probe_pod(runner, config, login_name, "login", "uname -m") @@ -361,6 +374,7 @@ def _require_fixture(runner: Any, config: Any) -> Fixture: slurm_addresses=(slurm_addresses[0], slurm_addresses[1]), architecture=host_arch, ssh_home_mode=str(state["ssh_home_mode"]), + storage_backend=str(state["storage_backend"]), ) @@ -443,10 +457,133 @@ def _download_archive(destination: Path, name: str, expected: str) -> None: temporary.replace(destination) -def _ensure_elbencho(config: Any, architecture: str) -> tuple[Path, str]: +def _extract_container_elbencho( + runner: Any, cache: Path, binary: Path, runtime: Path, architecture: str +) -> None: + """Build a portable wrapper from the digest-pinned upstream image.""" + runner.run(["docker", "pull", ELBENCHO_CONTAINER], timeout=300) + container = runner.run( + ["docker", "create", "--entrypoint", "sleep", ELBENCHO_CONTAINER, "infinity"] + ).stdout.strip() + if not container: + raise IntegrationTestError("Docker did not return an elbencho container ID") + archive = cache / f".{binary.name}.runtime.tar" + try: + bundle = """ +set -eu +rm -rf /tmp/elbencho-runtime /tmp/elbencho-runtime.tar +mkdir -p /tmp/elbencho-runtime +libraries=$(ldd /usr/bin/elbencho | awk '$3 ~ /^\\// {print $3} $1 ~ /^\\// {print $1}') +loader=$(ldd /usr/bin/elbencho | awk '{for (i=1; i<=NF; i++) if ($i ~ /^\\/.*ld-linux.*\\.so/) {print $i; exit}}') +test -n "$loader" +cp -L --parents /usr/bin/elbencho $libraries /tmp/elbencho-runtime +printf '%s\n' "$loader" > /tmp/elbencho-runtime/.loader-path +cd /tmp/elbencho-runtime +tar -cf /tmp/elbencho-runtime.tar . +""".strip() + runner.run(["docker", "start", container]) + runner.run(["docker", "exec", container, "sh", "-ec", bundle]) + runner.run(["docker", "cp", f"{container}:/tmp/elbencho-runtime.tar", archive]) + temporary_runtime = cache / f".{runtime.name}.new" + shutil.rmtree(temporary_runtime, ignore_errors=True) + temporary_runtime.mkdir() + with tarfile.open(archive) as tar: + tar.extractall(temporary_runtime, filter="data") + loader_marker = temporary_runtime / ".loader-path" + loader_path = Path(loader_marker.read_text(encoding="utf-8").strip()) + if not loader_path.is_absolute() or ".." in loader_path.parts: + raise IntegrationTestError( + f"container reported an invalid dynamic loader path: {loader_path}" + ) + loader_relative = loader_path.relative_to("/") + if not (temporary_runtime / loader_relative).is_file(): + raise IntegrationTestError( + f"container runtime is missing its dynamic loader: {loader_path}" + ) + loader_marker.unlink() + shutil.rmtree(runtime, ignore_errors=True) + temporary_runtime.replace(runtime) + library_arch = ( + "aarch64-linux-gnu" if architecture == "aarch64" else "x86_64-linux-gnu" + ) + wrapper = "\n".join( + ( + "#!/usr/bin/env bash", + "set -euo pipefail", + 'runtime_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/' + 'elbencho-runtime" && pwd)', + f'exec "$runtime_dir/{loader_relative.as_posix()}" \\', + f' --library-path "$runtime_dir/usr/lib/{library_arch}:' + f'$runtime_dir/lib/{library_arch}" \\', + ' "$runtime_dir/usr/bin/elbencho" "$@"', + "", + ) + ) + binary.write_text(wrapper, encoding="utf-8") + binary.chmod(0o755) + finally: + archive.unlink(missing_ok=True) + runner.run(["docker", "rm", "--force", container], check=False) + + +def _sbx_bundle_document(architecture: str, binary_name: str) -> dict[str, object]: + """Return the exact recipe identity for a cached SBX Elbencho bundle.""" + return { + "schema": 1, + "recipe": SBX_ELBENCHO_BUNDLE_RECIPE, + "container": ELBENCHO_CONTAINER, + "architecture": architecture, + "binary_name": binary_name, + } + + +def _write_sbx_bundle_marker(path: Path, document: dict[str, object]) -> None: + """Atomically record a successfully built SBX Elbencho bundle.""" + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, delete=False + ) as handle: + json.dump(document, handle, sort_keys=True) + handle.write("\n") + temporary = Path(handle.name) + temporary.chmod(0o640) + temporary.replace(path) + + +def _sbx_bundle_is_current( + binary: Path, + runtime: Path, + marker: Path, + expected: dict[str, object], +) -> bool: + """Return whether all cached bundle artifacts match the current recipe.""" + if not binary.is_file() or not runtime.is_dir() or not marker.is_file(): + return False + try: + document = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return document == expected + + +def _ensure_elbencho( + runner: Any, config: Any, architecture: str, storage_backend: str +) -> tuple[Path, str, Path | None]: """Return a verified, extracted pinned elbencho binary and staged name.""" archive_name, expected, binary_name = ELBENCHO_ARCHIVES[architecture] cache = config.state_dir / "test-cache" + cache.mkdir(parents=True, exist_ok=True) + binary = cache / f"{ELBENCHO_VERSION}-{binary_name}" + if storage_backend == "sbx-shared": + runtime = cache / f"{ELBENCHO_VERSION}-{binary_name}.runtime" + marker = cache / f"{ELBENCHO_VERSION}-{binary_name}.bundle.json" + bundle_document = _sbx_bundle_document(architecture, binary_name) + if not _sbx_bundle_is_current(binary, runtime, marker, bundle_document): + LOG.info("Extracting pinned elbencho %s container", ELBENCHO_VERSION) + _extract_container_elbencho(runner, cache, binary, runtime, architecture) + _write_sbx_bundle_marker(marker, bundle_document) + else: + LOG.info("Using cached elbencho from the pinned upstream container") + return binary, binary_name, runtime archive = cache / f"{ELBENCHO_VERSION}-{archive_name}" if not archive.is_file() or _sha256(archive) != expected: LOG.info( @@ -455,7 +592,6 @@ def _ensure_elbencho(config: Any, architecture: str) -> tuple[Path, str]: _download_archive(archive, archive_name, expected) else: LOG.info("Using cached pinned elbencho archive for %s", architecture) - binary = cache / f"{ELBENCHO_VERSION}-{binary_name}" with tarfile.open(archive, "r:gz") as tar: members = [ member @@ -474,7 +610,7 @@ def _ensure_elbencho(config: Any, architecture: str) -> tuple[Path, str]: shutil.copyfileobj(source, handle) temporary.chmod(0o755) temporary.replace(binary) - return binary, binary_name + return binary, binary_name, None def _shell(value: str | Path) -> str: @@ -503,7 +639,7 @@ def _override_block( "export ELBENCHO_FILE_SIZE_MULTIPLIER=1", 'export ELBENCHO_FILE_LAYOUT="shared-directory"', "export ELBENCHO_FILES_PER_NODE=1", - 'export ELBENCHO_FILE_SIZE="4K"', + 'export ELBENCHO_FILE_SIZE="16M"', 'export ELBENCHO_SCALE_IO_SIZES=("4K")', 'export ELBENCHO_IODEPTH_LIST=("1")', "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", @@ -597,6 +733,7 @@ def _validate_deployment_archive( binary_digest: str, architecture: str, runner: Any, + bundled_runtime: bool, ) -> Path: """Validate and safely extract the deployment tarball.""" if not archive.is_file() or archive.stat().st_size > MAX_DEPLOYMENT_ARCHIVE_BYTES: @@ -611,6 +748,8 @@ def _validate_deployment_archive( "storage-scale-test/storage-tests/fs/nv-elbencho-sweep.sh", f"storage-scale-test/utils/{binary_name}", } + if bundled_runtime: + required.add("storage-scale-test/utils/elbencho-runtime/usr/bin/elbencho") names: set[str] = set() content_bytes = 0 with tarfile.open(archive, "r:gz") as tar: @@ -662,7 +801,10 @@ def _validate_deployment_archive( raise IntegrationTestError( f"packaged elbencho failed identity checks: {packaged_binary}" ) - description = runner.run(["file", packaged_binary], timeout=30).stdout + inspected_binary = packaged_binary + if bundled_runtime: + inspected_binary = root / "utils" / "elbencho-runtime" / "usr/bin/elbencho" + description = runner.run(["file", inspected_binary], timeout=30).stdout expected_arch = "x86-64" if architecture == "x86_64" else "aarch64" if expected_arch not in description: raise IntegrationTestError( @@ -679,6 +821,7 @@ def _build_deployment_archive( binary: Path, binary_name: str, architecture: str, + runtime: Path | None, ) -> tuple[Path, Path]: """Build and inspect one filesystem-only deployment tarball.""" snapshot = build_root / "source" @@ -686,6 +829,8 @@ def _build_deployment_archive( seeded_binary = snapshot / "utils" / binary_name shutil.copy2(binary, seeded_binary) seeded_binary.chmod(0o755) + if runtime is not None: + shutil.copytree(runtime, snapshot / "utils" / "elbencho-runtime") LOG.info("Building deployment tarball from a tracked-files-only snapshot") result = runner.run( [ @@ -708,6 +853,7 @@ def _build_deployment_archive( _sha256(binary), architecture, runner, + runtime is not None, ) return archive, extracted @@ -761,6 +907,40 @@ def _stream_to_login( ) +def _stage_ssh_runtime( + runner: Any, config: Any, runtime: Path, pods: list[dict[str, Any]] +) -> None: + """Install the container-derived runtime beside each SSH wrapper target.""" + ssh_pods = sorted( + _pods_with_container(pods, "sshd"), + key=lambda item: item["metadata"]["name"], + ) + with tempfile.NamedTemporaryFile(suffix=".tar") as stream: + with tarfile.open(fileobj=stream, mode="w") as archive: + archive.add(runtime, arcname="elbencho-runtime") + stream.flush() + for pod in ssh_pods: + stream.seek(0) + command = [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + "-i", + pod["metadata"]["name"], + "-c", + "sshd", + "--", + ), + "bash", + "-ec", + "rm -rf -- /home/tester/elbencho-runtime && " + "tar --no-same-owner --no-same-permissions -xf - -C /home/tester", + ] + runner.run(command, stdin=stream, timeout=180) + + def _stage_workspace( runner: Any, config: Any, @@ -847,7 +1027,7 @@ def _test_command( "--kill-after=10s", f"{timeout}s", "bash", - "-lc", + "-c", host_command, ] return _pod_command( @@ -887,7 +1067,7 @@ def _run_step( f"full output: {log_path}\n{detail}" ) LOG.info("Passed %s filesystem step: %s", selector, name) - return output + return result.stdout def _assert_results( @@ -953,12 +1133,10 @@ def _assert_results( test -s "$result/executions/$id.log" test -s "$result/executions/$id.workload.tsv" done -grep -qx $'nodes\t1' "$result/executions/0001.workload.tsv" -grep -qx $'nodes\t2' "$result/executions/0002.workload.tsv" grep -qx $'dataset_files_total\t1' "$result/executions/0001.workload.tsv" grep -qx $'dataset_files_total\t2' "$result/executions/0002.workload.tsv" -grep -qx $'dataset_bytes_total\t4096' "$result/executions/0001.workload.tsv" -grep -qx $'dataset_bytes_total\t8192' "$result/executions/0002.workload.tsv" +grep -qx $'dataset_bytes_total\t16777216' "$result/executions/0001.workload.tsv" +grep -qx $'dataset_bytes_total\t33554432' "$result/executions/0002.workload.tsv" grep -qx $'completion_state\tcompleted' "$result/executions/0001.workload.tsv" grep -qx $'completion_state\tcompleted' "$result/executions/0002.workload.tsv" test "$(find "$result" -type f -name '*.csv' -size +0c | wc -l)" -ge 2 @@ -1155,7 +1333,12 @@ def run_filesystem_tests( selected = _selected_tests(selectors) LOG.info("Requiring an already-running integration setup") fixture = _require_fixture(runner, config) - binary, binary_name = _ensure_elbencho(config, fixture.architecture) + binary, binary_name, runtime = _ensure_elbencho( + runner, config, fixture.architecture, fixture.storage_backend + ) + if runtime is not None and "ssh" in selected: + LOG.info("Staging the pinned Elbencho container runtime in SSH worker homes") + _stage_ssh_runtime(runner, config, runtime, _pod_inventory(runner, config)) run_id = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + f"-{os.getpid()}" log_dir = config.state_dir / "test-runs" / run_id log_dir.mkdir(parents=True, exist_ok=False) @@ -1170,6 +1353,7 @@ def run_filesystem_tests( binary, binary_name, fixture.architecture, + runtime, ) shutil.copy2(build_root / "build-tarball.log", log_dir / "build-tarball.log") report_workspace = build_root / "report-workspace" diff --git a/integration-tests/manifests/kind-sbx-shared.yaml.tmpl b/integration-tests/manifests/kind-sbx-shared.yaml.tmpl new file mode 100644 index 0000000..4e0aade --- /dev/null +++ b/integration-tests/manifests/kind-sbx-shared.yaml.tmpl @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: @@CLUSTER_NAME@@ +nodes: + - role: control-plane + labels: + storage-scale-test/login: "true" + extraMounts: + - hostPath: @@SHARED_ROOT@@ + containerPath: /var/local/storage-scale-shared +@@KMSG_MOUNT@@ + - role: worker + labels: + storage-scale-test/target: "true" + extraMounts: + - hostPath: @@SHARED_ROOT@@ + containerPath: /var/local/storage-scale-shared +@@KMSG_MOUNT@@ + - role: worker + labels: + storage-scale-test/target: "true" + extraMounts: + - hostPath: @@SHARED_ROOT@@ + containerPath: /var/local/storage-scale-shared +@@KMSG_MOUNT@@ diff --git a/integration-tests/manifests/sbx-storage.yaml.tmpl b/integration-tests/manifests/sbx-storage.yaml.tmpl new file mode 100644 index 0000000..590a5c0 --- /dev/null +++ b/integration-tests/manifests/sbx-storage.yaml.tmpl @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: sbx-shared +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: Immediate +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storage-scale-sbx-data +spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: sbx-shared + claimRef: + namespace: @@NAMESPACE@@ + name: storage-test-rwx + hostPath: + path: /var/local/storage-scale-shared/storage-test + type: Directory +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-test-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: sbx-shared + volumeName: storage-scale-sbx-data + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storage-scale-sbx-home +spec: + capacity: + storage: 64Mi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: sbx-shared + claimRef: + namespace: @@NAMESPACE@@ + name: ssh-home-rwx + hostPath: + path: /var/local/storage-scale-shared/ssh-home + type: Directory +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ssh-home-rwx + namespace: @@NAMESPACE@@ +spec: + accessModes: + - ReadWriteMany + storageClassName: sbx-shared + volumeName: storage-scale-sbx-home + resources: + requests: + storage: 64Mi diff --git a/lib/_elbencho_functions.sh b/lib/_elbencho_functions.sh index 37e3e78..9fa79d8 100644 --- a/lib/_elbencho_functions.sh +++ b/lib/_elbencho_functions.sh @@ -238,6 +238,13 @@ kill_elbencho_by_pid() { kill_all_elbencho_processes() { local port="${1:-1611}" + # A launcher or dynamic loader can give the service process a name other + # than "elbencho". Prefer the exact listening port when fuser is present, + # then retain the name-based cleanup for other processes and platforms. + if type -P fuser >/dev/null 2>&1; then + fuser -k "${port}/tcp" >/dev/null 2>&1 || true + fi + # Determine kill tool (prefer pkill over killall) local kill_tool="" if type -P pkill >/dev/null 2>&1; then @@ -1458,7 +1465,8 @@ _elbencho_workload_update_failure_cleanup_state() { _elbencho_workload_write } -# Parse elbencho newline-delimited JSON without a runtime JSON dependency. +# Parse elbencho streamed JSON without a runtime JSON dependency. Releases may +# delimit top-level phase objects with newlines or write them adjacently. # Prints canonical "entriesbytes-or-nullelapsed-ms" for one phase. # WRITE/READ require bytes; RMFILES must not contain a bytes counter. _elbencho_parse_phase_json() { @@ -1558,19 +1566,23 @@ _elbencho_parse_phase_json() { { if ($0 ~ /^[ \t\r]*$/) { fail(); next } s = $0; p = 1; n = length(s); records++ - phase_count = last_count = entries_count = bytes_count = elapsed_count = 0 - phase = entries = bytes = elapsed = "" - ws(); object("top"); ws() - if (bad || p <= n || phase_count != 1) { fail(); next } - if (phase == expected) { - matches++ - if (last_count != 1 || entries_count != 1 || elapsed_count != 1) fail() - if (expected == "RMFILES" && bytes_count != 0) fail() - if (expected != "RMFILES" && bytes_count != 1) fail() - found_entries = entries - found_bytes = expected == "RMFILES" ? "null" : bytes - found_elapsed = elapsed - } else if (phase != "SYNC") fail() + ws() + while (p <= n && !bad) { + phase_count = last_count = entries_count = bytes_count = elapsed_count = 0 + phase = entries = bytes = elapsed = "" + object("top"); ws() + if (bad || phase_count != 1) { fail(); break } + if (phase == expected) { + matches++ + if (last_count != 1 || entries_count != 1 || elapsed_count != 1) fail() + if (expected == "RMFILES" && bytes_count != 0) fail() + if (expected != "RMFILES" && bytes_count != 1) fail() + found_entries = entries + found_bytes = expected == "RMFILES" ? "null" : bytes + found_elapsed = elapsed + } else if (phase != "SYNC") fail() + if (p <= n) records++ + } } END { if (records == 0 || matches != 1 || bad) exit 1 diff --git a/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh b/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh index 6ee5a38..3cdfc2b 100755 --- a/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh +++ b/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh @@ -94,9 +94,31 @@ _elbencho_adopt_slurm_dispatch_lock \ "$EXECUTIONS_DIR" "$DISPATCH_LOCK_TOKEN" "$SLURM_JOB_ID" || exit 1 COORDINATOR_LOCK_OWNED=1 -# Build the allocation IPv4 nodelist (same approach as the previous sbatch -# driver: some clusters intermittently fail to resolve short hostnames in -# elbencho's resolver, so we resolve once here and propagate as IPs). +# Build the allocation IPv4 nodelist. Slurm may canonicalize SLURM_JOB_NODELIST +# instead of preserving the configured include-list order, so restore that +# order when ORDER_NODES is active before choosing per-execution prefixes. +mapfile -t allocation_nodes < <( + scontrol show hostname "${SLURM_JOB_NODELIST:-}" +) +if [[ -n "${ORDER_NODES_ENABLED:-}" && ${#SLURM_ORDERED_NODES[@]} -gt 0 ]]; then + ordered_allocation_nodes=() + for ordered_node in "${SLURM_ORDERED_NODES[@]}"; do + for allocated_node in "${allocation_nodes[@]}"; do + if [[ "$ordered_node" == "$allocated_node" ]]; then + ordered_allocation_nodes+=("$allocated_node") + break + fi + done + done + if [[ ${#ordered_allocation_nodes[@]} -ne ${#allocation_nodes[@]} ]]; then + echo "Error: configured node order does not match the Slurm allocation" >&2 + exit 1 + fi + allocation_nodes=("${ordered_allocation_nodes[@]}") +fi + +# Some clusters intermittently fail to resolve short hostnames in elbencho's +# resolver, so resolve once here and propagate IPs. nodelist_ips=() while read -r node; do ip=$(getent ahostsv4 "$node" | awk '{print $1; exit}') @@ -105,7 +127,7 @@ while read -r node; do ip="$node" fi nodelist_ips+=("$ip") -done < <(scontrol show hostname "${SLURM_JOB_NODELIST:-}") +done < <(printf '%s\n' "${allocation_nodes[@]}") ALLOC_HOSTS_CSV=$(IFS=,; printf '%s' "${nodelist_ips[*]}") # nodelist_expanded_comma_separated is read by run_an_elbencho (a bash # function in this same shell scope) as a fallback when no per-execution diff --git a/tests/test_elbencho_shared_directory_shell.py b/tests/test_elbencho_shared_directory_shell.py index 15ea2b1..a40d611 100644 --- a/tests/test_elbencho_shared_directory_shell.py +++ b/tests/test_elbencho_shared_directory_shell.py @@ -124,6 +124,8 @@ def test_json_parser_accepts_sync_and_rejects_duplicate_counter(self) -> None: '{{"phase_type":"WRITE","last_done":{{"elapsed_time_ms":"7","entries":"2","bytes":"8192"}}}}' \ '{{"phase_type":"SYNC"}}' >"$tmp/good" [[ "$(_elbencho_parse_phase_json "$tmp/good" WRITE)" == $'2\t8192\t7' ]] + tr -d '\n' <"$tmp/good" >"$tmp/adjacent" + [[ "$(_elbencho_parse_phase_json "$tmp/adjacent" WRITE)" == $'2\t8192\t7' ]] printf '%s\n' \ '{{"phase_type":"WRITE","last_done":{{"elapsed_time_ms":"7","entries":"2","entries":"2","bytes":"8192"}}}}' \ >"$tmp/bad" diff --git a/tests/test_extract_elbencho_scale_efficiency.py b/tests/test_extract_elbencho_scale_efficiency.py new file mode 100644 index 0000000..0705a61 --- /dev/null +++ b/tests/test_extract_elbencho_scale_efficiency.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for Elbencho scaling-efficiency calculations.""" + +from tests.extract_elbencho_test_support import load_extract_elbencho_module + +_MODULE = load_extract_elbencho_module("extract_elbencho_scale_efficiency_under_test") +_scale_efficiency = getattr(_MODULE, "_scale_efficiency") + + +def test_scale_efficiency_preserves_relative_values(): + """The fastest per-unit result is the 100-percent reference.""" + assert _scale_efficiency([2.0, 1.0]) == [100.0, 50.0] + + +def test_scale_efficiency_accepts_all_zero_results(): + """Valid rounded-zero reports do not divide by zero.""" + assert _scale_efficiency([0.0, 0.0]) == [0.0, 0.0] diff --git a/tests/test_integration_driver_safety.py b/tests/test_integration_driver_safety.py new file mode 100644 index 0000000..d985b32 --- /dev/null +++ b/tests/test_integration_driver_safety.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Safety and rollout regression tests for the integration driver.""" + +import importlib.util +import json +import sys +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_DRIVER_PATH = _REPO_ROOT / "integration-tests" / "bin" / "integration-test.py" +_SPEC = importlib.util.spec_from_file_location( + "integration_driver_under_test", _DRIVER_PATH +) +assert _SPEC and _SPEC.loader +_DRIVER = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _DRIVER +_SPEC.loader.exec_module(_DRIVER) +_FILESYSTEM = sys.modules["filesystem_integration"] +_ensure_export_marker = getattr(_DRIVER, "_ensure_export_marker") +_login_pod = getattr(_DRIVER, "_login_pod") +_select_storage_backend = getattr(_DRIVER, "_select_storage_backend") +_storage_backend_document = getattr(_DRIVER, "_storage_backend_document") +_validate_lifecycle_paths = getattr(_DRIVER, "_validate_lifecycle_paths") +_ensure_elbencho = getattr(_FILESYSTEM, "_ensure_elbencho") +_pods_with_container = getattr(_FILESYSTEM, "_pods_with_container") + + +def _config(state_dir: Path, export_dir: Path) -> object: + """Return a minimal real driver configuration.""" + return _DRIVER.Config( + cluster_name="test-cluster", + namespace="test-namespace", + state_dir=state_dir, + export_dir=export_dir, + storage_backend="sbx-shared", + sbx_shared_root=_REPO_ROOT / "tmp" / "test-shared", + ssh_home_mode="separate", + test_user="tester", + test_uid=2000, + test_gid=2000, + verbose=False, + ) + + +def test_existing_state_requires_ownership_marker(tmp_path): + """Bootstrap cannot adopt an arbitrary existing directory.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + config = _config(state_dir, tmp_path / "export") + + with pytest.raises(_DRIVER.ProvisionError, match="unowned setup state"): + _validate_lifecycle_paths(config) + + +def test_existing_state_accepts_matching_ownership_marker(tmp_path): + """A correctly marked state directory remains reusable.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + marker = state_dir / _DRIVER.STATE_MARKER + marker.write_text( + json.dumps({"schema": _DRIVER.STATE_SCHEMA, "cluster_name": "test-cluster"}), + encoding="utf-8", + ) + config = _config(state_dir, tmp_path / "export") + + _validate_lifecycle_paths(config) + + +def _write_backend_state(config, backend): + """Write the retained backend identity used by repeated setup.""" + path = config.state_dir / "storage-backend.json" + path.write_text( + json.dumps(_storage_backend_document(config, backend)), encoding="utf-8" + ) + + +def _write_completed_state(config, backend): + """Write the immutable subset of a successful setup summary.""" + state = { + "schema": _DRIVER.STATE_SCHEMA, + "cluster_name": config.cluster_name, + "namespace": config.namespace, + "export_dir": str(config.export_dir), + "storage_backend": backend, + } + (config.state_dir / "state.json").write_text(json.dumps(state), encoding="utf-8") + + +def test_retained_setup_rejects_changed_namespace(tmp_path): + """Repeated setup cannot create a second namespace workload stack.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + original = replace(_config(state_dir, tmp_path / "export"), storage_backend="nfs") + _write_completed_state(original, "nfs") + _write_backend_state(original, "nfs") + changed = replace(original, namespace="other-namespace") + + with pytest.raises(_DRIVER.ProvisionError, match="teardown first"): + _select_storage_backend(changed) + + +def test_retained_nfs_setup_rejects_changed_export(tmp_path): + """Repeated setup cannot mount retained NFS data at a second path.""" + state_dir = tmp_path / "state" + state_dir.mkdir() + original = replace(_config(state_dir, tmp_path / "export"), storage_backend="nfs") + _write_completed_state(original, "nfs") + _write_backend_state(original, "nfs") + changed = replace(original, export_dir=tmp_path / "other-export") + + with pytest.raises(_DRIVER.ProvisionError, match="teardown first"): + _select_storage_backend(changed) + + +def test_system_directory_cannot_be_used_as_state(tmp_path): + """An existing broad system directory cannot be marked during setup.""" + config = _config(Path("/usr"), tmp_path / "export") + + with pytest.raises(_DRIVER.ProvisionError, match="unowned setup state"): + _validate_lifecycle_paths(config) + + +class _UnownedExportRunner: + """Record commands while presenting an existing unmarked export.""" + + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + """Answer ownership probes without executing privileged commands.""" + command = [str(item) for item in arguments] + self.commands.append(command) + if "test" in command and "-e" in command: + return SimpleNamespace(returncode=0, stdout="", stderr="") + if "cat" in command: + return SimpleNamespace(returncode=1, stdout="", stderr="missing") + raise AssertionError(f"unexpected mutation command: {command}") + + +def test_existing_export_requires_marker_before_mutation(tmp_path): + """An unmarked export is rejected before install or chown runs.""" + config = _config(tmp_path / "state", tmp_path / "export") + runner = _UnownedExportRunner() + + with pytest.raises(_DRIVER.ProvisionError, match="unowned export directory"): + _ensure_export_marker(runner, config) + + assert not any( + "install" in command or "chown" in command for command in runner.commands + ) + + +class _PodRunner: + """Return one terminating and one active ready login pod.""" + + def run(self, _arguments, **_kwargs): + """Return a fixed rollout-overlap pod list.""" + container = {"name": "login"} + ready = {"phase": "Running", "containerStatuses": [{"ready": True}]} + items = [ + { + "metadata": { + "name": "old-login", + "deletionTimestamp": "2026-09-19T00:00:00Z", + }, + "spec": {"containers": [container]}, + "status": ready, + }, + { + "metadata": {"name": "new-login"}, + "spec": {"containers": [container]}, + "status": ready, + }, + ] + return SimpleNamespace( + returncode=0, stdout=json.dumps({"items": items}), stderr="" + ) + + +def test_login_selection_ignores_terminating_rollout_pod(tmp_path): + """A terminating predecessor does not look like a second login node.""" + config = _config(tmp_path / "state", tmp_path / "export") + + assert _login_pod(_PodRunner(), config) == "new-login" + + +def test_fixture_discovery_ignores_terminating_ready_pod(): + """Test preflight observes only ready pods that are not terminating.""" + container = {"name": "login"} + ready = {"phase": "Running", "containerStatuses": [{"ready": True}]} + pods = [ + { + "metadata": {"name": "old", "deletionTimestamp": "now"}, + "spec": {"containers": [container]}, + "status": ready, + }, + { + "metadata": {"name": "new"}, + "spec": {"containers": [container]}, + "status": ready, + }, + ] + + selected = _pods_with_container(pods, "login") + + assert [pod["metadata"]["name"] for pod in selected] == ["new"] + + +def test_markerless_sbx_elbencho_bundle_is_rebuilt(tmp_path, monkeypatch): + """A pre-recipe cached wrapper cannot survive a repository update.""" + cache = tmp_path / "test-cache" + cache.mkdir() + binary_name = "elbencho.aarch64" + binary = cache / f"v3.1-11-{binary_name}" + runtime = cache / f"v3.1-11-{binary_name}.runtime" + binary.write_text("stale wrapper\n", encoding="utf-8") + runtime.mkdir() + calls = [] + + def fake_extract(_runner, _cache, target_binary, target_runtime, architecture): + calls.append(architecture) + target_binary.write_text("current wrapper\n", encoding="utf-8") + target_runtime.mkdir(exist_ok=True) + + monkeypatch.setattr(_FILESYSTEM, "_extract_container_elbencho", fake_extract) + config = SimpleNamespace(state_dir=tmp_path) + + _ensure_elbencho(object(), config, "aarch64", "sbx-shared") + _ensure_elbencho(object(), config, "aarch64", "sbx-shared") + + assert calls == ["aarch64"] + marker = cache / f"v3.1-11-{binary_name}.bundle.json" + document = json.loads(marker.read_text(encoding="utf-8")) + assert document["container"] == _FILESYSTEM.ELBENCHO_CONTAINER + assert document["recipe"] == _FILESYSTEM.SBX_ELBENCHO_BUNDLE_RECIPE diff --git a/utils/extract-elbencho.py b/utils/extract-elbencho.py index a18e156..a46f56c 100644 --- a/utils/extract-elbencho.py +++ b/utils/extract-elbencho.py @@ -2330,6 +2330,14 @@ def plot_performance_metrics( ) +def _scale_efficiency(throughput_per_unit: List[float]) -> List[float]: + """Return relative efficiency, including a defined all-zero result.""" + maximum = max(throughput_per_unit) + if maximum <= 0: + return [0.0] * len(throughput_per_unit) + return [value / maximum * 100 for value in throughput_per_unit] + + def plot_throughput_scale_efficiency( annotated_size_group: AnnotatedSizeGroup, output_dir: str, @@ -2434,15 +2442,9 @@ def plot_throughput_scale_efficiency( # Calculate throughput per unit (node/thread) for each data point bw_per_unit = [bw_val / x_val for bw_val, x_val in zip(bw, x_data)] - # Find the maximum throughput per unit to use as the reference (100%) - max_bw_per_unit = max(bw_per_unit) - # Calculate efficiency values relative to the maximum throughput per unit # Formula: (bw_per_unit / max_bw_per_unit) * 100 - efficiency = [ - (bw_val / x_val) / max_bw_per_unit * 100 - for bw_val, x_val in zip(bw, x_data) - ] + efficiency = _scale_efficiency(bw_per_unit) # Select color and marker using the pre-computed mapping color_index = color_key_to_index[color_key] From 2638c1447b48162d2f9e24bd8d07eec15a52db6c Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sat, 19 Sep 2026 20:43:12 -0700 Subject: [PATCH 04/10] Refactor filesystem integration scenario harness Separate substrate and scenario selection, schedule scenarios deterministically, and cache deployment archives from one immutable source snapshot consumed by the existing archive builder. Add crash-recoverable SSH home transitions and fast regression coverage for selection, caching, reconciliation, and extensible assertions. --- .github/workflows/integration.yml | 4 +- docs/CONTEXT.md | 20 +- integration-tests/README.md | 41 +- integration-tests/bin/integration-test.py | 322 ++++++++++-- integration-tests/lib/deployment_cache.py | 401 +++++++++++++++ .../lib/filesystem_integration.py | 175 ++++--- integration-tests/lib/scenario_planner.py | 356 ++++++++++++++ integration-tests/lib/ssh_home_transition.py | 465 ++++++++++++++++++ .../manifests/ssh-workers.yaml.tmpl | 6 + tests/test_filesystem_scenarios.py | 229 +++++++++ tests/test_integration_deployment_cache.py | 292 +++++++++++ tests/test_integration_driver_safety.py | 20 +- tests/test_ssh_home_transition.py | 303 ++++++++++++ 13 files changed, 2467 insertions(+), 167 deletions(-) create mode 100644 integration-tests/lib/deployment_cache.py create mode 100644 integration-tests/lib/scenario_planner.py create mode 100644 integration-tests/lib/ssh_home_transition.py create mode 100644 tests/test_filesystem_scenarios.py create mode 100644 tests/test_integration_deployment_cache.py create mode 100644 tests/test_ssh_home_transition.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1657b4e..97fce2c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -72,13 +72,13 @@ jobs: run: sudo "$(command -v python)" integration-tests/bin/integration-test.py start - name: Verify root test execution is rejected run: | - if sudo "$(command -v python)" integration-tests/bin/integration-test.py test filesystem; then + if sudo "$(command -v python)" integration-tests/bin/integration-test.py test; then echo "integration test action unexpectedly accepted root" >&2 exit 1 fi - name: Run all integration tests run: | - "$(command -v python)" integration-tests/bin/integration-test.py test all + "$(command -v python)" integration-tests/bin/integration-test.py test - name: Tear down the integration environment if: ${{ always() }} run: | diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 4ee3448..039b988 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -71,15 +71,17 @@ directory until teardown. The SBX compatibility profile pairs kind/Kubernetes when their pinned image or bundle recipe changes. Slurm coordinators restore configured `ORDER_NODES` include-list order after Slurm canonicalizes an allocation's node list. -The test action builds and validates a deployment tarball, -derives environments from the packaged `env.sh.template`, and runs bounded -one-node and two-node filesystem sweeps through SSH and Slurm as the non-root -account recorded by setup. It verifies ordered worker selection, exact workload -totals, cleanup, and report extraction for both node counts. The SSH entry point -runs on the host; the Slurm entry point runs from the archive extracted by the -LoginSet on shared storage. The on-demand integration workflow runs the full -lifecycle concurrently on amd64 and arm64. It does not add Kubernetes dispatch -to the benchmark entry points. +The test action selects execution substrate and named scenario independently. +Its deterministic planner batches the one shared-home SSH scenario behind a +crash-recoverable StatefulSet transition; separate SSH homes are canonical. +Deployment archives remain products of `utils/build_tarball.sh`, but the +harness caches them by the exact immutable tracked-source snapshot, build +options, architecture, and seeded Elbencho/runtime identity. Each scenario +extracts that artifact into isolated state. Tests run as the non-root account +recorded by setup and validate real SSH or Slurm dispatch, workload results, +cleanup, and reporting. The on-demand integration workflow runs the lifecycle +concurrently on amd64 and arm64; Kubernetes remains fixture infrastructure and +is not a benchmark execution substrate. GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. diff --git a/integration-tests/README.md b/integration-tests/README.md index 09016fa..0b7d5f8 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -74,12 +74,11 @@ access to the private kubeconfig, key, and test-run workspace, and verifies it can use Docker, kind, and kubectl. A root shell without `SUDO_USER` must name the account explicitly with `--test-user USER`. -The default SSH homes are separate `emptyDir` volumes. To mount the dedicated -RWX shared-home claim at `/home/tester` in both workers instead, use: - -```bash -integration-tests/bin/integration-test.py --ssh-home-mode shared setup -``` +SSH workers normally use separate `emptyDir` homes. A scenario that requires +the RWX shared-home claim owns a bounded StatefulSet transition and restores +separate homes afterward. Setup and SSH test preflight recover an interrupted +transition before allowing more SSH work; the kind and Slurm fixtures remain +running throughout. The generated host SSH key, strict known-hosts file, and two worker addresses are kept under `/var/lib/storage-scale-test-integration/`. Re-running setup @@ -89,21 +88,27 @@ or retained backend data. After setup succeeds, run the bounded filesystem regression cases with: ```bash -integration-tests/bin/integration-test.py test all +integration-tests/bin/integration-test.py test +integration-tests/bin/integration-test.py test --substrate ssh +integration-tests/bin/integration-test.py test --scenario baseline +integration-tests/bin/integration-test.py test --list-scenarios ``` -`all` and `filesystem` select both substrates. `ssh` and `slurm` may be used -individually, or together as two arguments. The `test` action never installs -or reconciles the fixture, and it refuses to run as root. It requires the saved -setup state to match the current non-root account, verifies that the live -topology is healthy, generates each test environment from the packaged -`env.sh.template`, and runs `validate_env.sh` before the sweep. +With no options, `test` runs every available scenario on each applicable +substrate. `--substrate` accepts `all`, `ssh`, or `slurm`; repeatable +`--scenario` options select named cases independently. Scenario listing needs +no setup state or privileges. Actual tests refuse root execution, require the +saved non-root identity, validate the live topology, generate environments +from the packaged `env.sh.template`, and run `validate_env.sh` before a sweep. Each substrate runs one 4 KiB buffered execution on one node and one on two -nodes through the real filesystem sweep entry point. The harness builds a real -deployment archive from a tracked-files-only snapshot with -`utils/build_tarball.sh`, validates its contents, and runs from the extracted -archive. The SSH case launches `validate_env.sh` and `nv-elbencho-sweep.sh` on +nodes through the real filesystem sweep entry point. The harness materializes +one immutable tracked-source snapshot and builds a real deployment archive from +it with `utils/build_tarball.sh`. It caches the validated archive by snapshot +manifest, architecture, builder options, and seeded Elbencho/runtime identity. +Targeted reruns extract that artifact into isolated workspaces instead of +rebuilding it. The SSH case launches `validate_env.sh` and +`nv-elbencho-sweep.sh` on the host and reaches the two worker pods over SSH. The Slurm case streams the same archive to the LoginSet, extracts it in the shared storage filesystem, and launches both commands there. @@ -123,7 +128,7 @@ of each result and requires the report to contain both node counts. The `Filesystem integration` GitHub Actions workflow runs independent amd64 and arm64 jobs concurrently. Each job runs setup twice, stops and restarts the -fixture, proves that root test execution is rejected, runs `test all` as the +fixture, proves that root test execution is rejected, runs `test` as the ordinary runner account, and tears down twice. A final status job requires both architectures to pass. The workflow is deliberately absent from ordinary pull-request and default-branch events. diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py index f5f6aae..9c63885 100755 --- a/integration-tests/bin/integration-test.py +++ b/integration-tests/bin/integration-test.py @@ -45,9 +45,20 @@ from filesystem_integration import ( # pylint: disable=wrong-import-position IntegrationTestError, - TEST_SELECTORS, run_filesystem_tests, ) +from scenario_planner import ( # pylint: disable=wrong-import-position + SUBSTRATES, + format_scenario_listing, +) +from ssh_home_transition import ( # pylint: disable=wrong-import-position + CANONICAL_HOME_MODE, + SHARED_HOME_MODE, + PoolObservation, + SshHomeTransitionError, + SshHomeTransitionManager, + statefulset_checksum, +) KIND_VERSION = "v0.33.0" SBX_KIND_VERSION = "v0.30.0" @@ -89,6 +100,8 @@ NFS_DAEMON_CONFIG = Path("/etc/nfs.conf.d/storage-scale-test-integration.conf") EXPORT_MARKER = ".storage-scale-test-integration.json" STATE_MARKER = "state-owner.json" +SSH_HOME_ANNOTATION = "storage-scale-test/ssh-home-mode" +SSH_CONFIG_ANNOTATION = "storage-scale-test/ssh-config-checksum" UFW_COMMENT = "storage-scale-test integration NFSv4" LOG = logging.getLogger("storage-scale-integration") @@ -115,7 +128,6 @@ class Config: export_dir: Path storage_backend: str sbx_shared_root: Path - ssh_home_mode: str test_user: str test_uid: int test_gid: int @@ -1497,16 +1509,130 @@ def _install_ssh_workers(runner: Runner, config: Config) -> None: "storage-ssh-identity", {"id_ed25519": private_key, "authorized_keys": public_key}, ) - home_volume = ( - "persistentVolumeClaim:\n claimName: ssh-home-rwx" - if config.ssh_home_mode == "shared" - else "emptyDir:\n sizeLimit: 64Mi" - ) - manifest = _render_resource( + _ensure_ssh_home_mode( + runner, config, - "manifests/ssh-workers.yaml.tmpl", - {"NAMESPACE": config.namespace, "SSH_HOME_VOLUME": home_volume}, + CANONICAL_HOME_MODE, + run_id="setup", + scenario_id="setup-preflight", + ) + + +def _ssh_home_volume(mode: str) -> str: + """Return the manifest fragment for one supported SSH home mode.""" + if mode == SHARED_HOME_MODE: + return "persistentVolumeClaim:\n claimName: ssh-home-rwx" + if mode == CANONICAL_HOME_MODE: + return "emptyDir:\n sizeLimit: 64Mi" + raise ProvisionError(f"unsupported SSH home mode: {mode}") + + +def _render_ssh_workers(config: Config, mode: str) -> tuple[Path, str]: + """Render one canonical SSH StatefulSet form and return its checksum.""" + source = _resource_path("manifests/ssh-workers.yaml.tmpl") + rendered = source.read_text(encoding="utf-8") + replacements = { + "NAMESPACE": config.namespace, + "SSH_HOME_MODE": mode, + "SSH_HOME_VOLUME": _ssh_home_volume(mode), + } + for token, value in replacements.items(): + rendered = rendered.replace(f"@@{token}@@", value) + checksum = statefulset_checksum(rendered.replace("@@SSH_CONFIG_CHECKSUM@@", "")) + rendered = rendered.replace("@@SSH_CONFIG_CHECKSUM@@", checksum) + unresolved = [word for word in rendered.split() if "@@" in word] + if unresolved: + raise ProvisionError( + f"unresolved SSH manifest token in {source}: {unresolved[0]}" + ) + destination = config.manifests_dir / f"ssh-workers-{mode}.yaml" + _write_text(destination, rendered) + return destination, checksum + + +def _inspect_ssh_home_pool(runner: Runner, config: Config) -> PoolObservation: + """Return the live SSH StatefulSet form and rollout state.""" + statefulset = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "statefulset/ssh-worker", + "-o", + "json", + ), + check=False, + timeout=30, ) + if statefulset.returncode: + return PoolObservation("absent", "", 0, 0, statefulset.stderr.strip()) + document = json.loads(statefulset.stdout) + annotations = document.get("metadata", {}).get("annotations", {}) + pods = _ssh_pods(runner, config) + terminating = sum( + bool(pod.get("metadata", {}).get("deletionTimestamp")) for pod in pods + ) + ready = sum( + _pod_ready(pod) and not pod.get("metadata", {}).get("deletionTimestamp") + for pod in pods + ) + names = ", ".join(str(pod.get("metadata", {}).get("name", "?")) for pod in pods) + return PoolObservation( + str(annotations.get(SSH_HOME_ANNOTATION, "unknown")), + str(annotations.get(SSH_CONFIG_ANNOTATION, "")), + ready, + terminating, + f"pods=[{names}]", + ) + + +def _pod_ready(pod: dict[str, object]) -> bool: + """Return whether every container in a running pod is ready.""" + status = pod.get("status", {}) + if not isinstance(status, dict) or status.get("phase") != "Running": + return False + containers = status.get("containerStatuses", []) + return bool(containers) and all( + isinstance(item, dict) and item.get("ready") for item in containers + ) + + +def _wait_for_no_ssh_pods(runner: Runner, config: Config) -> None: + """Wait until the host-network SSH port has no owning pod.""" + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + if not _ssh_pods(runner, config): + return + time.sleep(2) + names = [str(pod["metadata"]["name"]) for pod in _ssh_pods(runner, config)] + raise ProvisionError(f"SSH pods did not terminate before transition: {names}") + + +def _reconcile_ssh_home_pool( + runner: Runner, config: Config, mode: str, expected_checksum: str +) -> None: + """Replace only the SSH StatefulSet with one validated home mode.""" + manifest, checksum = _render_ssh_workers(config, mode) + if checksum != expected_checksum: + raise ProvisionError( + "SSH StatefulSet checksum changed during reconciliation: " + f"expected {expected_checksum}, rendered {checksum}" + ) + existing = runner.run( + _kubectl( + config, + "-n", + config.namespace, + "get", + "statefulset/ssh-worker", + ), + check=False, + timeout=30, + ) + if existing.returncode == 0: + _scale_ssh(runner, config, replicas=0) + _wait_for_no_ssh_pods(runner, config) runner.run( _kubectl( config, @@ -1518,8 +1644,54 @@ def _install_ssh_workers(runner: Runner, config: Config) -> None: manifest, ) ) + _scale_ssh(runner, config, replicas=2) _wait_for_ssh(runner, config) - _validate_ssh_workers(runner, config, private_key) + _validate_ssh_workers(runner, config, config.keys_dir / "id_ed25519", mode) + + +def _ssh_transition_manager(runner: Runner, config: Config) -> SshHomeTransitionManager: + """Return a transition manager backed by the live Kubernetes fixture.""" + return SshHomeTransitionManager( + config.state_dir, + lambda: _inspect_ssh_home_pool(runner, config), + lambda mode, checksum: _reconcile_ssh_home_pool(runner, config, mode, checksum), + ) + + +def _ensure_ssh_home_mode( + runner: Runner, + config: Config, + mode: str, + *, + run_id: str, + scenario_id: str, +) -> None: + """Enter a validated SSH home mode through persistent transition state.""" + separate_checksum = _render_ssh_workers(config, CANONICAL_HOME_MODE)[1] + shared_checksum = _render_ssh_workers(config, SHARED_HOME_MODE)[1] + manager = _ssh_transition_manager(runner, config) + if mode == CANONICAL_HOME_MODE: + manager.preflight(separate_checksum, run_id=run_id, scenario_id=scenario_id) + return + if mode == SHARED_HOME_MODE: + manager.enter_shared( + run_id=run_id, + scenario_id=scenario_id, + separate_checksum=separate_checksum, + shared_checksum=shared_checksum, + ) + return + raise ProvisionError(f"unsupported SSH home mode: {mode}") + + +def _restore_ssh_home_mode( + runner: Runner, config: Config, *, run_id: str, scenario_id: str +) -> None: + """Restore canonical SSH homes after a shared-home scenario batch.""" + checksum = _render_ssh_workers(config, CANONICAL_HOME_MODE)[1] + _ssh_transition_manager(runner, config).restore_separate( + checksum, run_id=run_id, scenario_id=scenario_id + ) def _ssh_pods(runner: Runner, config: Config) -> list[dict[str, object]]: @@ -1542,7 +1714,9 @@ def _ssh_pods(runner: Runner, config: Config) -> list[dict[str, object]]: ) -def _validate_ssh_workers(runner: Runner, config: Config, private_key: Path) -> None: +def _validate_ssh_workers( + runner: Runner, config: Config, private_key: Path, home_mode: str +) -> None: """Validate placement, host SSH, home semantics, and RWX visibility.""" pods = _ssh_pods(runner, config) if len(pods) != 2: @@ -1612,7 +1786,7 @@ def _validate_ssh_workers(runner: Runner, config: Config, private_key: Path) -> "-o StrictHostKeyChecking=accept-new " f"tester@{addresses[destination]} true", ) - _validate_ssh_storage(runner, config, pods) + _validate_ssh_storage(runner, config, pods, home_mode) def _pod_exec( @@ -1637,7 +1811,10 @@ def _pod_exec( def _validate_ssh_storage( - runner: Runner, config: Config, pods: list[dict[str, object]] + runner: Runner, + config: Config, + pods: list[dict[str, object]], + home_mode: str, ) -> None: """Validate the selected home mode and shared storage claim.""" names = [str(pod["metadata"]["name"]) for pod in pods] @@ -1680,7 +1857,7 @@ def _validate_ssh_storage( or host_probe.read_text(encoding="utf-8").strip() != token ): raise ProvisionError("SBX shared data is not visible from the agent") - expected_home_rc = 0 if config.ssh_home_mode == "shared" else 1 + expected_home_rc = 0 if home_mode == "shared" else 1 if home_probe.returncode != expected_home_rc or rwx_probe.returncode: raise ProvisionError("SSH home or RWX visibility validation failed") cleanup = ( @@ -2087,7 +2264,7 @@ def _write_state_summary( "cluster_name": config.cluster_name, "namespace": config.namespace, "export_dir": str(config.export_dir), - "ssh_home_mode": config.ssh_home_mode, + "ssh_home_mode": "separate", "storage_backend": backend, "test_user": config.test_user, "test_uid": config.test_uid, @@ -2160,7 +2337,7 @@ def setup_environment(runner: Runner, config: Config) -> None: _scale_ssh(runner, config, replicas=2) _wait_for_ssh(runner, config) private_key = config.keys_dir / "id_ed25519" - _validate_ssh_workers(runner, config, private_key) + _validate_ssh_workers(runner, config, private_key, "separate") _write_state_summary(config, backend, subnet, gateway) LOG.info("Integration environment is provisioned and running") @@ -2229,19 +2406,30 @@ def _scale_ssh(runner: Runner, config: Config, replicas: int) -> None: def _wait_for_ssh(runner: Runner, config: Config) -> None: - """Wait for both SSH workers after restoring the running fixture.""" - runner.run( - _kubectl( - config, - "-n", - config.namespace, - "rollout", - "status", - "statefulset/ssh-worker", - "--timeout=180s", - ), - timeout=210, - ) + """Wait for exactly two ready, nonterminating SSH workers.""" + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + pods = _ssh_pods(runner, config) + ready = [ + pod + for pod in pods + if _pod_ready(pod) and not pod.get("metadata", {}).get("deletionTimestamp") + ] + terminating = [ + pod for pod in pods if pod.get("metadata", {}).get("deletionTimestamp") + ] + if len(ready) == 2 and not terminating and len(pods) == 2: + return + time.sleep(2) + details = [ + { + "name": pod.get("metadata", {}).get("name"), + "ready": _pod_ready(pod), + "terminating": bool(pod.get("metadata", {}).get("deletionTimestamp")), + } + for pod in _ssh_pods(runner, config) + ] + raise ProvisionError(f"SSH worker rollout did not become healthy: {details}") def stop_environment(runner: Runner, config: Config) -> None: @@ -2797,18 +2985,29 @@ def _parser() -> argparse.ArgumentParser: "SUDO_USER identifies it" ), ) - parser.add_argument( - "--ssh-home-mode", choices=("separate", "shared"), default="separate" - ) parser.add_argument("--verbose", action="store_true") - parser.add_argument( - "action", choices=("setup", "start", "stop", "teardown", "test") - ) - parser.add_argument( - "tests", - nargs="*", - metavar="TEST", - help="test selectors for the test action: " + ", ".join(TEST_SELECTORS), + actions = parser.add_subparsers(dest="action", required=True) + for action in ("setup", "start", "stop", "teardown"): + actions.add_parser(action) + test_parser = actions.add_parser("test") + test_parser.add_argument( + "--substrate", + choices=SUBSTRATES, + default="all", + help="execution substrate to test (default: all)", + ) + test_parser.add_argument( + "--scenario", + dest="scenarios", + action="append", + default=[], + metavar="NAME", + help="scenario to run; repeat to select more than one", + ) + test_parser.add_argument( + "--list-scenarios", + action="store_true", + help="list scenarios without inspecting or changing fixture state", ) return parser @@ -2823,7 +3022,6 @@ def _config(arguments: argparse.Namespace) -> Config: export_dir=arguments.export_dir.resolve(), storage_backend=arguments.storage_backend, sbx_shared_root=arguments.sbx_shared_root.resolve(), - ssh_home_mode=arguments.ssh_home_mode, test_user=account.pw_name, test_uid=account.pw_uid, test_gid=account.pw_gid, @@ -2861,10 +3059,43 @@ def _test_account(explicit_user: str | None) -> pwd.struct_passwd: return account +def _run_filesystem_action( + runner: Runner, config: Config, arguments: argparse.Namespace +) -> None: + """Run selected scenarios with crash-recoverable SSH home transitions.""" + run_id = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) + f"-{os.getpid()}" + + def transition(mode: str, scenario_id: str) -> None: + if mode == CANONICAL_HOME_MODE and scenario_id != "preflight": + _restore_ssh_home_mode( + runner, config, run_id=run_id, scenario_id=scenario_id + ) + return + _ensure_ssh_home_mode( + runner, + config, + mode, + run_id=run_id, + scenario_id=scenario_id, + ) + + run_filesystem_tests( + runner, + config, + _repository_root(), + arguments.substrate, + arguments.scenarios, + transition, + ) + + def main() -> int: """Run one integration environment lifecycle action.""" _require_python() arguments = _parser().parse_args() + if arguments.action == "test" and arguments.list_scenarios: + print(format_scenario_listing()) + return 0 try: if arguments.action == "test" and os.geteuid() == 0: raise ProvisionError( @@ -2873,8 +3104,6 @@ def main() -> int: ) config = _config(arguments) _validate_lifecycle_paths(config) - if arguments.action != "test" and arguments.tests: - raise ProvisionError("test selectors are valid only with the test action") if arguments.action == "test" and not config.state_dir.is_dir(): raise ProvisionError( f"setup state directory is absent at {config.state_dir}; run setup first" @@ -2889,9 +3118,7 @@ def main() -> int: elif arguments.action == "teardown": teardown_environment(runner, config) elif arguments.action == "test": - run_filesystem_tests( - runner, config, _repository_root(), arguments.tests - ) + _run_filesystem_action(runner, config, arguments) else: setup_environment(runner, config) _grant_test_user_access(runner, config) @@ -2900,6 +3127,7 @@ def main() -> int: except ( ProvisionError, IntegrationTestError, + SshHomeTransitionError, OSError, subprocess.TimeoutExpired, json.JSONDecodeError, diff --git a/integration-tests/lib/deployment_cache.py b/integration-tests/lib/deployment_cache.py new file mode 100644 index 0000000..ea5b8e0 --- /dev/null +++ b/integration-tests/lib/deployment_cache.py @@ -0,0 +1,401 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Content-addressed deployment archives for the integration harness.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import tempfile +import uuid +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +CACHE_SCHEMA = 1 +DEFAULT_RECIPE = 1 +ARCHIVE_NAME = "storage-scale-test.tar.gz" +MANIFEST_NAME = "manifest.json" +BUILD_LOG_NAME = "build-tarball.log" + + +class DeploymentCacheError(RuntimeError): + """A deployment snapshot or cache entry is unsafe or inconsistent.""" + + +@dataclass(frozen=True) +class DeploymentCacheRequest: + """Inputs needed to build one content-addressed deployment archive.""" + + repo_root: Path + cache_root: Path + architecture: str + binary: Path + binary_name: str + runtime: Path | None = None + build_options: tuple[str, ...] = ("--skip-object-tools",) + recipe: int = DEFAULT_RECIPE + build_timeout: int = 600 + + +@dataclass(frozen=True) +class CachedDeployment: + """One verified archive returned from the deployment cache.""" + + key: str + archive: Path + manifest: Path + build_log: Path + cache_hit: bool + + +def _sha256(path: Path) -> str: + """Return the SHA-256 digest of one regular file.""" + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_digest(document: object) -> str: + """Hash one JSON-compatible document in a stable representation.""" + encoded = json.dumps( + document, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _safe_relative(name: str) -> Path: + """Validate and convert a Git-provided repository-relative path.""" + relative = PurePosixPath(name) + if relative.is_absolute() or not relative.parts or ".." in relative.parts: + raise DeploymentCacheError(f"unsafe tracked path from git: {name!r}") + return Path(*relative.parts) + + +def _tracked_names(runner: Any, repo_root: Path) -> tuple[str, ...]: + """Return tracked paths in a deterministic order.""" + result = runner.run( + ["git", "ls-files", "-z", "--cached"], cwd=repo_root, timeout=30 + ) + names = tuple(name for name in result.stdout.split("\0") if name) + if not names: + raise DeploymentCacheError(f"repository has no tracked files: {repo_root}") + return tuple(sorted(names)) + + +def _copy_tracked_file(source: Path, target: Path) -> None: + """Copy one regular file or safe relative symbolic link.""" + mode = source.lstat().st_mode + target.parent.mkdir(parents=True, exist_ok=True) + if stat.S_ISREG(mode): + shutil.copy2(source, target, follow_symlinks=False) + return + if stat.S_ISLNK(mode): + link = os.readlink(source) + link_path = PurePosixPath(link) + if link_path.is_absolute() or ".." in link_path.parts: + raise DeploymentCacheError( + f"unsafe tracked symbolic link: {source} -> {link}" + ) + target.symlink_to(link) + return + raise DeploymentCacheError( + f"deployment snapshot requires a regular file or relative symlink: {source}" + ) + + +def _manifest_entry(root: Path, path: Path) -> dict[str, object]: + """Describe one file-system entry below a snapshot root.""" + mode = path.lstat().st_mode + relative = path.relative_to(root).as_posix() + if stat.S_ISREG(mode): + kind = "file" + digest = _sha256(path) + elif stat.S_ISDIR(mode): + kind = "directory" + digest = hashlib.sha256(b"").hexdigest() + elif stat.S_ISLNK(mode): + kind = "symlink" + digest = hashlib.sha256(os.readlink(path).encode("utf-8")).hexdigest() + else: + raise DeploymentCacheError(f"unsupported snapshot entry: {path}") + return { + "path": relative, + "mode": stat.S_IMODE(mode), + "type": kind, + "sha256": digest, + } + + +def _manifest_for_paths(root: Path, paths: tuple[Path, ...]) -> dict[str, object]: + """Describe an explicit ordered set of paths.""" + entries = [_manifest_entry(root, path) for path in paths] + document: dict[str, object] = {"entries": entries} + document["sha256"] = _canonical_digest(entries) + return document + + +def _copy_tracked_snapshot( + runner: Any, repo_root: Path, destination: Path +) -> dict[str, object]: + """Materialize and describe the exact tracked working-tree snapshot.""" + names = _tracked_names(runner, repo_root) + destination.mkdir(parents=True) + copied: list[Path] = [] + for name in names: + relative = _safe_relative(name) + source = repo_root / relative + if not source.exists() and not source.is_symlink(): + raise DeploymentCacheError(f"tracked path is absent: {source}") + target = destination / relative + _copy_tracked_file(source, target) + copied.append(target) + for directory in sorted( + (path for path in destination.rglob("*") if stat.S_ISDIR(path.lstat().st_mode)), + reverse=True, + ): + directory.chmod(0o755) + destination.chmod(0o755) + return _manifest_for_paths(destination, tuple(copied)) + + +def _tree_paths(root: Path) -> tuple[Path, ...]: + """Return all entries in a tree without following symbolic links.""" + paths: list[Path] = [] + for directory, dirnames, filenames in os.walk(root, followlinks=False): + base = Path(directory) + linked_directories = [name for name in dirnames if (base / name).is_symlink()] + dirnames[:] = sorted( + name for name in dirnames if name not in linked_directories + ) + paths.extend(base / name for name in sorted(linked_directories)) + paths.extend(base / name for name in dirnames) + paths.extend(base / name for name in sorted(filenames)) + return tuple(sorted(paths, key=lambda item: item.relative_to(root).as_posix())) + + +def _stage_binary_and_runtime( + request: DeploymentCacheRequest, snapshot: Path +) -> tuple[dict[str, object], dict[str, object] | None]: + """Seed external build inputs and return their staged identities.""" + if not request.binary.is_file() or request.binary.is_symlink(): + raise DeploymentCacheError( + f"deployment binary is not a regular file: {request.binary}" + ) + binary_name = PurePosixPath(request.binary_name) + if ( + binary_name.is_absolute() + or len(binary_name.parts) != 1 + or binary_name.name in ("", ".", "..") + ): + raise DeploymentCacheError( + f"deployment binary name is not a basename: {request.binary_name!r}" + ) + binary_target = snapshot / "utils" / binary_name.name + binary_target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(request.binary, binary_target) + binary_target.chmod(0o755) + binary_identity = _manifest_entry(snapshot, binary_target) + + if request.runtime is None: + return binary_identity, None + if not request.runtime.is_dir() or request.runtime.is_symlink(): + raise DeploymentCacheError( + f"deployment runtime is not a directory: {request.runtime}" + ) + runtime_target = snapshot / "utils" / "elbencho-runtime" + shutil.copytree(request.runtime, runtime_target) + return binary_identity, _manifest_for_paths( + runtime_target, _tree_paths(runtime_target) + ) + + +def _identity_document( + request: DeploymentCacheRequest, + source_manifest: dict[str, object], + binary_identity: dict[str, object], + runtime_identity: dict[str, object] | None, +) -> dict[str, object]: + """Build the cache-key document for one immutable snapshot.""" + return { + "schema": CACHE_SCHEMA, + "recipe": request.recipe, + "architecture": request.architecture, + "build_options": list(request.build_options), + "source": source_manifest, + "binary": binary_identity, + "runtime": runtime_identity, + } + + +def _entry_paths(cache_root: Path, key: str) -> tuple[Path, Path, Path, Path]: + """Return the entry and its fixed files.""" + entry = cache_root / key + return ( + entry, + entry / ARCHIVE_NAME, + entry / MANIFEST_NAME, + entry / BUILD_LOG_NAME, + ) + + +def _load_json(path: Path) -> dict[str, object] | None: + """Read a JSON object, returning None for malformed input.""" + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + return document if isinstance(document, dict) else None + + +def _valid_entry(cache_root: Path, key: str, identity: dict[str, object]) -> bool: + """Return whether a cache entry exactly matches its expected identity.""" + entry, archive, manifest_path, build_log = _entry_paths(cache_root, key) + if not entry.is_dir() or entry.is_symlink(): + return False + if any( + path.is_symlink() or not path.is_file() + for path in (archive, manifest_path, build_log) + ): + return False + manifest = _load_json(manifest_path) + if manifest is None or manifest.get("identity") != identity: + return False + archive_record = manifest.get("archive") + if not isinstance(archive_record, dict) or archive.stat().st_size <= 0: + return False + return ( + archive_record.get("bytes") == archive.stat().st_size + and archive_record.get("sha256") == _sha256(archive) + and manifest.get("input_digest") == key + ) + + +def _run_builder( + runner: Any, request: DeploymentCacheRequest, staging: Path, snapshot: Path +) -> tuple[Path, str]: + """Invoke the repository's existing deployment builder from the snapshot.""" + builder = snapshot / "utils" / "build_tarball.sh" + if not builder.is_file() or builder.is_symlink(): + raise DeploymentCacheError(f"deployment builder is absent: {builder}") + arguments: list[str | Path] = [ + builder, + "--arch", + request.architecture, + *request.build_options, + ] + result = runner.run(arguments, cwd=snapshot, timeout=request.build_timeout) + archive = staging / ARCHIVE_NAME + if not archive.is_file() or archive.is_symlink() or archive.stat().st_size <= 0: + raise DeploymentCacheError(f"deployment builder did not create {archive}") + return archive, result.stdout + result.stderr + + +def _publish( + cache_root: Path, + key: str, + identity: dict[str, object], + archive: Path, + build_log: str, +) -> bool: + """Atomically publish one completed entry; return whether ours won.""" + publish: Path | None = cache_root / f".publish-{key}-{uuid.uuid4().hex}" + publish.mkdir(mode=0o700) + try: + published_archive = publish / ARCHIVE_NAME + shutil.copy2(archive, published_archive) + (publish / BUILD_LOG_NAME).write_text(build_log, encoding="utf-8") + manifest = { + "schema": CACHE_SCHEMA, + "input_digest": key, + "identity": identity, + "archive": { + "bytes": published_archive.stat().st_size, + "sha256": _sha256(published_archive), + }, + } + (publish / MANIFEST_NAME).write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if _valid_entry(cache_root, key, identity): + return False + _replace_invalid_entry(cache_root, key, publish) + publish = None + return True + finally: + if publish is not None: + shutil.rmtree(publish, ignore_errors=True) + + +def _replace_invalid_entry(cache_root: Path, key: str, publish: Path) -> None: + """Replace an invalid entry while keeping the new entry atomically visible.""" + target = cache_root / key + quarantine = cache_root / f".invalid-{key}-{uuid.uuid4().hex}" + moved_invalid = False + if target.exists() or target.is_symlink(): + os.replace(target, quarantine) + moved_invalid = True + try: + os.replace(publish, target) + except OSError: + if moved_invalid and not target.exists(): + os.replace(quarantine, target) + raise + finally: + if moved_invalid: + if quarantine.is_dir() and not quarantine.is_symlink(): + shutil.rmtree(quarantine, ignore_errors=True) + else: + quarantine.unlink(missing_ok=True) + + +def get_or_build_deployment( + runner: Any, request: DeploymentCacheRequest +) -> CachedDeployment: + """Return a verified cached deployment built from one exact snapshot.""" + request.cache_root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".deployment-staging-", dir=request.cache_root + ) as temporary: + staging = Path(temporary) + snapshot = staging / "source" + source_manifest = _copy_tracked_snapshot(runner, request.repo_root, snapshot) + binary_identity, runtime_identity = _stage_binary_and_runtime(request, snapshot) + build_input_manifest = _manifest_for_paths(snapshot, _tree_paths(snapshot)) + identity = _identity_document( + request, source_manifest, binary_identity, runtime_identity + ) + key = _canonical_digest(identity) + entry, archive, manifest, build_log = _entry_paths(request.cache_root, key) + if _valid_entry(request.cache_root, key, identity): + return CachedDeployment(key, archive, manifest, build_log, True) + + built_archive, output = _run_builder(runner, request, staging, snapshot) + if _manifest_for_paths(snapshot, _tree_paths(snapshot)) != build_input_manifest: + raise DeploymentCacheError( + "deployment builder modified its immutable source snapshot" + ) + published = _publish(request.cache_root, key, identity, built_archive, output) + if not _valid_entry(request.cache_root, key, identity): + raise DeploymentCacheError( + f"published deployment cache is invalid: {entry}" + ) + return CachedDeployment(key, archive, manifest, build_log, not published) diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py index 6d062e6..5337c4b 100644 --- a/integration-tests/lib/filesystem_integration.py +++ b/integration-tests/lib/filesystem_integration.py @@ -34,6 +34,18 @@ from pathlib import Path, PurePosixPath from typing import Any +from deployment_cache import ( + DeploymentCacheError, + DeploymentCacheRequest, + get_or_build_deployment, +) +from scenario_planner import ( + ScenarioPlanningError, + SshHomeTransition, + WorkItem, + plan_scenarios, +) + LOG = logging.getLogger("storage-scale-integration") ELBENCHO_VERSION = "v3.1-11" @@ -63,7 +75,6 @@ } REMOTE_BASE = "/mnt/storage-test/integration-regression" VALIDATION_SUCCESS = "All validation checks passed successfully" -TEST_SELECTORS = ("all", "filesystem", "ssh", "slurm") class IntegrationTestError(RuntimeError): @@ -282,7 +293,9 @@ def _probe_pod( return result.stdout.strip() -def _require_fixture(runner: Any, config: Any) -> Fixture: +def _require_fixture( + runner: Any, config: Any, ssh_home_mode: str = "separate" +) -> Fixture: """Validate setup without reconciling or installing anything.""" state = _load_state(config) required_host_tools = ( @@ -373,7 +386,7 @@ def _require_fixture(runner: Any, config: Any) -> Fixture: slurm_nodes=(slurm_nodes[0], slurm_nodes[1]), slurm_addresses=(slurm_addresses[0], slurm_addresses[1]), architecture=host_arch, - ssh_home_mode=str(state["ssh_home_mode"]), + ssh_home_mode=ssh_home_mode, storage_backend=str(state["storage_backend"]), ) @@ -698,34 +711,6 @@ def _render_env( return rendered, support -def _copy_tracked_snapshot(runner: Any, repo_root: Path, destination: Path) -> None: - """Copy only tracked working-tree files into an isolated packaging tree.""" - required = (repo_root / "utils" / "build_tarball.sh", repo_root / "NOTICE") - missing = [str(path) for path in required if not path.is_file()] - if missing: - raise IntegrationTestError( - "deployment tarball sources are absent: " + ", ".join(missing) - ) - result = runner.run( - ["git", "ls-files", "-z", "--cached"], cwd=repo_root, timeout=30 - ) - destination.mkdir(parents=True) - for name in result.stdout.split("\0"): - if not name: - continue - relative = PurePosixPath(name) - if relative.is_absolute() or ".." in relative.parts: - raise IntegrationTestError(f"unsafe tracked path from git: {name!r}") - source = repo_root / Path(*relative.parts) - target = destination / Path(*relative.parts) - if not source.is_file() or source.is_symlink(): - raise IntegrationTestError( - f"deployment snapshot requires a regular tracked file: {source}" - ) - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, target) - - def _validate_deployment_archive( archive: Path, destination: Path, @@ -816,6 +801,7 @@ def _validate_deployment_archive( def _build_deployment_archive( runner: Any, + config: Any, repo_root: Path, build_root: Path, binary: Path, @@ -823,29 +809,26 @@ def _build_deployment_archive( architecture: str, runtime: Path | None, ) -> tuple[Path, Path]: - """Build and inspect one filesystem-only deployment tarball.""" - snapshot = build_root / "source" - _copy_tracked_snapshot(runner, repo_root, snapshot) - seeded_binary = snapshot / "utils" / binary_name - shutil.copy2(binary, seeded_binary) - seeded_binary.chmod(0o755) - if runtime is not None: - shutil.copytree(runtime, snapshot / "utils" / "elbencho-runtime") - LOG.info("Building deployment tarball from a tracked-files-only snapshot") - result = runner.run( - [ - snapshot / "utils" / "build_tarball.sh", - "--arch", - architecture, - "--skip-object-tools", - ], - cwd=snapshot, - timeout=600, + """Reuse or build and inspect one filesystem-only deployment tarball.""" + request = DeploymentCacheRequest( + repo_root=repo_root, + cache_root=config.state_dir / "test-cache" / "deployments", + architecture=architecture, + binary=binary, + binary_name=binary_name, + runtime=runtime, ) - (build_root / "build-tarball.log").write_text( - result.stdout + result.stderr, encoding="utf-8" + try: + cached = get_or_build_deployment(runner, request) + except DeploymentCacheError as error: + raise IntegrationTestError(str(error)) from error + LOG.info( + "%s deployment cache entry %s", + "Reusing" if cached.cache_hit else "Built", + cached.key, ) - archive = build_root / "storage-scale-test.tar.gz" + shutil.copy2(cached.build_log, build_root / "build-tarball.log") + archive = cached.archive extracted = _validate_deployment_archive( archive, build_root / "extracted", @@ -1303,34 +1286,25 @@ def _run_substrate( _assert_report(runner, report_workspace, local_result, selector, log_dir) -def _selected_tests(selectors: list[str]) -> tuple[str, ...]: - """Normalize public selectors to ordered substrate names.""" - requested = selectors or ["all"] - unknown = sorted(set(requested) - set(TEST_SELECTORS)) - if unknown: - raise IntegrationTestError( - f"unknown test selector(s): {', '.join(unknown)}; " - f"choose from {', '.join(TEST_SELECTORS)}" - ) - duplicates = sorted(name for name in set(requested) if requested.count(name) > 1) - if duplicates: - raise IntegrationTestError( - f"duplicate test selector(s): {', '.join(duplicates)}" - ) - if "all" in requested and len(requested) > 1: - raise IntegrationTestError("test selector 'all' cannot be combined") - if "filesystem" in requested and len(requested) > 1: - raise IntegrationTestError("test selector 'filesystem' cannot be combined") - if requested in (["all"], ["filesystem"]): - return ("ssh", "slurm") - return tuple(name for name in ("ssh", "slurm") if name in requested) - - def run_filesystem_tests( - runner: Any, config: Any, repo_root: Path, selectors: list[str] + runner: Any, + config: Any, + repo_root: Path, + substrate: str, + scenarios: list[str], + transition_ssh_home: Any | None = None, ) -> None: """Run selected filesystem regression cases against an existing setup.""" - selected = _selected_tests(selectors) + try: + plan = plan_scenarios(substrate=substrate, requested=scenarios) + except ScenarioPlanningError as error: + raise IntegrationTestError(str(error)) from error + work = [step for step in plan if isinstance(step, WorkItem)] + selected = {step.substrate.value for step in work} + if "ssh" in selected: + if transition_ssh_home is None: + raise IntegrationTestError("SSH scenarios require a home transition hook") + transition_ssh_home("separate", "preflight") LOG.info("Requiring an already-running integration setup") fixture = _require_fixture(runner, config) binary, binary_name, runtime = _ensure_elbencho( @@ -1348,6 +1322,7 @@ def run_filesystem_tests( build_root = Path(temporary) archive, extracted = _build_deployment_archive( runner, + config, repo_root, build_root, binary, @@ -1365,16 +1340,36 @@ def run_filesystem_tests( fixture, template=extracted / "env.sh.template", ) - for selector in selected: - _run_substrate( - runner, - config, - fixture, - selector, - archive, - extracted, - build_root, - report_workspace, - log_dir, - ) - LOG.info("Filesystem integration tests passed: %s", ", ".join(selected)) + home_mode = "separate" + try: + for step in plan: + if isinstance(step, SshHomeTransition): + transition_ssh_home(step.target.value, "ssh-shared-home") + home_mode = step.target.value + fixture = _require_fixture(runner, config, home_mode) + continue + scenario = step.scenario.name + if scenario != "baseline": + raise IntegrationTestError( + f"integration scenario is not implemented: {scenario}" + ) + scenario_root = build_root / f"{scenario}-{step.substrate.value}" + scenario_root.mkdir() + scenario_logs = log_dir / f"{scenario}-{step.substrate.value}" + scenario_logs.mkdir() + _run_substrate( + runner, + config, + fixture, + step.substrate.value, + archive, + extracted, + scenario_root, + report_workspace, + scenario_logs, + ) + finally: + if home_mode == "shared" and transition_ssh_home is not None: + transition_ssh_home("separate", "ssh-shared-home-restore") + names = ", ".join(f"{step.scenario.name}/{step.substrate.value}" for step in work) + LOG.info("Filesystem integration tests passed: %s", names) diff --git a/integration-tests/lib/scenario_planner.py b/integration-tests/lib/scenario_planner.py new file mode 100644 index 0000000..d068d6f --- /dev/null +++ b/integration-tests/lib/scenario_planner.py @@ -0,0 +1,356 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declare and schedule filesystem integration scenarios.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntEnum, StrEnum +from typing import Iterable, TypeAlias + + +class ScenarioPlanningError(ValueError): + """An invalid scenario selection or registry definition.""" + + +class Substrate(StrEnum): + """Supported integration execution substrates.""" + + ALL = "all" + SSH = "ssh" + SLURM = "slurm" + + +SUBSTRATES = tuple(item.value for item in Substrate) + + +class SshHomeMode(StrEnum): + """SSH worker home configurations used by scenarios.""" + + SEPARATE = "separate" + SHARED = "shared" + + +class SchedulePhase(IntEnum): + """Stable phases surrounding the single shared-home SSH batch.""" + + BEFORE_SHARED_HOME = 10 + SHARED_HOME = 20 + AFTER_SHARED_HOME = 30 + + +@dataclass(frozen=True) +class Scenario: + """Stable metadata for one filesystem integration scenario.""" + + name: str + description: str + substrates: frozenset[Substrate] + order: int + phase: SchedulePhase = SchedulePhase.BEFORE_SHARED_HOME + ssh_home_mode: SshHomeMode | None = None + + def metadata(self) -> dict[str, object]: + """Return the stable machine-readable scenario metadata.""" + return { + "name": self.name, + "substrates": sorted(item.value for item in self.substrates), + "ssh_home_mode": ( + self.ssh_home_mode.value if self.ssh_home_mode is not None else None + ), + "schedule_phase": self.phase.name.lower(), + "order": self.order, + } + + +@dataclass(frozen=True) +class WorkItem: + """One scenario execution on one concrete substrate.""" + + scenario: Scenario + substrate: Substrate + + +@dataclass(frozen=True) +class SshHomeTransition: + """A required transition of the singleton SSH worker pool.""" + + target: SshHomeMode + + +PlanStep: TypeAlias = WorkItem | SshHomeTransition + + +SCENARIO_CATALOG = ( + Scenario( + "baseline", + "Shared-directory buffered-I/O baseline and report extraction", + frozenset({Substrate.SSH, Substrate.SLURM}), + 10, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "default-dio", + "Default worker-directory direct-I/O lifecycle", + frozenset({Substrate.SSH, Substrate.SLURM}), + 20, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "failure-resume", + "Real failure, cleanup, and resume lifecycle", + frozenset({Substrate.SSH, Substrate.SLURM}), + 30, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "retained-lifecycle", + "Write-only, repeated read-from, and delete-only lifecycle", + frozenset({Substrate.SSH, Substrate.SLURM}), + 40, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "live-capture", + "Extended live-data collection and reporting", + frozenset({Substrate.SSH, Substrate.SLURM}), + 50, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "slurm-cartesian", + "Representative multidimensional Slurm sweep", + frozenset({Substrate.SLURM}), + 60, + ), + Scenario( + "ssh-single-big-file", + "Cooperative and staged single-file SSH workloads", + frozenset({Substrate.SSH}), + 70, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "ssh-weighted-roots", + "Weighted-root SSH workload with active sizing", + frozenset({Substrate.SSH}), + 80, + ssh_home_mode=SshHomeMode.SEPARATE, + ), + Scenario( + "ssh-shared-home", + "SSH deployment and sweep using a shared worker home", + frozenset({Substrate.SSH}), + 90, + phase=SchedulePhase.SHARED_HOME, + ssh_home_mode=SshHomeMode.SHARED, + ), + Scenario( + "slurm-scheduling", + "Slurm allocation, include/exclude, and exclusive-user behavior", + frozenset({Substrate.SLURM}), + 100, + phase=SchedulePhase.AFTER_SHARED_HOME, + ), +) + +# The harness refactor initially preserves the real baseline. Later commits +# enable catalog entries as their substrate implementations land. +SCENARIOS = SCENARIO_CATALOG[:1] + + +def _concrete_substrates(substrate: Substrate) -> frozenset[Substrate]: + """Return concrete substrates selected by *substrate*.""" + if substrate is Substrate.ALL: + return frozenset({Substrate.SSH, Substrate.SLURM}) + return frozenset({substrate}) + + +def _validate_scenario(scenario: Scenario) -> None: + """Validate one registry entry.""" + if not scenario.name or scenario.name.strip() != scenario.name: + raise ScenarioPlanningError(f"invalid scenario name: {scenario.name!r}") + if not scenario.substrates or Substrate.ALL in scenario.substrates: + raise ScenarioPlanningError( + f"scenario {scenario.name!r} must declare concrete substrates" + ) + supports_ssh = Substrate.SSH in scenario.substrates + if supports_ssh != (scenario.ssh_home_mode is not None): + raise ScenarioPlanningError( + f"scenario {scenario.name!r} must declare an SSH home mode exactly " + "when it supports SSH" + ) + if scenario.ssh_home_mode is SshHomeMode.SHARED: + if scenario.substrates != frozenset({Substrate.SSH}): + raise ScenarioPlanningError( + f"shared-home scenario {scenario.name!r} must be SSH-only" + ) + if scenario.phase is not SchedulePhase.SHARED_HOME: + raise ScenarioPlanningError( + f"shared-home scenario {scenario.name!r} must use the shared phase" + ) + elif supports_ssh and scenario.phase is not SchedulePhase.BEFORE_SHARED_HOME: + raise ScenarioPlanningError( + f"separate-home scenario {scenario.name!r} must run before shared home" + ) + elif scenario.phase is SchedulePhase.SHARED_HOME: + raise ScenarioPlanningError( + f"scenario {scenario.name!r} cannot enter the shared phase" + ) + + +def _registry_by_name(registry: Iterable[Scenario]) -> dict[str, Scenario]: + """Validate *registry* and index it by scenario name.""" + indexed: dict[str, Scenario] = {} + for scenario in registry: + _validate_scenario(scenario) + if scenario.name in indexed: + raise ScenarioPlanningError( + f"duplicate scenario in registry: {scenario.name!r}" + ) + indexed[scenario.name] = scenario + return indexed + + +def _requested_names(requested: Iterable[str]) -> tuple[str, ...]: + """Validate requested scenario names while preserving diagnostics.""" + names = tuple(requested) + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise ScenarioPlanningError( + "duplicate requested scenario(s): " + ", ".join(duplicates) + ) + return names + + +def select_scenarios( + *, + substrate: Substrate | str = Substrate.ALL, + requested: Iterable[str] = (), + registry: Iterable[Scenario] = SCENARIOS, +) -> tuple[Scenario, ...]: + """Return selected scenarios in deterministic metadata order.""" + try: + selected_substrate = Substrate(substrate) + except ValueError as error: + choices = ", ".join(item.value for item in Substrate) + raise ScenarioPlanningError( + f"unknown substrate {substrate!r}; choose one of: {choices}" + ) from error + indexed = _registry_by_name(registry) + names = _requested_names(requested) + unknown = sorted(set(names) - set(indexed)) + if unknown: + raise ScenarioPlanningError("unknown scenario(s): " + ", ".join(unknown)) + candidates = [indexed[name] for name in names] if names else list(indexed.values()) + concrete = _concrete_substrates(selected_substrate) + incompatible = [ + scenario.name + for scenario in candidates + if names and scenario.substrates.isdisjoint(concrete) + ] + if incompatible: + raise ScenarioPlanningError( + f"scenario(s) incompatible with {selected_substrate.value}: " + + ", ".join(sorted(incompatible)) + ) + selected = [ + scenario + for scenario in candidates + if not scenario.substrates.isdisjoint(concrete) + ] + return tuple(sorted(selected, key=lambda item: (item.phase, item.order, item.name))) + + +def _work_items( + scenarios: Iterable[Scenario], substrate: Substrate +) -> tuple[WorkItem, ...]: + """Expand scenarios across selected concrete substrates.""" + selected_substrates = _concrete_substrates(substrate) + items = [ + WorkItem(scenario, concrete) + for scenario in scenarios + for concrete in scenario.substrates & selected_substrates + ] + return tuple( + sorted( + items, + key=lambda item: ( + item.scenario.phase, + item.scenario.order, + item.scenario.name, + item.substrate.value, + ), + ) + ) + + +def plan_scenarios( + *, + substrate: Substrate | str = Substrate.ALL, + requested: Iterable[str] = (), + registry: Iterable[Scenario] = SCENARIOS, +) -> tuple[PlanStep, ...]: + """Build a deterministic execution plan with one shared-home batch.""" + scenarios = select_scenarios( + substrate=substrate, requested=requested, registry=registry + ) + selected_substrate = Substrate(substrate) + items = _work_items(scenarios, selected_substrate) + before = [ + item + for item in items + if item.scenario.phase is SchedulePhase.BEFORE_SHARED_HOME + ] + shared = [ + item for item in items if item.scenario.phase is SchedulePhase.SHARED_HOME + ] + after = [ + item for item in items if item.scenario.phase is SchedulePhase.AFTER_SHARED_HOME + ] + steps: list[PlanStep] = list(before) + if shared: + steps.append(SshHomeTransition(SshHomeMode.SHARED)) + steps.extend(shared) + steps.append(SshHomeTransition(SshHomeMode.SEPARATE)) + steps.extend(after) + return tuple(steps) + + +def scenario_metadata( + registry: Iterable[Scenario] = SCENARIOS, +) -> tuple[dict[str, object], ...]: + """Return deterministic machine-readable metadata for scenario listing.""" + indexed = _registry_by_name(registry) + return tuple( + scenario.metadata() + for scenario in sorted( + indexed.values(), key=lambda item: (item.phase, item.order, item.name) + ) + ) + + +def format_scenario_listing(registry: Iterable[Scenario] = SCENARIOS) -> str: + """Format a concise human-readable scenario listing.""" + indexed = _registry_by_name(registry) + scenarios = sorted( + indexed.values(), key=lambda item: (item.phase, item.order, item.name) + ) + return "\n".join( + f"{scenario.name}\t{','.join(sorted(item.value for item in scenario.substrates))}" + f"\t{scenario.description}" + for scenario in scenarios + ) diff --git a/integration-tests/lib/ssh_home_transition.py b/integration-tests/lib/ssh_home_transition.py new file mode 100644 index 0000000..9bdd607 --- /dev/null +++ b/integration-tests/lib/ssh_home_transition.py @@ -0,0 +1,465 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Crash-recoverable state for temporary SSH shared-home transitions.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Callable, Mapping +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +CANONICAL_HOME_MODE = "separate" +SHARED_HOME_MODE = "shared" +UNKNOWN_HOME_MODE = "unknown" +STATE_FILENAME = "ssh-home-transition.json" +STATE_SCHEMA = 1 +EXPECTED_POD_COUNT = 2 +MAX_FAILURE_DIAGNOSTICS = 8000 + +HomeMode = Literal["separate", "shared"] +PoolMode = Literal["separate", "shared", "unknown"] +InspectPool = Callable[[], "PoolObservation"] +ReconcilePool = Callable[[HomeMode, str], None] +Clock = Callable[[], str] + +_PHASES = { + "applying-shared", + "shared-ready", + "restoring-separate", + "unhealthy", +} +_HOME_MODES = {CANONICAL_HOME_MODE, SHARED_HOME_MODE} +_POOL_MODES = {*_HOME_MODES, UNKNOWN_HOME_MODE} + + +class SshHomeTransitionError(RuntimeError): + """An actionable SSH home-mode transition failure.""" + + +@dataclass(frozen=True) +class PoolObservation: + """Relevant live state of the SSH worker StatefulSet and its pods.""" + + home_mode: str + configuration_checksum: str + ready_nonterminating_pods: int + terminating_pods: int + diagnostics: str = "" + + def matches(self, mode: HomeMode, checksum: str) -> bool: + """Return whether the live pool is ready in the requested form.""" + return ( + self.home_mode == mode + and self.configuration_checksum == checksum + and self.ready_nonterminating_pods == EXPECTED_POD_COUNT + and self.terminating_pods == 0 + ) + + +@dataclass(frozen=True) +class TransitionState: + """Persistent identity and progress for one SSH pool transition.""" + + schema: int + run_id: str + scenario_id: str + prior_mode: PoolMode + requested_mode: HomeMode + expected_statefulset_checksum: str + phase: str + updated_at: str + failure_diagnostics: str = "" + + @classmethod + def from_document(cls, document: Mapping[str, Any]) -> "TransitionState": + """Validate and decode a persistent transition document.""" + required = { + "schema", + "run_id", + "scenario_id", + "prior_mode", + "requested_mode", + "expected_statefulset_checksum", + "phase", + "updated_at", + "failure_diagnostics", + } + if set(document) != required: + raise SshHomeTransitionError( + "invalid SSH home transition state fields: " + f"expected {sorted(required)}, found {sorted(document)}" + ) + state = cls(**document) + state.validate() + return state + + def validate(self) -> None: + """Reject a state document that cannot safely drive reconciliation.""" + if self.schema != STATE_SCHEMA: + raise SshHomeTransitionError( + f"unsupported SSH home transition schema: {self.schema!r}" + ) + if self.prior_mode not in _POOL_MODES or self.requested_mode not in _HOME_MODES: + raise SshHomeTransitionError("invalid SSH home mode in transition state") + if self.phase not in _PHASES: + raise SshHomeTransitionError( + f"invalid SSH home transition phase: {self.phase!r}" + ) + text_fields = { + "run_id": self.run_id, + "scenario_id": self.scenario_id, + "expected_statefulset_checksum": self.expected_statefulset_checksum, + "updated_at": self.updated_at, + } + empty = [name for name, value in text_fields.items() if not _valid_text(value)] + if empty: + raise SshHomeTransitionError( + "empty or invalid SSH home transition fields: " + ", ".join(empty) + ) + + +class TransitionStore: + """Atomically persist the SSH transition record in an existing state dir.""" + + def __init__(self, state_dir: Path): + self.state_dir = state_dir + self.path = state_dir / STATE_FILENAME + + def load(self) -> TransitionState | None: + """Return validated transition state, if present.""" + if not self.path.exists(): + return None + try: + document = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SshHomeTransitionError( + f"cannot read SSH home transition state {self.path}: {error}" + ) from error + if not isinstance(document, dict): + raise SshHomeTransitionError( + f"invalid SSH home transition state in {self.path}: expected object" + ) + try: + return TransitionState.from_document(document) + except (TypeError, SshHomeTransitionError) as error: + raise SshHomeTransitionError( + f"invalid SSH home transition state in {self.path}: {error}" + ) from error + + def save(self, state: TransitionState) -> None: + """Atomically replace the transition document.""" + state.validate() + self._require_state_dir() + payload = json.dumps(asdict(state), indent=2, sort_keys=True) + "\n" + descriptor, temporary_name = tempfile.mkstemp( + dir=self.state_dir, prefix=f".{STATE_FILENAME}." + ) + temporary = Path(temporary_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self.path) + _sync_directory(self.state_dir) + finally: + temporary.unlink(missing_ok=True) + + def clear(self) -> None: + """Remove a completed transition record durably.""" + if not self.path.exists(): + return + self.path.unlink() + _sync_directory(self.state_dir) + + def _require_state_dir(self) -> None: + """Refuse to create or adopt lifecycle state implicitly.""" + if not self.state_dir.is_dir(): + raise SshHomeTransitionError( + f"integration state directory is absent: {self.state_dir}" + ) + + +class SshHomeTransitionManager: + """Drive and recover SSH home transitions through injected live hooks. + + ``reconcile_pool`` must apply the requested StatefulSet form, wait for two + ready nonterminating pods, regenerate known-host data, and validate SSH and + home semantics. Callers remain responsible for serializing fixture actions. + """ + + def __init__( + self, + state_dir: Path, + inspect_pool: InspectPool, + reconcile_pool: ReconcilePool, + *, + clock: Clock | None = None, + ): + self.store = TransitionStore(state_dir) + self.inspect_pool = inspect_pool + self.reconcile_pool = reconcile_pool + self.clock = clock or _utc_now + + def preflight( + self, + separate_checksum: str, + *, + run_id: str, + scenario_id: str, + ) -> PoolObservation: + """Ensure canonical separate-home state before an SSH setup or test.""" + retained = self.store.load() + try: + observed = self.inspect_pool() + except Exception as error: + self._save_unhealthy( + retained, + run_id=run_id, + scenario_id=scenario_id, + prior_mode=UNKNOWN_HOME_MODE, + requested_mode=CANONICAL_HOME_MODE, + checksum=separate_checksum, + error=error, + ) + raise self._blocked_error(error) from error + if retained is None and observed.matches( + CANONICAL_HOME_MODE, separate_checksum + ): + return observed + identity = retained or _new_state( + run_id, + scenario_id, + _known_mode(observed.home_mode), + CANONICAL_HOME_MODE, + separate_checksum, + "restoring-separate", + self.clock(), + ) + return self._reconcile_separate(identity, separate_checksum) + + def enter_shared( + self, + *, + run_id: str, + scenario_id: str, + separate_checksum: str, + shared_checksum: str, + ) -> PoolObservation: + """Enter shared-home mode after first recovering canonical state.""" + self.preflight(separate_checksum, run_id=run_id, scenario_id=scenario_id) + state = _new_state( + run_id, + scenario_id, + CANONICAL_HOME_MODE, + SHARED_HOME_MODE, + shared_checksum, + "applying-shared", + self.clock(), + ) + self.store.save(state) + try: + self.reconcile_pool(SHARED_HOME_MODE, shared_checksum) + observed = self.inspect_pool() + _require_observation(observed, SHARED_HOME_MODE, shared_checksum) + except Exception as error: + self._save_unhealthy_from_state(state, error) + raise self._blocked_error(error) from error + self.store.save(replace(state, phase="shared-ready", updated_at=self.clock())) + return observed + + def restore_separate( + self, + separate_checksum: str, + *, + run_id: str, + scenario_id: str, + ) -> PoolObservation: + """Restore canonical state after the complete shared-home batch.""" + retained = self.store.load() + identity = retained or _new_state( + run_id, + scenario_id, + SHARED_HOME_MODE, + CANONICAL_HOME_MODE, + separate_checksum, + "restoring-separate", + self.clock(), + ) + return self._reconcile_separate(identity, separate_checksum) + + def _reconcile_separate( + self, identity: TransitionState, checksum: str + ) -> PoolObservation: + state = replace( + identity, + requested_mode=CANONICAL_HOME_MODE, + expected_statefulset_checksum=checksum, + phase="restoring-separate", + updated_at=self.clock(), + failure_diagnostics="", + ) + self.store.save(state) + try: + self.reconcile_pool(CANONICAL_HOME_MODE, checksum) + observed = self.inspect_pool() + _require_observation(observed, CANONICAL_HOME_MODE, checksum) + except Exception as error: + self._save_unhealthy_from_state(state, error) + raise self._blocked_error(error) from error + self.store.clear() + return observed + + def _save_unhealthy_from_state( + self, state: TransitionState, error: Exception + ) -> None: + self.store.save( + replace( + state, + phase="unhealthy", + updated_at=self.clock(), + failure_diagnostics=_failure_text(error), + ) + ) + + def _save_unhealthy( + self, + retained: TransitionState | None, + *, + run_id: str, + scenario_id: str, + prior_mode: PoolMode, + requested_mode: HomeMode, + checksum: str, + error: Exception, + ) -> None: + state = ( + replace( + retained, + requested_mode=requested_mode, + expected_statefulset_checksum=checksum, + ) + if retained + else _new_state( + run_id, + scenario_id, + prior_mode, + requested_mode, + checksum, + "unhealthy", + self.clock(), + ) + ) + self._save_unhealthy_from_state(state, error) + + def _blocked_error(self, error: Exception) -> SshHomeTransitionError: + return SshHomeTransitionError( + "SSH worker pool reconciliation failed; further SSH scenarios are " + f"blocked. Retry setup to reconcile separate-home mode or teardown " + f"the fixture. State and diagnostics: {self.store.path}: {error}" + ) + + +def statefulset_checksum(manifest: str | bytes) -> str: + """Return a stable digest for one rendered StatefulSet configuration.""" + data = manifest.encode("utf-8") if isinstance(manifest, str) else manifest + return hashlib.sha256(data).hexdigest() + + +def _new_state( + run_id: str, + scenario_id: str, + prior_mode: PoolMode, + requested_mode: HomeMode, + checksum: str, + phase: str, + updated_at: str, +) -> TransitionState: + """Build and validate one transition state value.""" + state = TransitionState( + schema=STATE_SCHEMA, + run_id=run_id, + scenario_id=scenario_id, + prior_mode=prior_mode, + requested_mode=requested_mode, + expected_statefulset_checksum=checksum, + phase=phase, + updated_at=updated_at, + ) + state.validate() + return state + + +def _known_mode(value: str) -> PoolMode: + """Use a safe prior mode when live inspection finds a partial rollout.""" + if value == CANONICAL_HOME_MODE: + return CANONICAL_HOME_MODE + if value == SHARED_HOME_MODE: + return SHARED_HOME_MODE + return UNKNOWN_HOME_MODE + + +def _require_observation( + observed: PoolObservation, mode: HomeMode, checksum: str +) -> None: + """Require reconciliation to produce the exact healthy target form.""" + if observed.matches(mode, checksum): + return + detail = f"; {observed.diagnostics}" if observed.diagnostics else "" + raise SshHomeTransitionError( + "SSH worker pool did not reach the requested state: " + f"mode={observed.home_mode!r}, checksum=" + f"{observed.configuration_checksum!r}, ready=" + f"{observed.ready_nonterminating_pods}, terminating=" + f"{observed.terminating_pods}{detail}" + ) + + +def _failure_text(error: Exception) -> str: + """Return bounded persistent diagnostics for a reconciliation failure.""" + text = f"{type(error).__name__}: {error}".strip() + return text[-MAX_FAILURE_DIAGNOSTICS:] + + +def _valid_text(value: object) -> bool: + """Return whether a required document field is a nonempty string.""" + return isinstance(value, str) and bool(value.strip()) + + +def _utc_now() -> str: + """Return an ISO-8601 timestamp for persistent diagnostics.""" + return datetime.now(timezone.utc).isoformat() + + +def _sync_directory(directory: Path) -> None: + """Best-effort fsync of a directory after atomic state mutation.""" + try: + descriptor = os.open(directory, os.O_RDONLY) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) diff --git a/integration-tests/manifests/ssh-workers.yaml.tmpl b/integration-tests/manifests/ssh-workers.yaml.tmpl index 541341f..17f0c4e 100644 --- a/integration-tests/manifests/ssh-workers.yaml.tmpl +++ b/integration-tests/manifests/ssh-workers.yaml.tmpl @@ -32,6 +32,9 @@ kind: StatefulSet metadata: name: ssh-worker namespace: @@NAMESPACE@@ + annotations: + storage-scale-test/ssh-home-mode: "@@SSH_HOME_MODE@@" + storage-scale-test/ssh-config-checksum: "@@SSH_CONFIG_CHECKSUM@@" spec: serviceName: ssh-workers replicas: 2 @@ -42,6 +45,9 @@ spec: metadata: labels: app.kubernetes.io/name: storage-ssh-worker + annotations: + storage-scale-test/ssh-home-mode: "@@SSH_HOME_MODE@@" + storage-scale-test/ssh-config-checksum: "@@SSH_CONFIG_CHECKSUM@@" spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet diff --git a/tests/test_filesystem_scenarios.py b/tests/test_filesystem_scenarios.py new file mode 100644 index 0000000..bdf82af --- /dev/null +++ b/tests/test_filesystem_scenarios.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for filesystem integration scenario selection and scheduling.""" + +import importlib.util +import itertools +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_MODULE_PATH = _REPO_ROOT / "integration-tests" / "lib" / "scenario_planner.py" +_SPEC = importlib.util.spec_from_file_location( + "filesystem_scenarios_under_test", _MODULE_PATH +) +assert _SPEC and _SPEC.loader +_SCENARIOS = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _SCENARIOS +_SPEC.loader.exec_module(_SCENARIOS) + +Scenario = _SCENARIOS.Scenario +ScenarioPlanningError = _SCENARIOS.ScenarioPlanningError +SchedulePhase = _SCENARIOS.SchedulePhase +SshHomeMode = _SCENARIOS.SshHomeMode +SshHomeTransition = _SCENARIOS.SshHomeTransition +Substrate = _SCENARIOS.Substrate +WorkItem = _SCENARIOS.WorkItem +plan_scenarios = _SCENARIOS.plan_scenarios +format_scenario_listing = _SCENARIOS.format_scenario_listing +scenario_metadata = _SCENARIOS.scenario_metadata +select_scenarios = _SCENARIOS.select_scenarios + + +def _step_identity(step): + """Return a compact identity for a plan step.""" + if isinstance(step, WorkItem): + return ("work", step.scenario.name, step.substrate.value) + return ("transition", step.target.value) + + +def _plan_identity(**kwargs): + """Return compact identities for a scenario plan.""" + kwargs.setdefault("registry", _SCENARIOS.SCENARIO_CATALOG) + return tuple(_step_identity(step) for step in plan_scenarios(**kwargs)) + + +def test_catalog_exposes_stable_machine_metadata(): + """Listing metadata describes selection without freezing prose.""" + metadata = scenario_metadata(_SCENARIOS.SCENARIO_CATALOG) + + assert [item["name"] for item in metadata] == [ + "baseline", + "default-dio", + "failure-resume", + "retained-lifecycle", + "live-capture", + "slurm-cartesian", + "ssh-single-big-file", + "ssh-weighted-roots", + "ssh-shared-home", + "slurm-scheduling", + ] + shared = next(item for item in metadata if item["name"] == "ssh-shared-home") + assert shared["substrates"] == ["ssh"] + assert shared["ssh_home_mode"] == "shared" + assert shared["schedule_phase"] == "shared_home" + + +def test_human_listing_exposes_names_and_substrate_compatibility(): + """Human listing includes stable selection facts without freezing prose.""" + rows = { + fields[0]: fields[1] + for line in format_scenario_listing(_SCENARIOS.SCENARIO_CATALOG).splitlines() + if (fields := line.split("\t", 2)) + } + + assert rows["baseline"] == "slurm,ssh" + assert rows["ssh-shared-home"] == "ssh" + assert rows["slurm-scheduling"] == "slurm" + + +def test_default_plan_expands_substrates_and_batches_shared_home(): + """The full plan has one bounded shared-home SSH transition.""" + plan = _plan_identity() + + shared_start = plan.index(("transition", "shared")) + shared_stop = plan.index(("transition", "separate")) + assert plan[shared_start + 1 : shared_stop] == (("work", "ssh-shared-home", "ssh"),) + assert plan[shared_stop + 1 :] == (("work", "slurm-scheduling", "slurm"),) + assert all( + substrate != "ssh" + for kind, _name, substrate in plan[:shared_start] + if kind == "work" and _name == "ssh-shared-home" + ) + + +def test_ssh_selection_excludes_slurm_work(): + """An SSH plan contains only SSH work and still restores home mode.""" + plan = plan_scenarios(substrate="ssh", registry=_SCENARIOS.SCENARIO_CATALOG) + + work = [step for step in plan if isinstance(step, WorkItem)] + assert work + assert {item.substrate for item in work} == {Substrate.SSH} + assert isinstance(plan[-1], SshHomeTransition) + assert plan[-1].target is SshHomeMode.SEPARATE + + +def test_slurm_selection_needs_no_home_transition(): + """A Slurm-only plan cannot mutate the SSH worker pool.""" + plan = plan_scenarios( + substrate=Substrate.SLURM, registry=_SCENARIOS.SCENARIO_CATALOG + ) + + assert plan + assert all(isinstance(step, WorkItem) for step in plan) + assert {step.substrate for step in plan} == {Substrate.SLURM} + + +def test_requested_scenarios_are_order_independent(): + """Repeatable CLI selections do not determine execution order.""" + names = ("slurm-scheduling", "baseline", "ssh-shared-home") + expected = _plan_identity(requested=names) + + for permutation in itertools.permutations(names): + assert _plan_identity(requested=permutation) == expected + + +def test_registry_order_does_not_determine_execution_order(): + """Registry refactoring cannot silently change execution order.""" + expected = _plan_identity() + + assert _plan_identity(registry=reversed(_SCENARIOS.SCENARIO_CATALOG)) == expected + + +@pytest.mark.parametrize("substrate", ("ssh", "slurm")) +def test_explicit_incompatible_scenario_is_rejected(substrate): + """An explicit scenario/substrate mismatch is actionable.""" + scenario = "slurm-cartesian" if substrate == "ssh" else "ssh-shared-home" + + with pytest.raises(ScenarioPlanningError, match="incompatible"): + plan_scenarios( + substrate=substrate, + requested=(scenario,), + registry=_SCENARIOS.SCENARIO_CATALOG, + ) + + +def test_duplicate_and_unknown_requests_are_rejected(): + """Typos and duplicate repeatable options do not produce surprising work.""" + with pytest.raises(ScenarioPlanningError, match="duplicate requested"): + select_scenarios(requested=("baseline", "baseline")) + with pytest.raises(ScenarioPlanningError, match="unknown scenario"): + select_scenarios(requested=("not-a-scenario",)) + + +def test_duplicate_registry_names_are_rejected(): + """The registry has one authoritative definition per name.""" + duplicate = replace(_SCENARIOS.SCENARIO_CATALOG[0], description="Duplicate") + + with pytest.raises(ScenarioPlanningError, match="duplicate scenario in registry"): + scenario_metadata((*_SCENARIOS.SCENARIO_CATALOG, duplicate)) + + +def test_shared_home_metadata_constraints_are_enforced(): + """Shared mode cannot leak outside one SSH-only scheduling phase.""" + invalid = Scenario( + "invalid-shared", + "Invalid", + frozenset({Substrate.SSH, Substrate.SLURM}), + 1, + phase=SchedulePhase.SHARED_HOME, + ssh_home_mode=SshHomeMode.SHARED, + ) + + with pytest.raises(ScenarioPlanningError, match="must be SSH-only"): + scenario_metadata((invalid,)) + + +def test_selected_scenarios_follow_explicit_priority_and_tiebreaker(): + """Same-phase scenarios use order then stable name as tie-breakers.""" + later_name = Scenario( + "zeta", + "Zeta", + frozenset({Substrate.SLURM}), + 5, + ) + earlier_name = replace(later_name, name="alpha", description="Alpha") + + selected = select_scenarios(registry=(later_name, earlier_name)) + + assert [scenario.name for scenario in selected] == ["alpha", "zeta"] + + +def test_all_selector_filters_no_scenarios(): + """The all selector is accepted only as a request, not scenario metadata.""" + selected = select_scenarios(substrate=Substrate.ALL) + + assert len(selected) == len(_SCENARIOS.SCENARIOS) + invalid = Scenario( + "all-substrate", + "Invalid", + frozenset({Substrate.ALL}), + 1, + ) + with pytest.raises(ScenarioPlanningError, match="concrete substrates"): + scenario_metadata((invalid,)) + + +def test_unknown_substrate_is_actionable(): + """Planner wraps invalid substrate values in its public error type.""" + with pytest.raises(ScenarioPlanningError, match="unknown substrate"): + plan_scenarios(substrate="kubernetes") diff --git a/tests/test_integration_deployment_cache.py b/tests/test_integration_deployment_cache.py new file mode 100644 index 0000000..7b8313a --- /dev/null +++ b/tests/test_integration_deployment_cache.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for content-addressed integration deployments.""" + +import importlib.util +import json +import os +import subprocess +import sys +import tarfile +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_MODULE_PATH = _REPO_ROOT / "integration-tests" / "lib" / "deployment_cache.py" +_SPEC = importlib.util.spec_from_file_location( + "integration_deployment_cache_under_test", _MODULE_PATH +) +assert _SPEC and _SPEC.loader +_CACHE = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _CACHE +_SPEC.loader.exec_module(_CACHE) + + +class _Runner: + """Run local commands and record deployment-builder invocations.""" + + def __init__(self): + self.builder_calls = 0 + + def run(self, arguments, *, cwd=None, timeout=None): + """Run one command using the integration runner's result shape.""" + command = [str(argument) for argument in arguments] + if command[0].endswith("build_tarball.sh"): + self.builder_calls += 1 + result = subprocess.run( + command, + cwd=cwd, + timeout=timeout, + text=True, + capture_output=True, + check=False, + ) + if result.returncode: + raise RuntimeError( + f"command failed ({result.returncode}): {command}\n{result.stderr}" + ) + return SimpleNamespace(stdout=result.stdout, stderr=result.stderr) + + +def _write(path: Path, content: str, mode: int = 0o644) -> None: + """Write one fixture file with a controlled mode.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + path.chmod(mode) + + +def _repository(tmp_path: Path) -> tuple[Path, Path]: + """Create a minimal tracked tree with a compatible archive builder.""" + repository = tmp_path / "repository" + repository.mkdir() + _write(repository / "NOTICE", "notice\n") + _write(repository / "payload.txt", "first\n") + _write( + repository / "utils" / "build_tarball.sh", + """#!/usr/bin/env bash +set -eu +while [[ $# -gt 0 ]]; do + case "$1" in + --arch) shift 2 ;; + --skip-object-tools|--fixture-option) shift ;; + *) exit 64 ;; + esac +done +tar -czf ../storage-scale-test.tar.gz . +""", + 0o755, + ) + subprocess.run(["git", "init", "-q"], cwd=repository, check=True) + subprocess.run(["git", "add", "."], cwd=repository, check=True) + binary = tmp_path / "elbencho" + _write(binary, "#!/usr/bin/env bash\nexit 0\n", 0o755) + return repository, binary + + +def _request(tmp_path: Path, repository: Path, binary: Path): + """Return a standard request for one fixture deployment.""" + return _CACHE.DeploymentCacheRequest( + repo_root=repository, + cache_root=tmp_path / "cache", + architecture="x86_64", + binary=binary, + binary_name="elbencho", + ) + + +def _manifest(deployment) -> dict[str, object]: + """Read a cached deployment's published manifest.""" + return json.loads(deployment.manifest.read_text(encoding="utf-8")) + + +def test_reuses_verified_archive_for_identical_inputs(tmp_path): + """A second request reuses the archive built from the same snapshot.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + + first = _CACHE.get_or_build_deployment(runner, request) + second = _CACHE.get_or_build_deployment(runner, request) + + assert not first.cache_hit + assert second.cache_hit + assert first.archive == second.archive + assert runner.builder_calls == 1 + document = _manifest(first) + assert document["input_digest"] == first.key + assert document["archive"]["sha256"] + assert document["identity"]["binary"]["path"] == "utils/elbencho" + + +def test_snapshot_uses_current_tracked_content_and_excludes_untracked(tmp_path): + """The archive reflects tracked edits without admitting untracked files.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + _write(repository / "payload.txt", "modified\n") + _write(repository / "untracked.txt", "do not package\n") + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + with tarfile.open(second.archive, "r:gz") as archive: + names = archive.getnames() + payload = archive.extractfile("./payload.txt") + assert payload is not None + assert payload.read() == b"modified\n" + assert "./untracked.txt" not in names + + +def test_file_mode_participates_in_cache_identity(tmp_path): + """Changing a tracked mode invalidates an otherwise identical snapshot.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + (repository / "payload.txt").chmod(0o600) + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + entries = _manifest(second)["identity"]["source"]["entries"] + payload = next(entry for entry in entries if entry["path"] == "payload.txt") + assert payload["mode"] == 0o600 + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("architecture", "aarch64"), + ("build_options", ("--skip-object-tools", "--fixture-option")), + ("recipe", 2), + ), +) +def test_build_identity_options_invalidate_cache(tmp_path, field, value): + """Architecture, builder options, and recipe are cache-key inputs.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + + second = _CACHE.get_or_build_deployment(runner, replace(request, **{field: value})) + + assert first.key != second.key + assert runner.builder_calls == 2 + + +def test_binary_and_runtime_content_invalidate_cache(tmp_path): + """External executable and runtime identities participate in the key.""" + repository, binary = _repository(tmp_path) + runtime = tmp_path / "runtime" + _write(runtime / "lib" / "loader.so", "first runtime\n") + runner = _Runner() + request = replace(_request(tmp_path, repository, binary), runtime=runtime) + first = _CACHE.get_or_build_deployment(runner, request) + _write(binary, "#!/usr/bin/env bash\necho changed\n", 0o755) + + second = _CACHE.get_or_build_deployment(runner, request) + _write(runtime / "lib" / "loader.so", "second runtime\n") + third = _CACHE.get_or_build_deployment(runner, request) + + assert len({first.key, second.key, third.key}) == 3 + + +def test_runtime_directory_mode_participates_in_cache_identity(tmp_path): + """Runtime directory layout and modes cannot alias one cache entry.""" + repository, binary = _repository(tmp_path) + runtime = tmp_path / "runtime" + (runtime / "empty").mkdir(parents=True) + runner = _Runner() + request = replace(_request(tmp_path, repository, binary), runtime=runtime) + first = _CACHE.get_or_build_deployment(runner, request) + (runtime / "empty").chmod(0o700) + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + + +def test_unsafe_binary_name_is_rejected(tmp_path): + """A configured binary name cannot escape the snapshot's utils directory.""" + repository, binary = _repository(tmp_path) + request = replace( + _request(tmp_path, repository, binary), binary_name="../../outside" + ) + + with pytest.raises(_CACHE.DeploymentCacheError, match="not a basename"): + _CACHE.get_or_build_deployment(_Runner(), request) + + +def test_corrupt_archive_is_rebuilt_under_same_key(tmp_path): + """Checksum validation prevents reuse of a damaged cached archive.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + first.archive.write_bytes(b"corrupt") + + rebuilt = _CACHE.get_or_build_deployment(runner, request) + + assert rebuilt.key == first.key + assert not rebuilt.cache_hit + assert runner.builder_calls == 2 + assert rebuilt.archive.read_bytes() != b"corrupt" + assert not any( + path.name.startswith(".invalid-") for path in request.cache_root.iterdir() + ) + + +def test_builder_receives_snapshot_not_working_tree(tmp_path): + """The existing builder executes only from the manifested snapshot.""" + repository, binary = _repository(tmp_path) + + class SnapshotRunner(_Runner): + """Record the working directory used for the archive builder.""" + + def __init__(self): + super().__init__() + self.builder_cwd = None + + def run(self, arguments, *, cwd=None, timeout=None): + if str(arguments[0]).endswith("build_tarball.sh"): + self.builder_cwd = Path(cwd) + assert self.builder_cwd != repository + assert self.builder_cwd.name == "source" + return super().run(arguments, cwd=cwd, timeout=timeout) + + runner = SnapshotRunner() + + _CACHE.get_or_build_deployment(runner, _request(tmp_path, repository, binary)) + + assert runner.builder_cwd is not None + + +def test_unsafe_tracked_symlink_is_rejected(tmp_path): + """A tracked symbolic link cannot escape the immutable snapshot.""" + repository, binary = _repository(tmp_path) + os.symlink("../outside", repository / "escape") + subprocess.run(["git", "add", "escape"], cwd=repository, check=True) + + with pytest.raises(_CACHE.DeploymentCacheError, match="unsafe tracked symbolic"): + _CACHE.get_or_build_deployment( + _Runner(), _request(tmp_path, repository, binary) + ) diff --git a/tests/test_integration_driver_safety.py b/tests/test_integration_driver_safety.py index d985b32..e280a66 100644 --- a/tests/test_integration_driver_safety.py +++ b/tests/test_integration_driver_safety.py @@ -19,6 +19,7 @@ import importlib.util import json +import os import sys from dataclasses import replace from pathlib import Path @@ -45,6 +46,24 @@ _pods_with_container = getattr(_FILESYSTEM, "_pods_with_container") +def test_scenario_listing_short_circuits_before_privileged_state(monkeypatch, capsys): + """Listing scenarios needs no account, state, fixture, or root exception.""" + monkeypatch.setattr( + sys, + "argv", + [str(_DRIVER_PATH), "test", "--list-scenarios"], + ) + monkeypatch.setattr(os, "geteuid", lambda: 0) + monkeypatch.setattr( + _DRIVER, + "_config", + lambda _arguments: pytest.fail("scenario listing resolved an account"), + ) + + assert _DRIVER.main() == 0 + assert capsys.readouterr().out.startswith("baseline\t") + + def _config(state_dir: Path, export_dir: Path) -> object: """Return a minimal real driver configuration.""" return _DRIVER.Config( @@ -54,7 +73,6 @@ def _config(state_dir: Path, export_dir: Path) -> object: export_dir=export_dir, storage_backend="sbx-shared", sbx_shared_root=_REPO_ROOT / "tmp" / "test-shared", - ssh_home_mode="separate", test_user="tester", test_uid=2000, test_gid=2000, diff --git a/tests/test_ssh_home_transition.py b/tests/test_ssh_home_transition.py new file mode 100644 index 0000000..5355f39 --- /dev/null +++ b/tests/test_ssh_home_transition.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for crash-recoverable SSH home transitions.""" + +import json +import sys +from pathlib import Path + +import pytest + +_REPOSITORY_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_REPOSITORY_ROOT / "integration-tests" / "lib")) + +from ssh_home_transition import ( # pylint: disable=wrong-import-position + STATE_FILENAME, + PoolObservation, + SshHomeTransitionError, + SshHomeTransitionManager, + TransitionState, + TransitionStore, + statefulset_checksum, +) + +SEPARATE_CHECKSUM = "separate-checksum" +SHARED_CHECKSUM = "shared-checksum" +FIXED_TIME = "2026-09-19T00:00:00+00:00" + + +def _observation( + mode="separate", + checksum=SEPARATE_CHECKSUM, + ready=2, + terminating=0, + diagnostics="", +): + """Return one compact live-pool observation.""" + return PoolObservation(mode, checksum, ready, terminating, diagnostics) + + +class _Controller: + """Injectable live pool with observable reconciliation calls.""" + + def __init__(self, state_dir, observed=None): + self.state_dir = state_dir + self.observed = observed or _observation() + self.calls = [] + self.fail = None + self.state_during_reconcile = None + + def inspect(self): + """Return the current simulated StatefulSet and pod state.""" + return self.observed + + def reconcile(self, mode, checksum): + """Record and apply one simulated StatefulSet reconciliation.""" + self.calls.append((mode, checksum)) + state_path = self.state_dir / STATE_FILENAME + self.state_during_reconcile = json.loads(state_path.read_text(encoding="utf-8")) + if self.fail: + raise self.fail + self.observed = _observation(mode, checksum) + + +def _manager(tmp_path, controller): + """Return a deterministic transition manager.""" + return SshHomeTransitionManager( + tmp_path, + controller.inspect, + controller.reconcile, + clock=lambda: FIXED_TIME, + ) + + +def _state( + *, + phase="applying-shared", + requested_mode="shared", + checksum=SHARED_CHECKSUM, +): + """Return one valid retained state document.""" + return TransitionState( + schema=1, + run_id="run-1", + scenario_id="ssh-shared-home", + prior_mode="separate", + requested_mode=requested_mode, + expected_statefulset_checksum=checksum, + phase=phase, + updated_at=FIXED_TIME, + ) + + +def test_clean_canonical_preflight_does_not_roll_out_workers(tmp_path): + """A healthy canonical pool is accepted without replacing SSH pods.""" + controller = _Controller(tmp_path) + + observed = _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="setup", scenario_id="preflight" + ) + + assert observed.matches("separate", SEPARATE_CHECKSUM) + assert controller.calls == [] + assert not (tmp_path / STATE_FILENAME).exists() + + +@pytest.mark.parametrize( + "observed", + [ + _observation("shared", SHARED_CHECKSUM), + _observation("separate", "stale-separate-checksum"), + _observation("separate", SEPARATE_CHECKSUM, terminating=1), + _observation("unknown", "partial", ready=1), + ], +) +def test_preflight_reconciles_noncanonical_live_state(tmp_path, observed): + """Setup and SSH preflight restore any partial or shared pool.""" + controller = _Controller(tmp_path, observed) + + result = _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-2", scenario_id="preflight" + ) + + assert result.matches("separate", SEPARATE_CHECKSUM) + assert controller.calls == [("separate", SEPARATE_CHECKSUM)] + assert controller.state_during_reconcile["phase"] == "restoring-separate" + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_interrupted_transition_forces_validation_rollout(tmp_path): + """Retained intent forces reconciliation even if pods look canonical.""" + TransitionStore(tmp_path).save(_state()) + controller = _Controller(tmp_path) + + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="new-run", scenario_id="preflight" + ) + + assert controller.calls == [("separate", SEPARATE_CHECKSUM)] + assert controller.state_during_reconcile["run_id"] == "run-1" + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_enter_shared_persists_intent_before_rollout(tmp_path): + """A crash during rollout leaves enough state for the next preflight.""" + controller = _Controller(tmp_path) + + observed = _manager(tmp_path, controller).enter_shared( + run_id="run-3", + scenario_id="ssh-shared-home", + separate_checksum=SEPARATE_CHECKSUM, + shared_checksum=SHARED_CHECKSUM, + ) + + assert observed.matches("shared", SHARED_CHECKSUM) + assert controller.calls == [("shared", SHARED_CHECKSUM)] + assert controller.state_during_reconcile["phase"] == "applying-shared" + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "shared-ready" + assert retained.run_id == "run-3" + + +def test_restore_clears_state_only_after_canonical_validation(tmp_path): + """The transition record remains present while restoration runs.""" + TransitionStore(tmp_path).save(_state(phase="shared-ready")) + controller = _Controller(tmp_path, _observation("shared", SHARED_CHECKSUM)) + + observed = _manager(tmp_path, controller).restore_separate( + SEPARATE_CHECKSUM, run_id="run-3", scenario_id="ssh-shared-home" + ) + + assert observed.matches("separate", SEPARATE_CHECKSUM) + assert controller.state_during_reconcile["phase"] == "restoring-separate" + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_failed_reconciliation_persists_actionable_unhealthy_state(tmp_path): + """A rollout failure blocks SSH work and survives another invocation.""" + controller = _Controller(tmp_path, _observation("shared", SHARED_CHECKSUM)) + controller.fail = RuntimeError("rollout timed out") + + with pytest.raises(SshHomeTransitionError, match="further SSH scenarios"): + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-4", scenario_id="preflight" + ) + + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "unhealthy" + assert retained.prior_mode == "shared" + assert retained.requested_mode == "separate" + assert "rollout timed out" in retained.failure_diagnostics + + +def test_later_preflight_retries_unhealthy_reconciliation(tmp_path): + """A later setup can recover an SSH fixture marked unhealthy.""" + TransitionStore(tmp_path).save( + _state( + phase="unhealthy", + requested_mode="separate", + checksum=SEPARATE_CHECKSUM, + ) + ) + controller = _Controller(tmp_path) + + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-5", scenario_id="preflight" + ) + + assert controller.calls == [("separate", SEPARATE_CHECKSUM)] + assert not (tmp_path / STATE_FILENAME).exists() + + +def test_inspection_failure_retains_canonical_recovery_target(tmp_path): + """A failed preflight still records that separate mode must be restored.""" + + def failed_inspection(): + raise RuntimeError("Kubernetes API unavailable") + + manager = SshHomeTransitionManager( + tmp_path, + failed_inspection, + lambda _mode, _checksum: None, + clock=lambda: FIXED_TIME, + ) + + with pytest.raises(SshHomeTransitionError, match="blocked"): + manager.preflight(SEPARATE_CHECKSUM, run_id="run-6", scenario_id="preflight") + + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "unhealthy" + assert retained.prior_mode == "unknown" + assert retained.requested_mode == "separate" + assert retained.expected_statefulset_checksum == SEPARATE_CHECKSUM + assert "Kubernetes API unavailable" in retained.failure_diagnostics + + +def test_post_reconcile_observation_must_be_ready_and_nonterminating(tmp_path): + """A callback cannot claim success while rollout overlap remains.""" + controller = _Controller(tmp_path, _observation("shared", SHARED_CHECKSUM)) + + def incomplete_reconcile(_mode, _checksum): + controller.observed = _observation( + "separate", SEPARATE_CHECKSUM, ready=2, terminating=1 + ) + + manager = SshHomeTransitionManager( + tmp_path, + controller.inspect, + incomplete_reconcile, + clock=lambda: FIXED_TIME, + ) + + with pytest.raises(SshHomeTransitionError, match="blocked"): + manager.preflight(SEPARATE_CHECKSUM, run_id="run-6", scenario_id="preflight") + + retained = TransitionStore(tmp_path).load() + assert retained is not None + assert retained.phase == "unhealthy" + assert "terminating=1" in retained.failure_diagnostics + + +def test_invalid_state_is_not_silently_replaced(tmp_path): + """Malformed ownership state requires diagnosis instead of blind cleanup.""" + (tmp_path / STATE_FILENAME).write_text('{"schema": 99}\n', encoding="utf-8") + controller = _Controller(tmp_path) + + with pytest.raises(SshHomeTransitionError, match="invalid SSH home transition"): + _manager(tmp_path, controller).preflight( + SEPARATE_CHECKSUM, run_id="run-7", scenario_id="preflight" + ) + + assert controller.calls == [] + + +def test_state_store_requires_existing_state_directory(tmp_path): + """Transition code never creates or adopts lifecycle state directories.""" + absent = tmp_path / "absent" + + with pytest.raises(SshHomeTransitionError, match="state directory is absent"): + TransitionStore(absent).save(_state()) + + +def test_statefulset_checksum_uses_rendered_bytes(): + """Configuration identities change with the rendered StatefulSet form.""" + assert statefulset_checksum("emptyDir: {}") == statefulset_checksum(b"emptyDir: {}") + assert statefulset_checksum("emptyDir: {}") != statefulset_checksum( + "persistentVolumeClaim: {}" + ) From b0f97a36bee3b0b076163a39feaf88596e3e27e2 Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sat, 19 Sep 2026 21:40:13 -0700 Subject: [PATCH 05/10] Expand real filesystem integration coverage Exercise bounded Elbencho workloads through real SSH and Slurm paths on both CI architectures. Cover buffered and direct I/O, failure and resume, retained data, live capture, multidimensional sweeps, single-file and weighted-root behavior, shared SSH homes, and Slurm scheduling controls. Stage failure injection independently for each substrate and preserve its one-time marker through resume. Validate execution coordinates, lifecycle state, workload totals, native command semantics, reporting output, and scheduler evidence while allowing extensible artifacts and report fields. Isolate and clean scenario-owned storage, restage the SBX runtime after SSH home transitions, select NFS explicitly in CI, and document the expanded scenario catalog. Add specification and failure-staging regression tests. --- .github/workflows/integration.yml | 16 +- docs/CONTEXT.md | 14 +- integration-tests/README.md | 38 +- integration-tests/lib/failure_injection.py | 355 +++++ .../lib/filesystem_integration.py | 1379 ++++++++++++++++- .../lib/filesystem_scenario_specs.py | 666 ++++++++ integration-tests/lib/scenario_planner.py | 4 +- tests/test_filesystem_scenario_specs.py | 268 ++++ tests/test_integration_failure_injection.py | 302 ++++ 9 files changed, 2996 insertions(+), 46 deletions(-) create mode 100644 integration-tests/lib/failure_injection.py create mode 100644 integration-tests/lib/filesystem_scenario_specs.py create mode 100644 tests/test_filesystem_scenario_specs.py create mode 100644 tests/test_integration_failure_injection.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 97fce2c..fc391c0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -63,27 +63,27 @@ jobs: sudo -n true docker info >/dev/null - name: Set up the integration environment - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py setup + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup - name: Verify repeated setup - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py setup + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup - name: Stop the integration environment - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py stop + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs stop - name: Restart the integration environment - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py start + run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs start - name: Verify root test execution is rejected run: | - if sudo "$(command -v python)" integration-tests/bin/integration-test.py test; then + if sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs test; then echo "integration test action unexpectedly accepted root" >&2 exit 1 fi - name: Run all integration tests run: | - "$(command -v python)" integration-tests/bin/integration-test.py test + "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs test - name: Tear down the integration environment if: ${{ always() }} run: | - sudo "$(command -v python)" integration-tests/bin/integration-test.py teardown - sudo "$(command -v python)" integration-tests/bin/integration-test.py teardown + sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown + sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown integration-status: name: Filesystem integration status diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 039b988..904c147 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -77,11 +77,15 @@ crash-recoverable StatefulSet transition; separate SSH homes are canonical. Deployment archives remain products of `utils/build_tarball.sh`, but the harness caches them by the exact immutable tracked-source snapshot, build options, architecture, and seeded Elbencho/runtime identity. Each scenario -extracts that artifact into isolated state. Tests run as the non-root account -recorded by setup and validate real SSH or Slurm dispatch, workload results, -cleanup, and reporting. The on-demand integration workflow runs the lifecycle -concurrently on amd64 and arm64; Kubernetes remains fixture infrastructure and -is not a benchmark execution substrate. +extracts that artifact into isolated state. The real catalog covers baseline +and default I/O, failure/resume, retained datasets, live capture, Cartesian +sweeps, single-file and weighted-root behavior, shared SSH homes, and Slurm +scheduling. Tests run as the non-root account recorded by setup and validate +real SSH or Slurm dispatch, workload results, cleanup, and reporting. Scenario +storage is isolated and removed after host-side evidence is retained. The +on-demand integration workflow explicitly selects NFS and runs the complete +catalog concurrently on amd64 and arm64; Kubernetes remains fixture +infrastructure and is not a benchmark execution substrate. GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. diff --git a/integration-tests/README.md b/integration-tests/README.md index 0b7d5f8..26a673d 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -101,17 +101,27 @@ no setup state or privileges. Actual tests refuse root execution, require the saved non-root identity, validate the live topology, generate environments from the packaged `env.sh.template`, and run `validate_env.sh` before a sweep. -Each substrate runs one 4 KiB buffered execution on one node and one on two -nodes through the real filesystem sweep entry point. The harness materializes -one immutable tracked-source snapshot and builds a real deployment archive from -it with `utils/build_tarball.sh`. It caches the validated archive by snapshot -manifest, architecture, builder options, and seeded Elbencho/runtime identity. -Targeted reruns extract that artifact into isolated workspaces instead of -rebuilding it. The SSH case launches `validate_env.sh` and -`nv-elbencho-sweep.sh` on -the host and reaches the two worker pods over SSH. The Slurm case streams the -same archive to the LoginSet, extracts it in the shared storage filesystem, and -launches both commands there. +The real scenario catalog covers buffered and direct I/O, one- and two-node +selection, failure and resume, retained write/read/delete data, extended live +CSV capture, and result reporting on both SSH and Slurm. Focused cases add a +multidimensional Slurm sweep, Slurm include/exclude and exclusive-user +allocation behavior, SSH weighted roots, generated and staged single-file +work, and shared SSH homes. Workloads stay deliberately small; assertions +check execution coordinates and state transitions, phase and workload +evidence, dataset totals, required native flags, relevant scheduling evidence, +and semantic report rows and plot families without treating incidental output +or performance values as contracts. + +The harness materializes one immutable tracked-source snapshot and builds a +real deployment archive from it with `utils/build_tarball.sh`. It caches the +validated archive by snapshot manifest, architecture, builder options, and +seeded Elbencho/runtime identity. Every scenario extracts that artifact into an +isolated workspace, adds only its own environment and inputs, and cleans its +remote data afterward; host-side results and diagnostics remain under the +state directory. The SSH cases launch `validate_env.sh` and +`nv-elbencho-sweep.sh` on the host and reach the two worker pods over SSH. The +Slurm cases stream the same archive to the LoginSet, extract it in shared +storage, and launch both commands there. The NFS profile uses a size-limited, checksum-verified upstream benchmark archive. Docker SBX, where GitHub release assets may be unavailable, extracts @@ -120,9 +130,9 @@ the binary and runtime libraries from the digest-pinned upstream deployment. Timestamped build and step logs are retained below the state directory's `test-runs/` directory. The test also requires successful execution records, exact one- and two-node workload totals, ordered worker selection, -nonempty benchmark output, environment snapshots, and cleanup of its generated -data directories. It then runs `utils/extract-elbencho.sh` on a host-side copy -of each result and requires the report to contain both node counts. +nonempty benchmark output, environment snapshots, and cleanup of generated +data directories. It runs `utils/extract-elbencho.sh` on host-side result +copies and checks the semantic report content applicable to each scenario. ## On-demand CI diff --git a/integration-tests/lib/failure_injection.py b/integration-tests/lib/failure_injection.py new file mode 100644 index 0000000..e8dc706 --- /dev/null +++ b/integration-tests/lib/failure_injection.py @@ -0,0 +1,355 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Scenario-owned Elbencho failure injection for integration tests. + +The generated wrapper is deliberately independent of production deployment +code. Callers stage its real-binary delegate through injected operations, set +``ELBENCHO`` to the wrapper for a scenario, run the failed attempt and resume +inside one :func:`staged_failure_injection` context, and copy diagnostics before +leaving that outer context. +""" + +from __future__ import annotations + +import re +import shlex +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +DEFAULT_INJECTED_EXIT_CODE = 97 +MAX_SCENARIO_ID_LENGTH = 64 +_SCENARIO_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\Z") + + +class FailureInjectionError(RuntimeError): + """A failure-injection plan is unsafe or internally inconsistent.""" + + +@dataclass(frozen=True) +class FailureInjectionTarget: + """One machine or shared filesystem on which artifacts are staged. + + ``endpoint`` is opaque to this module. An SSH adapter can use a hostname, + while a Slurm adapter can name its shared login/compute filesystem. + ``install_wrapper`` is false for SSH workers because production code copies + the selected coordinator wrapper to them; they only need its delegate and + optional runtime at the same absolute paths. + """ + + endpoint: str + install_wrapper: bool + + +@dataclass(frozen=True) +class FailureInjectionLayout: + """Exact POSIX paths used on every staging endpoint.""" + + root: PurePosixPath + wrapper: PurePosixPath + delegate: PurePosixPath + runtime: PurePosixPath + marker: PurePosixPath + + +@dataclass(frozen=True) +class FailureInjectionPlan: + """Complete immutable staging and wrapper plan for one scenario.""" + + scenario_id: str + source_binary: Path + source_runtime: Path | None + target_argument: str + injected_exit_code: int + layout: FailureInjectionLayout + targets: tuple[FailureInjectionTarget, ...] + wrapper_script: str + + +MakeDirectory = Callable[[str, PurePosixPath], None] +CopyFile = Callable[[Path, str, PurePosixPath, int], None] +CopyTree = Callable[[Path, str, PurePosixPath], None] +WriteText = Callable[[str, PurePosixPath, str, int], None] +RemoveTree = Callable[[str, PurePosixPath], None] + + +@dataclass(frozen=True) +class FailureInjectionOperations: + """Caller-provided local, SSH, or Slurm staging operations.""" + + make_directory: MakeDirectory + copy_file: CopyFile + copy_tree: CopyTree + write_text: WriteText + remove_tree: RemoveTree + + +def _validated_root(root: str | PurePosixPath, scenario_id: str) -> PurePosixPath: + """Return a safe scenario-specific absolute staging root.""" + if ( + not scenario_id + or len(scenario_id) > MAX_SCENARIO_ID_LENGTH + or _SCENARIO_ID.fullmatch(scenario_id) is None + ): + raise FailureInjectionError(f"invalid scenario identifier: {scenario_id!r}") + base = PurePosixPath(root) + if not base.is_absolute() or base == PurePosixPath("/") or ".." in base.parts: + raise FailureInjectionError( + f"failure-injection root must be a dedicated absolute path: {root!s}" + ) + return base / scenario_id / "failure-injection" + + +def _layout(root: PurePosixPath) -> FailureInjectionLayout: + """Derive fixed wrapper, delegate, runtime, and marker paths.""" + delegate_root = root / "delegate" + return FailureInjectionLayout( + root=root, + wrapper=root / "elbencho-inject", + delegate=delegate_root / "elbencho", + runtime=delegate_root / "elbencho-runtime", + marker=root / "failure-injected.marker", + ) + + +def _validate_targets(targets: Sequence[FailureInjectionTarget]) -> None: + """Require unique endpoints and exactly one wrapper-capable coordinator.""" + if not targets: + raise FailureInjectionError("failure-injection plan has no staging targets") + endpoints = [target.endpoint for target in targets] + if any(not endpoint or "\n" in endpoint for endpoint in endpoints): + raise FailureInjectionError("failure-injection endpoint is empty or multiline") + if len(set(endpoints)) != len(endpoints): + raise FailureInjectionError("failure-injection endpoints must be unique") + wrappers = sum(target.install_wrapper for target in targets) + if wrappers != 1: + raise FailureInjectionError( + "failure-injection plan requires exactly one coordinator wrapper target" + ) + + +def _wrapper_script( + layout: FailureInjectionLayout, + target_argument: str, + injected_exit_code: int, +) -> str: + """Generate the exact Bash wrapper staged for one scenario.""" + delegate = shlex.quote(str(layout.delegate)) + marker = shlex.quote(str(layout.marker)) + target = shlex.quote(target_argument) + return f"""#!/usr/bin/env bash +set -euo pipefail + +readonly delegate={delegate} +readonly marker={marker} +readonly trace=$(dirname "$marker")/invocations.log +readonly target_argument={target} +readonly injected_exit_code={injected_exit_code} + +service_mode=0 +target_invocation=0 +for argument in "$@"; do + [[ "$argument" == "--service" ]] && service_mode=1 + [[ "$argument" == *"$target_argument"* ]] && target_invocation=1 +done + +if (( service_mode )); then + exec "$delegate" "$@" +fi +printf '%q ' "$@" >>"$trace" +printf '\n' >>"$trace" +if (( ! target_invocation )); then + exec "$delegate" "$@" +fi +if ! mkdir -- "$marker" 2>/dev/null; then + exec "$delegate" "$@" +fi + +child_pid="" +forward_signal() {{ + local signal=$1 + [[ -z "$child_pid" ]] || kill -s "$signal" "$child_pid" 2>/dev/null || true +}} +trap 'forward_signal HUP' HUP +trap 'forward_signal INT' INT +trap 'forward_signal TERM' TERM + +"$delegate" "$@" & +child_pid=$! +set +e +while true; do + wait "$child_pid" + delegate_rc=$? + if ! kill -0 "$child_pid" 2>/dev/null; then + break + fi +done +set -e +trap - HUP INT TERM + +if (( delegate_rc != 0 )); then + rm -rf -- "$marker" + exit "$delegate_rc" +fi +printf '%s\n' injected >"$marker/state" +exit "$injected_exit_code" +""" + + +def build_failure_injection_plan( + *, + scenario_id: str, + staging_root: str | PurePosixPath, + source_binary: Path, + source_runtime: Path | None, + target_argument: str, + targets: Sequence[FailureInjectionTarget], + injected_exit_code: int = DEFAULT_INJECTED_EXIT_CODE, +) -> FailureInjectionPlan: + """Build a substrate-neutral failure-injection plan. + + ``target_argument`` must be a distinctive argument fragment unique to the + cell that should fail, normally its execution-specific JSON path. Service + startup is handled separately before fragment matching. + """ + if not target_argument or "\x00" in target_argument: + raise FailureInjectionError( + "target argument must be nonempty and contain no NUL" + ) + if not 1 <= injected_exit_code <= 255: + raise FailureInjectionError("injected exit code must be between 1 and 255") + _validate_targets(targets) + plan_layout = _layout(_validated_root(staging_root, scenario_id)) + return FailureInjectionPlan( + scenario_id=scenario_id, + source_binary=source_binary, + source_runtime=source_runtime, + target_argument=target_argument, + injected_exit_code=injected_exit_code, + layout=plan_layout, + targets=tuple(targets), + wrapper_script=_wrapper_script( + plan_layout, target_argument, injected_exit_code + ), + ) + + +def build_ssh_failure_injection_plan( + *, + scenario_id: str, + staging_root: str | PurePosixPath, + source_binary: Path, + source_runtime: Path | None, + target_argument: str, + coordinator_endpoint: str, + worker_endpoints: Sequence[str], + injected_exit_code: int = DEFAULT_INJECTED_EXIT_CODE, +) -> FailureInjectionPlan: + """Build a plan that stages delegates on the coordinator and SSH workers.""" + targets = [FailureInjectionTarget(coordinator_endpoint, True)] + targets.extend( + FailureInjectionTarget(endpoint, False) for endpoint in worker_endpoints + ) + return build_failure_injection_plan( + scenario_id=scenario_id, + staging_root=staging_root, + source_binary=source_binary, + source_runtime=source_runtime, + target_argument=target_argument, + targets=targets, + injected_exit_code=injected_exit_code, + ) + + +def build_slurm_failure_injection_plan( + *, + scenario_id: str, + staging_root: str | PurePosixPath, + source_binary: Path, + source_runtime: Path | None, + target_argument: str, + shared_endpoint: str, + injected_exit_code: int = DEFAULT_INJECTED_EXIT_CODE, +) -> FailureInjectionPlan: + """Build a plan whose artifacts are visible to login and compute nodes.""" + return build_failure_injection_plan( + scenario_id=scenario_id, + staging_root=staging_root, + source_binary=source_binary, + source_runtime=source_runtime, + target_argument=target_argument, + targets=(FailureInjectionTarget(shared_endpoint, True),), + injected_exit_code=injected_exit_code, + ) + + +def stage_failure_injection( + plan: FailureInjectionPlan, operations: FailureInjectionOperations +) -> None: + """Stage a plan without assuming local, SSH, or Slurm command transport.""" + if not plan.source_binary.is_file() or plan.source_binary.is_symlink(): + raise FailureInjectionError( + f"real Elbencho delegate is not a regular file: {plan.source_binary}" + ) + if plan.source_runtime is not None and ( + not plan.source_runtime.is_dir() or plan.source_runtime.is_symlink() + ): + raise FailureInjectionError( + f"Elbencho runtime is not a directory: {plan.source_runtime}" + ) + for target in plan.targets: + operations.make_directory(target.endpoint, plan.layout.root) + operations.make_directory(target.endpoint, plan.layout.delegate.parent) + operations.copy_file( + plan.source_binary, target.endpoint, plan.layout.delegate, 0o755 + ) + if plan.source_runtime is not None: + operations.copy_tree( + plan.source_runtime, target.endpoint, plan.layout.runtime + ) + if target.install_wrapper: + operations.write_text( + target.endpoint, + plan.layout.wrapper, + plan.wrapper_script, + 0o755, + ) + + +def cleanup_failure_injection( + plan: FailureInjectionPlan, operations: FailureInjectionOperations +) -> None: + """Remove all scenario-owned staging, including the persistent marker.""" + for target in reversed(plan.targets): + operations.remove_tree(target.endpoint, plan.layout.root) + + +@contextmanager +def staged_failure_injection( + plan: FailureInjectionPlan, operations: FailureInjectionOperations +) -> Iterator[FailureInjectionPlan]: + """Stage once and clean only after the whole failed-attempt/resume scenario. + + Callers should execute the initial expected failure, copy intermediate + diagnostics, execute resume, and copy final diagnostics inside the context. + Cleanup also runs if the overall scenario aborts. + """ + stage_failure_injection(plan, operations) + try: + yield plan + finally: + cleanup_failure_injection(plan, operations) diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py index 5337c4b..f73495a 100644 --- a/integration-tests/lib/filesystem_integration.py +++ b/integration-tests/lib/filesystem_integration.py @@ -39,6 +39,21 @@ DeploymentCacheRequest, get_or_build_deployment, ) +from failure_injection import ( + FailureInjectionOperations, + build_slurm_failure_injection_plan, + build_ssh_failure_injection_plan, + staged_failure_injection, +) +from filesystem_scenario_specs import ( + SCENARIO_SPECS_BY_NAME, + CommandKind, + DatasetExpectation, + ExecutionStatus, + FilesystemScenarioSpec, + ScenarioStep, + WorkloadPhase, +) from scenario_planner import ( ScenarioPlanningError, SshHomeTransition, @@ -75,6 +90,8 @@ } REMOTE_BASE = "/mnt/storage-test/integration-regression" VALIDATION_SUCCESS = "All validation checks passed successfully" +POD_TEST_UID = 2000 +POD_TEST_GID = 2000 class IntegrationTestError(RuntimeError): @@ -95,6 +112,29 @@ class Fixture: storage_backend: str +@dataclass +class ScenarioRuntime: + """Mutable state shared by a scenario's ordered command steps.""" + + scenario: FilesystemScenarioSpec + selector: str + workspace: str + local_workspace: Path + artifact_root: Path + data_root: str + values: dict[str, str] + copied_results: list[Path] + + +@dataclass(frozen=True) +class StepOutcome: + """Result locations and output for one scenario command.""" + + output: str + remote_result: str | None + local_result: Path | None + + def _kubectl(config: Any, *arguments: str | Path) -> list[str | Path]: """Build a kubectl command using the fixture's private kubeconfig.""" return ["kubectl", "--kubeconfig", config.kubeconfig, *arguments] @@ -632,14 +672,20 @@ def _shell(value: str | Path) -> str: def _override_block( - selector: str, remote_root: str, fixture: Fixture + selector: str, + remote_root: str, + fixture: Fixture, + *, + results_dir: str | None = None, + logs_dir: str | None = None, + extra_env: str = "", ) -> tuple[str, dict[str, str]]: """Return template overrides and small support-file contents.""" data = "/mnt/storage-test" lines = [ "# Bounded integration regression overrides.", - f"export RESULTS_DIR={_shell(remote_root + '/results')}", - f"export LOGS_DIR={_shell(remote_root + '/logs')}", + f"export RESULTS_DIR={_shell(results_dir or remote_root + '/results')}", + f"export LOGS_DIR={_shell(logs_dir or remote_root + '/logs')}", "ORDER_NODES=1", 'client_type="cpu"', f"client_arch={_shell(fixture.architecture)}", @@ -693,11 +739,20 @@ def _override_block( ) support["slurm-nodes"] = "\n".join(fixture.slurm_nodes) + "\n" support["slurm-ignore"] = "" + if extra_env: + lines.extend(("# Scenario-specific integration overrides.", extra_env.rstrip())) return "\n".join(lines) + "\n", support def _render_env( - template: Path, selector: str, remote_root: str, fixture: Fixture + template: Path, + selector: str, + remote_root: str, + fixture: Fixture, + *, + results_dir: str | None = None, + logs_dir: str | None = None, + extra_env: str = "", ) -> tuple[str, dict[str, str]]: """Render one runtime env from the repository's real user template.""" text = template.read_text(encoding="utf-8") @@ -706,7 +761,14 @@ def _render_env( raise IntegrationTestError( f"expected exactly one env_base source anchor in {template}" ) - overrides, support = _override_block(selector, remote_root, fixture) + overrides, support = _override_block( + selector, + remote_root, + fixture, + results_dir=results_dir, + logs_dir=logs_dir, + extra_env=extra_env, + ) rendered = text.replace(anchor, overrides + "\n" + anchor) return rendered, support @@ -848,16 +910,31 @@ def _write_runtime_files( fixture: Fixture, *, template: Path | None = None, + results_dir: str | None = None, + logs_dir: str | None = None, + extra_env: str = "", + extra_support: dict[str, str] | None = None, ) -> None: """Add the generated environment and support files to a deployment.""" rendered, support = _render_env( - template or workspace / "env.sh.template", selector, runtime_root, fixture + template or workspace / "env.sh.template", + selector, + runtime_root, + fixture, + results_dir=results_dir, + logs_dir=logs_dir, + extra_env=extra_env, ) (workspace / "env.sh").write_text(rendered, encoding="utf-8") (workspace / "env.sh").chmod(0o640) for name, content in support.items(): (workspace / name).write_text(content, encoding="utf-8") (workspace / name).chmod(0o640) + for name, content in (extra_support or {}).items(): + path = workspace / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + path.chmod(0o640) def _stream_to_login( @@ -932,6 +1009,7 @@ def _stage_workspace( archive: Path, extracted: Path, build_root: Path, + workspace_id: str, ) -> str: """Stage a packaged deployment for host SSH or LoginSet Slurm execution.""" if selector == "ssh": @@ -940,7 +1018,7 @@ def _stage_workspace( _write_runtime_files(workspace, selector, str(workspace), fixture) return str(workspace) - remote_base = f"{REMOTE_BASE}/{selector}" + remote_base = f"{REMOTE_BASE}/workspaces/{workspace_id}" remote_root = f"{remote_base}/storage-scale-test" reset = f"rm -rf -- {_shell(remote_base)} && mkdir -p -- {_shell(remote_base)}" runner.run( @@ -1031,6 +1109,8 @@ def _run_step( command: str, log_dir: Path, timeout: int, + *, + expected_failure: bool = False, ) -> str: """Run one bounded substrate step and preserve diagnostic output.""" LOG.info("Running %s filesystem step: %s", selector, name) @@ -1043,14 +1123,14 @@ def _run_step( log_path = log_dir / f"{selector}-{name}.log" log_path.write_text(output, encoding="utf-8") log_path.chmod(0o640) - if result.returncode: + if result.returncode and not expected_failure: detail = output.strip()[-8000:] raise IntegrationTestError( f"{selector} {name} failed with exit code {result.returncode}; " f"full output: {log_path}\n{detail}" ) LOG.info("Passed %s filesystem step: %s", selector, name) - return result.stdout + return output def _assert_results( @@ -1167,7 +1247,7 @@ def _copy_result_for_reporting( destination: Path, ) -> Path: """Bring one completed result tree to the host for report validation.""" - destination.mkdir(parents=True) + destination.mkdir(parents=True, exist_ok=True) if selector == "ssh": shutil.copytree(result_dir, destination / Path(result_dir).name) else: @@ -1224,6 +1304,1234 @@ def _assert_report( ) +def _login_shell( + runner: Any, + config: Any, + fixture: Fixture, + command: str, + *, + timeout: int = 60, +) -> str: + """Run a storage-inspection command in the PVC-mounted login pod.""" + return runner.run( + _pod_command( + config, + fixture.login_pod, + fixture.login_container, + command, + timeout=timeout, + ), + timeout=timeout + 15, + ).stdout + + +def _prepare_scenario_data( + runner: Any, + config: Any, + fixture: Fixture, + data_root: str, +) -> None: + """Reset one marker-owned scenario subtree on the shared test storage.""" + command = f""" +set -euo pipefail +rm -rf -- {_shell(data_root)} +mkdir -p -- {_shell(data_root + '/primary')} {_shell(data_root + '/secondary')} +printf 'scenario-owned\n' > {_shell(data_root + '/primary/.integration-sentinel')} +printf 'scenario-owned\n' > {_shell(data_root + '/secondary/.integration-sentinel')} +chown -R {config.test_uid}:{config.test_gid} -- {_shell(data_root)} +""".strip() + _login_shell(runner, config, fixture, command) + + +def _sync_step_runtime( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + template: Path, + result_base: str, +) -> None: + """Render and install one step's env and support files.""" + extra_env = step.render_env(runtime.values) + extra_support = { + item.relative_path: item.content + for item in step.render_support_files(runtime.values) + } + if runtime.selector == "ssh": + _write_runtime_files( + Path(runtime.workspace), + runtime.selector, + runtime.workspace, + fixture, + template=template, + results_dir=result_base, + logs_dir=f"{runtime.workspace}/logs/{step.name}", + extra_env=extra_env, + extra_support=extra_support, + ) + return + stage = runtime.local_workspace / f"runtime-{step.name}" + if stage.exists(): + shutil.rmtree(stage) + stage.mkdir() + _write_runtime_files( + stage, + runtime.selector, + runtime.workspace, + fixture, + template=template, + results_dir=result_base, + logs_dir=f"{runtime.workspace}/logs/{step.name}", + extra_env=extra_env, + extra_support=extra_support, + ) + archive_path = runtime.local_workspace / f"runtime-{step.name}.tar" + with tarfile.open(archive_path, "w") as archive: + for child in sorted(stage.rglob("*")): + archive.add(child, arcname=child.relative_to(stage)) + _stream_to_login( + runner, + config, + fixture, + archive_path, + [ + "tar", + "--no-same-owner", + "--no-same-permissions", + "-xf", + "-", + "-C", + runtime.workspace, + ], + ) + + +def _create_generated_inputs( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, +) -> None: + """Create bounded scenario inputs through the shared PVC mount.""" + for generated in step.generated_inputs: + path = f"{runtime.values['test_root']}/{generated.relative_path}" + command = f""" +set -euo pipefail +mkdir -p -- {_shell(str(PurePosixPath(path).parent))} +truncate -s {generated.size_bytes} -- {_shell(path)} +chown -R {config.test_uid}:{config.test_gid} -- \ + {_shell(str(PurePosixPath(path).parent))} +test "$(stat -c %s -- {_shell(path)})" -eq {generated.size_bytes} +""".strip() + _login_shell(runner, config, fixture, command) + + +def _discover_result( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + result_base: str, + log_dir: Path, +) -> str: + """Find the single normal sweep result below an invocation-owned base.""" + output = _run_step( + runner, + config, + fixture, + runtime.selector, + f"{step.name}-discover", + f"find {_shell(result_base)} -mindepth 1 -maxdepth 1 -type d " + "-name 'elbencho-*' -printf '%p\\n'", + log_dir, + 30, + ) + directories = [line for line in output.splitlines() if line.strip()] + if len(directories) != 1: + raise IntegrationTestError( + f"{runtime.selector} {step.name} produced {len(directories)} " + f"normal result directories under {result_base}: {directories}" + ) + return directories[0] + + +def _copy_scenario_result( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + remote_result: str, +) -> Path: + """Copy one immutable result snapshot into the retained test-run log.""" + destination = runtime.artifact_root / "copied-results" / step.name + destination.mkdir(parents=True, exist_ok=True) + if runtime.selector == "ssh": + target = destination / Path(remote_result).name + if target.exists(): + shutil.rmtree(target) + shutil.copytree(remote_result, target) + else: + target = _copy_result_for_reporting( + runner, + config, + fixture, + runtime.selector, + remote_result, + destination, + ) + runtime.copied_results.append(target) + return target + + +def _shell_assignments(path: Path) -> dict[str, str]: + """Read simple exported scalar assignments from a reified execution.""" + assignments: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"export ([A-Za-z_][A-Za-z0-9_]*)=(.*)", line) + if not match: + continue + values = shlex.split(match.group(2), posix=True) + assignments[match.group(1)] = values[0] if values else "" + return assignments + + +def _coordinate_from_execution(path: Path) -> tuple[int, str, int, int]: + """Return a reified execution's stable sweep coordinate.""" + values = _shell_assignments(path) + try: + return ( + int(values["nodes"]), + values["io_size"], + int(values["thread_count"]), + int(values["io_depth"]), + ) + except (KeyError, ValueError) as error: + raise IntegrationTestError(f"invalid execution metadata in {path}") from error + + +def _workload_values(path: Path) -> dict[str, str]: + """Load a workload TSV while rejecting duplicate or malformed keys.""" + values: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + fields = line.split("\t") + if len(fields) != 2 or fields[0] in values: + raise IntegrationTestError(f"malformed workload metadata: {path}") + values[fields[0]] = fields[1] + return values + + +def _expected_dataset_totals( + scenario: str, step: ScenarioStep, nodes: int +) -> tuple[int, int] | None: + """Return exact bounded totals for workloads with a metadata contract.""" + if scenario in {"baseline", "ssh-shared-home", "failure-resume"}: + return nodes, nodes * 16 * 1024 * 1024 + if scenario == "live-capture": + return nodes * 2, nodes * 2 * 16 * 1024 * 1024 + if scenario == "slurm-cartesian": + return nodes * 2, nodes * 2 * 1024 * 1024 + if scenario == "retained-lifecycle" and step.kind is not CommandKind.DELETE: + return 1, 16 * 1024 * 1024 + return None + + +def _assert_phase_artifacts( + execution_root: Path, + execution_id: str, + step: ScenarioStep, + status: ExecutionStatus, +) -> None: + """Require phase evidence without freezing complete native arguments.""" + phase_files = { + WorkloadPhase.WRITE: "write.json", + WorkloadPhase.READ: "read.json", + WorkloadPhase.REMOVE_FILES: "delete.json", + } + for phase, suffix in phase_files.items(): + if phase not in step.required_phases: + continue + if status is ExecutionStatus.FAILED and phase is WorkloadPhase.REMOVE_FILES: + continue + path = execution_root / f"{execution_id}.{suffix}" + if not path.is_file() or path.stat().st_size == 0: + raise IntegrationTestError(f"missing {phase.value} evidence: {path}") + + +def _assert_execution_contract( + scenario: FilesystemScenarioSpec, + step: ScenarioStep, + result: Path, +) -> None: + """Validate exact coordinates, states, totals, and required phases.""" + execution_root = result / "executions" + scripts = sorted(execution_root.glob("[0-9][0-9][0-9][0-9].sh")) + if len(scripts) != len(step.executions): + raise IntegrationTestError( + f"{scenario.name}/{step.name}: expected {len(step.executions)} " + f"executions, found {len(scripts)}" + ) + actual_coordinates = [_coordinate_from_execution(path) for path in scripts] + expected_coordinates = [ + ( + item.coordinate.nodes, + item.coordinate.io_size, + item.coordinate.threads, + item.coordinate.io_depth, + ) + for item in step.executions + ] + if actual_coordinates != expected_coordinates: + raise IntegrationTestError( + f"{scenario.name}/{step.name}: coordinates {actual_coordinates!r} " + f"do not match {expected_coordinates!r}" + ) + for index, expected in enumerate(step.executions, start=1): + execution_id = f"{index:04d}" + status_path = execution_root / f"{execution_id}.status" + status = status_path.read_text(encoding="utf-8").strip() + if status != expected.status.value: + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: expected " + f"{expected.status.value}, found {status}" + ) + if expected.status is ExecutionStatus.PENDING: + continue + exitcode = ( + (execution_root / f"{execution_id}.exitcode") + .read_text(encoding="utf-8") + .strip() + ) + expected_exit = "97" if expected.status is ExecutionStatus.FAILED else "0" + if exitcode != expected_exit: + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: expected exit " + f"{expected_exit}, found {exitcode}" + ) + if scenario.name not in { + "default-dio", + "ssh-single-big-file", + "ssh-weighted-roots", + } and not ( + scenario.name == "retained-lifecycle" + and step.name.startswith("read-cache-") + ): + _assert_phase_artifacts(execution_root, execution_id, step, expected.status) + workload_path = execution_root / f"{execution_id}.workload.tsv" + totals = _expected_dataset_totals( + scenario.name, step, expected.coordinate.nodes + ) + if totals is not None and workload_path.is_file(): + workload = _workload_values(workload_path) + if workload.get("dataset_files_total") != str(totals[0]) or workload.get( + "dataset_bytes_total" + ) != str(totals[1]): + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: invalid " + f"dataset totals in {workload_path}" + ) + valid_completion = {"completed"} + if scenario.name == "retained-lifecycle" and step.name.startswith( + "read-cache-" + ): + valid_completion.add("not_applicable_time_based") + if ( + expected.status is ExecutionStatus.SUCCESS + and workload.get("completion_state") not in valid_completion + ): + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: workload did " + "not complete" + ) + + +def _execution_targets(result: Path) -> tuple[str, ...]: + """Return every exact generated target recorded by a result tree.""" + targets: list[str] = [] + for script in sorted((result / "executions").glob("[0-9][0-9][0-9][0-9].sh")): + value = _shell_assignments(script).get( + "ELBENCHO_RUN_GENERATED_TEST_DIRS_CSV", "" + ) + targets.extend(item for item in value.split(",") if item) + return tuple(targets) + + +def _assert_dataset_state( + runner: Any, + config: Any, + fixture: Fixture, + step: ScenarioStep, + result: Path | None, + retained_path: str | None, +) -> None: + """Validate cleanup or retention only within scenario-owned paths.""" + if step.dataset is DatasetExpectation.PRESERVED: + if retained_path: + _login_shell(runner, config, fixture, f"test -e {_shell(retained_path)}") + return + paths = list(_execution_targets(result)) if result is not None else [] + if step.dataset is DatasetExpectation.REMOVED and retained_path: + paths.append(retained_path) + if not paths: + return + probes = "\n".join(f"test ! -e {_shell(path)}" for path in paths) + _login_shell(runner, config, fixture, f"set -euo pipefail\n{probes}") + + +def _assert_semantic_flags(scenario: str, step: ScenarioStep, result: Path) -> None: + """Check required command semantics while allowing future optional flags.""" + command_lines: list[str] = [] + for path in result.glob("executions/*.log"): + command_lines.extend( + line + for line in path.read_text(encoding="utf-8", errors="replace").splitlines() + if line.startswith("# elbencho ") + ) + evidence = "\n".join(command_lines) + if not evidence: + raise IntegrationTestError( + f"{scenario}/{step.name}: execution logs omitted native commands" + ) + required: tuple[str, ...] = () + forbidden: tuple[str, ...] = () + if scenario == "baseline" or scenario == "ssh-shared-home": + required = ("--norandalign",) + elif scenario == "default-dio": + required, forbidden = ("--direct",), ("--norandalign",) + elif scenario == "live-capture": + required = ("--livecsv", "--livecsvex", "--liveint=10") + elif scenario == "ssh-single-big-file" and step.name == "inferred-extent-read": + required, forbidden = ("--nosvcshare", "--read"), ( + "--size", + "--treescan", + "--treefile", + ) + elif scenario == "ssh-weighted-roots": + required = ("--files=1",) + missing = [flag for flag in required if flag not in evidence] + present = [ + flag + for flag in forbidden + if re.search(rf"(? None: + """Exercise reporting and require semantic rows and plot families.""" + report_dir = log_dir / f"report-{step.name}" + report_dir.mkdir() + command: list[str | Path] = [ + report_workspace / "utils" / "extract-elbencho.sh", + "--markdown", + ] + if runtime.scenario.name == "live-capture": + command.extend( + ( + "--per-client-plots", + "--client-min-underperform-segments", + "2", + "--client-max-timeseries-lines", + "2", + "--client-max-heatmap-rows", + "2", + ) + ) + command.append(result) + completed = runner.run( + command, cwd=report_dir, timeout=step.timeout_seconds, check=False + ) + output = completed.stdout + completed.stderr + log_path = report_dir / "extract-elbencho.log" + log_path.write_text(output, encoding="utf-8") + if completed.returncode: + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: reporting " + f"failed; full output: {log_path}\n{output[-8000:]}" + ) + node_counts = sorted({item.coordinate.nodes for item in step.executions}) + missing_rows = [ + nodes + for nodes in node_counts + if not re.search(rf"^\|\s*{nodes}\s*\|", output, re.MULTILINE) + ] + required_operations = [] + if WorkloadPhase.WRITE in step.required_phases: + required_operations.append("WRITE Operation") + if WorkloadPhase.READ in step.required_phases: + required_operations.append("READ Operation") + missing_operations = [item for item in required_operations if item not in output] + if missing_rows or missing_operations: + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: report " + f"missing node rows {missing_rows} or operations {missing_operations}" + ) + if not any(path.stat().st_size for path in result.glob("*.png")): + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: no " + "nonempty report plot was generated" + ) + + +def _reset_result_base( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + result_base: str, +) -> None: + """Create an empty invocation-owned result base.""" + if runtime.selector == "ssh": + path = Path(result_base) + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True) + return + _login_shell( + runner, + config, + fixture, + f"rm -rf -- {_shell(result_base)} && mkdir -p -- {_shell(result_base)} " + f"&& chown {config.test_uid}:{config.test_gid} -- {_shell(result_base)}", + ) + + +def _validate_step_environment( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + log_dir: Path, +) -> None: + """Run the repository's validator for a rendered scenario environment.""" + output = _run_step( + runner, + config, + fixture, + runtime.selector, + f"{step.name}-validate-env", + f"cd -- {_shell(runtime.workspace)} && ./validate_env.sh", + log_dir, + 180, + ) + if VALIDATION_SUCCESS not in output: + raise IntegrationTestError( + f"{runtime.scenario.name}/{runtime.selector}/{step.name}: " + "validate_env.sh omitted its success marker" + ) + + +def _retained_path(result: Path) -> str: + """Return the sole generated path from a one-execution retained result.""" + targets = _execution_targets(result) + if len(targets) != 1: + raise IntegrationTestError( + f"expected one retained generated target, found {targets!r}" + ) + return targets[0] + + +def _assert_read_cache(step: ScenarioStep, result: Path) -> None: + """Validate the cache miss/hit contract for retained read invocations.""" + if step.name not in {"read-cache-miss", "read-cache-hit"}: + return + workload = _workload_values(result / "executions" / "0001.workload.tsv") + expected = ( + ("cache_miss_scan", "created") + if step.name == "read-cache-miss" + else ("cache_hit", "reused") + ) + actual = ( + workload.get("treefile_source"), + workload.get("treefile_cache_publish_outcome"), + ) + if actual != expected: + raise IntegrationTestError( + f"{step.name}: expected treefile cache {expected!r}, found {actual!r}" + ) + + +def _assert_live_capture(result: Path) -> None: + """Require a stable live counter series without comparing performance.""" + files = [path for path in result.rglob("*.live.csv") if path.stat().st_size] + if not files: + raise IntegrationTestError(f"live-capture result has no live CSV: {result}") + stable_series = False + for path in files: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + if len(lines) < 3: + continue + header = lines[0] + if "DoneBytes" not in header: + raise IntegrationTestError(f"live CSV lacks DoneBytes column: {path}") + stable_series = True + if not stable_series: + raise IntegrationTestError( + "live capture has no service/phase series with at least two samples" + ) + + +def _assert_slurm_scheduling( + runner: Any, + config: Any, + fixture: Fixture, + result: Path, +) -> None: + """Verify the real allocation's name, node constraint, CPUs, and state.""" + job_id = (result / "executions" / "0001.jobid").read_text(encoding="utf-8").strip() + command = ( + f"sacct -X -j {_shell(job_id)} -n -P " + "--format=JobName,NodeList,AllocCPUS,State" + ) + output = _login_shell(runner, config, fixture, command) + records = [line.split("|") for line in output.splitlines() if line.strip()] + matches = [record for record in records if len(record) >= 4 and record[0]] + if not matches: + raise IntegrationTestError(f"Slurm accounting omitted job {job_id}") + name, nodes, cpus, state = matches[0][:4] + if ( + not name.startswith("itest-elbencho-") + or fixture.slurm_nodes[0] not in nodes + or fixture.slurm_nodes[1] in nodes + or not cpus.isdigit() + or int(cpus) < 1 + or not state.startswith("COMPLETED") + ): + raise IntegrationTestError( + f"unexpected Slurm scheduling evidence for {job_id}: {matches[0]!r}" + ) + + +def _assert_step_specials( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + result: Path, +) -> None: + """Apply focused assertions unique to individual real scenarios.""" + _assert_read_cache(step, result) + if runtime.scenario.name == "live-capture": + _assert_live_capture(result) + if runtime.scenario.name == "slurm-scheduling": + _assert_slurm_scheduling(runner, config, fixture, result) + if runtime.scenario.name == "ssh-single-big-file" and step.name == ( + "inferred-extent-read" + ): + path = f"{runtime.values['test_root']}/staged-input/integration-bigfile" + output = _login_shell( + runner, + config, + fixture, + f"stat -c %s -- {_shell(path)} && sha256sum -- {_shell(path)}", + ) + if not output.startswith(f"{16 * 1024 * 1024}\n"): + raise IntegrationTestError("staged single-file input changed size") + + +def _run_regular_step( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + step: ScenarioStep, + template: Path, + report_workspace: Path, + log_dir: Path, +) -> StepOutcome: + """Render, execute, collect, and validate one ordinary scenario step.""" + if step.kind is CommandKind.RESUME: + raise IntegrationTestError("resume steps require the failure scenario runner") + result_base = f"{runtime.workspace}/results/{step.name}" + _reset_result_base(runner, config, fixture, runtime, result_base) + _sync_step_runtime( + runner, + config, + fixture, + runtime, + step, + template, + result_base, + ) + _create_generated_inputs(runner, config, fixture, runtime, step) + _validate_step_environment(runner, config, fixture, runtime, step, log_dir) + arguments = shlex.join(step.render_arguments(runtime.values)) + command = ( + f"cd -- {_shell(runtime.workspace)} && " + f"./storage-tests/fs/nv-elbencho-sweep.sh {arguments}" + ) + output = _run_step( + runner, + config, + fixture, + runtime.selector, + step.name, + command, + log_dir, + step.timeout_seconds, + ) + if step.kind is CommandKind.DELETE: + _assert_dataset_state( + runner, + config, + fixture, + step, + None, + runtime.values.get("retained_data_dir") + or f"{runtime.values['test_root']}/staged-input", + ) + return StepOutcome(output, None, None) + remote_result = _discover_result( + runner, + config, + fixture, + runtime, + step, + result_base, + log_dir, + ) + local_result = _copy_scenario_result( + runner, config, fixture, runtime, step, remote_result + ) + _assert_execution_contract(runtime.scenario, step, local_result) + _assert_semantic_flags(runtime.scenario.name, step, local_result) + if "retained_data_dir" in step.exports: + runtime.values["retained_data_dir"] = _retained_path(local_result) + _assert_dataset_state( + runner, + config, + fixture, + step, + local_result, + runtime.values.get("retained_data_dir"), + ) + _assert_step_specials(runner, config, fixture, runtime, step, local_result) + _assert_scenario_report( + runner, + report_workspace, + local_result, + runtime, + step, + log_dir, + ) + if {item.coordinate.nodes for item in step.executions} == {1, 2}: + _assert_ordered_workers(fixture, runtime.selector, output) + return StepOutcome(output, remote_result, local_result) + + +def _make_scenario_runtime( + runner: Any, + config: Any, + fixture: Fixture, + scenario: FilesystemScenarioSpec, + selector: str, + archive: Path, + extracted: Path, + build_root: Path, + artifact_root: Path, + run_id: str, +) -> ScenarioRuntime: + """Stage one isolated deployment and storage subtree for a scenario.""" + workspace_id = f"{run_id}-{scenario.name}-{selector}" + workspace = _stage_workspace( + runner, + config, + fixture, + selector, + archive, + extracted, + build_root, + workspace_id, + ) + data_root = f"{REMOTE_BASE}/test-data/{workspace_id}" + _prepare_scenario_data(runner, config, fixture, data_root) + values = { + "workspace": workspace, + "test_root": f"{data_root}/primary", + "test_root_secondary": f"{data_root}/secondary", + "slurm_node_1": fixture.slurm_nodes[0], + "slurm_node_2": fixture.slurm_nodes[1], + } + return ScenarioRuntime( + scenario, + selector, + workspace, + build_root, + artifact_root, + data_root, + values, + [], + ) + + +def _run_regular_scenario( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + template: Path, + report_workspace: Path, + log_dir: Path, +) -> None: + """Run all ordered steps for a non-failure scenario.""" + for step in runtime.scenario.steps: + _run_regular_step( + runner, + config, + fixture, + runtime, + step, + template, + report_workspace, + log_dir, + ) + + +def _cleanup_ssh_remote_results(runner: Any, config: Any) -> None: + """Remove retrieved per-scenario output trees from bounded SSH homes.""" + for pod in _pods_with_container(_pod_inventory(runner, config), "sshd"): + runner.run( + [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + pod["metadata"]["name"], + "-c", + "sshd", + "--", + ), + "bash", + "-c", + "rm -rf -- /home/tester/elbencho-[0-9]*", + ], + check=False, + timeout=60, + ) + + +def _cleanup_scenario_storage( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, +) -> None: + """Remove one scenario's isolated PVC data and Slurm deployment.""" + data_root = PurePosixPath(runtime.data_root) + data_base = PurePosixPath(REMOTE_BASE) / "test-data" + targets = [data_root] + if runtime.selector == "slurm": + workspace_root = PurePosixPath(runtime.workspace).parent + workspace_base = PurePosixPath(REMOTE_BASE) / "workspaces" + if workspace_root.parent != workspace_base: + raise IntegrationTestError( + f"refusing to remove unexpected Slurm workspace: {workspace_root}" + ) + targets.append(workspace_root) + if data_root.parent != data_base: + raise IntegrationTestError( + f"refusing to remove unexpected scenario data path: {data_root}" + ) + command = "rm -rf -- " + " ".join(_shell(path) for path in targets) + result = runner.run( + _pod_command( + config, + fixture.login_pod, + fixture.login_container, + command, + timeout=120, + ), + check=False, + timeout=135, + ) + if result.returncode: + LOG.warning( + "Could not clean scenario-owned remote storage for %s/%s", + runtime.scenario.name, + runtime.selector, + ) + + +def _remote_staging_operations( + runner: Any, + config: Any, + fixture: Fixture, + selector: str, + local_wrapper: Path | None, + restore_binary: Path | None, + remote_wrapper: PurePosixPath | None = None, +) -> FailureInjectionOperations: + """Return concrete PVC or SSH-pod operations for failure staging.""" + + def _container(_endpoint: str) -> str: + return fixture.login_container if selector == "slurm" else "sshd" + + def _command(endpoint: str, command: str, *, stdin: Any = None) -> None: + if endpoint == "local": + return + runner.run( + [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + "-i", + endpoint, + "-c", + _container(endpoint), + "--", + ), + "bash", + "-ec", + command, + ], + stdin=stdin, + timeout=180, + ) + + def make_directory(endpoint: str, path: PurePosixPath) -> None: + _command( + endpoint, + f"mkdir -p -- {_shell(path)} && chown {POD_TEST_UID}:{POD_TEST_GID} " + f"-- {_shell(path)}", + ) + + def copy_file( + source: Path, endpoint: str, destination: PurePosixPath, mode: int + ) -> None: + if endpoint == "local": + return + with tempfile.TemporaryFile() as stream: + stream.write(source.read_bytes()) + stream.seek(0) + _command( + endpoint, + f"cat > {_shell(destination)} && chmod {mode:o} -- " + f"{_shell(destination)} && chown {POD_TEST_UID}:{POD_TEST_GID} " + f"-- {_shell(destination)}", + stdin=stream, + ) + + def copy_tree(source: Path, endpoint: str, destination: PurePosixPath) -> None: + if endpoint == "local": + return + if selector == "ssh": + _command( + endpoint, + f"rm -rf -- {_shell(destination)} && ln -s -- " + f"/home/tester/elbencho-runtime {_shell(destination)}", + ) + return + with tempfile.TemporaryFile() as stream: + with tarfile.open(fileobj=stream, mode="w") as archive: + for child in sorted(source.rglob("*")): + archive.add(child, arcname=child.relative_to(source)) + stream.seek(0) + _command( + endpoint, + f"rm -rf -- {_shell(destination)} && mkdir -p -- " + f"{_shell(destination)} && tar -xf - -C {_shell(destination)} " + f"&& chown -R {POD_TEST_UID}:{POD_TEST_GID} -- " + f"{_shell(destination)}", + stdin=stream, + ) + + def write_text( + endpoint: str, destination: PurePosixPath, content: str, mode: int + ) -> None: + if endpoint == "local": + if local_wrapper is None: + raise IntegrationTestError("local failure wrapper path is absent") + local_wrapper.write_text(content, encoding="utf-8") + local_wrapper.chmod(mode) + return + if remote_wrapper is not None: + destination = remote_wrapper + with tempfile.TemporaryFile() as stream: + stream.write(content.encode()) + stream.seek(0) + _command( + endpoint, + f"cat > {_shell(destination)} && chmod {mode:o} -- " + f"{_shell(destination)} && chown {POD_TEST_UID}:{POD_TEST_GID} " + f"-- {_shell(destination)}", + stdin=stream, + ) + + def remove_tree(endpoint: str, path: PurePosixPath) -> None: + if endpoint == "local": + if local_wrapper is not None and restore_binary is not None: + shutil.copy2(restore_binary, local_wrapper) + return + if remote_wrapper is not None: + _command( + endpoint, + f"cp -- {_shell(path / 'delegate' / 'elbencho')} " + f"{_shell(remote_wrapper)}", + ) + _command(endpoint, f"rm -rf -- {_shell(path)}") + + return FailureInjectionOperations( + make_directory, copy_file, copy_tree, write_text, remove_tree + ) + + +def _failure_plan( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + binary_name: str, + source_binary: Path, + bundled_runtime: Path | None, +) -> tuple[Any, FailureInjectionOperations]: + """Build a feasible substrate-specific failure staging plan.""" + packaged_binary = ( + Path(runtime.workspace) / "utils" / binary_name + if runtime.selector == "ssh" + else runtime.local_workspace / "source-elbencho" + ) + if runtime.selector == "slurm": + raise IntegrationTestError("internal Slurm failure source was not prepared") + pods = sorted( + _pods_with_container(_pod_inventory(runner, config), "sshd"), + key=lambda item: item["metadata"]["name"], + ) + plan = build_ssh_failure_injection_plan( + scenario_id=f"{runtime.scenario.name}-{os.getpid()}", + staging_root="/home/tester/.storage-scale-test-failure", + source_binary=source_binary, + source_runtime=bundled_runtime, + target_argument="executions/0002.write.json", + coordinator_endpoint="local", + worker_endpoints=[item["metadata"]["name"] for item in pods], + ) + operations = _remote_staging_operations( + runner, + config, + fixture, + runtime.selector, + packaged_binary, + source_binary, + ) + return plan, operations + + +def _slurm_failure_plan( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + source_binary: Path, + binary_name: str, + bundled_runtime: Path | None, +) -> tuple[Any, FailureInjectionOperations]: + """Build failure staging on the Slurm deployment's shared PVC.""" + plan = build_slurm_failure_injection_plan( + scenario_id=f"{runtime.scenario.name}-{os.getpid()}", + staging_root=f"{runtime.workspace}/.storage-scale-test-failure", + source_binary=source_binary, + source_runtime=bundled_runtime, + target_argument="executions/0002.write.json", + shared_endpoint=fixture.login_pod, + ) + operations = _remote_staging_operations( + runner, + config, + fixture, + runtime.selector, + None, + None, + PurePosixPath(runtime.workspace) / "utils" / binary_name, + ) + return plan, operations + + +def _hash_execution_contract(result: Path, execution_id: str) -> dict[str, str]: + """Hash stable evidence that resume must not replace for successful cells.""" + evidence: dict[str, str] = {} + for path in sorted((result / "executions").glob(f"{execution_id}.*")): + if path.is_file(): + evidence[path.name] = _sha256(path) + return evidence + + +def _capture_failure_diagnostics( + runner: Any, + config: Any, + fixture: Fixture, + plan: Any, + selector: str, + log_dir: Path, +) -> None: + """Copy wrapper invocation traces before scenario-level cleanup.""" + chunks: list[str] = [] + container = fixture.login_container if selector == "slurm" else "sshd" + for target in plan.targets: + if target.endpoint == "local": + continue + result = runner.run( + [ + *_kubectl( + config, + "-n", + config.namespace, + "exec", + target.endpoint, + "-c", + container, + "--", + ), + "bash", + "-c", + f"cat -- {_shell(plan.layout.root / 'invocations.log')} " + "2>/dev/null || true", + ], + check=False, + timeout=30, + ) + chunks.append(f"## {target.endpoint}\n{result.stdout}") + (log_dir / "failure-wrapper-invocations.log").write_text( + "\n".join(chunks), encoding="utf-8" + ) + + +def _run_failure_resume( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + template: Path, + report_workspace: Path, + log_dir: Path, + binary_name: str, + source_binary: Path, + bundled_runtime: Path | None, +) -> None: + """Run one real injected failure and resume without restaging the marker.""" + first, resume = runtime.scenario.steps + if runtime.selector == "ssh": + plan, operations = _failure_plan( + runner, + config, + fixture, + runtime, + binary_name, + source_binary, + bundled_runtime, + ) + injected_first = first + else: + plan, operations = _slurm_failure_plan( + runner, + config, + fixture, + runtime, + source_binary, + binary_name, + bundled_runtime, + ) + injected_first = first + result_base = f"{runtime.workspace}/results/{first.name}" + with staged_failure_injection(plan, operations): + _reset_result_base(runner, config, fixture, runtime, result_base) + _sync_step_runtime( + runner, + config, + fixture, + runtime, + injected_first, + template, + result_base, + ) + _validate_step_environment( + runner, config, fixture, runtime, injected_first, log_dir + ) + arguments = shlex.join(first.render_arguments(runtime.values)) + command = ( + f"cd -- {_shell(runtime.workspace)} && " + f"./storage-tests/fs/nv-elbencho-sweep.sh {arguments}" + ) + _run_step( + runner, + config, + fixture, + runtime.selector, + first.name, + command, + log_dir, + first.timeout_seconds, + expected_failure=True, + ) + remote_result = _discover_result( + runner, + config, + fixture, + runtime, + first, + result_base, + log_dir, + ) + failed_result = _copy_scenario_result( + runner, config, fixture, runtime, first, remote_result + ) + _capture_failure_diagnostics( + runner, config, fixture, plan, runtime.selector, log_dir + ) + _assert_execution_contract(runtime.scenario, first, failed_result) + runtime.values["failed_results_dir"] = remote_result + _assert_dataset_state(runner, config, fixture, first, failed_result, None) + preserved = _hash_execution_contract(failed_result, "0001") + resume_command = ( + f"cd -- {_shell(runtime.workspace)} && " + "./storage-tests/fs/nv-elbencho-sweep.sh " + f"{shlex.join(resume.render_arguments(runtime.values))}" + ) + _run_step( + runner, + config, + fixture, + runtime.selector, + resume.name, + resume_command, + log_dir, + resume.timeout_seconds, + ) + resumed_result = _copy_scenario_result( + runner, config, fixture, runtime, resume, remote_result + ) + _assert_execution_contract(runtime.scenario, resume, resumed_result) + if _hash_execution_contract(resumed_result, "0001") != preserved: + raise IntegrationTestError("resume replaced successful execution 0001") + _assert_dataset_state(runner, config, fixture, resume, resumed_result, None) + _assert_scenario_report( + runner, + report_workspace, + resumed_result, + runtime, + resume, + log_dir, + ) + + def _run_substrate( runner: Any, config: Any, @@ -1244,6 +2552,7 @@ def _run_substrate( archive, extracted, build_root, + f"baseline-{selector}", ) prefix = f"cd -- {_shell(remote_root)} && " validation = _run_step( @@ -1347,27 +2656,65 @@ def run_filesystem_tests( transition_ssh_home(step.target.value, "ssh-shared-home") home_mode = step.target.value fixture = _require_fixture(runner, config, home_mode) + if runtime is not None: + _stage_ssh_runtime( + runner, + config, + runtime, + _pod_inventory(runner, config), + ) continue scenario = step.scenario.name - if scenario != "baseline": - raise IntegrationTestError( - f"integration scenario is not implemented: {scenario}" - ) scenario_root = build_root / f"{scenario}-{step.substrate.value}" scenario_root.mkdir() scenario_logs = log_dir / f"{scenario}-{step.substrate.value}" scenario_logs.mkdir() - _run_substrate( + specification = SCENARIO_SPECS_BY_NAME[scenario] + runtime_state = _make_scenario_runtime( runner, config, fixture, + specification, step.substrate.value, archive, extracted, scenario_root, - report_workspace, scenario_logs, + run_id, ) + try: + if scenario == "failure-resume": + _run_failure_resume( + runner, + config, + fixture, + runtime_state, + extracted / "env.sh.template", + report_workspace, + scenario_logs, + binary_name, + binary, + runtime, + ) + else: + _run_regular_scenario( + runner, + config, + fixture, + runtime_state, + extracted / "env.sh.template", + report_workspace, + scenario_logs, + ) + finally: + if step.substrate.value == "ssh": + _cleanup_ssh_remote_results(runner, config) + _cleanup_scenario_storage( + runner, + config, + fixture, + runtime_state, + ) finally: if home_mode == "shared" and transition_ssh_home is not None: transition_ssh_home("separate", "ssh-shared-home-restore") diff --git a/integration-tests/lib/filesystem_scenario_specs.py b/integration-tests/lib/filesystem_scenario_specs.py new file mode 100644 index 0000000..7068cbc --- /dev/null +++ b/integration-tests/lib/filesystem_scenario_specs.py @@ -0,0 +1,666 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Declarative workload contracts for filesystem integration scenarios. + +The driver owns path rendering, substrate setup, and semantic assertions. This +module deliberately describes only stable workload intent. Template fields in +environment lines, command arguments, and support files are resolved from the +scenario workspace by the driver. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from itertools import product +from typing import Iterable, Mapping + + +class ScenarioSpecError(ValueError): + """A scenario specification is internally inconsistent.""" + + +class CommandKind(StrEnum): + """Kinds of sweep command represented by a scenario step.""" + + SWEEP = "sweep" + RESUME = "resume" + DELETE = "delete" + + +class DatasetExpectation(StrEnum): + """Expected state of scenario-owned benchmark data after a step.""" + + CLEANED = "cleaned" + PRESERVED = "preserved" + REMOVED = "removed" + + +class ExecutionStatus(StrEnum): + """Expected terminal or interrupted state of one reified execution.""" + + SUCCESS = "SUCCESS" + FAILED = "FAILED" + PENDING = "PENDING" + + +class WorkloadPhase(StrEnum): + """Semantic phases whose evidence must be present for a step.""" + + DIRECTORY_CREATE = "directory-create" + WRITE = "write" + TREE_SCAN = "tree-scan" + READ = "read" + REMOVE_FILES = "remove-files" + DELETE_PATH = "delete-path" + + +class FailureInjection(StrEnum): + """Integration-only overlays used by deliberately failing scenarios.""" + + NONE = "none" + FAIL_AFTER_WRITE_ONCE = "fail-after-write-once" + + +@dataclass(frozen=True, order=True) +class ExecutionCoordinate: + """One expected point in an Elbencho sweep.""" + + nodes: int + io_size: str + threads: int + io_depth: int + + +@dataclass(frozen=True) +class ExpectedExecution: + """Expected state of one execution after a command finishes.""" + + coordinate: ExecutionCoordinate + status: ExecutionStatus = ExecutionStatus.SUCCESS + + +@dataclass(frozen=True) +class SupportFile: + """A small text file rendered relative to the scenario workspace.""" + + relative_path: str + content: str + + +@dataclass(frozen=True) +class GeneratedInput: + """A bounded input file the harness creates without storing its contents.""" + + relative_path: str + size_bytes: int + + +@dataclass(frozen=True) +class ScenarioStep: + """One command and its stable semantic expectations.""" + + name: str + kind: CommandKind + arguments: tuple[str, ...] + env_lines: tuple[str, ...] + support_files: tuple[SupportFile, ...] + generated_inputs: tuple[GeneratedInput, ...] + timeout_seconds: int + executions: tuple[ExpectedExecution, ...] + required_phases: tuple[WorkloadPhase, ...] + dataset: DatasetExpectation + requires: tuple[str, ...] = () + exports: tuple[str, ...] = () + failure_injection: FailureInjection = FailureInjection.NONE + preserve_failure_staging: bool = False + + def render_arguments(self, values: Mapping[str, str]) -> tuple[str, ...]: + """Render driver-controlled placeholders in command arguments.""" + return tuple(argument.format_map(values) for argument in self.arguments) + + def render_env(self, values: Mapping[str, str]) -> str: + """Render the environment override block for this step.""" + return "\n".join(line.format_map(values) for line in self.env_lines) + "\n" + + def render_support_files( + self, values: Mapping[str, str] + ) -> tuple[SupportFile, ...]: + """Render paths and contents of small scenario support files.""" + return tuple( + SupportFile( + item.relative_path.format_map(values), item.content.format_map(values) + ) + for item in self.support_files + ) + + +@dataclass(frozen=True) +class FilesystemScenarioSpec: + """All command steps for one named real-infrastructure scenario.""" + + name: str + substrates: frozenset[str] + steps: tuple[ScenarioStep, ...] + + +_SHARED_ENV = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=1)', + 'export ELBENCHO_FILE_LAYOUT="shared-directory"', + "export ELBENCHO_FILES_PER_NODE=1", + 'export ELBENCHO_FILE_SIZE="16M"', + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_LIVE_CSV_EXTENDED=0", + "export ELBENCHO_SINGLE_BIG_FILE=0", +) + +_WORKER_ENV = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=2)', + 'export ELBENCHO_FILE_LAYOUT="worker-directories"', + "export ELBENCHO_FILES_PER_NODE=", + "export ELBENCHO_FILE_SIZE=", + "export ELBENCHO_FILE_SIZE_MULTIPLIER=4096", + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_LIVE_CSV_EXTENDED=0", + "export ELBENCHO_SINGLE_BIG_FILE=0", +) + +_NORMAL_PHASES = ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.READ, + WorkloadPhase.REMOVE_FILES, +) + +_BOTH_SUBSTRATES = frozenset({"ssh", "slurm"}) +_SSH_ONLY = frozenset({"ssh"}) +_SLURM_ONLY = frozenset({"slurm"}) + + +def _coordinates( + nodes: Iterable[int], + io_sizes: Iterable[str], + threads: Iterable[int], + depths: Iterable[int], + *, + statuses: Mapping[ExecutionCoordinate, ExecutionStatus] | None = None, +) -> tuple[ExpectedExecution, ...]: + """Create expected coordinates in the sweep's documented nesting order.""" + status_by_coordinate = statuses or {} + return tuple( + ExpectedExecution( + coordinate, + status_by_coordinate.get(coordinate, ExecutionStatus.SUCCESS), + ) + for values in product(nodes, io_sizes, threads, depths) + for coordinate in (ExecutionCoordinate(*values),) + ) + + +def _step( + name: str, + arguments: tuple[str, ...], + env_lines: tuple[str, ...], + executions: tuple[ExpectedExecution, ...], + phases: tuple[WorkloadPhase, ...], + dataset: DatasetExpectation = DatasetExpectation.CLEANED, + *, + timeout_seconds: int = 300, + requires: tuple[str, ...] = (), + exports: tuple[str, ...] = (), + failure_injection: FailureInjection = FailureInjection.NONE, + preserve_failure_staging: bool = False, +) -> ScenarioStep: + """Build a sweep step with bounded defaults.""" + return ScenarioStep( + name=name, + kind=CommandKind.SWEEP, + arguments=arguments, + env_lines=env_lines, + support_files=(), + generated_inputs=(), + timeout_seconds=timeout_seconds, + executions=executions, + required_phases=phases, + dataset=dataset, + requires=requires, + exports=exports, + failure_injection=failure_injection, + preserve_failure_staging=preserve_failure_staging, + ) + + +def _baseline() -> FilesystemScenarioSpec: + return FilesystemScenarioSpec( + "baseline", + _BOTH_SUBSTRATES, + ( + _step( + "buffered-sweep", + ("--bio", "--nodes", "1,2"), + _SHARED_ENV, + _coordinates((1, 2), ("4K",), (1,), (1,)), + _NORMAL_PHASES, + ), + ), + ) + + +def _default_dio() -> FilesystemScenarioSpec: + return FilesystemScenarioSpec( + "default-dio", + _BOTH_SUBSTRATES, + ( + _step( + "direct-worker-directories", + ("--nodes", "1,2"), + _WORKER_ENV, + _coordinates((1, 2), ("4K",), (1,), (1,)), + ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.TREE_SCAN, + WorkloadPhase.READ, + WorkloadPhase.REMOVE_FILES, + ), + ), + ), + ) + + +def _failure_resume() -> FilesystemScenarioSpec: + coordinates = tuple( + execution.coordinate + for execution in _coordinates((1, 2), ("4K", "8K"), (1,), (1,)) + ) + statuses = { + coordinates[1]: ExecutionStatus.FAILED, + coordinates[2]: ExecutionStatus.PENDING, + coordinates[3]: ExecutionStatus.PENDING, + } + initial = _coordinates((1, 2), ("4K", "8K"), (1,), (1,), statuses=statuses) + env_lines = tuple( + line.replace( + 'ELBENCHO_SCALE_IO_SIZES=("4K")', + 'ELBENCHO_SCALE_IO_SIZES=("4K" "8K")', + ) + for line in _SHARED_ENV + ) + first = _step( + "inject-one-failure", + ("--write-no-read", "--nodes", "1,2"), + env_lines, + initial, + ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.REMOVE_FILES, + ), + exports=("failed_results_dir",), + failure_injection=FailureInjection.FAIL_AFTER_WRITE_ONCE, + preserve_failure_staging=True, + ) + resume = ScenarioStep( + name="resume", + kind=CommandKind.RESUME, + arguments=("--resume", "{failed_results_dir}"), + env_lines=(), + support_files=(), + generated_inputs=(), + timeout_seconds=300, + executions=_coordinates((1, 2), ("4K", "8K"), (1,), (1,)), + required_phases=(WorkloadPhase.WRITE, WorkloadPhase.REMOVE_FILES), + dataset=DatasetExpectation.CLEANED, + requires=("failed_results_dir",), + preserve_failure_staging=True, + ) + return FilesystemScenarioSpec("failure-resume", _BOTH_SUBSTRATES, (first, resume)) + + +def _retained_lifecycle() -> FilesystemScenarioSpec: + coordinate = _coordinates((1,), ("4K",), (1,), (1,)) + write = _step( + "write-only", + ("--write-only", "--nodes", "1"), + _SHARED_ENV, + coordinate, + (WorkloadPhase.DIRECTORY_CREATE, WorkloadPhase.WRITE), + DatasetExpectation.PRESERVED, + exports=("retained_data_dir",), + ) + read_miss = _step( + "read-cache-miss", + ("--read-from", "{retained_data_dir}", "--nodes", "1"), + _SHARED_ENV, + coordinate, + (WorkloadPhase.TREE_SCAN, WorkloadPhase.READ), + DatasetExpectation.PRESERVED, + requires=("retained_data_dir",), + ) + read_hit = _step( + "read-cache-hit", + ("--read-from", "{retained_data_dir}", "--nodes", "1"), + _SHARED_ENV, + coordinate, + (WorkloadPhase.READ,), + DatasetExpectation.PRESERVED, + requires=("retained_data_dir",), + ) + delete = ScenarioStep( + name="delete-only", + kind=CommandKind.DELETE, + arguments=("--delete-only", "{retained_data_dir}"), + env_lines=_SHARED_ENV, + support_files=(), + generated_inputs=(), + timeout_seconds=180, + executions=(), + required_phases=(WorkloadPhase.DELETE_PATH,), + dataset=DatasetExpectation.REMOVED, + requires=("retained_data_dir",), + ) + return FilesystemScenarioSpec( + "retained-lifecycle", _BOTH_SUBSTRATES, (write, read_miss, read_hit, delete) + ) + + +def _live_capture() -> FilesystemScenarioSpec: + env_lines = tuple( + line.replace( + "ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "ELBENCHO_SCALE_READ_WRITE_DURATION=3", + ) + .replace("ELBENCHO_FILES_PER_NODE=1", "ELBENCHO_FILES_PER_NODE=2") + .replace("ELBENCHO_LIVE_CSV_EXTENDED=0", "ELBENCHO_LIVE_CSV_EXTENDED=1") + for line in _SHARED_ENV + ) + ("export ELBENCHO_LIVEINT=10",) + return FilesystemScenarioSpec( + "live-capture", + _BOTH_SUBSTRATES, + ( + _step( + "extended-live-csv", + ("--nodes", "2"), + env_lines, + _coordinates((2,), ("4K",), (1,), (1,)), + ( + WorkloadPhase.DIRECTORY_CREATE, + WorkloadPhase.WRITE, + WorkloadPhase.READ, + WorkloadPhase.REMOVE_FILES, + ), + ), + ), + ) + + +def _slurm_cartesian() -> FilesystemScenarioSpec: + env_lines = tuple( + line.replace("ELBENCHO_FILES_PER_NODE=1", "ELBENCHO_FILES_PER_NODE=2") + .replace('ELBENCHO_FILE_SIZE="16M"', 'ELBENCHO_FILE_SIZE="1M"') + .replace( + 'ELBENCHO_SCALE_THREAD_LIST=("1")', + 'ELBENCHO_SCALE_THREAD_LIST=("1" "2")', + ) + .replace( + 'ELBENCHO_SCALE_IO_SIZES=("4K")', + 'ELBENCHO_SCALE_IO_SIZES=("4K" "4K,r8K")', + ) + .replace( + 'ELBENCHO_IODEPTH_LIST=("1")', + 'ELBENCHO_IODEPTH_LIST=("1" "2")', + ) + for line in _SHARED_ENV + ) + return FilesystemScenarioSpec( + "slurm-cartesian", + _SLURM_ONLY, + ( + _step( + "representative-matrix", + ("--nodes", "1,2"), + env_lines, + _coordinates((1, 2), ("4K", "4K,r8K"), (1, 2), (1, 2)), + _NORMAL_PHASES, + timeout_seconds=600, + ), + ), + ) + + +def _ssh_single_big_file() -> FilesystemScenarioSpec: + env_lines = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=1)', + 'export ELBENCHO_FILE_LAYOUT="worker-directories"', + "export ELBENCHO_FILES_PER_NODE=", + "export ELBENCHO_FILE_SIZE=", + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export ELBENCHO_SINGLE_BIG_FILE=1", + 'export ELBENCHO_SINGLE_BIG_FILE_BASENAME="integration-bigfile"', + 'export ELBENCHO_SINGLE_BIG_FILE_SIZE="16M"', + "export ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0", + ) + coordinate = _coordinates((2,), ("4K",), (1,), (1,)) + generated = _step( + "cooperative-readwrite", + ("--bio", "--nodes", "1,2"), + env_lines, + _coordinates((1, 2), ("4K",), (1,), (1,)), + (WorkloadPhase.WRITE, WorkloadPhase.READ, WorkloadPhase.REMOVE_FILES), + ) + read_env = tuple( + line.replace( + "ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0", + "ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1", + ) + for line in env_lines + ) + read = ScenarioStep( + name="inferred-extent-read", + kind=CommandKind.SWEEP, + arguments=( + "--read-from", + "{test_root}/staged-input/integration-bigfile", + "--bio", + "--nodes", + "2", + ), + env_lines=read_env, + support_files=(), + generated_inputs=( + GeneratedInput("staged-input/integration-bigfile", 16 * 1024 * 1024), + ), + timeout_seconds=300, + executions=coordinate, + required_phases=(WorkloadPhase.READ,), + dataset=DatasetExpectation.PRESERVED, + ) + delete = ScenarioStep( + name="delete-retained-directory", + kind=CommandKind.DELETE, + arguments=("--delete-only", "{test_root}/staged-input"), + env_lines=env_lines, + support_files=(), + generated_inputs=(), + timeout_seconds=180, + executions=(), + required_phases=(WorkloadPhase.DELETE_PATH,), + dataset=DatasetExpectation.REMOVED, + ) + return FilesystemScenarioSpec( + "ssh-single-big-file", _SSH_ONLY, (generated, read, delete) + ) + + +def _ssh_weighted_roots() -> FilesystemScenarioSpec: + env_lines = ( + "unset TEST_DIRS", + 'declare -A TEST_DIRS=(["{test_root}"]=1 ["{test_root_secondary}"]=2)', + 'export ELBENCHO_FILE_LAYOUT="worker-directories"', + "export ELBENCHO_FILES_PER_NODE=", + 'export ELBENCHO_FILE_SIZE="1M"', + 'export ELBENCHO_SCALE_THREAD_LIST=("1")', + 'export ELBENCHO_SCALE_IO_SIZES=("4K")', + 'export ELBENCHO_IODEPTH_LIST=("1")', + "export ELBENCHO_SCALE_READ_WRITE_DURATION=1", + "export FS_MAX_AGG_THROUGHPUT=1", + "export FS_MAX_NODE_THROUGHPUT_GBPS=1", + "export FS_MAX_NODE_IOPS=100", + "export ELBENCHO_SINGLE_BIG_FILE=0", + ) + return FilesystemScenarioSpec( + "ssh-weighted-roots", + _SSH_ONLY, + ( + _step( + "active-single-sizing", + ("--bio", "--single", "--nodes", "1"), + env_lines, + _coordinates((1,), ("4K",), (1,), (1,)), + _NORMAL_PHASES, + ), + ), + ) + + +def _ssh_shared_home() -> FilesystemScenarioSpec: + return FilesystemScenarioSpec( + "ssh-shared-home", + _SSH_ONLY, + ( + _step( + "shared-home-smoke", + ("--bio", "--nodes", "1,2"), + _SHARED_ENV + ("export SSH_HOMEDIR_SHARED=1",), + _coordinates((1, 2), ("4K",), (1,), (1,)), + _NORMAL_PHASES, + ), + ), + ) + + +def _slurm_scheduling() -> FilesystemScenarioSpec: + support = ( + SupportFile("slurm-scenario-includes", "{slurm_node_1}\n{slurm_node_2}\n"), + SupportFile("slurm-scenario-ignores", "{slurm_node_2}\n"), + ) + env_lines = tuple( + line.replace('ELBENCHO_FILE_SIZE="16M"', 'ELBENCHO_FILE_SIZE="4M"') + for line in _SHARED_ENV + ) + ( + "SLURM_EXCLUSIVE_USER=1", + 'SLURM_JOB_NAME_PREFIX="itest-"', + 'export SLURM_NODE_INCLUDES="{workspace}/slurm-scenario-includes"', + 'export SLURM_NODE_IGNORES="{workspace}/slurm-scenario-ignores"', + ) + step = ScenarioStep( + name="allocation-options", + kind=CommandKind.SWEEP, + arguments=("--nodes", "1"), + env_lines=env_lines, + support_files=support, + generated_inputs=(), + timeout_seconds=300, + executions=_coordinates((1,), ("4K",), (1,), (1,)), + required_phases=_NORMAL_PHASES, + dataset=DatasetExpectation.CLEANED, + ) + return FilesystemScenarioSpec("slurm-scheduling", _SLURM_ONLY, (step,)) + + +SCENARIO_SPECS = ( + _baseline(), + _default_dio(), + _failure_resume(), + _retained_lifecycle(), + _live_capture(), + _slurm_cartesian(), + _ssh_single_big_file(), + _ssh_weighted_roots(), + _ssh_shared_home(), + _slurm_scheduling(), +) + +SCENARIO_SPECS_BY_NAME = {spec.name: spec for spec in SCENARIO_SPECS} + + +def validate_scenario_specs( + specifications: Iterable[FilesystemScenarioSpec] = SCENARIO_SPECS, +) -> None: + """Reject inconsistent catalog entries before a real fixture is touched.""" + specs = tuple(specifications) + names = [spec.name for spec in specs] + if len(names) != len(set(names)): + raise ScenarioSpecError("scenario specification names must be unique") + for spec in specs: + _validate_spec(spec) + + +def _validate_spec(spec: FilesystemScenarioSpec) -> None: + """Validate one scenario and its sequence dependencies.""" + if not spec.name or not spec.steps: + raise ScenarioSpecError("scenario names and step sequences must be nonempty") + if not spec.substrates or not spec.substrates <= _BOTH_SUBSTRATES: + raise ScenarioSpecError(f"{spec.name}: invalid substrates") + available: set[str] = set() + step_names: set[str] = set() + for step in spec.steps: + if step.name in step_names: + raise ScenarioSpecError(f"{spec.name}: duplicate step {step.name}") + step_names.add(step.name) + missing = set(step.requires) - available + if missing: + raise ScenarioSpecError( + f"{spec.name}/{step.name}: unavailable values: {sorted(missing)}" + ) + _validate_step(spec.name, step) + available.update(step.exports) + + +def _validate_step(scenario_name: str, step: ScenarioStep) -> None: + """Validate bounded execution and expectation invariants for one step.""" + label = f"{scenario_name}/{step.name}" + if not 1 <= step.timeout_seconds <= 600: + raise ScenarioSpecError(f"{label}: timeout must be between 1 and 600 seconds") + coordinates = [execution.coordinate for execution in step.executions] + if len(coordinates) != len(set(coordinates)): + raise ScenarioSpecError(f"{label}: duplicate execution coordinates") + if step.kind is CommandKind.DELETE and step.executions: + raise ScenarioSpecError(f"{label}: delete steps cannot reify executions") + if step.kind is not CommandKind.DELETE and not step.executions: + raise ScenarioSpecError(f"{label}: sweep and resume steps require executions") + if step.failure_injection is not FailureInjection.NONE: + if not any( + execution.status is ExecutionStatus.FAILED for execution in step.executions + ): + raise ScenarioSpecError(f"{label}: failure injection lacks failed state") + if len(step.required_phases) != len(set(step.required_phases)): + raise ScenarioSpecError(f"{label}: duplicate required phases") + + +validate_scenario_specs() diff --git a/integration-tests/lib/scenario_planner.py b/integration-tests/lib/scenario_planner.py index d068d6f..a2af45b 100644 --- a/integration-tests/lib/scenario_planner.py +++ b/integration-tests/lib/scenario_planner.py @@ -167,9 +167,7 @@ class SshHomeTransition: ), ) -# The harness refactor initially preserves the real baseline. Later commits -# enable catalog entries as their substrate implementations land. -SCENARIOS = SCENARIO_CATALOG[:1] +SCENARIOS = SCENARIO_CATALOG def _concrete_substrates(substrate: Substrate) -> frozenset[Substrate]: diff --git a/tests/test_filesystem_scenario_specs.py b/tests/test_filesystem_scenario_specs.py new file mode 100644 index 0000000..9cc315b --- /dev/null +++ b/tests/test_filesystem_scenario_specs.py @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Contract tests for declarative filesystem integration workloads.""" + +import importlib.util +from dataclasses import replace +from pathlib import Path +import sys + +import pytest + +_MODULE_PATH = ( + Path(__file__).resolve().parents[1] + / "integration-tests" + / "lib" + / "filesystem_scenario_specs.py" +) +_SPEC = importlib.util.spec_from_file_location( + "filesystem_scenario_specs", _MODULE_PATH +) +assert _SPEC is not None and _SPEC.loader is not None +_SCENARIOS = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = _SCENARIOS +_SPEC.loader.exec_module(_SCENARIOS) + +_PLANNER_PATH = _MODULE_PATH.with_name("scenario_planner.py") +_PLANNER_SPEC = importlib.util.spec_from_file_location( + "filesystem_spec_scenario_planner", _PLANNER_PATH +) +assert _PLANNER_SPEC is not None and _PLANNER_SPEC.loader is not None +_PLANNER = importlib.util.module_from_spec(_PLANNER_SPEC) +sys.modules[_PLANNER_SPEC.name] = _PLANNER +_PLANNER_SPEC.loader.exec_module(_PLANNER) + +CommandKind = _SCENARIOS.CommandKind +DatasetExpectation = _SCENARIOS.DatasetExpectation +ExecutionStatus = _SCENARIOS.ExecutionStatus +FailureInjection = _SCENARIOS.FailureInjection +ScenarioSpecError = _SCENARIOS.ScenarioSpecError +WorkloadPhase = _SCENARIOS.WorkloadPhase +validate_scenario_specs = _SCENARIOS.validate_scenario_specs + + +EXPECTED_NAMES = { + "baseline", + "default-dio", + "failure-resume", + "retained-lifecycle", + "live-capture", + "slurm-cartesian", + "ssh-single-big-file", + "ssh-weighted-roots", + "ssh-shared-home", + "slurm-scheduling", +} + + +def _scenario(name): + return _SCENARIOS.SCENARIO_SPECS_BY_NAME[name] + + +def test_catalog_defines_every_planned_real_scenario(): + """The execution catalog and workload catalog have matching stable names.""" + assert set(_SCENARIOS.SCENARIO_SPECS_BY_NAME) == EXPECTED_NAMES + assert {scenario.name for scenario in _PLANNER.SCENARIO_CATALOG} == EXPECTED_NAMES + validate_scenario_specs() + + +def test_each_step_is_bounded_and_has_semantic_expectations(): + """Real workloads remain bounded without freezing incidental artifacts.""" + for scenario in _SCENARIOS.SCENARIO_SPECS: + for step in scenario.steps: + assert 0 < step.timeout_seconds <= 600 + assert step.required_phases + assert step.dataset in DatasetExpectation + assert not hasattr(step, "expected_filenames") + assert not hasattr(step, "expected_native_arguments") + + +@pytest.mark.parametrize("name", ("baseline", "default-dio", "ssh-shared-home")) +def test_basic_real_sweeps_cover_one_and_two_nodes(name): + """Transport smokes exercise node subset changes on their real substrates.""" + step = _scenario(name).steps[0] + assert {execution.coordinate.nodes for execution in step.executions} == {1, 2} + assert all( + execution.status is ExecutionStatus.SUCCESS for execution in step.executions + ) + + +def test_default_dio_uses_worker_layout_and_derived_file_size(): + """The default-path scenario does not accidentally reuse the BIO fixture.""" + step = _scenario("default-dio").steps[0] + environment = "\n".join(step.env_lines) + + assert step.arguments == ("--nodes", "1,2") + assert 'ELBENCHO_FILE_LAYOUT="worker-directories"' in environment + assert "ELBENCHO_FILE_SIZE=" in environment + assert "ELBENCHO_FILE_SIZE_MULTIPLIER=4096" in environment + assert WorkloadPhase.TREE_SCAN in step.required_phases + + +def test_failure_resume_preserves_overlay_through_resume(): + """One atomic injected failure is not restaged between attempts.""" + initial, resume = _scenario("failure-resume").steps + statuses = [execution.status for execution in initial.executions] + + assert statuses == [ + ExecutionStatus.SUCCESS, + ExecutionStatus.FAILED, + ExecutionStatus.PENDING, + ExecutionStatus.PENDING, + ] + assert initial.failure_injection is FailureInjection.FAIL_AFTER_WRITE_ONCE + assert initial.preserve_failure_staging + assert resume.kind is CommandKind.RESUME + assert resume.requires == ("failed_results_dir",) + assert resume.preserve_failure_staging + assert all( + execution.status is ExecutionStatus.SUCCESS for execution in resume.executions + ) + + +def test_retained_lifecycle_expresses_cache_miss_hit_and_deletion(): + """The chained dataset contract survives reads and is deleted last.""" + steps = _scenario("retained-lifecycle").steps + + assert [step.name for step in steps] == [ + "write-only", + "read-cache-miss", + "read-cache-hit", + "delete-only", + ] + assert [step.dataset for step in steps] == [ + DatasetExpectation.PRESERVED, + DatasetExpectation.PRESERVED, + DatasetExpectation.PRESERVED, + DatasetExpectation.REMOVED, + ] + assert WorkloadPhase.TREE_SCAN in steps[1].required_phases + assert WorkloadPhase.TREE_SCAN not in steps[2].required_phases + assert steps[-1].kind is CommandKind.DELETE + assert not steps[-1].executions + + +def test_live_capture_has_stable_series_sampling_window(): + """The real live-data case requests several intervals without a huge run.""" + step = _scenario("live-capture").steps[0] + environment = "\n".join(step.env_lines) + + assert "ELBENCHO_SCALE_READ_WRITE_DURATION=3" in environment + assert "ELBENCHO_LIVE_CSV_EXTENDED=1" in environment + assert "ELBENCHO_LIVEINT=10" in environment + assert {execution.coordinate.nodes for execution in step.executions} == {2} + + +def test_representative_cartesian_sweep_has_all_sixteen_coordinates(): + """The real Slurm matrix is representative rather than exhaustive.""" + scenario = _scenario("slurm-cartesian") + coordinates = [execution.coordinate for execution in scenario.steps[0].executions] + + assert scenario.substrates == {"slurm"} + assert len(coordinates) == 16 + assert {coordinate.nodes for coordinate in coordinates} == {1, 2} + assert {coordinate.io_size for coordinate in coordinates} == {"4K", "4K,r8K"} + assert {coordinate.threads for coordinate in coordinates} == {1, 2} + assert {coordinate.io_depth for coordinate in coordinates} == {1, 2} + + +def test_single_file_sequence_uses_inferred_read_extent(): + """A retained cooperative file feeds the all-node staged read.""" + generated, read, delete = _scenario("ssh-single-big-file").steps + read_environment = "\n".join(read.env_lines) + + assert generated.dataset is DatasetExpectation.CLEANED + assert "{test_root}/staged-input/integration-bigfile" in read.arguments + assert 'ELBENCHO_SINGLE_BIG_FILE_SIZE="16M"' in read_environment + assert "export ELBENCHO_SINGLE_BIG_FILE_SIZE=" in read_environment + assert "ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1" in read_environment + assert read.dataset is DatasetExpectation.PRESERVED + assert delete.dataset is DatasetExpectation.REMOVED + + +def test_weighted_roots_activates_single_sizing_with_bounded_limits(): + """The SSH sizing case uses both weighted roots and deliberately low limits.""" + step = _scenario("ssh-weighted-roots").steps[0] + environment = "\n".join(step.env_lines) + + assert step.arguments == ("--bio", "--single", "--nodes", "1") + assert '["{test_root}"]=1' in environment + assert '["{test_root_secondary}"]=2' in environment + assert "FS_MAX_NODE_IOPS=100" in environment + + +def test_slurm_scheduling_renders_include_and_ignore_files(): + """The scheduling scenario constrains a real one-node allocation.""" + step = _scenario("slurm-scheduling").steps[0] + rendered = step.render_support_files( + { + "slurm_node_1": "compute-0", + "slurm_node_2": "compute-1", + "workspace": "/tmp/work", + "test_root": "/mnt/test", + } + ) + + assert step.arguments == ("--nodes", "1") + assert rendered[0].content == "compute-0\ncompute-1\n" + assert rendered[1].content == "compute-1\n" + assert "SLURM_EXCLUSIVE_USER=1" in step.env_lines + assert 'SLURM_JOB_NAME_PREFIX="itest-"' in step.env_lines + + +def test_template_rendering_resolves_driver_owned_paths(): + """The driver can render env and argument templates without shell discovery.""" + step = _scenario("retained-lifecycle").steps[1] + values = {"test_root": "/mnt/test", "retained_data_dir": "/mnt/test/data"} + + assert step.render_arguments(values) == ( + "--read-from", + "/mnt/test/data", + "--nodes", + "1", + ) + assert '["/mnt/test"]=1' in step.render_env(values) + + +def test_validation_rejects_missing_sequence_dependency(): + """A chained command cannot consume an artifact no prior step exported.""" + scenario = _scenario("retained-lifecycle") + broken_first = replace(scenario.steps[0], exports=()) + broken = replace(scenario, steps=(broken_first, *scenario.steps[1:])) + + with pytest.raises(ScenarioSpecError, match="unavailable values"): + validate_scenario_specs((broken,)) + + +def test_validation_rejects_duplicate_coordinates(): + """One command cannot claim two result records for the same coordinate.""" + scenario = _scenario("baseline") + step = scenario.steps[0] + broken_step = replace(step, executions=(step.executions[0],) * 2) + broken = replace(scenario, steps=(broken_step,)) + + with pytest.raises(ScenarioSpecError, match="duplicate execution"): + validate_scenario_specs((broken,)) + + +def test_validation_rejects_unbounded_timeout(): + """Every real command has an explicit upper wall-time bound.""" + scenario = _scenario("baseline") + broken_step = replace(scenario.steps[0], timeout_seconds=601) + broken = replace(scenario, steps=(broken_step,)) + + with pytest.raises(ScenarioSpecError, match="timeout"): + validate_scenario_specs((broken,)) diff --git a/tests/test_integration_failure_injection.py b/tests/test_integration_failure_injection.py new file mode 100644 index 0000000..49020c9 --- /dev/null +++ b/tests/test_integration_failure_injection.py @@ -0,0 +1,302 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for scenario-owned Elbencho failure injection.""" + +import os +import shutil +import signal +import subprocess +import sys +from pathlib import Path + +import pytest + +_REPOSITORY_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_REPOSITORY_ROOT / "integration-tests" / "lib")) + +from failure_injection import ( # pylint: disable=wrong-import-position + FailureInjectionError, + FailureInjectionOperations, + build_slurm_failure_injection_plan, + build_ssh_failure_injection_plan, + cleanup_failure_injection, + stage_failure_injection, + staged_failure_injection, +) + +TARGET_ARGUMENT = "/mnt/storage-test/results-e0002" + + +def _write_delegate(path, body): + """Create one executable delegate used by wrapper process tests.""" + path.write_text(f"#!/usr/bin/env bash\nset -u\n{body}\n", encoding="utf-8") + path.chmod(0o755) + + +def _local_plan(tmp_path, *, runtime=False, exit_code=97): + """Return a staged local Slurm-shaped plan.""" + source = tmp_path / "source-ussegl" + _write_delegate(source, 'printf "stdout:%s\\n" "$*"; printf "stderr\\n" >&2') + source_runtime = None + if runtime: + source_runtime = tmp_path / "source-runtime" + source_runtime.mkdir() + (source_runtime / "library.so").write_text("runtime", encoding="utf-8") + plan = build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path.as_posix(), + source_binary=source, + source_runtime=source_runtime, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + injected_exit_code=exit_code, + ) + operations = _local_operations(tmp_path) + stage_failure_injection(plan, operations) + return plan, operations + + +def _local_operations(base): + """Return staging operations that require the expected local endpoint.""" + + def _check(endpoint, path): + assert endpoint == "local" + assert Path(path).is_relative_to(base) + return Path(path) + + def _make(endpoint, path): + _check(endpoint, path).mkdir(parents=True, exist_ok=True) + + def _copy_file(source, endpoint, path, mode): + destination = _check(endpoint, path) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + destination.chmod(mode) + + def _copy_tree(source, endpoint, path): + shutil.copytree(source, _check(endpoint, path)) + + def _write(endpoint, path, content, mode): + destination = _check(endpoint, path) + destination.write_text(content, encoding="utf-8") + destination.chmod(mode) + + def _remove(endpoint, path): + shutil.rmtree(_check(endpoint, path), ignore_errors=True) + + return FailureInjectionOperations(_make, _copy_file, _copy_tree, _write, _remove) + + +def test_target_succeeds_then_fails_once_and_marker_survives_resume(tmp_path): + """A successful target fails once, while resume delegates successfully.""" + plan, _ = _local_plan(tmp_path) + command = [str(plan.layout.wrapper), "--write", TARGET_ARGUMENT] + + first = subprocess.run(command, text=True, capture_output=True, check=False) + second = subprocess.run(command, text=True, capture_output=True, check=False) + + assert first.returncode == 97 + assert first.stdout == f"stdout:--write {TARGET_ARGUMENT}\n" + assert first.stderr == "stderr\n" + assert (Path(plan.layout.marker) / "state").read_text( + encoding="utf-8" + ) == "injected\n" + assert second.returncode == 0 + assert second.stdout == first.stdout + assert second.stderr == first.stderr + assert Path(plan.layout.marker).is_dir() + + +def test_service_mode_execs_delegate_with_same_process_identity(tmp_path): + """Service mode replaces the wrapper instead of leaving a parent process.""" + plan, _ = _local_plan(tmp_path) + _write_delegate(Path(plan.layout.delegate), 'printf "%s\\n" "$BASHPID"') + + process = subprocess.Popen( + [str(plan.layout.wrapper), "--service", TARGET_ARGUMENT], + text=True, + stdout=subprocess.PIPE, + ) + output = process.communicate(timeout=5)[0] + + assert process.returncode == 0 + assert int(output.strip()) == process.pid + assert not Path(plan.layout.marker).exists() + + +def test_nontarget_and_real_failure_are_not_replaced(tmp_path): + """Only a successful target is eligible for the injected exit status.""" + plan, _ = _local_plan(tmp_path) + + nontarget = subprocess.run( + [str(plan.layout.wrapper), "--write", "/another/path"], check=False + ) + assert nontarget.returncode == 0 + assert not Path(plan.layout.marker).exists() + + _write_delegate(Path(plan.layout.delegate), "exit 23") + failed = subprocess.run([str(plan.layout.wrapper), TARGET_ARGUMENT], check=False) + assert failed.returncode == 23 + assert not Path(plan.layout.marker).exists() + + +def test_coordinator_forwards_signal_and_preserves_child_status(tmp_path): + """Signals reach the real coordinator and its failure remains authoritative.""" + plan, _ = _local_plan(tmp_path) + _write_delegate( + Path(plan.layout.delegate), + "trap 'exit 42' TERM\nprintf '%s\\n' \"$BASHPID\"\nwhile true; do sleep 1; done", + ) + process = subprocess.Popen( + [str(plan.layout.wrapper), TARGET_ARGUMENT], + text=True, + stdout=subprocess.PIPE, + ) + assert process.stdout is not None + child_pid = int(process.stdout.readline().strip()) + + process.send_signal(signal.SIGTERM) + process.wait(timeout=5) + + assert process.returncode == 42 + with pytest.raises(ProcessLookupError): + os.kill(child_pid, 0) + assert not Path(plan.layout.marker).exists() + + +class _Recorder: + """Record substrate-neutral staging operations without touching hosts.""" + + def __init__(self): + self.calls = [] + + def operations(self): + """Return callbacks backed by this recorder.""" + return FailureInjectionOperations( + lambda endpoint, path: self.calls.append(("mkdir", endpoint, path)), + lambda source, endpoint, path, mode: self.calls.append( + ("copy-file", source, endpoint, path, mode) + ), + lambda source, endpoint, path: self.calls.append( + ("copy-tree", source, endpoint, path) + ), + lambda endpoint, path, content, mode: self.calls.append( + ("write", endpoint, path, content, mode) + ), + lambda endpoint, path: self.calls.append(("remove", endpoint, path)), + ) + + +def test_ssh_staging_places_delegate_runtime_on_every_worker(tmp_path): + """SSH workers receive delegate trees but production copies their wrapper.""" + source = tmp_path / "elbencho" + _write_delegate(source, "exit 0") + runtime = tmp_path / "elbencho-runtime" + runtime.mkdir() + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root="/var/tmp/storage-scale-test", + source_binary=source, + source_runtime=runtime, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="coordinator", + worker_endpoints=("worker-1", "worker-2"), + ) + recorder = _Recorder() + + stage_failure_injection(plan, recorder.operations()) + + file_endpoints = {call[2] for call in recorder.calls if call[0] == "copy-file"} + tree_endpoints = {call[2] for call in recorder.calls if call[0] == "copy-tree"} + writes = [call for call in recorder.calls if call[0] == "write"] + assert file_endpoints == {"coordinator", "worker-1", "worker-2"} + assert tree_endpoints == file_endpoints + assert len(writes) == 1 + assert writes[0][1] == "coordinator" + assert f"readonly delegate={plan.layout.delegate}" in writes[0][3] + + +def test_outer_context_retains_artifacts_through_resume_then_cleans(tmp_path): + """No per-attempt cleanup removes the marker needed by resume.""" + source = tmp_path / "source-elbencho" + _write_delegate(source, "exit 0") + plan = build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path.as_posix(), + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + ) + operations = _local_operations(tmp_path) + + with staged_failure_injection(plan, operations): + command = [str(plan.layout.wrapper), TARGET_ARGUMENT] + assert subprocess.run(command, check=False).returncode == 97 + assert Path(plan.layout.marker).is_dir() + assert subprocess.run(command, check=False).returncode == 0 + assert Path(plan.layout.marker).is_dir() + + assert not Path(plan.layout.root).exists() + + +def test_staging_rejects_missing_or_unsafe_inputs(tmp_path): + """Plans reject unsafe roots and staging rejects missing delegates.""" + source = tmp_path / "missing" + with pytest.raises(FailureInjectionError, match="dedicated absolute"): + build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root="relative", + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + ) + + plan = build_slurm_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path.as_posix(), + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + shared_endpoint="local", + ) + with pytest.raises(FailureInjectionError, match="not a regular file"): + stage_failure_injection(plan, _local_operations(tmp_path)) + + +def test_cleanup_runs_in_reverse_target_order(tmp_path): + """Cleanup removes workers before the coordinator marker owner.""" + source = tmp_path / "elbencho" + _write_delegate(source, "exit 0") + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root="/var/tmp/storage-scale-test", + source_binary=source, + source_runtime=None, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="coordinator", + worker_endpoints=("worker-1", "worker-2"), + ) + recorder = _Recorder() + + cleanup_failure_injection(plan, recorder.operations()) + + assert [call[1] for call in recorder.calls] == [ + "worker-2", + "worker-1", + "coordinator", + ] From 7452d720727b0e0a4a2e509e270e8d6d320a13d7 Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sat, 19 Sep 2026 21:49:46 -0700 Subject: [PATCH 06/10] Add isolated filesystem sweep contract coverage Cover node-range parsing, Cartesian reification, configuration precedence, SSH host selection, destructive path guards, single-file constraints, weighted sizing, and Slurm argument boundaries with fast shell tests. Exercise reporting filters, CSV round trips, output modes, and live-report option validation without requiring a running fixture. Fix associative TEST_DIRS detection so legacy TEST_DIR cannot override an explicit map. Reject partially invalid node specifications before any execution is reified, and coerce cached report fields using resolved type annotations so numeric filters work after --from-csv. Make CSV input exclusive with raw result directories and reject malformed or no-match filters to prevent successful but misleading reports. Document the resulting reporting and fast-test contracts. --- docs/CONTEXT.md | 7 + docs/DESIGN.md | 5 +- lib/_elbencho_functions.sh | 6 +- lib/env_base.sh | 5 +- storage-tests/fs/nv-elbencho-sweep.sh | 4 +- tests/test_elbencho_sweep_contracts_shell.py | 431 +++++++++++++++++++ tests/test_extract_elbencho_cli_contracts.py | 236 ++++++++++ utils/extract-elbencho.py | 127 ++++-- 8 files changed, 775 insertions(+), 46 deletions(-) create mode 100644 tests/test_elbencho_sweep_contracts_shell.py create mode 100644 tests/test_extract_elbencho_cli_contracts.py diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 904c147..39313ef 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -86,6 +86,13 @@ storage is isolated and removed after host-side evidence is retained. The on-demand integration workflow explicitly selects NFS and runs the complete catalog concurrently on amd64 and arm64; Kubernetes remains fixture infrastructure and is not a benchmark execution substrate. +Fast shell contract tests cover node-range parsing and Cartesian order, +configuration precedence, SSH host parsing and selection, workload-mode and +path safety, sizing limits, and Slurm argument boundaries without a live +fixture. Elbencho CSV reload resolves postponed dataclass annotations before +coercing types, so cached metrics remain filterable. Its reporting CLI treats +`--from-csv` as exclusive with raw directories and rejects malformed or +no-match filters. GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 0a47fb0..539a9c4 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -692,7 +692,10 @@ All analysis scripts support filtering to reduce clutter: Intermediate data can be cached (`--to-csv`, `--to-json`) and reloaded (`--from-csv`, `--from-json`) to avoid re-parsing raw files, enabling fast -iterative report refinement. +iterative report refinement. For Elbencho, `--from-csv` is an alternative input +source and cannot be combined with raw result directories. Malformed filters +and filters that match neither aggregate nor live metrics fail instead of +silently producing an unfiltered or empty report. ### 9.5 Plot Design Principles diff --git a/lib/_elbencho_functions.sh b/lib/_elbencho_functions.sh index 9fa79d8..ba08d87 100644 --- a/lib/_elbencho_functions.sh +++ b/lib/_elbencho_functions.sh @@ -3494,11 +3494,13 @@ reify_all_elbencho_executions() { local sweep_write_no_read="$7" local sweep_read_from="$8" - local -a node_counts - if ! mapfile -t node_counts < <(parse_range_specification "$nodes_spec"); then + local node_counts_output="" + if ! node_counts_output=$(parse_range_specification "$nodes_spec"); then echo "Error: Invalid node specification: $nodes_spec" >&2 return 1 fi + local -a node_counts + mapfile -t node_counts <<<"$node_counts_output" if [[ ${#node_counts[@]} -eq 0 ]]; then echo "Error: Node specification produced no node counts" >&2 return 1 diff --git a/lib/env_base.sh b/lib/env_base.sh index 8881916..208be84 100644 --- a/lib/env_base.sh +++ b/lib/env_base.sh @@ -58,9 +58,10 @@ export PATH # If TEST_DIR is defined and TEST_DIRS is empty, use TEST_DIR as the only test directory if [[ -n "${TEST_DIR:-}" ]]; then # Check if TEST_DIRS is not defined or is empty - if [[ -z "${TEST_DIRS+x}" ]] || [[ ${#TEST_DIRS[@]} -eq 0 ]]; then + if ! declare -p TEST_DIRS &>/dev/null || [[ ${#TEST_DIRS[@]} -eq 0 ]]; then # Initialize TEST_DIRS as an associative array if not already defined - declare -A TEST_DIRS + unset TEST_DIRS + declare -gA TEST_DIRS # Set TEST_DIR as the only test directory with weight 1 TEST_DIRS["$TEST_DIR"]=1 fi diff --git a/storage-tests/fs/nv-elbencho-sweep.sh b/storage-tests/fs/nv-elbencho-sweep.sh index 8a0e618..64d46d2 100755 --- a/storage-tests/fs/nv-elbencho-sweep.sh +++ b/storage-tests/fs/nv-elbencho-sweep.sh @@ -445,10 +445,12 @@ if [[ "${ELBENCHO_SINGLE_BIG_FILE:-0}" == "1" && "$rand_option" == "1" ]]; then fi # Parse --nodes specification into array -if ! mapfile -t node_counts < <(parse_range_specification "$nodes_spec"); then +node_counts_output="" +if ! node_counts_output=$(parse_range_specification "$nodes_spec"); then echo "Error: Invalid node specification: $nodes_spec" >&2 exit 1 fi +mapfile -t node_counts <<<"$node_counts_output" if [[ ${#node_counts[@]} -eq 0 ]]; then echo "Error: Node specification produced no node counts" >&2 exit 1 diff --git a/tests/test_elbencho_sweep_contracts_shell.py b/tests/test_elbencho_sweep_contracts_shell.py new file mode 100644 index 0000000..5cd2d45 --- /dev/null +++ b/tests/test_elbencho_sweep_contracts_shell.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fast decision-partition tests for the filesystem sweep contract.""" + +import os +import shutil +import subprocess +import textwrap +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_ENV_BASE = _REPO_ROOT / "lib" / "env_base.sh" +_ENV_FUNCTIONS = _REPO_ROOT / "lib" / "env_functions.sh" +_ELBENCHO_FUNCTIONS = _REPO_ROOT / "lib" / "_elbencho_functions.sh" +_SWEEP = _REPO_ROOT / "storage-tests" / "fs" / "nv-elbencho-sweep.sh" +_BASH = shutil.which("bash") or "bash" + + +def _run_bash(script: str) -> subprocess.CompletedProcess[str]: + """Run a dedented Bash contract harness.""" + return subprocess.run( + [_BASH, "-c", textwrap.dedent(script)], + check=False, + cwd=_REPO_ROOT, + text=True, + capture_output=True, + ) + + +def test_node_specification_equivalence_classes_and_boundaries(): + """Lists and stepped ranges retain order while malformed forms fail.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + diff -u <(printf '3\n5\n7\n9\n10\n13\n16\n8\n8\n') \ + <(parse_range_specification '3-10+2,13,16,8,8') + for invalid in '' ',1' '1,' '1,,2' '0' '3-1' '1-3+0' '1-x'; do + ! parse_range_specification "$invalid" >/dev/null 2>&1 + done + """) + assert result.returncode == 0, result.stderr + + +def test_cartesian_reification_has_stable_order_and_rotation(tmp_path): + """Every coordinate is unique and numbered in documented loop order.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + source "{_ELBENCHO_FUNCTIONS}" + declare -A TEST_DIRS=(["{tmp_path}/data"]=1) + generate_fs_test_directories() {{ printf '%s/target\n' "{tmp_path}"; }} + DS=20260920Z010203 + nodes_spec='2,1-3+2' + rand_option=0 + ELBENCHO_SCALE_IO_SIZES=(4K r8K) + ELBENCHO_SCALE_THREAD_LIST=(1 2) + ELBENCHO_IODEPTH_LIST=(1 4) + ELBENCHO_SCALE_READ_WRITE_DURATION=1 + ELBENCHO_LIVE_CSV_EXTENDED=0 + ELBENCHO_FILE_SIZE_MULTIPLIER=1 + ELBENCHO_FILE_LAYOUT=worker-directories + ELBENCHO_FILES_PER_NODE= + ELBENCHO_FILE_SIZE=1M + ELBENCHO_READ_AFTER_WRITE_PAUSE=0 + FS_MAX_AGG_THROUGHPUT=1 + FS_MAX_NODE_THROUGHPUT_GBPS=1 + FS_MAX_NODE_IOPS=1 + ELBENCHO_SINGLE_BIG_FILE=0 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + mkdir -p "{tmp_path}/result" + reify_all_elbencho_executions \ + "{tmp_path}/result" "$nodes_spec" dio 0 0 0 0 '' + [[ $(find "{tmp_path}/result/executions" -name '*.sh' | wc -l) -eq 24 ]] + [[ $(find "{tmp_path}/result/executions" -name '*.status' | wc -l) -eq 24 ]] + source "{tmp_path}/result/executions/0001.sh" + [[ "$nodes:$io_size:$thread_count:$io_depth:$ELBENCHO_READ_HOST_ROTATE_STEPS" == \ + '2:4K:1:1:0' ]] + source "{tmp_path}/result/executions/0008.sh" + [[ "$nodes:$io_size:$thread_count:$io_depth:$ELBENCHO_READ_HOST_ROTATE_STEPS" == \ + '2:r8K:2:4:7' ]] + source "{tmp_path}/result/executions/0024.sh" + [[ "$nodes:$io_size:$thread_count:$io_depth:$ELBENCHO_READ_HOST_ROTATE_STEPS" == \ + '3:r8K:2:4:23' ]] + [[ $(sort -u "{tmp_path}/result/executions/"*.status) == PENDING ]] + """) + assert result.returncode == 0, result.stderr + + +def test_env_precedence_selects_test_dirs_ssh_and_order_aliases(tmp_path): + """Populated TEST_DIRS wins and SSH disables otherwise valid Slurm state.""" + hosts = tmp_path / "hosts" + hosts.write_text("# comment\nhost-a, host-b\n\nhost-a\n", encoding="utf-8") + result = _run_bash(f""" + set -e + SCALE_TEST_BASE="{_REPO_ROOT}" + RESULTS_DIR="{tmp_path}/results" + LOGS_DIR="{tmp_path}/logs" + TEST_DIR="{tmp_path}/legacy" + declare -A TEST_DIRS=(["{tmp_path}/preferred"]=2) + OBJ_BUCKET= + OBJ_AUTH_FILE="{tmp_path}/missing-auth" + SSH_HOST_LIST="{hosts}" + SSH_USER=tester + ORDER_NODES=YeS + client_type=cpu + client_arch=x86_64 + source "{_ENV_BASE}" + [[ "${{#TEST_DIRS[@]}}" -eq 1 ]] + [[ "${{TEST_DIRS[{tmp_path}/preferred]}}" == 2 ]] + [[ -z "${{TEST_DIRS[{tmp_path}/legacy]+x}}" ]] + [[ "$SSH_ENABLED" == 1 && -z "$SLURM_ENABLED" ]] + [[ "$ORDER_NODES_ENABLED" == 1 ]] + [[ "${{SSH_ALL_HOSTS[*]}}" == 'host-a host-b host-a' ]] + """) + assert result.returncode == 0, result.stderr + + +def test_env_fallbacks_apply_only_when_primary_values_are_empty(tmp_path): + """Legacy directory and throughput names fill gaps without overriding values.""" + hosts = tmp_path / "hosts" + hosts.write_text("host-a\n", encoding="utf-8") + result = _run_bash(f""" + set -e + SCALE_TEST_BASE="{_REPO_ROOT}" + RESULTS_DIR="{tmp_path}/results" + LOGS_DIR="{tmp_path}/logs" + TEST_DIR="{tmp_path}/legacy" + declare -A TEST_DIRS=() + OBJ_BUCKET= + OBJ_AUTH_FILE="{tmp_path}/missing-auth" + SSH_HOST_LIST="{hosts}" + ORDER_NODES=off + client_type=cpu + client_arch=x86_64 + IOR_FS_MAX_AGG_THROUGHPUT=27 + FS_MAX_AGG_THROUGHPUT=19 + source "{_ENV_BASE}" + [[ "${{#TEST_DIRS[@]}}" -eq 1 ]] + [[ "${{TEST_DIRS[{tmp_path}/legacy]}}" == 1 ]] + [[ -z "$ORDER_NODES_ENABLED" ]] + [[ "$FS_MAX_AGG_THROUGHPUT" == 19 ]] + """) + assert result.returncode == 0, result.stderr + + +def test_ssh_selection_contract_covers_ordered_and_unordered_modes(): + """Ordered selection is a prefix; unordered selection is unique and bounded.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + SSH_ENABLED=1 + SSH_ALL_HOSTS=(host-a host-b host-c host-d) + ORDER_NODES_ENABLED=1 + choose_N_ssh_hosts 2 + [[ "$SSH_NODELIST" == host-a,host-b ]] + ORDER_NODES_ENABLED= + RANDOM=7 + choose_N_ssh_hosts 3 + IFS=, read -ra selected <<<"$SSH_NODELIST" + [[ "${{#selected[@]}}" -eq 3 ]] + [[ $(printf '%s\n' "${{selected[@]}}" | sort -u | wc -l) -eq 3 ]] + for host in "${{selected[@]}}"; do + [[ " ${{SSH_ALL_HOSTS[*]}} " == *" $host "* ]] + done + ! choose_N_ssh_hosts 0 >/dev/null 2>&1 + ! choose_N_ssh_hosts 5 >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_read_and_delete_path_guards_resolve_symlinks(tmp_path): + """Read permits the root, while deletion requires a real strict descendant.""" + root = tmp_path / "root" + child = root / "child" + outside = tmp_path / "outside" + child.mkdir(parents=True) + outside.mkdir() + (root / "escape").symlink_to(outside, target_is_directory=True) + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + declare -A TEST_DIRS=(["{root}"]=1) + validate_elbencho_sweep_read_from_path "{root}" + validate_elbencho_sweep_read_from_path "{child}" + validate_elbencho_sweep_delete_only_path "{child}" + ! validate_elbencho_sweep_delete_only_path "{root}" >/dev/null 2>&1 + ! validate_elbencho_sweep_delete_only_path "{outside}" >/dev/null 2>&1 + ! validate_elbencho_sweep_delete_only_path \ + "{root}/escape/victim" >/dev/null 2>&1 + ! validate_elbencho_sweep_read_from_path \ + "{root}/escape/input" >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_single_big_file_validation_partitions(tmp_path): + """Single-file mode enforces one root, sequential I/O, and extent rules.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + declare -A TEST_DIRS=(["{tmp_path}"]=1) + ELBENCHO_SINGLE_BIG_FILE=1 + ELBENCHO_SCALE_IO_SIZES=(4K '1M,4K') + ELBENCHO_SINGLE_BIG_FILE_SIZE=16M + validate_elbencho_single_big_file_env '' + ELBENCHO_SINGLE_BIG_FILE_SIZE= + validate_elbencho_single_big_file_env "{tmp_path}/existing-file" + ! validate_elbencho_single_big_file_env '' >/dev/null 2>&1 + ELBENCHO_SINGLE_BIG_FILE_SIZE=16M + ELBENCHO_SCALE_IO_SIZES=(r4K) + ! validate_elbencho_single_big_file_env '' >/dev/null 2>&1 + ELBENCHO_SCALE_IO_SIZES=(4K) + TEST_DIRS["{tmp_path}/second"]=1 + ! validate_elbencho_single_big_file_env '' >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_workload_precedence_and_incompatible_values(tmp_path): + """Only active branch inputs affect sizing and workload validation.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + source "{_ELBENCHO_FUNCTIONS}" + declare -A TEST_DIRS=(["{tmp_path}"]=1) + ELBENCHO_SCALE_THREAD_LIST=(1) + ELBENCHO_SCALE_IO_SIZES=('6K,r4K') + ELBENCHO_FILE_LAYOUT=worker-directories + ELBENCHO_FILES_PER_NODE= + ELBENCHO_FILE_SIZE=6K + ELBENCHO_FILE_SIZE_MULTIPLIER=1024 + ELBENCHO_SINGLE_BIG_FILE=0 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + sweep_write_only=0 + sweep_write_no_read=0 + [[ $(_elbencho_resolve_generated_file_size 4K) == 6K ]] + ! validate_elbencho_sweep_workload_mode bio 0 '' >/dev/null 2>&1 + sweep_write_only=1 + validate_elbencho_sweep_workload_mode bio 0 '' + ELBENCHO_FILE_SIZE= + [[ $(_elbencho_resolve_generated_file_size 4K) == 4M ]] + sweep_write_only=0 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1 + ! validate_elbencho_sweep_workload_mode dio 0 '' >/dev/null 2>&1 + ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + ELBENCHO_FILE_LAYOUT=shared-directory + ELBENCHO_FILES_PER_NODE=3 + ELBENCHO_SCALE_THREAD_LIST=(2) + ELBENCHO_SCALE_IO_SIZES=(4K) + ELBENCHO_FILE_SIZE=4K + ! validate_elbencho_sweep_workload_mode dio 0 '' >/dev/null 2>&1 + ELBENCHO_FILES_PER_NODE=4 + validate_elbencho_sweep_workload_mode dio 0 '' + TEST_DIRS["{tmp_path}/second"]=1 + ! validate_elbencho_sweep_workload_mode dio 0 '' >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def test_weighted_sizing_uses_the_tightest_capacity_limit(): + """IOPS and aggregate bandwidth limits independently bound file counts.""" + result = _run_bash(f""" + set -e + source "{_ELBENCHO_FUNCTIONS}" + FS_MAX_NODE_THROUGHPUT_GBPS=8 + FS_MAX_AGG_THROUGHPUT=100 + FS_MAX_NODE_IOPS=512 + [[ $(compute_target_file_count_per_thread 4K 1M 2 1 1) -eq 2 ]] + FS_MAX_NODE_IOPS=999999999 + FS_MAX_AGG_THROUGHPUT=1 + [[ $(compute_target_file_count_per_thread 4K 1M 2 1 1) -eq 512 ]] + """) + assert result.returncode == 0, result.stderr + + +def test_slurm_command_boundaries_preserve_spaced_arguments(tmp_path): + """Scheduler arrays retain reservations, GPU requests, and one spaced value.""" + for name in ("sbatch", "srun"): + executable = tmp_path / name + executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + executable.chmod(0o755) + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + PATH="{tmp_path}:$PATH" + _SBATCH_OPTIONS_BASE='-A account-a -p batch --exclusive=user' + _SRUN_OPTIONS_BASE='-A account-a -p batch --exclusive=user' + SLURM_EXTRA_ARGS=(--qos=normal '--comment=integration scenario') + declare -a batch run + build_sbatch_cmd batch + build_srun_cmd run + [[ "${{batch[-1]}}" == '--comment=integration scenario' ]] + [[ "${{run[-1]}}" == '--comment=integration scenario' ]] + [[ "${{batch[-2]}}" == --qos=normal ]] + account=account-a + partition=gpu + run_time=00:05:00 + reservation=reserved-a + SLURM_GPUS_PER_NODE_OPT=--gpus-per-node=4 + build_srun_validate_scale_cmd run + [[ " ${{run[*]}} " == *' --reservation reserved-a '* ]] + [[ " ${{run[*]}} " == *' --gpus-per-node=4 '* ]] + [[ "${{run[-1]}}" == '--comment=integration scenario' ]] + SLURM_JOB_NAME_PREFIX='project-' + [[ $(make_sbatch_job_name elbencho 20260920Z010203 2-1) == \ + project-elbencho-20260920Z010203-2-1 ]] + """) + assert result.returncode == 0, result.stderr + + +def test_exclusive_user_cpu_discovery_uses_smallest_target(): + """Exclusive-user scheduling requests a capacity valid on every target.""" + result = _run_bash(f""" + set -e + source "{_ENV_FUNCTIONS}" + scontrol() {{ printf 'node-a\nnode-b\n'; }} + sinfo() {{ printf '16\n4\n8\n'; }} + SLURM_INCLUDE_COUNT=2 + SLURM_INCLUDES=--nodelist=node-a,node-b + [[ $(get_slurm_target_node_cpus) == 4 ]] + sinfo() {{ printf 'unknown\n'; }} + ! get_slurm_target_node_cpus >/dev/null 2>&1 + """) + assert result.returncode == 0, result.stderr + + +def _make_sweep_fixture(tmp_path: Path) -> Path: + """Create a minimal deployment for parser-only sweep invocations.""" + script_dir = tmp_path / "storage-tests" / "fs" + script_dir.mkdir(parents=True) + sweep = script_dir / _SWEEP.name + shutil.copy2(_SWEEP, sweep) + root = tmp_path / "data" + root.mkdir() + env = tmp_path / "env.sh" + env.write_text( + textwrap.dedent(f""" + export SCALE_TEST_BASE={_REPO_ROOT!s} + export RESULTS_DIR={tmp_path / 'results'!s} + export LOGS_DIR={tmp_path / 'logs'!s} + declare -A TEST_DIRS=([{root!s}]=1) + ELBENCHO_SCALE_THREAD_LIST=(1) + ELBENCHO_SCALE_IO_SIZES=(4K) + ELBENCHO_IODEPTH_LIST=(1) + export ELBENCHO_SCALE_READ_WRITE_DURATION=1 + export ELBENCHO_FILE_LAYOUT=worker-directories + export ELBENCHO_FILES_PER_NODE= + export ELBENCHO_FILE_SIZE=1M + export ELBENCHO_FILE_SIZE_MULTIPLIER=1 + export ELBENCHO_LIVE_CSV_EXTENDED=0 + export ELBENCHO_SINGLE_BIG_FILE=0 + export ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=0 + export FS_MAX_AGG_THROUGHPUT=1 + export FS_MAX_NODE_THROUGHPUT_GBPS=1 + export FS_MAX_NODE_IOPS=1 + export SSH_ENABLED= + export SLURM_ENABLED= + source "$SCALE_TEST_BASE/lib/env_functions.sh" + """), + encoding="utf-8", + ) + return sweep + + +@pytest.mark.parametrize( + "arguments, message", + [ + (("--nodes",), "requires an argument"), + (("--read-from",), "requires a path argument"), + (("--delete-only",), "requires a path argument"), + (("--resume",), "requires a path argument"), + (("--unknown",), "Unknown option"), + (("positional",), "Unexpected positional argument"), + ((), "--nodes is required"), + (("--nodes", "3-1"), "Invalid node specification"), + (("--nodes", "1,invalid"), "Invalid node specification"), + ( + ("--write-only", "--write-no-read", "--nodes", "1"), + "Use at most one", + ), + (("--resume", "/missing", "--bio"), "mutually exclusive"), + ], +) +def test_sweep_cli_rejects_invalid_equivalence_classes(tmp_path, arguments, message): + """Parser and mode errors fail before dispatching or creating a run.""" + sweep = _make_sweep_fixture(tmp_path) + result = subprocess.run( + [str(sweep), *arguments], + check=False, + cwd=tmp_path, + env={**os.environ, "SHELL": _BASH}, + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert message in result.stdout + result.stderr + assert not (tmp_path / "results").exists() + + +@pytest.mark.parametrize("flag", ["-h", "--help"]) +def test_sweep_help_is_environment_independent(tmp_path, flag): + """Both help aliases exit before workload validation or dispatch.""" + sweep = _make_sweep_fixture(tmp_path) + result = subprocess.run( + [str(sweep), flag], + check=False, + cwd=tmp_path, + env={**os.environ, "SHELL": _BASH}, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stderr + assert "Usage:" in result.stdout + assert "--resume" in result.stdout diff --git a/tests/test_extract_elbencho_cli_contracts.py b/tests/test_extract_elbencho_cli_contracts.py new file mode 100644 index 0000000..19a5036 --- /dev/null +++ b/tests/test_extract_elbencho_cli_contracts.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fast CLI contract tests for the Elbencho report extractor.""" + +import sys +from dataclasses import replace + +import pytest + +from tests.extract_elbencho_test_support import load_extract_elbencho_module + +_EXTRACT = load_extract_elbencho_module("extract_elbencho_cli_contracts") + + +def _metric(): + """Return one valid metric suitable for a CSV-only report invocation.""" + return _EXTRACT.ElbenchoMetrics( + nodes=1, + io_size="4K", + threads=1, + io_depth=1, + operation="WRITE", + datestamp="20260101Z000000", + is_multi_node=False, + command="elbencho --size 4K", + file_size_bytes=4096, + direct_io=0, + random_io=0, + iops=1.0, + throughput_mib_s=1.0, + throughput_mb_s=1.048576, + throughput_gbps=0.008, + min_lat_sec=0.001, + avg_lat_sec=0.002, + max_lat_sec=0.003, + lat_pct_1=0.001, + lat_pct_50=0.002, + lat_pct_75=0.003, + lat_pct_99=0.004, + ) + + +def _run_main(monkeypatch, *arguments): + """Run the extractor entry point with a temporary argv.""" + monkeypatch.setattr(sys, "argv", ["extract-elbencho.py", *arguments]) + return _EXTRACT.main() + + +def test_from_csv_rejects_raw_result_directories(monkeypatch, tmp_path): + """CSV input is an alternative source, never a second source to merge.""" + with pytest.raises(SystemExit) as raised: + _run_main( + monkeypatch, + "--from-csv", + str(tmp_path / "metrics.csv"), + str(tmp_path / "raw-results"), + ) + assert raised.value.code == 2 + + +@pytest.mark.parametrize( + "option, value", + [ + ("--only-threads", "1,,2"), + ("--only-nodes", "2-1"), + ("--only-iodepths", "not-an-integer"), + ("--only-sizes", "4K;;1M"), + ("--only-sizes", "not-a-size"), + ], +) +def test_filter_parser_rejects_malformed_syntax(option, value): + """Malformed filters fail instead of silently broadening a report.""" + if option == "--only-sizes": + with pytest.raises(ValueError): + _EXTRACT.parse_only_sizes_filter([value]) + else: + with pytest.raises(ValueError): + _EXTRACT.parse_int_values_with_ranges(value) + + +def test_valid_filter_matching_no_metrics_is_an_error(monkeypatch, tmp_path): + """A successful process must not hide an accidentally empty report.""" + csv_path = tmp_path / "metrics.csv" + _EXTRACT.write_csv(str(csv_path), [_metric()]) + + with pytest.raises(SystemExit) as raised: + _run_main( + monkeypatch, + "--from-csv", + str(csv_path), + "--only-nodes", + "99", + "--markdown", + ) + assert raised.value.code == 1 + + +def test_csv_round_trip_preserves_report_dimensions(tmp_path): + """Persisted metrics retain the dimensions needed by later filtering.""" + metrics = [ + _metric(), + replace( + _metric(), + nodes=2, + io_size="1M,r4K", + threads=8, + io_depth=4, + operation="READ", + random_io=1, + ), + ] + path = tmp_path / "metrics.csv" + _EXTRACT.write_csv(str(path), metrics) + loaded = _EXTRACT.read_csv(str(path)) + assert [ + ( + item.nodes, + item.io_size, + item.threads, + item.io_depth, + item.operation, + item.random_io, + ) + for item in loaded + ] == [ + (1, "4K", 1, 1, "WRITE", 0), + (2, "1M,r4K", 8, 4, "READ", 1), + ] + + +def test_csv_cli_filters_compound_size_and_forwards_plot_mode(monkeypatch, tmp_path): + """CLI filters preserve compound sizes and the single-axis request.""" + path = tmp_path / "metrics.csv" + _EXTRACT.write_csv( + str(path), + [ + _metric(), + replace( + _metric(), + nodes=2, + io_size="1M,r4K", + threads=8, + io_depth=4, + operation="READ", + ), + ], + ) + reported = [] + plotted = [] + monkeypatch.setattr( + _EXTRACT, + "print_markdown_table", + lambda metrics, single_axis: reported.append((metrics, single_axis)), + ) + monkeypatch.setattr( + _EXTRACT, + "plot_metrics", + lambda metrics, output_dir, single_axis: plotted.append( + (metrics, output_dir, single_axis) + ), + ) + + _run_main( + monkeypatch, + "--from-csv", + str(path), + "--only-nodes", + "2", + "--only-threads", + "8", + "--only-iodepths", + "4", + "--only-sizes", + "1M,r4K", + "--no-dual-y-axis", + "--markdown", + ) + + assert len(reported) == len(plotted) == 1 + assert reported[0][1] is True + assert plotted[0][2] is True + assert [item.operation for item in reported[0][0]] == ["READ"] + + +def test_csv_cli_can_reemit_cache_and_default_terminal_report(monkeypatch, tmp_path): + """CSV input supports cache output and the normal mirrored text report.""" + source = tmp_path / "source.csv" + _EXTRACT.write_csv(str(source), [_metric()]) + mirrored = [] + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + _EXTRACT, + "mirror_stdout_to_file", + lambda path, printer, metrics: mirrored.append((path, printer, metrics)), + ) + monkeypatch.setattr(_EXTRACT, "plot_metrics", lambda *_arguments: None) + + _run_main(monkeypatch, "--from-csv", str(source), "--to-csv") + + cache = tmp_path / "elbencho-metrics.csv" + assert cache.is_file() + assert len(_EXTRACT.read_csv(str(cache))) == 1 + assert len(mirrored) == 1 + assert mirrored[0][0] == str(tmp_path / _EXTRACT.REPORT_TXT_FILENAME) + assert [item.operation for item in mirrored[0][2]] == ["WRITE"] + + +@pytest.mark.parametrize( + "option", + [ + "--client-outlier-threshold", + "--client-min-underperform-segments", + "--client-max-timeseries-lines", + "--client-max-heatmap-rows", + ], +) +def test_live_report_limits_must_be_positive(monkeypatch, option): + """Invalid live-report limits fail at the CLI boundary.""" + with pytest.raises(SystemExit) as raised: + _run_main(monkeypatch, "--from-csv", "unused.csv", option, "0") + assert raised.value.code == 2 diff --git a/utils/extract-elbencho.py b/utils/extract-elbencho.py index a46f56c..d2c780f 100644 --- a/utils/extract-elbencho.py +++ b/utils/extract-elbencho.py @@ -53,6 +53,7 @@ Tuple, TypedDict, cast, + get_type_hints, ) import matplotlib.pyplot as plt @@ -81,7 +82,6 @@ write_client_summaries, ) from lib.join_datestamps import join_datestamps, join_datestamps_for_filename -from lib.parse_only_sizes import parse_only_sizes_arg from lib.reporting_common import histogram_axis_ranges from lib.stdout_report_file import ( # pylint: disable=wrong-import-position REPORT_TXT_FILENAME, @@ -267,6 +267,9 @@ class ElbenchoMetrics: histogram: Dict[float, int] = field(default_factory=dict) +_ELBENCHO_METRIC_FIELD_TYPES = get_type_hints(ElbenchoMetrics) + + @dataclass(frozen=True) class ExecutionCoordinates: """Coordinates and configured layout captured in one reified execution.""" @@ -2683,13 +2686,14 @@ def _elbencho_csv_coerce_field_types(row: Dict[str, Any]) -> None: for a_field in fields(ElbenchoMetrics): field_name = a_field.name field_value = row[field_name] + field_type = _ELBENCHO_METRIC_FIELD_TYPES[field_name] - if a_field.type == bool: + if field_type is bool: if isinstance(field_value, str): row[field_name] = field_value.lower() == "true" - elif a_field.type == int: + elif field_type is int: row[field_name] = int(float(field_value)) if field_value else 0 - elif a_field.type == float: + elif field_type is float: row[field_name] = float(field_value) if field_value else 0.0 @@ -3533,40 +3537,64 @@ def test_parse(base_filename: str) -> None: # Helper function to parse value lists with ranges -def parse_int_values_with_ranges(value_str): - """Parse a comma-separated list of integers or ranges. +def parse_int_values_with_ranges(value_str: str) -> Set[int]: + """Parse a strict comma-separated list of integers or inclusive ranges. - Ranges are specified as 'start-end'. Returns a set of integer values. - Example: '1,2,5-10,15' would return {1, 2, 5, 6, 7, 8, 9, 10, 15} + A report filter is an operator request, not a best-effort hint. Silently + dropping a malformed token can turn a narrow report into an unfiltered + report, so invalid syntax raises ``ValueError`` for the CLI to report. """ - result = set() - if not value_str: - return result + if not value_str or not value_str.strip(): + raise ValueError("filter must not be empty") + + result: Set[int] = set() + for raw_part in value_str.split(","): + part = raw_part.strip() + if not part: + raise ValueError("empty filter item") + pieces = part.split("-") + if len(pieces) > 2 or not all(piece.isdigit() for piece in pieces): + raise ValueError(f"invalid integer or range: {part}") + start = int(pieces[0]) + end = int(pieces[-1]) + if len(pieces) == 2 and start > end: + raise ValueError(f"range start exceeds end: {part}") + result.update(range(start, end + 1)) + return result - for part in value_str.split(","): - part = part.strip() - if "-" in part: - # Handle range (e.g., "1-10") - try: - start, end = map(int, part.split("-", 1)) - # Add all integers in the range (inclusive) - result.update(range(start, end + 1)) - except ValueError: - eprint( - f"Warning: Invalid range format: {part}. Using individual values only." - ) - continue - else: - # Handle individual value - try: - result.add(int(part)) - except ValueError: - eprint(f"Warning: Invalid integer value: {part}. Skipping.") - continue +_REPORT_SIZE_FILTER_RE = re.compile(r"^r?[0-9]+[KMG](,r?[0-9]+[KMG])?$") + + +def parse_only_sizes_filter(values: Optional[List[str]]) -> Optional[Set[str]]: + """Parse and validate repeated ``--only-sizes`` filter arguments.""" + if not values: + return None + + result: Set[str] = set() + for raw_value in values: + parts = raw_value.split(";") + if any(not part.strip() for part in parts): + raise ValueError("size filter contains an empty item") + for part in parts: + size = part.strip() + if not _REPORT_SIZE_FILTER_RE.fullmatch(size): + raise ValueError(f"invalid IO size: {size}") + result.add(size) return result +def _parse_report_integer_filter( + parser: argparse.ArgumentParser, option_name: str, value: str +) -> Set[int]: + """Parse one integer filter and convert syntax errors to CLI errors.""" + try: + return parse_int_values_with_ranges(value) + except ValueError as exc: + parser.error(f"invalid {option_name} filter: {exc}") + return set() # ``parser.error`` raises; keeps static type checkers happy. + + def parse_benchmark_filename(filename: str) -> Optional[Dict[str, Any]]: """Parse an elbencho benchmark filename to extract parameters. @@ -4509,6 +4537,9 @@ def main() -> None: ) args = parser.parse_args() + if args.from_csv and args.input_dirs: + parser.error("--from-csv cannot be combined with input directories") + positive_live_options = ( ("--client-outlier-threshold", args.client_outlier_threshold), ( @@ -4612,20 +4643,32 @@ def main() -> None: node_filter = None size_filter = None iodepth_filter = None - if args.only_threads or args.only_sizes or args.only_nodes or args.only_iodepths: + filter_requested = bool( + args.only_threads or args.only_sizes or args.only_nodes or args.only_iodepths + ) + if filter_requested: thread_filter = ( - parse_int_values_with_ranges(args.only_threads) - if args.only_threads - else None + None + if not args.only_threads + else _parse_report_integer_filter( + parser, "--only-threads", args.only_threads + ) ) node_filter = ( - parse_int_values_with_ranges(args.only_nodes) if args.only_nodes else None + None + if not args.only_nodes + else _parse_report_integer_filter(parser, "--only-nodes", args.only_nodes) ) - size_filter = parse_only_sizes_arg(args.only_sizes) + try: + size_filter = parse_only_sizes_filter(args.only_sizes) + except ValueError as exc: + parser.error(f"invalid --only-sizes filter: {exc}") iodepth_filter = ( - parse_int_values_with_ranges(args.only_iodepths) - if args.only_iodepths - else None + None + if not args.only_iodepths + else _parse_report_integer_filter( + parser, "--only-iodepths", args.only_iodepths + ) ) metrics = filter_metrics( @@ -4645,6 +4688,10 @@ def main() -> None: ] eprint(f"After filtering, {len(live_files)} live CSV files remain") + if not metrics and not live_files: + eprint("ERROR: Filters matched no report metrics") + sys.exit(1) + # Write to CSV if requested if args.to_csv: # Use the first input directory or current directory if none provided From cb8be2fbb15c9f303d82b9f7ea8bf049d82d9baa Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sun, 20 Sep 2026 09:15:29 -0700 Subject: [PATCH 07/10] Harden integration fixture identity and validation Keep host operator IDs out of pod-side storage setup and preserve all-squashed NFS ownership. Stage the SSH failure-injection delegate and runtime on the local coordinator so the wrapper can exercise the intended failure and resume lifecycle. Make fixture validation substrate-aware, including Slurm-only report extraction while SSH is unavailable. Remove only harness-owned NFS configuration during teardown and preserve pre-existing services and unrelated exports. Require workload metadata for bounded result contracts and validate aggregate and live report filtering independently. Make deployment cache mode coverage independent of umask and add focused regressions for these lifecycle and reporting boundaries. Document the integration invariants and require repository-pinned Python tooling for authoritative lint and test results. Run integration workloads as fixed non-root identity Provision a fixed tester UID/GID 2000 in the Slinky login and compute images, register its Slurm account, and require both the coordinator and two-node srun fan-out to prove the expected identity. Run Slurm sweeps, PVC staging, cleanup, and SSH worker staging as that account with restrictive umasks. Remove client-side ownership changes that fail against all-squashed NFS exports and keep host account IDs confined to local fixture state. Repair failure injection's cross-host staging location, avoid Slurm indices during SSH-only runs, and ignore terminating Slinky workers during rollout validation. Add sentinel-ID regression coverage, document ownership and teardown contracts, and validate the complete SSH and Slurm scenario catalog. Make Slurm account setup idempotent Query existing Slurm accounts, associations, and user defaults before mutating accounting state. Add only missing records, repair a stale default account, and verify the exact tester/storage-test relationship after reconciliation. Cover retained accounting state with a regression that runs the reconciler twice and proves the second pass performs no mutation. This allows repeated setup to accept the state created by its first pass. Run integration lifecycle as ordinary user Keep kubeconfig, keys, downloaded clients, caches, manifests, logs, and test runs user-owned from creation under the repository state tree. Reject root for every lifecycle action and let the NFS backend invoke sudo only for package installation and host-system operations. Make the Docker SBX path fully unprivileged. Use sticky shared roots instead of host-side chown, isolate Helm state from stale root-owned caches, and remove UID-mapped data through the private Docker engine. Install pinned client copies in the user state tree instead of globally. Run the GitHub lifecycle as the runner account and validate the root rejection boundary. Document the privilege model and cover idempotent state bootstrap, sudo-free SBX prerequisites, and diagnostics. Fix filesystem device validation Compare st_dev values from stat -c %d instead of interpreting statfs free-inode counts as filesystem identities. Apply the correction to SSH scriptlet/direct and Slurm sbatch/srun probes, preserve paths with spaces, and report the actual distinct-from-root contract. Add shell regressions that exercise every dispatch route against distinct and matching device identities. Align design, requirements, and repository context with the validation semantics. Direct agents to wrap future commit messages at about 72 characters. Create default integration state parent Allow the known repository-local state path to create its ignored tmp parent in a fresh checkout before bootstrapping user-owned state. Continue rejecting custom state paths unless their parent already exists, preserving the destructive-cleanup safety boundary. Cover clean-checkout bootstrap and custom-path rejection with focused driver safety tests, and document the distinction. Fix integration scenario validation and teardown Search standard sbin paths so ordinary-user NFS setup can find privileged utilities, and record NFS service ownership before package installation can activate it. Keep cleanup safe when private kind or exportfs clients are absent after interrupted or repeated teardown. Read Slurm worker-order evidence from copied execution logs and validate the real Elbencho binary before installing failure-injection wrappers. Use non-sticky writable SBX fixture directories so replacement pods can clean files when Docker SBX remaps bind-mounted ownership. Add focused regressions for tool lookup, ownership sequencing, copied Slurm evidence, wrapper ordering, SBX modes, and missing-client cleanup. Harden SSH fixture transition convergence Require StatefulSet generation and revision convergence before accepting ready SSH workers. Use unique home and RWX probes, retry bounded cross-node visibility, and clean probes from both pods on every exit. Guard NFS teardown when interrupted package installation leaves valid ownership state without exportfs, and cover both lifecycle boundaries with focused regression tests. --- .github/workflows/integration.yml | 28 +- AGENTS.md | 4 + docs/CONTEXT.md | 38 +- docs/DESIGN.md | 2 +- docs/REQUIREMENTS.md | 2 +- integration-tests/README.md | 62 +- integration-tests/bin/integration-test.py | 655 ++++++++++++------ .../lib/filesystem_integration.py | 390 +++++++---- .../manifests/slinky-slurm-values.yaml | 6 +- .../slinky-login-image.Dockerfile | 2 + .../slinky-slurmd-image.Dockerfile | 21 + tests/test_extract_elbencho_cli_contracts.py | 40 ++ tests/test_integration_deployment_cache.py | 1 + tests/test_integration_driver_safety.py | 610 +++++++++++++++- tests/test_integration_failure_injection.py | 115 +++ tests/test_validate_env_filesystem_shell.py | 148 ++++ utils/extract-elbencho.py | 13 +- validate_env.sh | 33 +- 18 files changed, 1768 insertions(+), 402 deletions(-) create mode 100644 integration-tests/slinky-slurmd-image.Dockerfile create mode 100644 tests/test_validate_env_filesystem_shell.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fc391c0..41c4768 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -63,27 +63,33 @@ jobs: sudo -n true docker info >/dev/null - name: Set up the integration environment - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup + run: | + "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup - name: Verify repeated setup - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup + run: | + "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs setup - name: Stop the integration environment - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs stop + run: | + "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs stop - name: Restart the integration environment - run: sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs start - - name: Verify root test execution is rejected run: | - if sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs test; then - echo "integration test action unexpectedly accepted root" >&2 - exit 1 - fi + "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs start + - name: Verify root lifecycle execution is rejected + run: | + for action in setup test; do + if sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs "$action"; then + echo "integration $action action unexpectedly accepted root" >&2 + exit 1 + fi + done - name: Run all integration tests run: | "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs test - name: Tear down the integration environment if: ${{ always() }} run: | - sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown - sudo "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown + "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown + "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs teardown integration-status: name: Filesystem integration status diff --git a/AGENTS.md b/AGENTS.md index fa0df59..5c42e0d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,8 @@ The script creates and reuses `.venv-ci` with the pinned tools. Pass `pylint`, or `pytest` to run one check. Set `CI_CHECK_JOBS=1` in a constrained sandbox. In a pre-provisioned, network-restricted sandbox, set `CI_BOOTSTRAP=0` and use `CI_PYTHON` or `CI_SHELLCHECK` to select installed tools. +Run tests and lint through this script or an environment populated from both +requirements files; never treat ambient Python tooling as authoritative. `black` must be 25.9.0+; `pylint` must score 10.00/10. Details and rationale: [docs/CODING_STANDARDS.md](docs/CODING_STANDARDS.md). @@ -96,6 +98,8 @@ and use `CI_PYTHON` or `CI_SHELLCHECK` to select installed tools. ## Pull requests +Wrap commit-message lines at about 72 characters. + This project is currently not accepting external contributions. For maintainer changes: keep PRs focused, ensure the checks above pass, and update `README.md`/`docs/` when behavior or configuration changes. diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 39313ef..54ab001 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -63,10 +63,25 @@ export and NFS CSI. The Docker-SBX-specific `sbx-shared` backend mounts a repository-backed directory into every kind node and uses static RWX volumes; both backends validate the repository's same shared-storage contract. Backend selection is explicit or capability-based and persists until teardown. The +SBX shared directories are non-sticky writable fixture paths because SBX can +remap bind-mounted file ownership between replacement pods. harness's state and export directories are dedicated leaves: an existing path -must carry the exact harness ownership marker before setup changes it or teardown -removes it. A completed setup also locks its namespace and, for NFS, export -directory until teardown. The SBX compatibility profile pairs kind/Kubernetes +must carry the exact harness ownership marker before setup changes it or +teardown removes it. Every lifecycle action runs as an ordinary user. Its +kubeconfig, keys, client tools, caches, manifests, logs, and runs are user-owned +under `tmp/integration-state`; the driver invokes `sudo` only for the NFS +profile's narrow host-system operations, while `sbx-shared` never invokes it. +The driver adds standard `sbin` locations to its ordinary-user tool search +path. It records NFS service ownership before package installation can start +the service, and repeated teardown remains valid after its private clients are +removed. Partial bootstrap teardown also tolerates `exportfs` not having been +installed yet. +Bootstrap creates the known default `tmp/` parent in a clean checkout, while a +custom state path must remain a leaf below an existing directory. +A completed setup also locks its namespace and, for NFS, export directory until +teardown. NFS teardown removes only the harness export and configuration; a +server or unrelated exports that predated the fixture remain active. The SBX +compatibility profile pairs kind/Kubernetes 1.34 with kubectl 1.34 and rebuilds cached container-derived Elbencho bundles when their pinned image or bundle recipe changes. Slurm coordinators restore configured `ORDER_NODES` include-list order after Slurm canonicalizes an @@ -74,6 +89,18 @@ allocation's node list. The test action selects execution substrate and named scenario independently. Its deterministic planner batches the one shared-home SSH scenario behind a crash-recoverable StatefulSet transition; separate SSH homes are canonical. +Transitions require the StatefulSet rollout to finish before accepting two +ready nonterminating pods. Cross-pod storage checks use unique probe paths, +bounded visibility retries, and unconditional cleanup. +Fixture preflight requires only the workloads selected by the plan, so Slurm +diagnosis remains available while SSH workers are unhealthy. Host operator +UIDs own local state only. LoginSet coordination, Slurm tasks, SSH workers, and +pod-side staging use the fixed `tester` UID/GID 2000 contract. Setup provisions +the matching Slurm account, verifies the coordinator and both real `srun` +tasks, and all-squashed NFS paths retain their server-assigned ownership. +Slurm worker-order assertions use the copied execution logs rather than the +asynchronous submission stream. Failure scenarios validate the real Elbencho +binary before replacing it with their scenario-owned wrapper. Deployment archives remain products of `utils/build_tarball.sh`, but the harness caches them by the exact immutable tracked-source snapshot, build options, architecture, and seeded Elbencho/runtime identity. Each scenario @@ -92,7 +119,7 @@ path safety, sizing limits, and Slurm argument boundaries without a live fixture. Elbencho CSV reload resolves postponed dataclass annotations before coercing types, so cached metrics remain filterable. Its reporting CLI treats `--from-csv` as exclusive with raw directories and rejects malformed or -no-match filters. +no-match filters independently for aggregate metrics and live CSV reports. GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. @@ -158,6 +185,9 @@ Important configuration relationships: inclusive ranges, and `+step` increments. Entry points validate the complete expanded list before dispatch. - `TEST_DIRS` is an associative array of filesystem test roots and weights. + `validate_env.sh` compares each path's `stat -c %d` device number with `/` + through both dispatch interfaces; this establishes distinct backing storage, + not that the configured path is itself the exact mountpoint. Object tests use a dedicated `OBJ_BUCKET`, endpoint settings, and credentials sourced from `OBJ_AUTH_FILE`. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 539a9c4..7aa9b3b 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -210,7 +210,7 @@ edit-validate loop: | Slurm connectivity | `sinfo`, `sbatch` a test job, wait for completion | | SSH connectivity | `ssh` command execution + scriptlet execution on each host | | Binary architecture match | `file` on binary vs. `uname -m` on remote | - | Filesystem paths are mountpoints | `mountpoint` on compute nodes (via Slurm/SSH) | + | Filesystem paths use storage distinct from `/` | Compare `stat -c %d` device IDs on compute nodes (via Slurm/SSH) | | Filesystem paths are writable | Touch test on compute nodes | | S3 credentials and bucket access | `s3test` binary | | S3 bucket emptiness | Object count check (warning if non-empty) | diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index ba0a66d..ded5fa9 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -219,7 +219,7 @@ These requirements address the infrastructure and runtime constraints the tool m | CV-2.3 | Validation shall verify that benchmark binaries are present, executable, and compiled for the correct architecture. | Yes | | CV-2.4 | Validation shall verify Slurm connectivity (if Slurm mode is enabled): partition access, account, reservation, sbatch and srun functionality. | Yes | | CV-2.5 | Validation shall verify SSH connectivity (if SSH mode is enabled): ability to run commands and scriptlets on remote hosts. | Yes | -| CV-2.6 | Validation shall verify that filesystem test paths are mountpoints and writable on compute nodes. | Yes | +| CV-2.6 | Validation shall verify that filesystem test paths reside on a filesystem device distinct from `/` and are writable on compute nodes. | Yes | | CV-2.7 | Validation shall verify S3 object storage credentials and bucket accessibility (if object tests are enabled). | Yes | | CV-2.8 | Validation shall warn if the target S3 bucket contains existing objects (warp deletes all objects). | Yes | | CV-2.9 | Validation shall validate elbencho configuration parameters (thread list contains integers, IO sizes are valid, duration is valid). | Yes | diff --git a/integration-tests/README.md b/integration-tests/README.md index 26a673d..5a18663 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -26,14 +26,17 @@ The current setup target is Ubuntu 24.04 on x86-64 or ARM64 with at least two CPUs, 8 GiB total RAM, 6 GiB available RAM, and 20 GiB free on the selected backend's filesystem. Python 3.12 and an accessible rootful Docker daemon are prerequisites. The driver installs its other host packages and pinned client -tools when needed. It also builds a small derived Slinky login image containing -the standard `file` package required by `validate_env.sh`. +tools when needed. It also builds small derived Slinky login and compute images +containing the fixed integration workload account; the login image additionally +provides the standard `file` package required by `validate_env.sh`. -Run setup (or its exact synonym, start) with: +Run setup (or its exact synonym, start) as the ordinary test user. The NFS +profile invokes passwordless `sudo` itself only for package installation and +the dedicated export, loop-device, firewall, and systemd operations: ```bash sudo -v -sudo integration-tests/bin/integration-test.py setup +integration-tests/bin/integration-test.py setup ``` `--storage-backend auto` is the default. It selects `nfs` when the host has the @@ -56,7 +59,7 @@ repository feature coverage. Select Docker SBX explicitly with: ```bash -sudo integration-tests/bin/integration-test.py \ +integration-tests/bin/integration-test.py \ --storage-backend sbx-shared setup ``` @@ -67,12 +70,24 @@ lacks that device. When Docker SBX exposes its proxy CA, setup installs that CA in the disposable kind nodes so containerd can pull the fixture images. The default shared root is `tmp/integration-sbx-shared`; an alternate path may be set with `--sbx-shared-root`, but must remain below the repository's `tmp/` -directory. - -Setup records the invoking pre-sudo account, gives that non-root account -access to the private kubeconfig, key, and test-run workspace, and verifies it -can use Docker, kind, and kubectl. A root shell without `SUDO_USER` must name -the account explicitly with `--test-user USER`. +directory. This profile never invokes `sudo`; required host packages and an +accessible Docker engine must already be present. Its two disposable backing +directories use sticky shared-directory permissions so UID 2000 workloads can +create their own restricted scenario trees without host-side ownership changes. + +Every lifecycle action runs as the ordinary test account and refuses root. +Kubeconfig, keys, downloaded clients, cached deployments, rendered manifests, +logs, and test runs remain user-owned from creation under the default +`tmp/integration-state` directory. Setup never recursively changes that +tree's ownership. It verifies that the caller can use Docker, kind, kubectl, +and the state directory after provisioning. + +The host account and in-cluster workload account are deliberately independent. +LoginSet coordination, Slurm jobs, SSH workers, and shared-storage staging run +as `tester` with UID/GID 2000; setup verifies that identity on the coordinator +and on both real `srun` tasks. This matches the NFS export's anonymous mapping, +so clients create scenario data directly with a restrictive umask and never +attempt to change ownership through an all-squashed mount. SSH workers normally use separate `emptyDir` homes. A scenario that requires the RWX shared-home claim owns a bounded StatefulSet transition and restores @@ -81,7 +96,7 @@ transition before allowing more SSH work; the kind and Slurm fixtures remain running throughout. The generated host SSH key, strict known-hosts file, and two worker addresses -are kept under `/var/lib/storage-scale-test-integration/`. Re-running setup +are kept under `tmp/integration-state/`. Re-running setup reconciles and validates the environment without replacing those credentials or retained backend data. @@ -138,7 +153,7 @@ copies and checks the semantic report content applicable to each scenario. The `Filesystem integration` GitHub Actions workflow runs independent amd64 and arm64 jobs concurrently. Each job runs setup twice, stops and restarts the -fixture, proves that root test execution is rejected, runs `test` as the +fixture, proves that root lifecycle execution is rejected, runs `test` as the ordinary runner account, and tears down twice. A final status job requires both architectures to pass. The workflow is deliberately absent from ordinary pull-request and default-branch events. @@ -180,12 +195,15 @@ integration-tests/bin/integration-test.py teardown Teardown is idempotent. It performs the disposable stop and deletes the fixture's generated data, keys, logs, and locally built image tags. For NFS it -also removes the dedicated export and configuration, disables and stops -`nfs-server`, removes any harness-owned UFW rule, and unmounts the verified -loop-backed filesystem. For Docker SBX it removes only the marker-owned shared -root and never invokes NFS, systemd, firewall, loop, or mount operations. It -refuses destructive cleanup when the applicable ownership and path checks do -not match the fixture. Operating-system packages, kind, kubectl, Helm, and -reusable upstream Docker image layers are not uninstalled. If a locally built -tag existed before setup, teardown restores that exact prior image ID instead -of deleting it. +also removes the dedicated export and configuration, removes any harness-owned +UFW rule, and unmounts the verified loop-backed filesystem. It disables and +stops `nfs-server` only when setup started it and no unrelated exports remain; +a pre-existing service is left running. For Docker SBX it removes only the +marker-owned shared root and never invokes NFS, systemd, firewall, loop, or +mount operations. It refuses destructive cleanup when the applicable ownership +and path checks do not match the fixture. Operating-system packages, +pre-existing client tools, and reusable upstream Docker image layers are not +uninstalled. Checksum-verified client copies that the harness downloaded into +its own state tree are removed with that tree. If a locally built tag existed +before setup, teardown restores that exact prior image ID instead of deleting +it. diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py index 9c63885..cc4e719 100755 --- a/integration-tests/bin/integration-test.py +++ b/integration-tests/bin/integration-test.py @@ -82,16 +82,26 @@ ) SLINKY_VERSION = "1.2.0" SLINKY_LOGIN_BASE_IMAGE = "ghcr.io/slinkyproject/login:26.05-ubuntu26.04" -SLINKY_LOGIN_IMAGE = "storage-scale-integration-login:slinky-26.05-file" +SLINKY_LOGIN_IMAGE = "storage-scale-integration-login:slinky-26.05-user" +SLINKY_SLURMD_BASE_IMAGE = "ghcr.io/slinkyproject/slurmd:26.05-ubuntu26.04" +SLINKY_SLURMD_IMAGE = "storage-scale-integration-slurmd:slinky-26.05-user" SSH_IMAGE = "storage-scale-integration-ssh:ubuntu-24.04" STATE_SCHEMA = 1 TARGET_LABEL = "storage-scale-test/target=true" LOGIN_LABEL = "storage-scale-test/login=true" -NFS_UID = 2000 -NFS_GID = 2000 +WORKLOAD_USER = "tester" +WORKLOAD_UID = 2000 +WORKLOAD_GID = 2000 +WORKLOAD_ACCOUNT = "storage-test" +NFS_UID = WORKLOAD_UID +NFS_GID = WORKLOAD_GID NFS_IMAGE_BYTES = 128 * 1024 * 1024 GIB = 1024**3 -DEFAULT_STATE_DIR = Path("/var/lib/storage-scale-test-integration") +STORAGE_BACKENDS = ("nfs", "sbx-shared") +SYSTEM_ADMIN_PATHS = ("/usr/local/sbin", "/usr/sbin", "/sbin") +SBX_SHARED_DIRECTORY_MODE = 0o777 +SSH_STORAGE_VISIBILITY_TIMEOUT_SECONDS = 30 +DEFAULT_STATE_DIR = Path(__file__).resolve().parents[2] / "tmp" / "integration-state" DEFAULT_EXPORT_DIR = Path("/srv/storage-scale-test-integration") DEFAULT_SBX_SHARED_ROOT = ( Path(__file__).resolve().parents[2] / "tmp" / "integration-sbx-shared" @@ -232,28 +242,21 @@ def _sudo_prefix() -> list[str]: def _bootstrap_state_dir(config: Config) -> None: - """Create the operator-owned state directory before file logging.""" + """Create user-owned state before file logging or privileged work.""" create_marker = not config.state_dir.exists() - command = [ - *_sudo_prefix(), - "install", - "-d", - "-m", - "0750", - "-o", - f"+{config.test_uid}", - "-g", - f"+{config.test_gid}", - config.state_dir, + config.state_dir.mkdir(mode=0o750, parents=True, exist_ok=True) + config.state_dir.chmod(0o750) + for directory in ( config.manifests_dir, config.keys_dir, config.state_dir / "logs", - ] - result = subprocess.run(command, check=False, capture_output=True, text=True) - if result.returncode: - raise ProvisionError( - f"cannot create state directory {config.state_dir}: {result.stderr.strip()}" - ) + config.state_dir / "bin", + config.state_dir / "tool-state" / "helm" / "cache", + config.state_dir / "tool-state" / "helm" / "config", + config.state_dir / "tool-state" / "helm" / "data", + ): + directory.mkdir(mode=0o750, parents=True, exist_ok=True) + directory.chmod(0o750) if create_marker: _write_text( config.state_dir / STATE_MARKER, @@ -265,6 +268,25 @@ def _bootstrap_state_dir(config: Config) -> None: ) +def _configure_user_tool_path(config: Config) -> None: + """Confine installed clients and their mutable state to user-owned paths.""" + tool_dir = str(config.state_dir / "bin") + current = os.environ.get("PATH", "") + entries = [entry for entry in current.split(os.pathsep) if entry] + entries = [entry for entry in entries if entry != tool_dir] + entries.insert(0, tool_dir) + entries.extend(path for path in SYSTEM_ADMIN_PATHS if path not in entries) + os.environ["PATH"] = os.pathsep.join(entries) + helm_root = config.state_dir / "tool-state" / "helm" + os.environ.update( + { + "HELM_CACHE_HOME": str(helm_root / "cache"), + "HELM_CONFIG_HOME": str(helm_root / "config"), + "HELM_DATA_HOME": str(helm_root / "data"), + } + ) + + def _configure_logging(config: Config, action: str) -> Path: """Configure console and timestamped file logging.""" timestamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) @@ -400,7 +422,7 @@ def _validate_retained_state_summary(config: Config) -> None: return state = json.loads(path.read_text(encoding="utf-8")) backend = state.get("storage_backend") - if backend not in {"nfs", "sbx-shared"}: + if backend not in STORAGE_BACKENDS: raise ProvisionError(f"invalid retained setup state: {path}") expected: dict[str, object] = { "schema": STATE_SCHEMA, @@ -427,7 +449,7 @@ def _select_storage_backend(config: Config) -> str: document = json.loads(path.read_text(encoding="utf-8")) backend = str(document.get("backend", "")) expected = _storage_backend_document(config, backend) - if backend not in {"nfs", "sbx-shared"} or document != expected: + if backend not in STORAGE_BACKENDS or document != expected: raise ProvisionError(f"invalid retained storage backend state: {path}") if config.storage_backend not in {"auto", backend}: raise ProvisionError( @@ -508,6 +530,11 @@ def _ensure_apt_packages(runner: Runner, backend: str) -> None: if not missing: LOG.info("Required operating-system packages are already installed") return + if backend == "sbx-shared": + raise ProvisionError( + "sbx-shared requires its host packages to be preinstalled and " + "never invokes sudo; missing: " + ", ".join(missing) + ) if not shutil.which("apt-get"): raise ProvisionError(f"missing packages and apt-get is unavailable: {missing}") LOG.info("Installing required packages: %s", ", ".join(missing)) @@ -527,6 +554,13 @@ def _ensure_apt_packages(runner: Runner, backend: str) -> None: ) +def _prepare_host_dependencies(runner: Runner, config: Config, backend: str) -> None: + """Install host packages after capturing service ownership.""" + if backend == "nfs": + _record_nfs_service_state(runner, config) + _ensure_apt_packages(runner, backend) + + def _ensure_docker(runner: Runner) -> None: """Require a working rootful Docker daemon.""" if not shutil.which("docker"): @@ -556,9 +590,12 @@ def _command_version(runner: Runner, command: str) -> str: def _ensure_client_tools( - runner: Runner, architecture: str, storage_backend: str + runner: Runner, + config: Config, + architecture: str, + storage_backend: str, ) -> None: - """Install checksum-verified kind, kubectl, and Helm when versions differ.""" + """Install checksum-verified clients into the user-owned state tree.""" expected = { "kind": KIND_VERSION, "kubectl": ( @@ -581,13 +618,17 @@ def _ensure_client_tools( if version in _command_version(runner, command): LOG.info("Using %s %s", command, version) continue - _install_client_tool(runner, command, version, architecture) + _install_client_tool(runner, config, command, version, architecture) def _install_client_tool( - runner: Runner, command: str, version: str, architecture: str + runner: Runner, + config: Config, + command: str, + version: str, + architecture: str, ) -> None: - """Download, verify, and install one client binary.""" + """Download, verify, and install one user-local client binary.""" LOG.info("Installing %s %s", command, version) with tempfile.TemporaryDirectory(prefix="storage-scale-tool-") as directory: target = Path(directory) @@ -598,7 +639,7 @@ def _install_client_tool( else: binary = _download_helm(runner, target, version, architecture) runner.run( - [*_sudo_prefix(), "install", "-m", "0755", binary, "/usr/local/bin/"] + ["install", "-m", "0755", binary, config.state_dir / "bin" / command] ) @@ -681,6 +722,9 @@ def _kubectl(config: Config, *arguments: str | Path) -> list[str | Path]: def _kind_clusters(runner: Runner) -> set[str]: """Return running kind cluster names.""" + if not shutil.which("kind"): + LOG.info("kind is unavailable; relying on Docker cluster discovery") + return set() result = runner.run(["kind", "get", "clusters"], check=False, timeout=30) return { line.strip() @@ -793,7 +837,10 @@ def _prepare_sbx_shared(runner: Runner, config: Config) -> None: ) for directory in (root / "storage-test", root / "ssh-home"): directory.mkdir(exist_ok=True) - directory.chmod(0o777) + # Docker SBX can remap one pod's file owner when the same bind mount is + # observed from a replacement pod. A sticky directory would then stop + # the fixed workload identity from deleting its own prior files. + directory.chmod(SBX_SHARED_DIRECTORY_MODE) token = secrets.token_hex(16) source = root / "agent-probe" source.write_text(token + "\n", encoding="utf-8") @@ -1568,7 +1615,19 @@ def _inspect_ssh_home_pool(runner: Runner, config: Config) -> PoolObservation: if statefulset.returncode: return PoolObservation("absent", "", 0, 0, statefulset.stderr.strip()) document = json.loads(statefulset.stdout) - annotations = document.get("metadata", {}).get("annotations", {}) + metadata = document.get("metadata", {}) + annotations = metadata.get("annotations", {}) + status = document.get("status", {}) + generation = metadata.get("generation") + observed_generation = status.get("observedGeneration") + revision_ready = ( + isinstance(generation, int) + and isinstance(observed_generation, int) + and observed_generation >= generation + and status.get("currentRevision") == status.get("updateRevision") + and status.get("updatedReplicas", 0) == 2 + and status.get("readyReplicas", 0) == 2 + ) pods = _ssh_pods(runner, config) terminating = sum( bool(pod.get("metadata", {}).get("deletionTimestamp")) for pod in pods @@ -1578,12 +1637,18 @@ def _inspect_ssh_home_pool(runner: Runner, config: Config) -> PoolObservation: for pod in pods ) names = ", ".join(str(pod.get("metadata", {}).get("name", "?")) for pod in pods) + details = ( + f"pods=[{names}], generation={generation}, " + f"observedGeneration={observed_generation}, " + f"currentRevision={status.get('currentRevision')}, " + f"updateRevision={status.get('updateRevision')}" + ) return PoolObservation( str(annotations.get(SSH_HOME_ANNOTATION, "unknown")), str(annotations.get(SSH_CONFIG_ANNOTATION, "")), - ready, + ready if revision_ready else 0, terminating, - f"pods=[{names}]", + details, ) @@ -1819,52 +1884,73 @@ def _validate_ssh_storage( """Validate the selected home mode and shared storage claim.""" names = [str(pod["metadata"]["name"]) for pod in pods] token = secrets.token_hex(16) - _pod_exec( - runner, - config, - names[0], - "touch /home/tester/.integration-home-probe; " - f"printf '%s\\n' {shlex.quote(token)} " - ">/mnt/storage-test/.integration-rwx-probe", - ) - home_probe = _pod_exec( - runner, - config, - names[1], - "test -e /home/tester/.integration-home-probe", - check=False, + home_probe_path = f"/home/tester/.integration-home-probe-{token}" + storage_probe_path = f"/mnt/storage-test/.integration-rwx-probe-{token}" + storage_check = f'test "$(cat {shlex.quote(storage_probe_path)})" = ' + shlex.quote( + token ) - storage_check = ( - f'test "$(cat /mnt/storage-test/.integration-rwx-probe)" = ' - f"{shlex.quote(token)}" - ) - if _select_storage_backend(config) == "nfs": + backend = _select_storage_backend(config) + if backend == "nfs": storage_check += ( " && case $(stat -f -c %T /mnt/storage-test) in " "nfs|nfs4) true;; *) false;; esac" ) - rwx_probe = _pod_exec( - runner, - config, - names[1], - storage_check, - check=False, - ) - if _select_storage_backend(config) == "sbx-shared": - host_probe = config.sbx_shared_root / "storage-test" / ".integration-rwx-probe" - if ( - not host_probe.is_file() - or host_probe.read_text(encoding="utf-8").strip() != token - ): - raise ProvisionError("SBX shared data is not visible from the agent") expected_home_rc = 0 if home_mode == "shared" else 1 - if home_probe.returncode != expected_home_rc or rwx_probe.returncode: - raise ProvisionError("SSH home or RWX visibility validation failed") - cleanup = ( - "rm -f /home/tester/.integration-home-probe " - "/mnt/storage-test/.integration-rwx-probe" - ) - _pod_exec(runner, config, names[0], cleanup) + cleanup = f"rm -f {shlex.quote(home_probe_path)} {shlex.quote(storage_probe_path)}" + host_visible = backend != "sbx-shared" + try: + _pod_exec( + runner, + config, + names[0], + f"touch {shlex.quote(home_probe_path)}; " + f"printf '%s\\n' {shlex.quote(token)} " + f">{shlex.quote(storage_probe_path)}", + ) + deadline = time.monotonic() + SSH_STORAGE_VISIBILITY_TIMEOUT_SECONDS + while True: + home_probe = _pod_exec( + runner, + config, + names[1], + f"test -e {shlex.quote(home_probe_path)}", + check=False, + ) + rwx_probe = _pod_exec( + runner, + config, + names[1], + storage_check, + check=False, + ) + if backend == "sbx-shared": + host_probe = ( + config.sbx_shared_root + / "storage-test" + / Path(storage_probe_path).name + ) + host_visible = ( + host_probe.is_file() + and host_probe.read_text(encoding="utf-8").strip() == token + ) + if ( + home_probe.returncode == expected_home_rc + and rwx_probe.returncode == 0 + and host_visible + ): + return + if time.monotonic() >= deadline: + raise ProvisionError( + "SSH home or RWX visibility did not converge: " + f"home_rc={home_probe.returncode}, " + f"expected_home_rc={expected_home_rc}, " + f"rwx_rc={rwx_probe.returncode}, " + f"host_visible={host_visible}" + ) + time.sleep(1) + finally: + for name in names: + _pod_exec(runner, config, name, cleanup, check=False) def _load_or_create_db_credentials(config: Config) -> dict[str, str]: @@ -1907,7 +1993,7 @@ def _ensure_mariadb_secret(runner: Runner, config: Config) -> None: def _install_slurm(runner: Runner, config: Config) -> None: """Install MariaDB, Slinky, and the two-node Slurm fixture.""" - _prepare_slinky_login_image(runner, config) + _prepare_slinky_images(runner, config) _ensure_mariadb_secret(runner, config) mariadb = _render_resource( config, @@ -1930,31 +2016,59 @@ def _install_slurm(runner: Runner, config: Config) -> None: _helm_slinky(runner, config) _wait_for_slurm(runner, config) _restart_slinky_login(runner, config) + _ensure_slurm_workload_account(runner, config) _validate_slurm(runner, config) -def _prepare_slinky_login_image(runner: Runner, config: Config) -> None: - """Build and preload the login image with declared test prerequisites.""" - _record_image_build_start(runner, config, SLINKY_LOGIN_IMAGE) +def _build_slinky_image( + runner: Runner, + config: Config, + *, + image: str, + base_image: str, + dockerfile: str, + nodes: list[str], +) -> None: + """Build, record, and preload one fixture-owned Slinky image.""" + _record_image_build_start(runner, config, image) runner.run( [ "docker", "build", "--tag", - SLINKY_LOGIN_IMAGE, + image, "--build-arg", - f"BASE_IMAGE={SLINKY_LOGIN_BASE_IMAGE}", + f"BASE_IMAGE={base_image}", "--file", - _resource_path("slinky-login-image.Dockerfile"), + _resource_path(dockerfile), _resource_path("."), ], timeout=600, ) - _record_image_build_complete(runner, config, SLINKY_LOGIN_IMAGE) - _load_image_into_nodes( + _record_image_build_complete(runner, config, image) + _load_image_into_nodes(runner, image, nodes) + + +def _prepare_slinky_images(runner: Runner, config: Config) -> None: + """Build and preload Slinky images with the fixed workload identity.""" + _build_slinky_image( runner, - SLINKY_LOGIN_IMAGE, - [f"{config.cluster_name}-control-plane"], + config, + image=SLINKY_LOGIN_IMAGE, + base_image=SLINKY_LOGIN_BASE_IMAGE, + dockerfile="slinky-login-image.Dockerfile", + nodes=[f"{config.cluster_name}-control-plane"], + ) + _build_slinky_image( + runner, + config, + image=SLINKY_SLURMD_IMAGE, + base_image=SLINKY_SLURMD_BASE_IMAGE, + dockerfile="slinky-slurmd-image.Dockerfile", + nodes=[ + f"{config.cluster_name}-worker", + f"{config.cluster_name}-worker2", + ], ) @@ -2076,11 +2190,12 @@ def _namespace_pods(runner: Runner, config: Config) -> list[dict[str, object]]: def _pods_with_container( pods: list[dict[str, object]], container: str ) -> list[dict[str, object]]: - """Select pods containing a named Slinky workload container.""" + """Select nonterminating pods containing a Slinky workload container.""" return [ pod for pod in pods - if container + if not pod["metadata"].get("deletionTimestamp") # type: ignore[index] + and container in {entry["name"] for entry in pod["spec"]["containers"]} # type: ignore[index] ] @@ -2143,6 +2258,99 @@ def _restart_slinky_login(runner: Runner, config: Config) -> None: ) +def _sacctmgr_rows( + runner: Runner, + prefix: list[str | Path], + entity: str, + fields: tuple[str, ...], +) -> set[tuple[str, ...]]: + """Return exact pipe-delimited rows from one Slurm accounting query.""" + result = runner.run( + [ + *prefix, + "sacctmgr", + "--noheader", + "--parsable2", + "show", + entity, + f"format={','.join(fields)}", + ] + ) + return { + tuple(line.removesuffix("|").split("|")) + for line in result.stdout.splitlines() + if line.strip() + } + + +def _ensure_slurm_workload_account(runner: Runner, config: Config) -> None: + """Reconcile the fixed non-root workload identity in Slurm accounting.""" + login = _login_pod(runner, config) + prefix = _kubectl(config, "-n", config.namespace, "exec", login, "--") + if (WORKLOAD_ACCOUNT,) not in _sacctmgr_rows( + runner, prefix, "account", ("Account",) + ): + runner.run( + [ + *prefix, + "sacctmgr", + "--immediate", + "add", + "account", + WORKLOAD_ACCOUNT, + "Description=storage scale integration workloads", + "Organization=storage-scale-test", + ] + ) + expected_association = (WORKLOAD_USER, WORKLOAD_ACCOUNT) + if expected_association not in _sacctmgr_rows( + runner, prefix, "association", ("User", "Account") + ): + runner.run( + [ + *prefix, + "sacctmgr", + "--immediate", + "add", + "user", + WORKLOAD_USER, + f"Account={WORKLOAD_ACCOUNT}", + f"DefaultAccount={WORKLOAD_ACCOUNT}", + ] + ) + expected_user = (WORKLOAD_USER, WORKLOAD_ACCOUNT) + if expected_user not in _sacctmgr_rows( + runner, prefix, "user", ("User", "DefaultAccount") + ): + runner.run( + [ + *prefix, + "sacctmgr", + "--immediate", + "modify", + "user", + "where", + f"Name={WORKLOAD_USER}", + "set", + f"DefaultAccount={WORKLOAD_ACCOUNT}", + ] + ) + account_ready = (WORKLOAD_ACCOUNT,) in _sacctmgr_rows( + runner, prefix, "account", ("Account",) + ) + association_ready = expected_association in _sacctmgr_rows( + runner, prefix, "association", ("User", "Account") + ) + user_ready = expected_user in _sacctmgr_rows( + runner, prefix, "user", ("User", "DefaultAccount") + ) + if not account_ready or not association_ready or not user_ready: + raise ProvisionError( + "Slurm workload accounting reconciliation did not produce the " + f"required {WORKLOAD_USER}/{WORKLOAD_ACCOUNT} association" + ) + + def _validate_slurm(runner: Runner, config: Config) -> None: """Validate LoginSet placement, storage, two-node fan-out, and accounting.""" login = _login_pod(runner, config) @@ -2185,6 +2393,13 @@ def _validate_slurm(runner: Runner, config: Config) -> None: f"targets={sorted(target_nodes)}" ) prefix = _kubectl(config, "-n", config.namespace, "exec", login, "--") + workload_prefix = [*prefix, "runuser", "-u", WORKLOAD_USER, "--"] + coordinator_uid = runner.run([*workload_prefix, "id", "-u"]).stdout.strip() + if coordinator_uid != str(WORKLOAD_UID) or coordinator_uid == "0": + raise ProvisionError( + "Slurm coordinator workload identity is invalid: " + f"expected {WORKLOAD_UID}, found {coordinator_uid!r}" + ) backend = _select_storage_backend(config) token = secrets.token_hex(16) storage_probe = ( @@ -2195,7 +2410,7 @@ def _validate_slurm(runner: Runner, config: Config) -> None: "; case $(stat -f -c %T /mnt/storage-test) in " "nfs|nfs4) true;; *) false;; esac" ) - runner.run([*prefix, "bash", "-c", storage_probe]) + runner.run([*workload_prefix, "bash", "-c", storage_probe]) if backend == "sbx-shared": host_probe = config.sbx_shared_root / "storage-test" / ".login-probe" if ( @@ -2203,28 +2418,38 @@ def _validate_slurm(runner: Runner, config: Config) -> None: or host_probe.read_text(encoding="utf-8").strip() != token ): raise ProvisionError("LoginSet SBX data is not visible from the agent") - runner.run([*prefix, "rm", "-f", "/mnt/storage-test/.login-probe"]) + runner.run([*workload_prefix, "rm", "-f", "/mnt/storage-test/.login-probe"]) fanout = runner.run( [ - *prefix, + *workload_prefix, "srun", + "--account", + WORKLOAD_ACCOUNT, "-p", "all", "-N2", "-n2", "--ntasks-per-node=1", - "hostname", + "sh", + "-c", + 'printf "%s %s\\n" "$(hostname)" "$(id -u)"', ], timeout=120, ).stdout.splitlines() - if len(set(fanout)) != 2: + identities = [line.split() for line in fanout if line.strip()] + hostnames = {fields[0] for fields in identities if len(fields) == 2} + task_uids = {fields[1] for fields in identities if len(fields) == 2} + if len(identities) != 2 or len(hostnames) != 2 or task_uids != {str(WORKLOAD_UID)}: raise ProvisionError( - f"Slurm fan-out did not reach two distinct nodes: {fanout}" + "Slurm fan-out did not reach two distinct nodes as the fixed " + f"workload identity {WORKLOAD_UID}: {fanout}" ) job = runner.run( [ - *prefix, + *workload_prefix, "sbatch", + "--account", + WORKLOAD_ACCOUNT, "--wait", "--parsable", "-p", @@ -2239,7 +2464,7 @@ def _validate_slurm(runner: Runner, config: Config) -> None: ).stdout.strip() accounting = runner.run( [ - *prefix, + *workload_prefix, "sacct", "-X", "-j", @@ -2269,6 +2494,10 @@ def _write_state_summary( "test_user": config.test_user, "test_uid": config.test_uid, "test_gid": config.test_gid, + "workload_user": WORKLOAD_USER, + "workload_uid": WORKLOAD_UID, + "workload_gid": WORKLOAD_GID, + "workload_account": WORKLOAD_ACCOUNT, "kind_version": (SBX_KIND_VERSION if backend == "sbx-shared" else KIND_VERSION), "kubectl_version": ( SBX_KUBECTL_VERSION if backend == "sbx-shared" else KUBECTL_VERSION @@ -2290,9 +2519,9 @@ def setup_environment(runner: Runner, config: Config) -> None: backend = _select_storage_backend(config) capacity_path = _repository_root() if backend == "sbx-shared" else Path("/") _check_host_capacity(capacity_path) - _ensure_apt_packages(runner, backend) + _prepare_host_dependencies(runner, config, backend) _ensure_docker(runner) - _ensure_client_tools(runner, architecture, backend) + _ensure_client_tools(runner, config, architecture, backend) running_clusters = _kind_clusters(runner) containers = _kind_containers(runner, config, running_only=False) running_containers = _kind_containers(runner, config, running_only=True) @@ -2342,53 +2571,25 @@ def setup_environment(runner: Runner, config: Config) -> None: LOG.info("Integration environment is provisioned and running") -def _grant_test_user_access(runner: Runner, config: Config) -> None: - """Give the non-root test identity access to its private setup state.""" - marker = config.state_dir / STATE_MARKER - if not marker.is_file() or json.loads(marker.read_text(encoding="utf-8")) != ( - _owner_document(config) - ): - raise ProvisionError( - f"refusing to change ownership of unverified setup state: {config.state_dir}" - ) - runner.run( - [ - *_sudo_prefix(), - "chown", - "-R", - f"{config.test_uid}:{config.test_gid}", - config.state_dir, - ] - ) - - -def _test_user_command(config: Config, *command: str | Path) -> list[str | Path]: - """Build a command that executes as the configured non-root test user.""" - if os.geteuid() != 0: - return list(command) - return ["runuser", "-u", config.test_user, "--", *command] - - -def _verify_test_user_access(runner: Runner, config: Config) -> None: - """Verify the test identity can use the provisioned cluster and state.""" +def _verify_user_access(runner: Runner, config: Config) -> None: + """Verify the current user can use the provisioned cluster and state.""" probe = config.state_dir / "test-runs" / ".access-probe" - runner.run(_test_user_command(config, "mkdir", "-p", probe.parent)) - runner.run(_test_user_command(config, "touch", probe)) - runner.run(_test_user_command(config, "rm", "--", probe)) - runner.run(_test_user_command(config, "docker", "info"), timeout=60) - runner.run(_test_user_command(config, "kind", "get", "clusters"), timeout=30) + runner.run(["mkdir", "-p", probe.parent]) + runner.run(["touch", probe]) + runner.run(["rm", "--", probe]) + runner.run(["docker", "info"], timeout=60) + runner.run(["kind", "get", "clusters"], timeout=30) runner.run( - _test_user_command( - config, + [ "kubectl", "--kubeconfig", config.kubeconfig, "get", "nodes", - ), + ], timeout=30, ) - LOG.info("Verified integration test access for non-root user %s", config.test_user) + LOG.info("Verified integration access for current user %s", config.test_user) def _scale_ssh(runner: Runner, config: Config, replicas: int) -> None: @@ -2407,6 +2608,18 @@ def _scale_ssh(runner: Runner, config: Config, replicas: int) -> None: def _wait_for_ssh(runner: Runner, config: Config) -> None: """Wait for exactly two ready, nonterminating SSH workers.""" + runner.run( + _kubectl( + config, + "-n", + config.namespace, + "rollout", + "status", + "statefulset/ssh-worker", + "--timeout=180s", + ), + timeout=210, + ) deadline = time.monotonic() + 180 while time.monotonic() < deadline: pods = _ssh_pods(runner, config) @@ -2468,7 +2681,7 @@ def teardown_environment(runner: Runner, config: Config) -> None: ) LOG.info( "Docker SBX integration environment torn down; installed host " - "packages and client tools were preserved" + "packages were preserved" ) return state_owned, setup_owned, export_owned, nfs_configured = ( @@ -2485,8 +2698,7 @@ def teardown_environment(runner: Runner, config: Config) -> None: runner, config, remove_state=state_owned, remove_export=export_owned ) LOG.info( - "Integration environment torn down; installed host packages and client " - "tools were preserved" + "Integration environment torn down; installed host packages were preserved" ) @@ -2547,7 +2759,22 @@ def _validate_sbx_teardown_ownership( def _remove_sbx_shared(runner: Runner, config: Config) -> None: """Remove only the validated marker-owned Docker SBX shared root.""" root = config.sbx_shared_root - runner.run(["find", root, "-xdev", "-depth", "-delete"], timeout=120) + runner.run( + [ + "docker", + "run", + "--rm", + "--entrypoint", + "sh", + "--mount", + f"type=bind,src={root},dst=/owned", + SBX_KIND_NODE_IMAGE, + "-ec", + "find /owned -mindepth 1 -xdev -depth -delete", + ], + timeout=120, + ) + root.rmdir() if root.exists(): raise ProvisionError(f"cleanup did not remove SBX shared root: {root}") @@ -2663,15 +2890,6 @@ def _validate_teardown_ownership( if cluster_owned: _validate_image_ownership(runner, config) - if nfs_configured: - unrelated = [ - path for path in _export_paths(runner) if path != str(config.export_dir) - ] - if unrelated: - raise ProvisionError( - "refusing to stop and disable nfs-server while unrelated exports " - "exist: " + ", ".join(unrelated) - ) return state_owned, cluster_owned, export_owned, nfs_configured @@ -2695,13 +2913,17 @@ def _validate_lifecycle_paths(config: Config) -> None: _validate_cleanup_paths(config) state_dir = config.state_dir if not state_dir.exists(): - if not state_dir.parent.is_dir(): + if not state_dir.parent.is_dir() and state_dir != DEFAULT_STATE_DIR: raise ProvisionError( f"state directory must be a leaf below an existing directory: {state_dir}" ) return if not state_dir.is_dir(): raise ProvisionError(f"state path is not a directory: {state_dir}") + if state_dir.stat().st_uid != os.getuid(): + raise ProvisionError( + f"setup state is not owned by the current user: {state_dir}" + ) marker = state_dir / STATE_MARKER if not marker.is_file(): raise ProvisionError(f"refusing to modify unowned setup state: {state_dir}") @@ -2729,6 +2951,8 @@ def _validate_installed_config( def _export_paths(runner: Runner) -> list[str]: """Return currently exported local paths.""" + if not shutil.which("exportfs"): + return [] exports = runner.run([*_sudo_prefix(), "exportfs", "-v"], check=False).stdout return [line.split()[0] for line in exports.splitlines() if line.startswith("/")] @@ -2819,15 +3043,16 @@ def _validate_image_ownership(runner: Runner, config: Config) -> None: def _remove_nfs_configuration(runner: Runner, config: Config) -> None: - """Unexport storage, disable NFS, and remove exact host configuration.""" + """Unexport storage and remove only the fixture's host configuration.""" LOG.info("Removing the dedicated NFS export and host configuration") + exportfs = shutil.which("exportfs") export_source = config.manifests_dir / "storage-scale-test.exports" - if export_source.exists(): + if exportfs and export_source.exists(): client = export_source.read_text(encoding="utf-8").split()[1].split("(", 1)[0] runner.run( [ *_sudo_prefix(), - "exportfs", + exportfs, "-u", f"{client}:{config.export_dir}", ], @@ -2835,10 +3060,14 @@ def _remove_nfs_configuration(runner: Runner, config: Config) -> None: ) for path in (NFS_EXPORT_CONFIG, NFS_DAEMON_CONFIG): runner.run([*_sudo_prefix(), "rm", "--force", "--", path]) - runner.run([*_sudo_prefix(), "exportfs", "-ra"]) + if exportfs: + runner.run([*_sudo_prefix(), exportfs, "-ra"]) + else: + LOG.info("exportfs is unavailable; skipping partial-bootstrap reload") if str(config.export_dir) in _export_paths(runner): raise ProvisionError(f"NFS export is still active: {config.export_dir}") - runner.run([*_sudo_prefix(), "systemctl", "disable", "--now", "nfs-server"]) + if _nfs_service_started_by_harness(config) and not _export_paths(runner): + runner.run([*_sudo_prefix(), "systemctl", "disable", "nfs-server"]) firewall_state = config.state_dir / "ufw-rule.json" if firewall_state.exists(): state = json.loads(firewall_state.read_text(encoding="utf-8")) @@ -2884,33 +3113,31 @@ def _remove_owned_directories( remove_state: bool, remove_export: bool, ) -> None: - """Remove the validated export and state directories without crossing mounts.""" - paths = [config.state_dir] if remove_state else [] + """Remove validated user and privileged roots without crossing mounts.""" + paths: list[tuple[Path, bool]] = [] + if remove_state: + paths.append((config.state_dir, False)) if remove_export: - paths.insert(0, config.export_dir) - for path in paths: - probe = runner.run([*_sudo_prefix(), "test", "-e", path], check=False) + paths.insert(0, (config.export_dir, True)) + for path, privileged in paths: + prefix = _sudo_prefix() if privileged else [] + probe = runner.run([*prefix, "test", "-e", path], check=False) if probe.returncode: continue runner.run( - [*_sudo_prefix(), "find", path, "-xdev", "-depth", "-delete"], + [*prefix, "find", path, "-xdev", "-depth", "-delete"], timeout=120, ) - if ( - runner.run([*_sudo_prefix(), "test", "-e", path], check=False).returncode - == 0 - ): + if runner.run([*prefix, "test", "-e", path], check=False).returncode == 0: raise ProvisionError(f"cleanup did not remove {path}") def _stop_owned_nfs(runner: Runner, config: Config) -> None: """Stop NFS only when this harness started the otherwise-dedicated service.""" - state_path = config.state_dir / "nfs-service.json" - if not state_path.exists(): + if not (config.state_dir / "nfs-service.json").exists(): LOG.info("NFS ownership state is absent; leaving nfs-server unchanged") return - state = json.loads(state_path.read_text(encoding="utf-8")) - if not state.get("started_by_harness", False): + if not _nfs_service_started_by_harness(config): LOG.info("nfs-server predated this fixture; leaving it running") return export_paths = _export_paths(runner) @@ -2930,17 +3157,54 @@ def _stop_owned_nfs(runner: Runner, config: Config) -> None: LOG.info("nfs-server is already stopped") +def _nfs_service_started_by_harness(config: Config) -> bool: + """Return whether setup recorded ownership of the NFS service lifecycle.""" + state_path = config.state_dir / "nfs-service.json" + if not state_path.exists(): + return False + return bool( + json.loads(state_path.read_text(encoding="utf-8")).get( + "started_by_harness", False + ) + ) + + +def _diagnostic_backend(config: Config) -> str | None: + """Return the configured or retained backend without probing or mutation.""" + if config.storage_backend != "auto": + return config.storage_backend + path = config.state_dir / "storage-backend.json" + if not path.is_file(): + return None + try: + backend = json.loads(path.read_text(encoding="utf-8")).get("backend") + except (OSError, json.JSONDecodeError): + return None + return backend if backend in STORAGE_BACKENDS else None + + def _collect_diagnostics(runner: Runner, config: Config) -> None: """Collect bounded troubleshooting state after a setup failure.""" LOG.error("Collecting troubleshooting diagnostics") - commands: tuple[Sequence[str | Path], ...] = ( + commands: list[Sequence[str | Path]] = [ ("free", "-h"), ("df", "-h", "/"), ("docker", "ps", "--all"), ("kind", "get", "clusters"), - (*_sudo_prefix(), "systemctl", "status", "nfs-server", "--no-pager"), - (*_sudo_prefix(), "exportfs", "-v"), - ) + ] + if _diagnostic_backend(config) == "nfs": + commands.extend( + ( + ( + *_sudo_prefix(), + "systemctl", + "status", + "nfs-server", + "--no-pager", + ), + (*_sudo_prefix(), "exportfs", "-v"), + ) + ) for command in commands: if not shutil.which(str(command[0])): LOG.error("diagnostic command is unavailable: %s", command[0]) @@ -2970,7 +3234,7 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--export-dir", type=Path, default=DEFAULT_EXPORT_DIR) parser.add_argument( "--storage-backend", - choices=("auto", "nfs", "sbx-shared"), + choices=("auto", *STORAGE_BACKENDS), default="auto", ) parser.add_argument( @@ -2978,13 +3242,6 @@ def _parser() -> argparse.ArgumentParser: type=Path, default=DEFAULT_SBX_SHARED_ROOT, ) - parser.add_argument( - "--test-user", - help=( - "non-root account that runs tests; required for root setup unless " - "SUDO_USER identifies it" - ), - ) parser.add_argument("--verbose", action="store_true") actions = parser.add_subparsers(dest="action", required=True) for action in ("setup", "start", "stop", "teardown"): @@ -3014,7 +3271,7 @@ def _parser() -> argparse.ArgumentParser: def _config(arguments: argparse.Namespace) -> Config: """Convert parsed arguments into immutable configuration.""" - account = _test_account(arguments.test_user) + account = _test_account() return Config( cluster_name=arguments.cluster_name, namespace=arguments.namespace, @@ -3029,32 +3286,18 @@ def _config(arguments: argparse.Namespace) -> Config: ) -def _test_account(explicit_user: str | None) -> pwd.struct_passwd: - """Resolve the non-root account that owns and runs integration tests.""" - requested = explicit_user - if os.geteuid() == 0 and requested is None: - requested = os.environ.get("SUDO_USER") - if requested is None: - try: - requested = pwd.getpwuid(os.getuid()).pw_name - except KeyError as error: - raise ProvisionError( - f"current uid has no password-database entry: {os.getuid()}" - ) from error +def _test_account() -> pwd.struct_passwd: + """Resolve the ordinary account running every lifecycle action.""" try: - account = pwd.getpwnam(requested) + account = pwd.getpwuid(os.getuid()) except KeyError as error: raise ProvisionError( - f"integration test user does not exist: {requested}" + f"current uid has no password-database entry: {os.getuid()}" ) from error if account.pw_uid == 0: raise ProvisionError( - "integration tests require a non-root account; use sudo from that " - "account or pass --test-user" - ) - if os.geteuid() != 0 and account.pw_uid != os.getuid(): - raise ProvisionError( - f"non-root caller cannot provision tests for another user: {requested}" + "integration lifecycle actions must run as an ordinary user; " + "the driver invokes sudo only for narrow host-system operations" ) return account @@ -3097,10 +3340,10 @@ def main() -> int: print(format_scenario_listing()) return 0 try: - if arguments.action == "test" and os.geteuid() == 0: + if os.geteuid() == 0: raise ProvisionError( - "refusing to run integration sweeps as root; rerun the test " - "action as the account provisioned by setup" + "refusing to run the integration lifecycle as root; rerun as " + "an ordinary user with sudo available for NFS host operations" ) config = _config(arguments) _validate_lifecycle_paths(config) @@ -3109,6 +3352,7 @@ def main() -> int: f"setup state directory is absent at {config.state_dir}; run setup first" ) _bootstrap_state_dir(config) + _configure_user_tool_path(config) log_path = _configure_logging(config, arguments.action) LOG.info("Detailed log: %s", log_path) with _acquire_lock(config): @@ -3121,8 +3365,7 @@ def main() -> int: _run_filesystem_action(runner, config, arguments) else: setup_environment(runner, config) - _grant_test_user_access(runner, config) - _verify_test_user_access(runner, config) + _verify_user_access(runner, config) return 0 except ( ProvisionError, diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py index f73495a..cbd5e8f 100644 --- a/integration-tests/lib/filesystem_integration.py +++ b/integration-tests/lib/filesystem_integration.py @@ -30,7 +30,7 @@ import time import urllib.error import urllib.request -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path, PurePosixPath from typing import Any @@ -89,9 +89,11 @@ ), } REMOTE_BASE = "/mnt/storage-test/integration-regression" +SSH_FAILURE_STAGING_BASE = "/tmp/storage-scale-test-integration-failure" VALIDATION_SUCCESS = "All validation checks passed successfully" -POD_TEST_UID = 2000 -POD_TEST_GID = 2000 +WORKLOAD_USER = "tester" +WORKLOAD_UID = 2000 +WORKLOAD_GID = 2000 class IntegrationTestError(RuntimeError): @@ -104,9 +106,9 @@ class Fixture: login_pod: str login_container: str - ssh_addresses: tuple[str, str] - slurm_nodes: tuple[str, str] - slurm_addresses: tuple[str, str] + ssh_addresses: tuple[str, ...] + slurm_nodes: tuple[str, ...] + slurm_addresses: tuple[str, ...] architecture: str ssh_home_mode: str storage_backend: str @@ -294,17 +296,22 @@ def _pod_inventory(runner: Any, config: Any) -> list[dict[str, Any]]: def _require_pods( pods: list[dict[str, Any]], + substrates: set[str], ) -> tuple[dict[str, Any], list[dict[str, Any]]]: - """Require one LoginSet and two SSH worker pods.""" + """Require only the live workloads needed by selected substrates.""" login = _pods_with_container(pods, "login") ssh = sorted( _pods_with_container(pods, "sshd"), key=lambda item: item["metadata"]["name"] ) slurmd = _pods_with_container(pods, "slurmd") - if len(login) != 1 or len(ssh) != 2 or len(slurmd) != 2: + invalid = len(login) != 1 + invalid = invalid or ("ssh" in substrates and len(ssh) != 2) + invalid = invalid or ("slurm" in substrates and len(slurmd) != 2) + if invalid: raise IntegrationTestError( "fixture workloads are incomplete: " - f"login={len(login)}, ssh={len(ssh)}, slurmd={len(slurmd)}; run setup" + f"required={sorted(substrates)}, login={len(login)}, ssh={len(ssh)}, " + f"slurmd={len(slurmd)}; run setup" ) return login[0], ssh @@ -334,22 +341,25 @@ def _probe_pod( def _require_fixture( - runner: Any, config: Any, ssh_home_mode: str = "separate" + runner: Any, + config: Any, + substrates: set[str], + ssh_home_mode: str = "separate", ) -> Fixture: """Validate setup without reconciling or installing anything.""" state = _load_state(config) - required_host_tools = ( + required_host_tools = { "bash", "file", "find", "git", - "ssh-add", - "ssh-agent", "tar", "timeout", - ) + } + if "ssh" in substrates: + required_host_tools.update(("ssh-add", "ssh-agent")) missing_host_tools = [ - tool for tool in required_host_tools if shutil.which(tool) is None + tool for tool in sorted(required_host_tools) if shutil.which(tool) is None ] if missing_host_tools: raise IntegrationTestError( @@ -361,70 +371,127 @@ def _require_fixture( ) _require_nodes(runner, config) _require_storage(runner, config) - login, ssh = _require_pods(_pod_inventory(runner, config)) + login, ssh = _require_pods(_pod_inventory(runner, config), substrates) login_name = str(login["metadata"]["name"]) - ssh_name = str(ssh[0]["metadata"]["name"]) - addresses = tuple(str(item["status"]["podIP"]) for item in ssh) - if len(set(addresses)) != 2: - raise IntegrationTestError(f"SSH workers lack distinct addresses: {addresses}") + addresses: tuple[str, ...] = () + if "ssh" in substrates: + addresses = tuple(str(item["status"]["podIP"]) for item in ssh) + if len(set(addresses)) != 2: + raise IntegrationTestError( + f"SSH workers lack distinct addresses: {addresses}" + ) tools = ( "for tool in bash file find tar timeout; do " 'command -v "$tool" >/dev/null || exit 1; done' ) - _probe_pod(runner, config, login_name, "login", tools) - _probe_pod(runner, config, ssh_name, "sshd", tools, as_user="tester") + _probe_pod( + runner, + config, + login_name, + "login", + tools, + as_user=WORKLOAD_USER, + ) + if "ssh" in substrates: + _probe_pod( + runner, + config, + str(ssh[0]["metadata"]["name"]), + "sshd", + tools, + as_user="tester", + ) mount_probe = "test -w /mnt/storage-test" if state["storage_backend"] == "nfs": mount_probe += ( " && case $(stat -f -c %T /mnt/storage-test) in " "nfs|nfs4) true;; *) false;; esac" ) - _probe_pod(runner, config, login_name, "login", mount_probe) - _probe_pod(runner, config, ssh_name, "sshd", mount_probe, as_user="tester") - login_arch = _probe_pod(runner, config, login_name, "login", "uname -m") - ssh_arch = _probe_pod( - runner, config, ssh_name, "sshd", "uname -m", as_user="tester" - ) - host_arch = platform.machine() - if login_arch != ssh_arch or login_arch != host_arch: - raise IntegrationTestError( - "fixture architecture mismatch: " - f"host={host_arch}, login={login_arch}, ssh={ssh_arch}" - ) - if host_arch not in ELBENCHO_ARCHIVES: - raise IntegrationTestError(f"unsupported fixture architecture: {host_arch}") - slurm_output = _probe_pod( + _probe_pod( runner, config, login_name, "login", - "sinfo -N -h -o %N | sort -u", + mount_probe, + as_user=WORKLOAD_USER, ) - slurm_nodes = tuple(line for line in slurm_output.splitlines() if line) - if len(slurm_nodes) != 2: - raise IntegrationTestError( - f"expected two Slurm compute nodes; found {slurm_nodes}" + if "ssh" in substrates: + _probe_pod( + runner, + config, + str(ssh[0]["metadata"]["name"]), + "sshd", + mount_probe, + as_user="tester", ) - slurm_address_output = _probe_pod( + login_arch = _probe_pod( runner, config, login_name, "login", - "for node in " - + " ".join(shlex.quote(node) for node in slurm_nodes) - + "; do getent ahostsv4 \"$node\" | awk 'NR == 1 {print $1}'; done", + "uname -m", + as_user=WORKLOAD_USER, ) - slurm_addresses = tuple(line for line in slurm_address_output.splitlines() if line) - if len(slurm_addresses) != 2 or len(set(slurm_addresses)) != 2: + host_arch = platform.machine() + fixture_architectures = {"host": host_arch, "login": login_arch} + if "ssh" in substrates: + fixture_architectures["ssh"] = _probe_pod( + runner, + config, + str(ssh[0]["metadata"]["name"]), + "sshd", + "uname -m", + as_user="tester", + ) + if len(set(fixture_architectures.values())) != 1: raise IntegrationTestError( - f"Slurm workers lack distinct IPv4 addresses: {slurm_addresses}" + "fixture architecture mismatch: " + + ", ".join( + f"{name}={architecture}" + for name, architecture in fixture_architectures.items() + ) + ) + if host_arch not in ELBENCHO_ARCHIVES: + raise IntegrationTestError(f"unsupported fixture architecture: {host_arch}") + slurm_nodes: tuple[str, ...] = () + slurm_addresses: tuple[str, ...] = () + if "slurm" in substrates: + slurm_output = _probe_pod( + runner, + config, + login_name, + "login", + "sinfo -N -h -o %N | sort -u", + as_user=WORKLOAD_USER, ) + slurm_nodes = tuple(line for line in slurm_output.splitlines() if line) + if len(slurm_nodes) != 2: + raise IntegrationTestError( + f"expected two Slurm compute nodes; found {slurm_nodes}" + ) + slurm_address_output = _probe_pod( + runner, + config, + login_name, + "login", + "for node in " + + " ".join(shlex.quote(node) for node in slurm_nodes) + + "; do getent ahostsv4 \"$node\" | awk 'NR == 1 {print $1}'; done", + as_user=WORKLOAD_USER, + ) + slurm_addresses = tuple( + line for line in slurm_address_output.splitlines() if line + ) + if len(slurm_addresses) != 2 or len(set(slurm_addresses)) != 2: + raise IntegrationTestError( + f"Slurm workers lack distinct IPv4 addresses: {slurm_addresses}" + ) return Fixture( login_pod=login_name, login_container="login", - ssh_addresses=(addresses[0], addresses[1]), - slurm_nodes=(slurm_nodes[0], slurm_nodes[1]), - slurm_addresses=(slurm_addresses[0], slurm_addresses[1]), + ssh_addresses=addresses, + slurm_nodes=slurm_nodes, + slurm_addresses=slurm_addresses, architecture=host_arch, ssh_home_mode=ssh_home_mode, storage_backend=str(state["storage_backend"]), @@ -728,7 +795,7 @@ def _override_block( lines.extend( ( "unset SSH_HOST_LIST SSH_USER SSH_HOMEDIR_SHARED", - 'account=""', + 'account="storage-test"', 'reservation=""', 'partition="all"', 'run_time="00:05:00"', @@ -943,28 +1010,27 @@ def _stream_to_login( fixture: Fixture, source: Path, command: list[str | Path], + *, + as_user: str | None = None, timeout: int = 180, ) -> None: """Stream one local archive to a command in the Slinky LoginSet.""" with source.open("rb") as stream: - runner.run( - [ - *_kubectl( - config, - "-n", - config.namespace, - "exec", - "-i", - fixture.login_pod, - "-c", - fixture.login_container, - "--", - ), - *command, - ], - stdin=stream, - timeout=timeout, + pod_command: list[str | Path] = _kubectl( + config, + "-n", + config.namespace, + "exec", + "-i", + fixture.login_pod, + "-c", + fixture.login_container, + "--", ) + if as_user: + pod_command.extend(("runuser", "-u", as_user, "--")) + pod_command.extend(command) + runner.run(pod_command, stdin=stream, timeout=timeout) def _stage_ssh_runtime( @@ -993,9 +1059,13 @@ def _stage_ssh_runtime( "sshd", "--", ), + "runuser", + "-u", + WORKLOAD_USER, + "--", "bash", "-ec", - "rm -rf -- /home/tester/elbencho-runtime && " + "umask 0007; rm -rf -- /home/tester/elbencho-runtime && " "tar --no-same-owner --no-same-permissions -xf - -C /home/tester", ] runner.run(command, stdin=stream, timeout=180) @@ -1020,27 +1090,35 @@ def _stage_workspace( remote_base = f"{REMOTE_BASE}/workspaces/{workspace_id}" remote_root = f"{remote_base}/storage-scale-test" - reset = f"rm -rf -- {_shell(remote_base)} && mkdir -p -- {_shell(remote_base)}" + reset = ( + f"umask 0007; rm -rf -- {_shell(remote_base)} && " + f"mkdir -p -- {_shell(remote_base)}" + ) runner.run( _pod_command( config, fixture.login_pod, fixture.login_container, reset, + as_user=WORKLOAD_USER, timeout=60, ), timeout=75, ) extract_command = [ - "tar", - "--no-same-owner", - "--no-same-permissions", - "-xzf", - "-", - "-C", - remote_base, + "bash", + "-ec", + "umask 0007; tar --no-same-owner --no-same-permissions " + f"-xzf - -C {_shell(remote_base)}", ] - _stream_to_login(runner, config, fixture, archive, extract_command) + _stream_to_login( + runner, + config, + fixture, + archive, + extract_command, + as_user=WORKLOAD_USER, + ) support_stage = build_root / "slurm-runtime" support_stage.mkdir() _write_runtime_files( @@ -1055,15 +1133,19 @@ def _stage_workspace( for child in sorted(support_stage.iterdir()): tar.add(child, arcname=child.name) support_command = [ - "tar", - "--no-same-owner", - "--no-same-permissions", - "-xf", - "-", - "-C", - remote_root, + "bash", + "-ec", + "umask 0007; tar --no-same-owner --no-same-permissions " + f"-xf - -C {_shell(remote_root)}", ] - _stream_to_login(runner, config, fixture, support_archive, support_command) + _stream_to_login( + runner, + config, + fixture, + support_archive, + support_command, + as_user=WORKLOAD_USER, + ) return remote_root @@ -1096,6 +1178,7 @@ def _test_command( fixture.login_pod, fixture.login_container, command, + as_user=WORKLOAD_USER, timeout=timeout, ) @@ -1219,7 +1302,20 @@ def _assert_results( return result_dir -def _assert_ordered_workers(fixture: Fixture, selector: str, sweep_output: str) -> None: +def _ordered_worker_evidence(selector: str, sweep_output: str, result: Path) -> str: + """Return substrate-appropriate durable worker-selection evidence.""" + if selector == "ssh": + return sweep_output + logs = sorted((result / "executions").glob("*.log")) + logs.extend(sorted(result.glob("coordinator-*.log"))) + return "\n".join( + path.read_text(encoding="utf-8", errors="replace") for path in logs + ) + + +def _assert_ordered_workers( + fixture: Fixture, selector: str, sweep_output: str, result: Path +) -> None: """Prove increasing cells use the configured workers in prefix order.""" workers = fixture.ssh_addresses if selector == "ssh" else fixture.slurm_addresses expected = { @@ -1227,7 +1323,8 @@ def _assert_ordered_workers(fixture: Fixture, selector: str, sweep_output: str) "2": ",".join(workers), } selected: dict[str, str] = {} - for line in sweep_output.splitlines(): + evidence = _ordered_worker_evidence(selector, sweep_output, result) + for line in evidence.splitlines(): match = re.search(r"starting execution \d+:.*nodes=(\d+).*hosts=([^ ]+)", line) if match: selected[match.group(1)] = match.group(2) @@ -1319,6 +1416,7 @@ def _login_shell( fixture.login_pod, fixture.login_container, command, + as_user=WORKLOAD_USER, timeout=timeout, ), timeout=timeout + 15, @@ -1334,11 +1432,11 @@ def _prepare_scenario_data( """Reset one marker-owned scenario subtree on the shared test storage.""" command = f""" set -euo pipefail +umask 0007 rm -rf -- {_shell(data_root)} mkdir -p -- {_shell(data_root + '/primary')} {_shell(data_root + '/secondary')} printf 'scenario-owned\n' > {_shell(data_root + '/primary/.integration-sentinel')} printf 'scenario-owned\n' > {_shell(data_root + '/secondary/.integration-sentinel')} -chown -R {config.test_uid}:{config.test_gid} -- {_shell(data_root)} """.strip() _login_shell(runner, config, fixture, command) @@ -1396,14 +1494,12 @@ def _sync_step_runtime( fixture, archive_path, [ - "tar", - "--no-same-owner", - "--no-same-permissions", - "-xf", - "-", - "-C", - runtime.workspace, + "bash", + "-ec", + "umask 0007; tar --no-same-owner --no-same-permissions " + f"-xf - -C {_shell(runtime.workspace)}", ], + as_user=WORKLOAD_USER, ) @@ -1419,10 +1515,9 @@ def _create_generated_inputs( path = f"{runtime.values['test_root']}/{generated.relative_path}" command = f""" set -euo pipefail +umask 0007 mkdir -p -- {_shell(str(PurePosixPath(path).parent))} truncate -s {generated.size_bytes} -- {_shell(path)} -chown -R {config.test_uid}:{config.test_gid} -- \ - {_shell(str(PurePosixPath(path).parent))} test "$(stat -c %s -- {_shell(path)})" -eq {generated.size_bytes} """.strip() _login_shell(runner, config, fixture, command) @@ -1624,7 +1719,12 @@ def _assert_execution_contract( totals = _expected_dataset_totals( scenario.name, step, expected.coordinate.nodes ) - if totals is not None and workload_path.is_file(): + if totals is not None: + if not workload_path.is_file(): + raise IntegrationTestError( + f"{scenario.name}/{step.name}/{execution_id}: missing " + f"required workload metadata {workload_path}" + ) workload = _workload_values(workload_path) if workload.get("dataset_files_total") != str(totals[0]) or workload.get( "dataset_bytes_total" @@ -1805,8 +1905,8 @@ def _reset_result_base( runner, config, fixture, - f"rm -rf -- {_shell(result_base)} && mkdir -p -- {_shell(result_base)} " - f"&& chown {config.test_uid}:{config.test_gid} -- {_shell(result_base)}", + f"umask 0007; rm -rf -- {_shell(result_base)} && " + f"mkdir -p -- {_shell(result_base)}", ) @@ -2031,7 +2131,7 @@ def _run_regular_step( log_dir, ) if {item.coordinate.nodes for item in step.executions} == {1, 2}: - _assert_ordered_workers(fixture, runtime.selector, output) + _assert_ordered_workers(fixture, runtime.selector, output, local_result) return StepOutcome(output, remote_result, local_result) @@ -2065,8 +2165,8 @@ def _make_scenario_runtime( "workspace": workspace, "test_root": f"{data_root}/primary", "test_root_secondary": f"{data_root}/secondary", - "slurm_node_1": fixture.slurm_nodes[0], - "slurm_node_2": fixture.slurm_nodes[1], + "slurm_node_1": fixture.slurm_nodes[0] if fixture.slurm_nodes else "", + "slurm_node_2": fixture.slurm_nodes[1] if fixture.slurm_nodes else "", } return ScenarioRuntime( scenario, @@ -2118,6 +2218,10 @@ def _cleanup_ssh_remote_results(runner: Any, config: Any) -> None: "sshd", "--", ), + "runuser", + "-u", + WORKLOAD_USER, + "--", "bash", "-c", "rm -rf -- /home/tester/elbencho-[0-9]*", @@ -2156,6 +2260,7 @@ def _cleanup_scenario_storage( fixture.login_pod, fixture.login_container, command, + as_user=WORKLOAD_USER, timeout=120, ), check=False, @@ -2199,25 +2304,35 @@ def _command(endpoint: str, command: str, *, stdin: Any = None) -> None: _container(endpoint), "--", ), + "runuser", + "-u", + WORKLOAD_USER, + "--", "bash", "-ec", - command, + f"umask 0007; {command}", ], stdin=stdin, timeout=180, ) def make_directory(endpoint: str, path: PurePosixPath) -> None: + if endpoint == "local": + Path(path).mkdir(parents=True, exist_ok=True) + return _command( endpoint, - f"mkdir -p -- {_shell(path)} && chown {POD_TEST_UID}:{POD_TEST_GID} " - f"-- {_shell(path)}", + f"mkdir -p -- {_shell(path)}", ) def copy_file( source: Path, endpoint: str, destination: PurePosixPath, mode: int ) -> None: if endpoint == "local": + local_destination = Path(destination) + local_destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, local_destination) + local_destination.chmod(mode) return with tempfile.TemporaryFile() as stream: stream.write(source.read_bytes()) @@ -2225,13 +2340,16 @@ def copy_file( _command( endpoint, f"cat > {_shell(destination)} && chmod {mode:o} -- " - f"{_shell(destination)} && chown {POD_TEST_UID}:{POD_TEST_GID} " - f"-- {_shell(destination)}", + f"{_shell(destination)}", stdin=stream, ) def copy_tree(source: Path, endpoint: str, destination: PurePosixPath) -> None: if endpoint == "local": + local_destination = Path(destination) + if local_destination.exists(): + shutil.rmtree(local_destination) + shutil.copytree(source, local_destination) return if selector == "ssh": _command( @@ -2249,8 +2367,7 @@ def copy_tree(source: Path, endpoint: str, destination: PurePosixPath) -> None: endpoint, f"rm -rf -- {_shell(destination)} && mkdir -p -- " f"{_shell(destination)} && tar -xf - -C {_shell(destination)} " - f"&& chown -R {POD_TEST_UID}:{POD_TEST_GID} -- " - f"{_shell(destination)}", + f"&& chmod -R u+rwX,g+rX,o-rwx -- {_shell(destination)}", stdin=stream, ) @@ -2271,8 +2388,7 @@ def write_text( _command( endpoint, f"cat > {_shell(destination)} && chmod {mode:o} -- " - f"{_shell(destination)} && chown {POD_TEST_UID}:{POD_TEST_GID} " - f"-- {_shell(destination)}", + f"{_shell(destination)}", stdin=stream, ) @@ -2280,6 +2396,7 @@ def remove_tree(endpoint: str, path: PurePosixPath) -> None: if endpoint == "local": if local_wrapper is not None and restore_binary is not None: shutil.copy2(restore_binary, local_wrapper) + shutil.rmtree(Path(path), ignore_errors=True) return if remote_wrapper is not None: _command( @@ -2317,7 +2434,7 @@ def _failure_plan( ) plan = build_ssh_failure_injection_plan( scenario_id=f"{runtime.scenario.name}-{os.getpid()}", - staging_root="/home/tester/.storage-scale-test-failure", + staging_root=SSH_FAILURE_STAGING_BASE, source_binary=source_binary, source_runtime=bundled_runtime, target_argument="executions/0002.write.json", @@ -2451,20 +2568,20 @@ def _run_failure_resume( ) injected_first = first result_base = f"{runtime.workspace}/results/{first.name}" + _reset_result_base(runner, config, fixture, runtime, result_base) + _sync_step_runtime( + runner, + config, + fixture, + runtime, + injected_first, + template, + result_base, + ) + _validate_step_environment( + runner, config, fixture, runtime, injected_first, log_dir + ) with staged_failure_injection(plan, operations): - _reset_result_base(runner, config, fixture, runtime, result_base) - _sync_step_runtime( - runner, - config, - fixture, - runtime, - injected_first, - template, - result_base, - ) - _validate_step_environment( - runner, config, fixture, runtime, injected_first, log_dir - ) arguments = shlex.join(first.render_arguments(runtime.values)) command = ( f"cd -- {_shell(runtime.workspace)} && " @@ -2580,7 +2697,6 @@ def _run_substrate( log_dir, 600, ) - _assert_ordered_workers(fixture, selector, sweep_output) result_dir = _assert_results( runner, config, fixture, selector, remote_root, log_dir ) @@ -2592,6 +2708,7 @@ def _run_substrate( result_dir, build_root / f"{selector}-report-input", ) + _assert_ordered_workers(fixture, selector, sweep_output, local_result) _assert_report(runner, report_workspace, local_result, selector, log_dir) @@ -2615,7 +2732,7 @@ def run_filesystem_tests( raise IntegrationTestError("SSH scenarios require a home transition hook") transition_ssh_home("separate", "preflight") LOG.info("Requiring an already-running integration setup") - fixture = _require_fixture(runner, config) + fixture = _require_fixture(runner, config, selected) binary, binary_name, runtime = _ensure_elbencho( runner, config, fixture.architecture, fixture.storage_backend ) @@ -2642,11 +2759,16 @@ def run_filesystem_tests( shutil.copy2(build_root / "build-tarball.log", log_dir / "build-tarball.log") report_workspace = build_root / "report-workspace" shutil.copytree(extracted, report_workspace) + report_fixture = fixture + if not report_fixture.ssh_addresses: + report_fixture = replace( + report_fixture, ssh_addresses=report_fixture.slurm_addresses + ) _write_runtime_files( report_workspace, "ssh", str(report_workspace), - fixture, + report_fixture, template=extracted / "env.sh.template", ) home_mode = "separate" @@ -2655,7 +2777,9 @@ def run_filesystem_tests( if isinstance(step, SshHomeTransition): transition_ssh_home(step.target.value, "ssh-shared-home") home_mode = step.target.value - fixture = _require_fixture(runner, config, home_mode) + fixture = _require_fixture( + runner, config, selected, ssh_home_mode=home_mode + ) if runtime is not None: _stage_ssh_runtime( runner, diff --git a/integration-tests/manifests/slinky-slurm-values.yaml b/integration-tests/manifests/slinky-slurm-values.yaml index 16f81c5..c13ef03 100644 --- a/integration-tests/manifests/slinky-slurm-values.yaml +++ b/integration-tests/manifests/slinky-slurm-values.yaml @@ -71,7 +71,7 @@ loginsets: login: image: repository: storage-scale-integration-login - tag: slinky-26.05-file + tag: slinky-26.05-user digest: null resources: requests: @@ -107,6 +107,10 @@ nodesets: scalingMode: DaemonSet workloadDisruptionProtection: false slurmd: + image: + repository: storage-scale-integration-slurmd + tag: slinky-26.05-user + digest: null resources: requests: cpu: 100m diff --git a/integration-tests/slinky-login-image.Dockerfile b/integration-tests/slinky-login-image.Dockerfile index 24236f2..f4d4645 100644 --- a/integration-tests/slinky-login-image.Dockerfile +++ b/integration-tests/slinky-login-image.Dockerfile @@ -20,4 +20,6 @@ USER root RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ file \ + && groupadd --gid 2000 storage-test \ + && useradd --uid 2000 --gid 2000 --create-home --shell /bin/bash tester \ && rm -rf /var/lib/apt/lists/* diff --git a/integration-tests/slinky-slurmd-image.Dockerfile b/integration-tests/slinky-slurmd-image.Dockerfile new file mode 100644 index 0000000..d943bf8 --- /dev/null +++ b/integration-tests/slinky-slurmd-image.Dockerfile @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +RUN groupadd --gid 2000 storage-test \ + && useradd --uid 2000 --gid 2000 --create-home --shell /bin/bash tester diff --git a/tests/test_extract_elbencho_cli_contracts.py b/tests/test_extract_elbencho_cli_contracts.py index 19a5036..2182b19 100644 --- a/tests/test_extract_elbencho_cli_contracts.py +++ b/tests/test_extract_elbencho_cli_contracts.py @@ -110,6 +110,46 @@ def test_valid_filter_matching_no_metrics_is_an_error(monkeypatch, tmp_path): assert raised.value.code == 1 +def _mixed_report_input(tmp_path, live_nodes): + """Create recognizable aggregate and live filenames for CLI tests.""" + stem = "elbencho-4K-c_001-s_001-d_001_20260101Z000000" + (tmp_path / f"{stem}.csv").write_text("aggregate\n", encoding="utf-8") + (tmp_path / f"{stem}.out").write_text("aggregate\n", encoding="utf-8") + live_stem = f"elbencho-4K-c_{live_nodes:03d}-s_001-d_001_20260101Z000000" + (tmp_path / f"{live_stem}.live.csv").write_text("live\n", encoding="utf-8") + + +def test_live_match_cannot_hide_empty_filtered_aggregate(monkeypatch, tmp_path): + """Each discovered report family must independently survive filtering.""" + _mixed_report_input(tmp_path, live_nodes=2) + monkeypatch.setattr(_EXTRACT, "parse_elbencho_files", lambda _base: [_metric()]) + + with pytest.raises(SystemExit) as raised: + _run_main( + monkeypatch, + str(tmp_path), + "--only-nodes", + "2", + "--markdown", + ) + + assert raised.value.code == 1 + + +def test_failed_live_reports_cannot_hide_behind_aggregate(monkeypatch, tmp_path): + """Aggregate output does not turn a failed live report into success.""" + _mixed_report_input(tmp_path, live_nodes=1) + monkeypatch.setattr(_EXTRACT, "parse_elbencho_files", lambda _base: [_metric()]) + monkeypatch.setattr(_EXTRACT, "print_markdown_table", lambda *_args: None) + monkeypatch.setattr(_EXTRACT, "plot_metrics", lambda *_args: None) + monkeypatch.setattr(_EXTRACT, "_report_one_live_file", lambda *_args: False) + + with pytest.raises(SystemExit) as raised: + _run_main(monkeypatch, str(tmp_path), "--markdown") + + assert raised.value.code == 1 + + def test_csv_round_trip_preserves_report_dimensions(tmp_path): """Persisted metrics retain the dimensions needed by later filtering.""" metrics = [ diff --git a/tests/test_integration_deployment_cache.py b/tests/test_integration_deployment_cache.py index 7b8313a..8a6dd09 100644 --- a/tests/test_integration_deployment_cache.py +++ b/tests/test_integration_deployment_cache.py @@ -215,6 +215,7 @@ def test_runtime_directory_mode_participates_in_cache_identity(tmp_path): repository, binary = _repository(tmp_path) runtime = tmp_path / "runtime" (runtime / "empty").mkdir(parents=True) + (runtime / "empty").chmod(0o755) runner = _Runner() request = replace(_request(tmp_path, repository, binary), runtime=runtime) first = _CACHE.get_or_build_deployment(runner, request) diff --git a/tests/test_integration_driver_safety.py b/tests/test_integration_driver_safety.py index e280a66..88a42db 100644 --- a/tests/test_integration_driver_safety.py +++ b/tests/test_integration_driver_safety.py @@ -22,7 +22,7 @@ import os import sys from dataclasses import replace -from pathlib import Path +from pathlib import Path, PurePosixPath from types import SimpleNamespace import pytest @@ -37,13 +37,34 @@ sys.modules[_SPEC.name] = _DRIVER _SPEC.loader.exec_module(_DRIVER) _FILESYSTEM = sys.modules["filesystem_integration"] +_bootstrap_state_dir = getattr(_DRIVER, "_bootstrap_state_dir") +_collect_diagnostics = getattr(_DRIVER, "_collect_diagnostics") +_configure_user_tool_path = getattr(_DRIVER, "_configure_user_tool_path") +_ensure_apt_packages = getattr(_DRIVER, "_ensure_apt_packages") _ensure_export_marker = getattr(_DRIVER, "_ensure_export_marker") +_ensure_slurm_workload_account = getattr(_DRIVER, "_ensure_slurm_workload_account") +_export_paths = getattr(_DRIVER, "_export_paths") +_kind_clusters = getattr(_DRIVER, "_kind_clusters") _login_pod = getattr(_DRIVER, "_login_pod") +_inspect_ssh_home_pool = getattr(_DRIVER, "_inspect_ssh_home_pool") +_prepare_host_dependencies = getattr(_DRIVER, "_prepare_host_dependencies") +_prepare_sbx_shared = getattr(_DRIVER, "_prepare_sbx_shared") +_driver_pods_with_container = getattr(_DRIVER, "_pods_with_container") +_remove_nfs_configuration = getattr(_DRIVER, "_remove_nfs_configuration") _select_storage_backend = getattr(_DRIVER, "_select_storage_backend") _storage_backend_document = getattr(_DRIVER, "_storage_backend_document") +_validate_teardown_ownership = getattr(_DRIVER, "_validate_teardown_ownership") _validate_lifecycle_paths = getattr(_DRIVER, "_validate_lifecycle_paths") +_validate_ssh_storage = getattr(_DRIVER, "_validate_ssh_storage") +_wait_for_ssh = getattr(_DRIVER, "_wait_for_ssh") +_assert_execution_contract = getattr(_FILESYSTEM, "_assert_execution_contract") +_assert_ordered_workers = getattr(_FILESYSTEM, "_assert_ordered_workers") _ensure_elbencho = getattr(_FILESYSTEM, "_ensure_elbencho") +_prepare_scenario_data = getattr(_FILESYSTEM, "_prepare_scenario_data") +_remote_staging_operations = getattr(_FILESYSTEM, "_remote_staging_operations") +_reset_result_base = getattr(_FILESYSTEM, "_reset_result_base") _pods_with_container = getattr(_FILESYSTEM, "_pods_with_container") +_require_pods = getattr(_FILESYSTEM, "_require_pods") def test_scenario_listing_short_circuits_before_privileged_state(monkeypatch, capsys): @@ -64,6 +85,19 @@ def test_scenario_listing_short_circuits_before_privileged_state(monkeypatch, ca assert capsys.readouterr().out.startswith("baseline\t") +def test_root_lifecycle_is_rejected_before_state_resolution(monkeypatch): + """Every mutating or executing lifecycle action rejects root up front.""" + monkeypatch.setattr(sys, "argv", [str(_DRIVER_PATH), "setup"]) + monkeypatch.setattr(os, "geteuid", lambda: 0) + monkeypatch.setattr( + _DRIVER, + "_config", + lambda _arguments: pytest.fail("root lifecycle resolved setup state"), + ) + + assert _DRIVER.main() == 1 + + def _config(state_dir: Path, export_dir: Path) -> object: """Return a minimal real driver configuration.""" return _DRIVER.Config( @@ -74,12 +108,186 @@ def _config(state_dir: Path, export_dir: Path) -> object: storage_backend="sbx-shared", sbx_shared_root=_REPO_ROOT / "tmp" / "test-shared", test_user="tester", - test_uid=2000, - test_gid=2000, + test_uid=42424, + test_gid=43434, verbose=False, ) +def test_state_bootstrap_is_user_owned_and_idempotent(tmp_path, monkeypatch): + """State starts under the caller and repeated bootstrap preserves it.""" + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setenv("PATH", "/usr/bin") + + _bootstrap_state_dir(config) + _configure_user_tool_path(config) + first_marker = (config.state_dir / _DRIVER.STATE_MARKER).read_text(encoding="utf-8") + _bootstrap_state_dir(config) + + assert config.state_dir.stat().st_uid == os.getuid() + assert (config.state_dir.stat().st_mode & 0o777) == 0o750 + assert (config.state_dir / _DRIVER.STATE_MARKER).read_text( + encoding="utf-8" + ) == first_marker + assert os.environ["PATH"].split(os.pathsep)[0] == str(config.state_dir / "bin") + assert all( + path in os.environ["PATH"].split(os.pathsep) + for path in _DRIVER.SYSTEM_ADMIN_PATHS + ) + assert os.environ["HELM_CACHE_HOME"] == str( + config.state_dir / "tool-state" / "helm" / "cache" + ) + + +def test_standard_admin_path_resolves_nfs_tools(tmp_path, monkeypatch): + """Ordinary-user setup finds tools installed outside its initial PATH.""" + admin_path = tmp_path / "usr-sbin" + admin_path.mkdir() + losetup = admin_path / "losetup" + losetup.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + losetup.chmod(0o755) + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setenv("PATH", "/usr/bin") + monkeypatch.setattr(_DRIVER, "SYSTEM_ADMIN_PATHS", (str(admin_path),)) + + _configure_user_tool_path(config) + + assert _DRIVER.shutil.which("losetup") == str(losetup) + + +def test_default_state_bootstrap_creates_missing_tmp_parent(tmp_path, monkeypatch): + """A clean checkout need not contain the ignored default tmp directory.""" + checkout = tmp_path / "checkout" + checkout.mkdir() + state_dir = checkout / "tmp" / "integration-state" + monkeypatch.setattr(_DRIVER, "DEFAULT_STATE_DIR", state_dir) + config = _config(state_dir, tmp_path / "export") + + _validate_lifecycle_paths(config) + _bootstrap_state_dir(config) + + assert state_dir.is_dir() + assert (state_dir / _DRIVER.STATE_MARKER).is_file() + + +def test_custom_state_still_requires_an_existing_parent(tmp_path): + """Parent creation is limited to the repository's known default path.""" + state_dir = tmp_path / "missing-parent" / "state" + config = _config(state_dir, tmp_path / "export") + + with pytest.raises(_DRIVER.ProvisionError, match="leaf below an existing"): + _validate_lifecycle_paths(config) + + +class _RecordingRunner: + """Record commands and return an empty successful process.""" + + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + """Record one command without executing it.""" + command = [str(item) for item in arguments] + self.commands.append(command) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_nfs_service_ownership_precedes_package_install(tmp_path, monkeypatch): + """Package activation cannot obscure who started the NFS service.""" + events = [] + config = _config(tmp_path / "state", tmp_path / "export") + runner = _RecordingRunner() + monkeypatch.setattr( + _DRIVER, + "_record_nfs_service_state", + lambda *_args: events.append("record"), + ) + monkeypatch.setattr( + _DRIVER, + "_ensure_apt_packages", + lambda *_args: events.append("packages"), + ) + + _prepare_host_dependencies(runner, config, "nfs") + + assert events == ["record", "packages"] + + +def test_missing_kind_is_an_empty_cluster_listing(monkeypatch): + """Repeated teardown does not require a deleted private kind client.""" + + class _UnexpectedRunner: + def run(self, *_args, **_kwargs): + pytest.fail("kind was invoked after its private client was removed") + + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: None) + + assert _kind_clusters(_UnexpectedRunner()) == set() + + +def test_missing_exportfs_is_an_empty_export_listing(monkeypatch): + """Cleanup after an interrupted package install needs no NFS client tool.""" + + class _UnexpectedRunner: + def run(self, *_args, **_kwargs): + pytest.fail("exportfs was invoked when it is unavailable") + + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: None) + + assert _export_paths(_UnexpectedRunner()) == [] + + +def test_sbx_shared_directories_allow_replacement_pod_cleanup(tmp_path, monkeypatch): + """SBX UID remapping cannot make prior pod files sticky and undeletable.""" + repository = tmp_path / "repository" + (repository / "tmp").mkdir(parents=True) + config = replace( + _config(tmp_path / "state", tmp_path / "export"), + sbx_shared_root=repository / "tmp" / "shared", + ) + + class _SbxProbeRunner: + def run(self, arguments, **_kwargs): + token = str(arguments[-1]) + (config.sbx_shared_root / "engine-probe").write_text( + token + "\n", encoding="utf-8" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(_DRIVER, "_repository_root", lambda: repository) + + _prepare_sbx_shared(_SbxProbeRunner(), config) + + for name in ("storage-test", "ssh-home"): + mode = (config.sbx_shared_root / name).stat().st_mode & 0o7777 + assert mode == _DRIVER.SBX_SHARED_DIRECTORY_MODE == 0o777 + + +def test_sbx_diagnostics_never_invoke_privileged_nfs_tools(tmp_path, monkeypatch): + """SBX failure reporting remains entirely within its unprivileged profile.""" + config = _config(tmp_path / "state", tmp_path / "export") + runner = _RecordingRunner() + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: "/bin/tool") + + _collect_diagnostics(runner, config) + + rendered = "\n".join(" ".join(command) for command in runner.commands) + assert "sudo" not in rendered + assert "systemctl" not in rendered + assert "exportfs" not in rendered + + +def test_sbx_missing_packages_fail_without_sudo(): + """The SBX profile reports host prerequisites instead of escalating.""" + runner = _RecordingRunner() + + with pytest.raises(_DRIVER.ProvisionError, match="never invokes sudo"): + _ensure_apt_packages(runner, "sbx-shared") + + assert runner.commands + assert all(command[0] == "dpkg-query" for command in runner.commands) + + def test_existing_state_requires_ownership_marker(tmp_path): """Bootstrap cannot adopt an arbitrary existing directory.""" state_dir = tmp_path / "state" @@ -154,7 +362,7 @@ def test_system_directory_cannot_be_used_as_state(tmp_path): """An existing broad system directory cannot be marked during setup.""" config = _config(Path("/usr"), tmp_path / "export") - with pytest.raises(_DRIVER.ProvisionError, match="unowned setup state"): + with pytest.raises(_DRIVER.ProvisionError, match="not owned by the current user"): _validate_lifecycle_paths(config) @@ -244,6 +452,400 @@ def test_fixture_discovery_ignores_terminating_ready_pod(): assert [pod["metadata"]["name"] for pod in selected] == ["new"] +def test_driver_worker_discovery_ignores_terminating_ready_pod(): + """Slinky worker rollout waits cannot count terminating pods as ready.""" + pods = [ + { + "metadata": {"name": "old", "deletionTimestamp": "now"}, + "spec": {"containers": [{"name": "slurmd"}]}, + }, + { + "metadata": {"name": "new"}, + "spec": {"containers": [{"name": "slurmd"}]}, + }, + ] + + selected = _driver_pods_with_container(pods, "slurmd") + + assert [pod["metadata"]["name"] for pod in selected] == ["new"] + + +def test_slurm_workload_account_reconciliation_is_idempotent(tmp_path, monkeypatch): + """Repeated setup does not add an existing Slurm account or association.""" + + class _AccountingRunner: + def __init__(self): + self.accounts = set() + self.associations = set() + self.users = {} + self.mutations = [] + + def run(self, command, **_kwargs): + arguments = [str(item) for item in command] + index = arguments.index("sacctmgr") + operation = arguments[index + 1 :] + if "show" in operation: + entity = operation[operation.index("show") + 1] + rows = { + "account": ((account,) for account in self.accounts), + "association": iter(self.associations), + "user": ((user, account) for user, account in self.users.items()), + }[entity] + output = "".join("|".join(row) + "|\n" for row in rows) + return SimpleNamespace(stdout=output, returncode=0) + self.mutations.append(operation) + action = operation[1] + if action == "add" and operation[2] == "account": + self.accounts.add(operation[3]) + elif action == "add" and operation[2] == "user": + user = operation[3] + account = operation[4].removeprefix("Account=") + self.associations.add((user, account)) + self.users[user] = operation[5].removeprefix("DefaultAccount=") + elif action == "modify": + user = operation[operation.index("where") + 1].removeprefix("Name=") + self.users[user] = operation[-1].removeprefix("DefaultAccount=") + return SimpleNamespace(stdout="", returncode=0) + + runner = _AccountingRunner() + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setattr(_DRIVER, "_login_pod", lambda *_args: "login") + + _ensure_slurm_workload_account(runner, config) + first_mutations = list(runner.mutations) + _ensure_slurm_workload_account(runner, config) + + assert len(first_mutations) == 2 + assert runner.mutations == first_mutations + + +def _ready_pod(name, container): + """Return one minimal ready, nonterminating fixture pod.""" + return { + "metadata": {"name": name}, + "spec": {"containers": [{"name": container}]}, + "status": {"phase": "Running", "containerStatuses": [{"ready": True}]}, + } + + +def test_ssh_rollout_waits_for_statefulset_revision(monkeypatch, tmp_path): + """Pod readiness is checked only after Kubernetes finishes its rollout.""" + runner = _RecordingRunner() + config = _config(tmp_path / "state", tmp_path / "export") + pods = [_ready_pod("ssh-worker-0", "sshd"), _ready_pod("ssh-worker-1", "sshd")] + monkeypatch.setattr(_DRIVER, "_ssh_pods", lambda *_args: pods) + + _wait_for_ssh(runner, config) + + assert runner.commands[0][-4:] == [ + "rollout", + "status", + "statefulset/ssh-worker", + "--timeout=180s", + ] + + +def test_ssh_pool_inspection_rejects_unobserved_revision(tmp_path): + """Ready old pods cannot satisfy a newly annotated StatefulSet form.""" + config = _config(tmp_path / "state", tmp_path / "export") + pods = [_ready_pod("ssh-worker-0", "sshd"), _ready_pod("ssh-worker-1", "sshd")] + statefulset = { + "metadata": { + "generation": 2, + "annotations": { + _DRIVER.SSH_HOME_ANNOTATION: "separate", + _DRIVER.SSH_CONFIG_ANNOTATION: "new-checksum", + }, + }, + "status": { + "observedGeneration": 1, + "currentRevision": "old-revision", + "updateRevision": "new-revision", + "updatedReplicas": 0, + "readyReplicas": 2, + }, + } + + class _PoolRunner: + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + document = ( + statefulset if "statefulset/ssh-worker" in command else {"items": pods} + ) + return SimpleNamespace( + returncode=0, + stdout=json.dumps(document), + stderr="", + ) + + observed = _inspect_ssh_home_pool(_PoolRunner(), config) + + assert not observed.matches("separate", "new-checksum") + assert observed.ready_nonterminating_pods == 0 + assert "old-revision" in observed.diagnostics + + +def test_ssh_storage_visibility_retries_unique_probes(tmp_path, monkeypatch): + """Transient NFS visibility cannot fail a healthy home-mode transition.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + pods = [_ready_pod("ssh-worker-0", "sshd"), _ready_pod("ssh-worker-1", "sshd")] + commands = [] + storage_attempts = 0 + + def pod_exec(_runner, _config, pod, script, *, check=True): + nonlocal storage_attempts + commands.append((pod, script, check)) + returncode = 0 + if script.startswith("test -e /home/tester/"): + returncode = 1 + elif script.startswith('test "$(cat /mnt/storage-test/'): + storage_attempts += 1 + returncode = 1 if storage_attempts == 1 else 0 + return SimpleNamespace(returncode=returncode, stdout="", stderr="") + + monkeypatch.setattr(_DRIVER, "_pod_exec", pod_exec) + monkeypatch.setattr(_DRIVER, "_select_storage_backend", lambda _config: "nfs") + monkeypatch.setattr(_DRIVER.secrets, "token_hex", lambda _length: "unique-token") + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + _validate_ssh_storage(runner=None, config=config, pods=pods, home_mode="separate") + + assert storage_attempts == 2 + assert all("unique-token" in script for _pod, script, _check in commands) + cleanup = [ + script for _pod, script, _check in commands if script.startswith("rm -f") + ] + assert len(cleanup) == 2 + + +def test_slurm_only_fixture_does_not_require_ssh_workers(): + """Slurm diagnosis remains available while the SSH pool is unhealthy.""" + pods = [ + _ready_pod("login", "login"), + _ready_pod("slurmd-1", "slurmd"), + _ready_pod("slurmd-2", "slurmd"), + ] + + login, ssh = _require_pods(pods, {"slurm"}) + + assert login["metadata"]["name"] == "login" + assert ssh == [] + with pytest.raises(_FILESYSTEM.IntegrationTestError, match=r"required=\['ssh'\]"): + _require_pods(pods, {"ssh"}) + + +def test_pod_storage_commands_use_only_the_workload_identity(tmp_path): + """Conspicuous host IDs never leak into pod-side storage operations.""" + + class _CaptureRunner: + def __init__(self): + self.commands = [] + + def run(self, command, **_kwargs): + self.commands.append([str(item) for item in command]) + return SimpleNamespace(stdout="", returncode=0) + + runner = _CaptureRunner() + config = _config(tmp_path / "state", tmp_path / "export") + fixture = SimpleNamespace(login_pod="login", login_container="login") + runtime = SimpleNamespace(selector="slurm") + _prepare_scenario_data( + runner, + config, + fixture, + "/mnt/storage-test/integration-regression/test-data/sentinel", + ) + _reset_result_base( + runner, + config, + fixture, + runtime, + "/mnt/storage-test/integration-regression/results/sentinel", + ) + operations = _remote_staging_operations( + runner, + config, + fixture, + "slurm", + None, + None, + ) + operations.make_directory( + "login", + PurePosixPath("/mnt/storage-test/integration-regression/failure/sentinel"), + ) + + rendered = "\n".join(" ".join(command) for command in runner.commands) + assert "42424" not in rendered + assert "43434" not in rendered + assert "chown" not in rendered + assert "runuser -u tester --" in rendered + assert "umask 0007" in rendered + + +def test_workload_totals_require_resume_metadata(tmp_path, monkeypatch): + """Expected dataset totals cannot silently pass without workload metadata.""" + execution_root = tmp_path / "result" / "executions" + execution_root.mkdir(parents=True) + (execution_root / "0001.sh").write_text("coordinates\n", encoding="utf-8") + (execution_root / "0001.status").write_text("SUCCESS\n", encoding="utf-8") + (execution_root / "0001.exitcode").write_text("0\n", encoding="utf-8") + coordinate = SimpleNamespace(nodes=1, io_size="4K", threads=1, io_depth=1) + expected = SimpleNamespace( + coordinate=coordinate, + status=_FILESYSTEM.ExecutionStatus.SUCCESS, + ) + step = SimpleNamespace(name="bounded", executions=(expected,), required_phases=()) + scenario = SimpleNamespace(name="baseline") + monkeypatch.setattr( + _FILESYSTEM, + "_coordinate_from_execution", + lambda _path: (1, "4K", 1, 1), + ) + + with pytest.raises( + _FILESYSTEM.IntegrationTestError, match="missing required workload metadata" + ): + _assert_execution_contract(scenario, step, tmp_path / "result") + + +def test_slurm_ordering_uses_copied_execution_logs(tmp_path): + """Asynchronous Slurm ordering comes from durable result evidence.""" + result = tmp_path / "result" + executions = result / "executions" + executions.mkdir(parents=True) + (executions / "0001.log").write_text( + "[coordinator] starting execution 1: nodes=1 hosts=10.0.0.1 io_size=4K\n", + encoding="utf-8", + ) + (executions / "0002.log").write_text( + "[coordinator] starting execution 2: " + "nodes=2 hosts=10.0.0.1,10.0.0.2 io_size=4K\n", + encoding="utf-8", + ) + fixture = SimpleNamespace(slurm_addresses=("10.0.0.1", "10.0.0.2")) + + _assert_ordered_workers( + fixture, + "slurm", + "submission output contains no coordinator execution lines", + result, + ) + + for path in executions.iterdir(): + path.unlink() + with pytest.raises(_FILESYSTEM.IntegrationTestError, match=r"was \{\}"): + _assert_ordered_workers(fixture, "slurm", "", result) + + +def test_teardown_validation_allows_unrelated_nfs_exports(tmp_path, monkeypatch): + """Owned fixture cleanup is valid while an unrelated export remains active.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + config.manifests_dir.mkdir() + owner = json.dumps( + {"schema": _DRIVER.STATE_SCHEMA, "cluster_name": config.cluster_name} + ) + (config.state_dir / _DRIVER.STATE_MARKER).write_text(owner, encoding="utf-8") + (config.state_dir / "cluster-owner.json").write_text(owner, encoding="utf-8") + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": False}), encoding="utf-8" + ) + export_config = "owned export\n" + daemon_config = "owned daemon config\n" + (config.manifests_dir / "storage-scale-test.exports").write_text( + export_config, encoding="utf-8" + ) + (config.manifests_dir / "storage-scale-test-nfs.conf").write_text( + daemon_config, encoding="utf-8" + ) + installed = { + config.export_dir / _DRIVER.EXPORT_MARKER: owner, + _DRIVER.NFS_EXPORT_CONFIG: export_config, + _DRIVER.NFS_DAEMON_CONFIG: daemon_config, + } + monkeypatch.setattr( + _DRIVER, "_read_system_file", lambda _runner, path: installed.get(path) + ) + monkeypatch.setattr(_DRIVER, "_export_mount_type", lambda *_args: "") + monkeypatch.setattr(_DRIVER, "_validate_loop_associations", lambda *_args: None) + monkeypatch.setattr(_DRIVER, "_validate_image_ownership", lambda *_args: None) + monkeypatch.setattr( + _DRIVER, "_export_paths", lambda _runner: ["/srv/unrelated-export"] + ) + + ownership = _validate_teardown_ownership(object(), config) + + assert ownership == (True, True, True, True) + + +class _NfsRemovalRunner: + """Record NFS cleanup while reporting one unrelated active export.""" + + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + """Record one cleanup command and return stable export state.""" + command = [str(item) for item in arguments] + self.commands.append(command) + stdout = ( + "/srv/unrelated-export 10.0.0.0/24(options)\n" if "-v" in command else "" + ) + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + +def test_nfs_cleanup_preserves_preexisting_service(tmp_path, monkeypatch): + """Removing owned NFS configuration never disables a pre-existing service.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + (config.manifests_dir / "storage-scale-test.exports").write_text( + f"{config.export_dir} 10.0.0.0/24(options)\n", encoding="utf-8" + ) + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": False}), encoding="utf-8" + ) + runner = _NfsRemovalRunner() + monkeypatch.setattr( + _DRIVER.shutil, + "which", + lambda command: f"/usr/sbin/{command}", + ) + + _remove_nfs_configuration(runner, config) + + assert not any("systemctl" in command for command in runner.commands) + assert any( + any(Path(item).name == "exportfs" for item in command) and "-u" in command + for command in runner.commands + ) + + +def test_nfs_cleanup_survives_missing_exportfs(tmp_path, monkeypatch): + """Interrupted package bootstrap leaves teardown able to remove state.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": True}), encoding="utf-8" + ) + runner = _NfsRemovalRunner() + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _command: None) + + _remove_nfs_configuration(runner, config) + + rendered = [" ".join(command) for command in runner.commands] + assert not any("exportfs" in command for command in rendered) + assert sum("rm --force" in command for command in rendered) == 2 + + def test_markerless_sbx_elbencho_bundle_is_rebuilt(tmp_path, monkeypatch): """A pre-recipe cached wrapper cannot survive a repository update.""" cache = tmp_path / "test-cache" diff --git a/tests/test_integration_failure_injection.py b/tests/test_integration_failure_injection.py index 49020c9..ab2facf 100644 --- a/tests/test_integration_failure_injection.py +++ b/tests/test_integration_failure_injection.py @@ -21,6 +21,7 @@ import subprocess import sys from pathlib import Path +from types import SimpleNamespace import pytest @@ -36,6 +37,7 @@ stage_failure_injection, staged_failure_injection, ) +import filesystem_integration as _INTEGRATION # pylint: disable=wrong-import-position TARGET_ARGUMENT = "/mnt/storage-test/results-e0002" @@ -229,6 +231,52 @@ def test_ssh_staging_places_delegate_runtime_on_every_worker(tmp_path): assert f"readonly delegate={plan.layout.delegate}" in writes[0][3] +def test_ssh_coordinator_adapter_stages_local_delegate_and_runtime(tmp_path): + """The concrete SSH adapter supplies every path referenced by its wrapper.""" + source = tmp_path / "source-elbencho" + _write_delegate(source, 'printf "delegate:%s\\n" "$*"') + runtime = tmp_path / "source-runtime" + runtime.mkdir() + (runtime / "library.so").write_text("runtime\n", encoding="utf-8") + packaged = tmp_path / "workspace" / "utils" / "elbencho" + packaged.parent.mkdir(parents=True) + shutil.copy2(source, packaged) + plan = build_ssh_failure_injection_plan( + scenario_id="failure-resume", + staging_root=tmp_path / "scenario-staging", + source_binary=source, + source_runtime=runtime, + target_argument=TARGET_ARGUMENT, + coordinator_endpoint="local", + worker_endpoints=(), + ) + operations = ( + _INTEGRATION._remote_staging_operations( # pylint: disable=protected-access + object(), + SimpleNamespace(namespace="unused"), + SimpleNamespace(login_container="login"), + "ssh", + packaged, + source, + ) + ) + + stage_failure_injection(plan, operations) + + assert Path(plan.layout.delegate).is_file() + assert (Path(plan.layout.runtime) / "library.so").is_file() + injected = subprocess.run( + [str(packaged), TARGET_ARGUMENT], text=True, capture_output=True, check=False + ) + assert injected.returncode == 97 + assert injected.stdout == f"delegate:{TARGET_ARGUMENT}\n" + + cleanup_failure_injection(plan, operations) + + assert packaged.read_bytes() == source.read_bytes() + assert not Path(plan.layout.root).exists() + + def test_outer_context_retains_artifacts_through_resume_then_cleans(tmp_path): """No per-attempt cleanup removes the marker needed by resume.""" source = tmp_path / "source-elbencho" @@ -300,3 +348,70 @@ def test_cleanup_runs_in_reverse_target_order(tmp_path): "worker-1", "coordinator", ] + + +def test_real_binary_validation_precedes_wrapper_staging(tmp_path, monkeypatch): + """The validator observes Elbencho before the failure wrapper replaces it.""" + events = [] + first = SimpleNamespace(name="inject-one-failure") + resume = SimpleNamespace(name="resume") + runtime = SimpleNamespace( + selector="ssh", + scenario=SimpleNamespace(steps=(first, resume)), + workspace=tmp_path.as_posix(), + ) + + class _StopAfterStaging: + def __enter__(self): + events.append("stage-wrapper") + raise RuntimeError("stop after staging") + + def __exit__(self, *_args): + return False + + monkeypatch.setattr( + _INTEGRATION, + "_failure_plan", + lambda *_args: (object(), object()), + ) + monkeypatch.setattr( + _INTEGRATION, + "_reset_result_base", + lambda *_args: events.append("reset"), + ) + monkeypatch.setattr( + _INTEGRATION, + "_sync_step_runtime", + lambda *_args: events.append("render-real-env"), + ) + monkeypatch.setattr( + _INTEGRATION, + "_validate_step_environment", + lambda *_args: events.append("validate-real-binary"), + ) + monkeypatch.setattr( + _INTEGRATION, + "staged_failure_injection", + lambda *_args: _StopAfterStaging(), + ) + + with pytest.raises(RuntimeError, match="stop after staging"): + _INTEGRATION._run_failure_resume( # pylint: disable=protected-access + object(), + object(), + object(), + runtime, + tmp_path / "env.sh.template", + tmp_path, + tmp_path, + "elbencho", + tmp_path / "elbencho", + None, + ) + + assert events == [ + "reset", + "render-real-env", + "validate-real-binary", + "stage-wrapper", + ] diff --git a/tests/test_validate_env_filesystem_shell.py b/tests/test_validate_env_filesystem_shell.py new file mode 100644 index 0000000..f16ea61 --- /dev/null +++ b/tests/test_validate_env_filesystem_shell.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shell-level regressions for validate_env filesystem identity checks.""" + +import shlex +import shutil +import subprocess +import textwrap +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_VALIDATE_ENV = _REPO_ROOT / "validate_env.sh" +_BASH = shutil.which("bash") or "bash" + + +def _filesystem_functions() -> str: + """Extract the production filesystem-check functions without running main.""" + source = _VALIDATE_ENV.read_text(encoding="utf-8") + start = source.index("check_fs() {") + end = source.index("check_obj() {", start) + return source[start:end] + + +def _write_fake_stat(directory: Path) -> None: + """Install a stat whose statfs value would collide but st_dev does not.""" + command = directory / "stat" + command.write_text( + textwrap.dedent("""\ + #!/bin/sh + if [ "$1" != -c ] || [ "$2" != %d ]; then + printf 'unexpected stat arguments: %s\\n' "$*" >&2 + exit 90 + fi + shift 2 + [ "${1-}" != -- ] || shift + if [ "${SAME_DEVICE-}" = 1 ] || [ "$1" = / ]; then + printf '11\\n' + else + printf '22\\n' + fi + """), + encoding="utf-8", + ) + command.chmod(0o755) + + +def _run_check( + tmp_path: Path, substrate: str, *, same_device: bool +) -> subprocess.CompletedProcess[str]: + """Run both validation paths for one substrate against the fake stat.""" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + _write_fake_stat(fake_bin) + target = tmp_path / "test directory" + target.mkdir() + trace = tmp_path / "trace" + script = _filesystem_functions() + textwrap.dedent(f""" + declare -a captured_errors=() + FAKE_BIN={shlex.quote(str(fake_bin))} + TRACE={shlex.quote(str(trace))} + TARGET={shlex.quote(str(target))} + export FAKE_BIN TRACE + {"export SAME_DEVICE=1" if same_device else "unset SAME_DEVICE"} + + register_error() {{ + captured_errors+=("$1") + }} + run_scriptlet() {{ + local scriptlet="$1" + shift + PATH="$FAKE_BIN:$PATH" bash -s -- "$@" <<< "$scriptlet" + }} + check_sbatch_with() {{ + local scriptlet="$1" + shift + printf 'sbatch\\n' >> "$TRACE" + run_scriptlet "$scriptlet" "$@" + }} + check_srun_with() {{ + local scriptlet="$1" + shift + printf 'srun\\n' >> "$TRACE" + PATH="$FAKE_BIN:$PATH" bash -c "$scriptlet" "$@" + }} + check_ssh_with() {{ + local scriptlet="$1" + shift + printf 'ssh\\n' >> "$TRACE" + if [[ -n "$scriptlet" ]]; then + run_scriptlet "$scriptlet" "$@" + else + PATH="$FAKE_BIN:$PATH" bash -c "$1" + fi + }} + + if [[ {shlex.quote(substrate)} == slurm ]]; then + check_fs 0 "$TARGET" + [[ $(grep -c '^sbatch$' "$TRACE") -eq 1 ]] + [[ $(grep -c '^srun$' "$TRACE") -eq 1 ]] + else + SSH_ENABLED=1 + check_fs 1 "$TARGET" + [[ $(grep -c '^ssh$' "$TRACE") -eq 2 ]] + fi + printf '%s\\n' "${{captured_errors[@]}}" + """) + return subprocess.run( + [_BASH, "-c", script], + check=False, + cwd=_REPO_ROOT, + text=True, + capture_output=True, + ) + + +@pytest.mark.parametrize("substrate", ("ssh", "slurm")) +def test_distinct_device_passes_every_remote_validation_path(tmp_path, substrate): + """SSH, sbatch, and srun compare st_dev rather than statfs free inodes.""" + result = _run_check(tmp_path, substrate, same_device=False) + + assert result.returncode == 0, result.stderr + assert result.stdout == "\n" + + +@pytest.mark.parametrize("substrate", ("ssh", "slurm")) +def test_root_device_is_rejected_by_every_substrate(tmp_path, substrate): + """A test directory sharing root's device remains a validation error.""" + result = _run_check(tmp_path, substrate, same_device=True) + + assert result.returncode == 0, result.stderr + assert "not on a filesystem distinct from /" in result.stdout diff --git a/utils/extract-elbencho.py b/utils/extract-elbencho.py index d2c780f..88c0192 100644 --- a/utils/extract-elbencho.py +++ b/utils/extract-elbencho.py @@ -4626,6 +4626,7 @@ def main() -> None: ) had_aggregate_metrics = bool(metrics) + had_live_files = bool(live_files) # Verify that all metrics have a datestamp if metrics: @@ -4688,8 +4689,11 @@ def main() -> None: ] eprint(f"After filtering, {len(live_files)} live CSV files remain") - if not metrics and not live_files: - eprint("ERROR: Filters matched no report metrics") + if had_aggregate_metrics and not metrics: + eprint("ERROR: Filters matched no aggregate metrics") + sys.exit(1) + if had_live_files and not live_files: + eprint("ERROR: Filters matched no live CSV files") sys.exit(1) # Write to CSV if requested @@ -4711,8 +4715,7 @@ def main() -> None: elif args.from_csv: output_dir = os.path.dirname(os.path.abspath(args.from_csv)) - # Preserve the pre-live behavior when filters remove all aggregate metrics. - if had_aggregate_metrics: + if metrics: if args.markdown: print_markdown_table(metrics, args.no_dual_y_axis) else: @@ -4747,7 +4750,7 @@ def main() -> None: except Exception as exc: # pylint: disable=broad-exception-caught eprint(f"Error reporting live CSV {metadata.path}: {exc}") traceback.print_exc() - if live_files and not successful_live_reports and not had_aggregate_metrics: + if live_files and not successful_live_reports: eprint("ERROR: No live CSV reports were generated") sys.exit(1) diff --git a/validate_env.sh b/validate_env.sh index 69653c7..9a20661 100755 --- a/validate_env.sh +++ b/validate_env.sh @@ -578,14 +578,17 @@ check_fs() { local slurm_rc="$1" # 0 if we can run slurm local fs_path="$2" # filesystem path to test - # This scriptlet is used to test if the filesystem is a mountpoint. - # It is used for both SLURM and SSH-based testing. + # This scriptlet verifies that the test path is a directory on a different + # filesystem device than /. It does not prove that the path itself is the + # exact mountpoint. It is used for both Slurm and SSH-based testing. local scriptlet scriptlet=$(cat << 'EOF' TEST_DIR="$1" -test_dir_inode=$(stat -f -c %d "$TEST_DIR") -root_inode=$(stat -f -c %d "/") -test -d "$TEST_DIR" && [ "$test_dir_inode" != "$root_inode" ] || (echo "failed_$(hostname -s)" && exit 1) +fs_device_failure() { echo "failed_$(hostname -s)"; exit 1; } +test -d "$TEST_DIR" || fs_device_failure +test_dir_device=$(stat -c %d -- "$TEST_DIR") || fs_device_failure +root_device=$(stat -c %d -- "/") || fs_device_failure +[ "$test_dir_device" != "$root_device" ] || fs_device_failure test -w "$TEST_DIR" || (echo "nowrite_$(hostname -s)" && exit 1) EOF ) @@ -614,23 +617,25 @@ check_fs_ssh() { ssh_rc=$? if [[ "$ssh_output" =~ failed_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" - register_error "ssh: ${fs_path} is not a mountpoint on compute node ${nodename}" + register_error "ssh: ${fs_path} is not on a filesystem distinct from / on compute node ${nodename}" elif [[ "$ssh_output" =~ nowrite_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" register_error "ssh: ${fs_path} is not writable on compute node ${nodename}" elif [[ $ssh_rc != 0 ]]; then - register_error "ssh: $fs_path is not a mountpoint on some node" + register_error "ssh: $fs_path is not on a filesystem distinct from / on some node" register_error "ssh: output: $ssh_output" fi + local fs_path_quoted + printf -v fs_path_quoted '%q' "$fs_path" # shellcheck disable=SC2016 - local fs_check_cmd='test -d '"$fs_path"' && [ $(stat -f -c %d '"$fs_path"') != $(stat -f -c %d /) ] || echo "failed_$(hostname -s)"' + local fs_check_cmd='TEST_DIR='"$fs_path_quoted"'; test -d "$TEST_DIR" && test_dir_device=$(stat -c %d -- "$TEST_DIR") && root_device=$(stat -c %d -- /) && [ "$test_dir_device" != "$root_device" ] || echo "failed_$(hostname -s)"' ssh_output=$(check_ssh_with "" "$fs_check_cmd") - # Check for mountpoint failure regardless of ssh exit status + # Check for filesystem-device failure regardless of ssh exit status if [[ "$ssh_output" =~ failed_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" - register_error "ssh: ${fs_path} is not a mountpoint on compute node ${nodename}" + register_error "ssh: ${fs_path} is not on a filesystem distinct from / on compute node ${nodename}" fi } @@ -641,17 +646,17 @@ check_fs_slurm() { sbatch_output=$(check_sbatch_with "$scriptlet" "$fs_path") sbatch_rc=$? if [[ $sbatch_rc != 0 ]]; then - register_error "sbatch: $fs_path is not a mountpoint on some node" + register_error "sbatch: $fs_path is not on a filesystem distinct from / on some node" register_error "sbatch: output: $sbatch_output" fi # shellcheck disable=SC2016 - srun_output=$(check_srun_with 'TEST_DIR="$1"; h=$(hostname -s); test -d ${TEST_DIR} && [ $(stat -f -c %d "$TEST_DIR") != $(stat -f -c %d /) ] || echo "failed_${h}"' _ "$fs_path") + srun_output=$(check_srun_with 'TEST_DIR="$1"; h=$(hostname -s); test -d "$TEST_DIR" && test_dir_device=$(stat -c %d -- "$TEST_DIR") && root_device=$(stat -c %d -- /) && [ "$test_dir_device" != "$root_device" ] || echo "failed_${h}"' _ "$fs_path") - # Check for mountpoint failure regardless of srun exit status + # Check for filesystem-device failure regardless of srun exit status if [[ "$srun_output" =~ failed_([[:alnum:]_-]+) ]]; then nodename="${BASH_REMATCH[1]}" - register_error "srun: ${fs_path} is not a mountpoint on compute node ${nodename}" + register_error "srun: ${fs_path} is not on a filesystem distinct from / on compute node ${nodename}" fi } From 36b5fb6c89311247e20003b8ea15f744f6efc14c Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sun, 20 Sep 2026 19:17:50 -0700 Subject: [PATCH 08/10] Harden integration reporting and NFS migration cleanup Preserve positive sub-0.1 ms latency histogram bounds instead of clamping them above the observed data. This avoids a collapsed log axis causing tight-layout plot rendering to allocate an enormous canvas and trigger the OOM killer during the SBX single-file scenario. Retry temporary NFS migration unmounts with a bounded delay and verify with findmnt that the mount is gone before TemporaryDirectory cleanup. Fail safely with actionable diagnostics when the mount remains busy or its removal cannot be verified. Add regression coverage for low-latency histogram bounds, transient busy unmounts, and permanently busy mounts. Document both operational invariants in the repository context. Simplify integration deployment packaging Remove integration-only flags from the deployment builder. Use the same zero-option path used by end users and tolerate normal warnings for unavailable object or cross-architecture tools. Keep cache identities tied to the exact tracked working-tree snapshot, seeded Elbencho binary, runtime, architecture, and recipe. Invoke the builder once from an exact disposable copy. Its download and s3test side effects cannot change the manifested input, and the verified archive is reused across scenarios and runs. Honor pending tracked-file deletions in deployment snapshots and license checks. Cover deleted inputs, builder side effects, zero-argument invocation, and cache invalidation with regression tests. Remove superseded integration feasibility and coverage-gap research notes. Condense CONTEXT.md additions while retaining fixture, privilege, lifecycle, cache, coverage, and CI invariants. --- docs/CONTEXT.md | 100 ++-- ...esystem-sweep-integration-coverage-gaps.md | 442 ------------------ .../single-host-integration-feasibility.md | 322 ------------- integration-tests/bin/integration-test.py | 43 +- integration-tests/lib/deployment_cache.py | 21 +- lib/reporting_common.py | 14 +- tests/test_check_license_headers.py | 9 + tests/test_integration_deployment_cache.py | 39 +- tests/test_integration_driver_safety.py | 59 +++ tests/test_reporting_common.py | 6 + utils/build_tarball.sh | 59 +-- utils/check_license_headers.py | 2 +- 12 files changed, 206 insertions(+), 910 deletions(-) delete mode 100644 docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md delete mode 100644 docs/research/single-host-integration-feasibility.md diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 54ab001..a4083ca 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -57,69 +57,36 @@ and are responsible for every binary they place in it. Slurm is the default execution substrate. Setting `SSH_HOST_LIST` selects passwordless SSH instead. Kubernetes execution is not implemented. The separate `integration-tests/` fixture provisions a three-node kind cluster, -a shared RWX storage backend, two passwordless-SSH workers, and a Slinky Slurm -environment on one Linux host. The `nfs` backend uses a loop-backed NFSv4 -export and NFS CSI. The Docker-SBX-specific `sbx-shared` backend mounts a -repository-backed directory into every kind node and uses static RWX volumes; -both backends validate the repository's same shared-storage contract. Backend -selection is explicit or capability-based and persists until teardown. The -SBX shared directories are non-sticky writable fixture paths because SBX can -remap bind-mounted file ownership between replacement pods. -harness's state and export directories are dedicated leaves: an existing path -must carry the exact harness ownership marker before setup changes it or -teardown removes it. Every lifecycle action runs as an ordinary user. Its -kubeconfig, keys, client tools, caches, manifests, logs, and runs are user-owned -under `tmp/integration-state`; the driver invokes `sudo` only for the NFS -profile's narrow host-system operations, while `sbx-shared` never invokes it. -The driver adds standard `sbin` locations to its ordinary-user tool search -path. It records NFS service ownership before package installation can start -the service, and repeated teardown remains valid after its private clients are -removed. Partial bootstrap teardown also tolerates `exportfs` not having been -installed yet. -Bootstrap creates the known default `tmp/` parent in a clean checkout, while a -custom state path must remain a leaf below an existing directory. -A completed setup also locks its namespace and, for NFS, export directory until -teardown. NFS teardown removes only the harness export and configuration; a -server or unrelated exports that predated the fixture remain active. The SBX -compatibility profile pairs kind/Kubernetes -1.34 with kubectl 1.34 and rebuilds cached container-derived Elbencho bundles -when their pinned image or bundle recipe changes. Slurm coordinators restore -configured `ORDER_NODES` include-list order after Slurm canonicalizes an -allocation's node list. -The test action selects execution substrate and named scenario independently. -Its deterministic planner batches the one shared-home SSH scenario behind a -crash-recoverable StatefulSet transition; separate SSH homes are canonical. -Transitions require the StatefulSet rollout to finish before accepting two -ready nonterminating pods. Cross-pod storage checks use unique probe paths, -bounded visibility retries, and unconditional cleanup. -Fixture preflight requires only the workloads selected by the plan, so Slurm -diagnosis remains available while SSH workers are unhealthy. Host operator -UIDs own local state only. LoginSet coordination, Slurm tasks, SSH workers, and -pod-side staging use the fixed `tester` UID/GID 2000 contract. Setup provisions -the matching Slurm account, verifies the coordinator and both real `srun` -tasks, and all-squashed NFS paths retain their server-assigned ownership. -Slurm worker-order assertions use the copied execution logs rather than the -asynchronous submission stream. Failure scenarios validate the real Elbencho -binary before replacing it with their scenario-owned wrapper. -Deployment archives remain products of `utils/build_tarball.sh`, but the -harness caches them by the exact immutable tracked-source snapshot, build -options, architecture, and seeded Elbencho/runtime identity. Each scenario -extracts that artifact into isolated state. The real catalog covers baseline -and default I/O, failure/resume, retained datasets, live capture, Cartesian -sweeps, single-file and weighted-root behavior, shared SSH homes, and Slurm -scheduling. Tests run as the non-root account recorded by setup and validate -real SSH or Slurm dispatch, workload results, cleanup, and reporting. Scenario -storage is isolated and removed after host-side evidence is retained. The -on-demand integration workflow explicitly selects NFS and runs the complete -catalog concurrently on amd64 and arm64; Kubernetes remains fixture -infrastructure and is not a benchmark execution substrate. -Fast shell contract tests cover node-range parsing and Cartesian order, -configuration precedence, SSH host parsing and selection, workload-mode and -path safety, sizing limits, and Slurm argument boundaries without a live -fixture. Elbencho CSV reload resolves postponed dataclass annotations before -coercing types, so cached metrics remain filterable. Its reporting CLI treats -`--from-csv` as exclusive with raw directories and rejects malformed or -no-match filters independently for aggregate metrics and live CSV reports. +a shared RWX backend, two passwordless-SSH workers, and Slinky Slurm. Its `nfs` +backend uses loop-backed NFSv4 and NFS CSI; Docker SBX instead shares a +repository path across kind nodes through static RWX volumes. Both validate the +same in-scope storage contract, and the selected backend persists until +teardown. The SBX profile pairs kind/Kubernetes 1.34 with kubectl 1.34. + +Lifecycle actions run as an ordinary user and keep state below +`tmp/integration-state`; only narrow NFS host operations use `sudo`. Dedicated +state and export leaves require exact ownership markers, and setup locks its +namespace and export path until teardown. Cleanup removes only harness-owned +resources, preserves pre-existing NFS services and exports, tolerates partial +bootstrap, and verifies temporary and final unmounts before deleting paths. + +The test CLI selects substrates and named scenarios independently. Its planner +uses substrate-aware preflight and batches the shared-home SSH scenario behind +a crash-recoverable transition, with separate homes as the canonical state. +All pod-side work uses the fixed `tester` UID/GID 2000 identity, matching the +all-squashed NFS export and Slurm account. + +The harness builds the ordinary deployment archive from an immutable tracked +source snapshot, caches it by source, architecture, and seeded Elbencho/runtime +identity, and extracts isolated scenario workspaces. The real catalog covers +baseline and default I/O, failure/resume, retained datasets, live capture, +Cartesian sweeps, single-file and weighted-root behavior, shared SSH homes, +and Slurm scheduling. Fast tests cover parsing, precedence, workload and path +safety, sizing, scheduler boundaries, failure contracts, and reporting. CI +runs the complete NFS-backed catalog concurrently on amd64 and arm64; SBX is a +supported local backend. Kubernetes remains fixture infrastructure rather than +a benchmark execution substrate. + GitHub Actions runs concurrent compliance, ShellCheck, Black, and Pylint checks alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python 3.14 unit tests run weekly and on manual request. @@ -143,8 +110,7 @@ alongside Python 3.12 unit tests for pull requests and pushes to `main`. Python | `utils/build_tarball.sh` | User-local deployment-tarball builder | | `utils/build/` | Helpers for building Warp and the in-tree s3test program | | `tests/` | Python and shell-behavior regression tests collected by `pytest` | -| `integration-tests/` | Single-host kind, RWX storage, SSH, and Slinky fixture provisioner and manifests | -| `docs/research/` | Feasibility studies and implementation handoffs for future integration work | +| `integration-tests/` | Single-host kind, RWX storage, SSH, and Slinky fixture | The checked-in benchmark entry points are: @@ -476,10 +442,6 @@ into a benchmark environment. It: - includes existing Warp binaries but does not download or automatically build them, warning when an architecture is missing. -By default the helper retains that full behavior. `--arch` can select only the -native elbencho architecture, and `--skip-object-tools` omits Warp checks and -s3test builds when creating a filesystem-only deployment archive. - `utils/build/build_s3test_from_source.sh` tries suitable local compilers, Docker, and Docker Buildx. Failure to produce one architecture warns and permits tarball creation, but object testing on that architecture will be unavailable. diff --git a/docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md b/docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md deleted file mode 100644 index c3a7d75..0000000 --- a/docs/research/elbencho-filesystem-sweep-integration-coverage-gaps.md +++ /dev/null @@ -1,442 +0,0 @@ - - -# Elbencho filesystem sweep integration coverage gaps - -## Scope and method - -This note compares the integration harness as of 2026-09-19 with the complete -filesystem sweep and reporting interfaces implemented by: - -- `env.sh.template` and `lib/env_base.sh`; -- `storage-tests/fs/nv-elbencho-sweep.sh`; -- `lib/env_functions.sh` and `lib/_elbencho_functions.sh`; and -- `utils/extract-elbencho.sh` and `utils/extract-elbencho.py`. - -The focus is functional integration coverage. Unit and shell tests reduce risk, -but tests with a mocked Elbencho or dispatcher are not counted as end-to-end SSH -or Slurm coverage. - -The configuration space is not a simple Cartesian product. Several variables -select a branch and make other values invalid, ignored, or operationally inert. -This note therefore distinguishes: - -- **covered**: the value changes behavior that the integration test executes - and meaningfully asserts; -- **executed but weakly asserted**: the code runs, but weak postconditions allow - a semantically wrong result to pass; -- **set but inert**: the harness sets a value that the selected workload does - not consume; and -- **not covered**: no integration case exercises the behavior. - -## Executive conclusion - -The harness covers deployment packaging, environment validation, -service/coordinator startup, one- and two-node worker selection, result -retrieval, and basic reporting over SSH and Slurm on amd64 and arm64. It does -**not** broadly cover sweep workloads. Every integration sweep is one buffered, -sequential, generated `shared-directory` workload with one 16 MiB file per node, -one thread, one I/O depth, and the default write/read/delete lifecycle. The -default `worker-directories` and direct-I/O path is absent, as are -failure/resume, retained or staged datasets, single-big-file mode, random and -split I/O, live CSV capture, weighted targets, and nearly all reporting options. - -The current report assertion is especially permissive: a successful command -with a `Nodes` header and rows beginning with `1` and `2` passes. It does not -prove that WRITE and READ were both reported, metrics are correct, workload -metadata survived extraction, or expected plots were created. - -## What the integration suite tests now - -The environment generator in -`integration-tests/lib/filesystem_integration.py:577-638` fixes one bounded -configuration. `_run_substrate()` invokes only: - -```text -./storage-tests/fs/nv-elbencho-sweep.sh -b --nodes 1,2 -``` - -The resulting effective matrix is: - -| Axis | Covered value | -|---|---| -| Execution substrate | Passwordless SSH and Slurm | -| CI architecture | amd64 and arm64 | -| Node count | Literal list `1,2` | -| Node selection | `ORDER_NODES=1`; first node, then both nodes | -| Access mode | Buffered I/O (`-b`) | -| I/O pattern | Sequential | -| Lifecycle | Generated mkdir, write, read, distributed delete | -| Layout | `ELBENCHO_FILE_LAYOUT=shared-directory` | -| Targets | One `TEST_DIRS` root with weight 1 | -| Dataset | One 16 MiB file per node | -| Sweep dimensions | One I/O size (`4K`), thread count (`1`), and depth (`1`) | -| Live capture | Disabled | -| Single-big-file mode | Disabled | -| SSH identity | Explicit user `tester` | -| Slurm selection | Two explicit include nodes, empty ignore list | -| Slurm allocation | CPU client, `partition=all`, bare `--exclusive` | -| Reporting | One raw result directory at a time, `--markdown` | - -The harness meaningfully verifies: - -- the deployment tarball and `env.sh.template` can be used from an extracted - deployment; -- `validate_env.sh` succeeds in both environments; -- the sweep dispatches through SSH and through a real Slurm coordinator; -- the one-node cell selects the first configured worker and the two-node cell - selects both workers; -- exactly two execution status files exist and both end in `SUCCESS` with exit - code zero; -- each execution has a log and workload record; -- dataset totals are one file/16777216 bytes and two files/33554432 bytes; -- at least two nonempty aggregate CSV and human-readable output files exist; -- generated target directories are removed after the default lifecycle; and -- the report wrapper exits successfully and emits a node header plus apparent - one- and two-node rows. - -The workflow also checks setup idempotence, stop/start, rejection of root test -execution, teardown idempotence, and both architecture jobs. These checks do not -expand the sweep matrix. CI uses separate SSH homes; shared homes are optional -in the harness but not a CI axis. - -## Configuration precedence and inactive-value semantics - -These relationships are important when selecting future cases. Merely changing -a value does not necessarily exercise it. - -1. A nonempty `SSH_HOST_LIST` selects SSH and disables Slurm. Slurm account, - reservation, partition, runtime, module, include/ignore, exclusivity, and - extra-argument settings cannot affect that run. This selection happens in - `lib/env_base.sh:90-105`. -2. `TEST_DIR` is a compatibility fallback only when `TEST_DIRS` is unset or - empty. A populated `TEST_DIRS` always wins (`lib/env_base.sh:58-67`). -3. `ORDER_NODES=1`, `yes`, or `true` selects deterministic prefixes. Other - values select the normal behavior. For Slurm, deterministic prefixing only - has an ordered list to use when `SLURM_NODE_INCLUDES` is nonempty - (`lib/env_base.sh:81-88,221-230`). -4. `ELBENCHO_FILE_SIZE` overrides size derivation from the write block size and - `ELBENCHO_FILE_SIZE_MULTIPLIER`. The integration harness sets both, so the - multiplier is inert (`lib/_elbencho_functions.sh:737-763`). -5. Generated `shared-directory` work is exact-completion work. Its duration is - reported as not applicable and it does not use `--timelimit` or `--infloop`. - The integration's one-second duration is therefore inert for its measured - phases (`lib/_elbencho_functions.sh:2901-2947`). -6. `-s/--single` is active only for a generated legacy - `worker-directories` workload. It is intentionally inactive for - `shared-directory`, `--read-from`, and single-big-file runs. -7. Multiple generated target paths, whether from multiple `TEST_DIRS` roots or - a weight above one, take the same computed fixed-file-count branch as - `--single`. That is where the `FS_MAX_*` estimates matter - (`lib/_elbencho_functions.sh:3161-3194`). -8. `--read-from` selects a staged-dataset reader before either generated - many-file layout. Current layout, files-per-node, generated file size, and - multiplier do not define the staged dataset. The exact treefile defines its - file and byte totals (`lib/_elbencho_functions.sh:2952-2968,2610-2643`). -9. `ELBENCHO_SINGLE_BIG_FILE=1` takes precedence over staged-directory and - generated-layout dispatch. It requires one root and sequential I/O and is - incompatible with `shared-directory`. With `--read-from `, the file - supplies the extent and `ELBENCHO_SINGLE_BIG_FILE_SIZE` is optional and - unused for that read (`lib/env_functions.sh:3360-3387`). -10. `ELBENCHO_ALL_NODES_ACCESS_ALL_DATA` is valid only in single-big-file mode; - value `1` adds Elbencho `--nosvcshare`. It is invalid elsewhere - (`lib/_elbencho_functions.sh:785-797,1100-1148`). -11. `-r/--rand` makes every applicable phase random. An `r` prefix inside one - `ELBENCHO_SCALE_IO_SIZES` component can independently force only WRITE or - READ random. There is no per-size marker that cancels a global `-r`. -12. Buffered timed reads omit `--infloop`; direct timed reads add `--direct` and - `--infloop` (`lib/_elbencho_functions.sh:3140-3159`). Upstream defines - `--infloop` as restarting completed worker workloads and `--direct` as - avoiding buffering/caching; see the upstream [changelog](https://github.com/breuner/elbencho/blob/master/CHANGELOG.md) - and [generated large-file help](https://github.com/breuner/elbencho/blob/master/docs/usage/help-large.md). -13. `--write-only`, `--write-no-read`, `--read-from`, and `--delete-only` are - mutually exclusive. A positive read-after-write pause is irrelevant to - write-only, write-no-read, and read-from operations. -14. `--resume` is exclusive with every other CLI flag. It restores the original - environment and CLI choices from `env_used.sh`; the caller's current sweep - values are not supposed to redefine the remaining cells. - -## Environment-variable coverage gaps - -### Dispatch, placement, and deployment variables - -| Variable or group | Current coverage | Gap | -|---|---|---| -| `RESULTS_DIR`, `LOGS_DIR` | Nondefault absolute fixture paths work. | Relative, unusual, and unwritable paths are not tested. | -| `SSH_HOST_LIST` | A simple newline-delimited two-IP file selects SSH. | Comma-separated and whitespace-separated entries, comments, blank lines, hostnames, duplicates, an empty file, unavailable hosts, and setting SSH alongside nonempty Slurm values are not integrated. | -| `SSH_USER` | Explicit `tester`. | Default/SSH-config identity and a wrong user are not tested. | -| `SSH_HOMEDIR_SHARED` | The harness can provision either mode, but CI uses separate homes. | No required two-mode matrix proves the different copy/staging behavior and a complete sweep in both modes. | -| `ORDER_NODES` | Only numeric true with Slurm includes and an SSH host list. | Disabled/random SSH selection, true/false aliases, Slurm without includes, changing subsets, and include ordering are absent. | -| `account`, `reservation`, `partition`, `run_time` | Empty account/reservation, `partition=all`, five-minute limit in Slurm only. | Explicit account/reservation, alternate or empty partition, time-limit propagation, and proof that all are ignored in SSH mode are absent. | -| `MODULES` | No custom module is required; defaults are attempted only if present. | Loading a real configured module, missing-module behavior, and SSH-mode irrelevance are not tested. | -| `SLURM_NODE_INCLUDES` | Simple two-line node list. | Unset/empty behavior, compressed hostlists, multiple entries per line, malformed names, ordering disabled, and capacity shorter than the requested node count are absent. | -| `SLURM_NODE_IGNORES` | Present but empty. | Real exclusions, compressed hostlists, overlap/conflict with includes, and resulting capacity checks are absent. | -| `SLURM_EXTRA_ARGS` | Empty array. | Multiple elements, an element containing spaces, duplicate/overriding scheduler options, invalid options, and consistent `sbatch`/`srun` propagation are absent. | -| `SLURM_EXCLUSIVE_USER` | `0`, producing bare `--exclusive`. | `1`/`yes`/`true`, target CPU discovery, `--cpus-per-task`, discovery failure, and execution inside an existing allocation are absent. | -| `SLURM_JOB_NAME_PREFIX` | Empty. | Nonempty naming and scheduler-safe unusual characters are absent. | -| `client_type` | `cpu`. | `gpu`, its four-GPU request, and GPU-only partition behavior are absent. | -| `client_arch` | amd64 and arm64 in separate CI jobs. | Configured/probed mismatch and wrong-binary validation are not integration cases. | - -### Filesystem target and workload variables - -| Variable or group | Current coverage | Gap and consequence | -|---|---|---| -| `TEST_DIRS` | One root, weight 1. | Multiple roots, weights above one, mixed weights, empty/invalid weights, and special-mode rejection are absent. These cases change targets and select computed file counts. | -| Legacy `TEST_DIR` | Not used. | Fallback when `TEST_DIRS` is empty and non-override when it is populated are not integrated. | -| `FS_MAX_AGG_THROUGHPUT` | Set to 1 but inert. | Aggregate-bandwidth limiting of computed file counts is absent. The compatibility fallback from `IOR_FS_MAX_AGG_THROUGHPUT` is also absent. | -| `FS_MAX_NODE_THROUGHPUT_GBPS` | Set to 1 but inert. | Per-node bandwidth limiting, scale with node count, and rounding are absent. | -| `FS_MAX_NODE_IOPS` | Set to 100 but inert. | IOPS-limited computed counts and the choice of IOPS versus bandwidth limit are absent. | -| `ELBENCHO_SCALE_THREAD_LIST` | One value, `1`. | Multiple values, Cartesian ordering, high counts, invalid/zero values, shared-file divisibility, and files-per-worker behavior are absent. | -| `ELBENCHO_SCALE_IO_SIZES` | One sequential `4K`. | Multiple values, `rSIZE`, split `WRITE,READ`, independently random components, malformed values, exact block divisibility, and Cartesian ordering are absent. | -| `ELBENCHO_IODEPTH_LIST` | One value, `1`. | Multiple depths, depth greater than one, invalid/zero values, and interaction with thread count and shared files are absent. | -| `ELBENCHO_SCALE_READ_WRITE_DURATION` | Set to 1 but inert in generated shared-directory mode. | Time-bounded worker-directory write/read, staged reads, single-file reads, validation, and actual timeout behavior are absent. | -| `ELBENCHO_READ_AFTER_WRITE_PAUSE` | Zero. | Positive pause, ordering around the pause, and interruption during the pause are absent. | -| `ELBENCHO_FILE_LAYOUT` | Only `shared-directory`. | The default `worker-directories` implementation, invalid values, and the layout's special interactions with read-from and single-file mode are absent. | -| `ELBENCHO_FILES_PER_NODE` | `1`. | Unset default, nontrivial counts, counts smaller than threads, indivisible counts, large boundary values, and rejection outside shared-directory are absent. | -| `ELBENCHO_FILE_SIZE` | Explicit `16M`. | Unset/derived size, alternate exact sizes, direct/random block divisibility, and invalid values are absent. | -| `ELBENCHO_FILE_SIZE_MULTIPLIER` | Set to 1 but shadowed by explicit file size. | Derived sizing and multiplier effects are entirely absent. | -| `ELBENCHO_LIVE_CSV_EXTENDED` | `0`. | Native live CSV production, retrieval, aggregate/service rows, large-artifact behavior, and report discovery are absent. | -| `ELBENCHO_LIVEINT` | Default value is parsed but live capture is off. | A nondefault interval, the below-250 ms warning, invalid values, and cadence in real output are absent. | -| `ELBENCHO_SINGLE_BIG_FILE` | `0`. | The whole generated and staged single-file branch is absent. | -| `ELBENCHO_SINGLE_BIG_FILE_BASENAME` | Default but inert. | Custom basename, path construction, cleanup, and unsafe/unusual names are absent. | -| `ELBENCHO_SINGLE_BIG_FILE_SIZE` | Empty and inert. | Required generated extent, exact size behavior, and optional/ignored read-from extent are absent. | -| `ELBENCHO_ALL_NODES_ACCESS_ALL_DATA` | `0` and inert outside single-file mode. | Both values and `--nosvcshare` propagation are absent. | - -Existing shell tests cover many validators, exact counters, cleanup traps, -treefile helpers, and single-file helpers with mocked calls. The integration -gap is entrypoint-to-substrate composition with the real Elbencho binary. - -## Sweep command-line coverage gaps - -The parser is defined in `storage-tests/fs/nv-elbencho-sweep.sh:53-205`. Only -`-b` and the literal `--nodes 1,2` form have integration coverage. - -| CLI interface | Semantics | Missing integration coverage | -|---|---|---| -| No `-b` (default DIO) | Adds `--direct` and, for timed reads, `--infloop`. | The default access path, alignment/exactness, direct-I/O failures, and report labeling. | -| `-b`, `--bio` | Adds `--norandalign` and omits read `--infloop`. | Short form is used, but native argv and report label are not asserted; the long alias is not tested. | -| `-r`, `--rand` | Makes applicable WRITE and READ phases random. | Both aliases, phase argv, interaction with per-size `r`, BIO/DIO, and rejection in single-big-file mode. | -| `-s`, `--single` | Forces computed file counts for one generated legacy target. | Active worker-directory behavior, `FS_MAX_*` sizing, cleanup, and documented no-op branches. | -| `--nodes X` | One node count. | Zero, negative, and nonnumeric rejection is absent. | -| `--nodes X-Y` | Inclusive ascending range. | Expansion, execution ordering, allocation size, and bad descending range. | -| `--nodes X-Y+Z` | Stepped range that always includes the stop. | Non-dividing steps, steps larger than the range, zero step, and ordering. | -| Comma node list | Preserves specified order, including a descending list. | Only `1,2`; mixed ranges, descending lists, duplicates, empty elements, and maximum-capacity rejection are absent. | -| `--write-only` | Retains one unique dataset per cell and emits its path. | Retention, emitted path, absence of READ/delete, safe later reuse/deletion, special one-root/weight rule, and both substrates. | -| `--write-no-read` | Writes then deletes without READ. | Missing-READ artifact/report semantics, distributed RMFILES, cleanup evidence, and both substrates. | -| `--read-from ` | Reads an operator dataset via scan or cached treefile. | Real cache miss, atomic publication, cache hit, stale-cache operator contract, mount-root fallback, exact totals, no mutation/deletion, random reads, and both substrates. | -| `--read-from ` with single-file mode | Reads file extent from metadata. | Optional size, no treescan, sequential enforcement, and both substrates. | -| `--delete-only ` | Deletes one strict descendant on one compute node; `--nodes` is irrelevant. | Successful SSH/Slurm deletion, preservation of root/siblings, root and outside-root rejection, symlink/realpath safety, and accepted-but-irrelevant `-b`/`-r`/`-s`/`--nodes`. | -| `--resume ` | Restores saved settings, resets stale RUNNING cells, skips SUCCESS, and retries remaining cells in order. | A real interrupted or failed sweep, mixed statuses, snapshot fidelity, lock ownership, Slurm live-job/accounting checks, SSH host reselection, repeated resume, malformed snapshots, and successful reporting of retry artifacts. | -| `-h`, `--help` | Prints usage without environment work. | Both aliases and stable documented interface. | -| Invalid CLI | Missing values, unknown options, positional arguments, missing `--nodes`, conflicting path modes, or resume plus another flag must fail. | No integration/entrypoint-level contract matrix. Many are cheap tests that do not require a live cluster. | -| Invocation inside Slurm | Direct invocation with `SLURM_JOB_ID` and no SSH is rejected. | Rejection behavior and the SSH-enabled exception. | - -The current two-cell run also does not exercise the full reification product. -Multiple node counts, I/O sizes, threads, and depths should demonstrate stable -execution numbering, unique test suffixes, read-host rotation, and result -association across more than one varying axis. - -## Failure, recovery, and lifecycle gaps - -Successful cells cover only the easiest state transition: -`PENDING -> RUNNING -> SUCCESS`. No real integration scenario covers: - -- a benchmark phase returning nonzero; -- incomplete exact counters despite process exit zero; -- stop-on-first-failure with later cells left pending; -- cleanup after mkdir, WRITE, READ, or RMFILES failure; -- a worker service dying between phases and Slurm service restart; -- an SSH worker failing startup and being pruned from the usable pool; -- signal handling while a shared-directory target is active; -- stale `RUNNING` recovery; -- dispatch-lock contention or stale lock recovery; -- Slurm coordinator disappearance versus a still-live allocation; -- result retrieval failing after remote benchmark success; or -- a successful resume that preserves earlier successes and removes duplicate - or stale artifacts from the retried cell. - -Several of these have focused shell tests in -`tests/test_elbencho_dispatch_shell.py` and -`tests/test_elbencho_shared_directory_signals.py`. Those tests use stubs and -synthetic sentinels. They do not establish the end-to-end contract among the -entrypoint, service processes, remote files, scheduler state, and reporter. - -## Reporting CLI coverage gaps - -The reporting parser is at `utils/extract-elbencho.py:4410-4507`. The only -wrapper-level integration call is `--markdown `. - -| Argument or input mode | Current status | Gap | -|---|---|---| -| Positional `input_dirs` | One directory per invocation. | Multiple directories, duplicate coordinates/datestamps, relative paths, empty/missing/unwritable directories, and first-directory output selection. | -| `--markdown` | Command succeeds; header and node-like rows are checked. | WRITE/READ sections, operation/config labels, metrics, workload metadata, representative command, image references, and exact row cardinality are not asserted. | -| Default terminal mode | Not invoked end to end. | Terminal table content, stdout mirroring to `report.txt`, and consistency with Markdown. | -| `--to-csv` | Not invoked through the CLI. | Creation of `elbencho-metrics.csv`, complete schema, histogram serialization, errors, and round-trip fidelity. | -| `--from-csv FILE` | Not invoked through the CLI. | CSV-only reports/plots, legacy optional columns, malformed required fields, filters, Markdown, and output-directory choice. | -| `--from-csv FILE` plus input dirs | Not covered and semantically unclear. | Help says “instead of” raw parsing, but implementation currently combines CSV metrics with parsed directories, risking duplicate rows. The intended contract needs a test or a validation error. | -| `--only-threads` | Not invoked. | Exact values, comma lists, inclusive ranges, malformed tokens, and empty matches. | -| `--only-nodes` | Not invoked. | Same, including proof that aggregate and live inputs are filtered consistently. | -| `--only-sizes` | Parser helper has unit tests; CLI is not invoked. | Repeated flags, semicolon lists, compound sizes such as `1M,r64K`, malformed input, and exact matching. | -| `--only-iodepths` | Not invoked. | Exact/list/range/error and empty-match behavior. | -| `--test-parse FILE` | Not invoked. | Base, `.csv`, and `.out` paths; missing partner; malformed files; histogram output; and intentional precedence over otherwise supplied report options. | -| `--no-dual-y-axis` | Default false path may generate plots, but no files are asserted. | Expected single-axis versus dual-axis filenames and graph content. | -| `--per-client-plots` | No live input exists. | CLI discovery, second-pass client selection, summary CSV/text, time series, heatmaps, missing clients, counter resets, and failover. | -| `--client-outlier-threshold` | Only the default is parsed; effect is inactive. | Nondefault threshold and zero/negative rejection through the CLI. | -| `--client-min-underperform-segments` | Only the default is parsed; effect is inactive. | Nondefault selection and zero/negative rejection. | -| `--client-max-timeseries-lines` | Only the default is parsed; effect is inactive. | Truncation/selection and zero/negative rejection. | -| `--client-max-heatmap-rows` | Only the default is parsed; effect is inactive. | Row limiting and zero/negative rejection. | - -The report also lacks integration inputs for every sweep branch absent above: -worker directories, direct/random or split I/O, multiple dimensions, -write-only, no-read, staged directory and file reads, single-big-file, -all-nodes-all-data, and resumed attempts. Those modes affect grouping, labels, -duration interpretation, deduplication, and workload metadata. - -Existing unit tests substantially reduce parser risk: - -- `tests/test_extract_elbencho_resume.py` covers workload joins, resumed CSV - deduplication, and last output sections; -- `tests/test_extract_elbencho_terminal_subgroups.py` covers subgroup keys and - some mode-sensitive display rules; -- `tests/test_extract_elbencho_treescan_scan.py` covers treescan artifact - matching; -- `tests/test_extract_elbencho_live_csv.py` covers the live-analysis library; - and -- `tests/test_parse_only_sizes.py` covers the size-filter parser. - -They mostly call Python functions directly. They do not cover `main()` argument -composition, shell-wrapper virtual-environment setup, real result discovery, -or end-to-end output artifacts for those options. - -## False-pass weaknesses in the current integration assertions - -The present assertions are useful smoke checks, but several regressions could -pass: - -1. Two successful statuses and plausible dataset totals do not prove the - intended native Elbencho argv. A regression could drop `--norandalign`, use - the wrong block size/depth/thread count, or omit host rotation. -2. The workload checks validate only total files, total bytes, and overall - completion. They do not assert WRITE, READ, and delete completion states and - counters, cleanup timing, files per worker, or layout/source fields. -3. `find ... | wc -l` requires at least two `.csv` and `.out` files, not the - exact expected set associated with execution IDs 0001 and 0002. -4. `env_used.yaml` and `env_used.sh` need only be nonempty. Snapshot omissions, - wrong CLI flags, wrong arrays, wrong `TEST_DIRS`, or values leaking from the - current `env.sh` would not be detected before a resume is attempted. -5. Ordered-worker checking parses “starting execution” log lines. It does not - independently inspect the actual SSH or `srun` command, and later duplicate - lines for a node count overwrite earlier entries in its dictionary. -6. Cleanup checks only top-level names matching - `elbencho-sweep-target-*`. A wrongly named leaked dataset or a retained - single-file artifact would escape the check. -7. The reporter needs only emit a `Nodes` header and any rows that begin with 1 - and 2. It could omit an operation, report wrong values, duplicate stale - metrics, lose workload metadata, or fail to create plots and still pass. -8. No deliberate bad artifact is injected to verify that result and report - assertions fail rather than accepting partial output. -9. The reporter logs a bad benchmark pair and continues. One valid pair can - therefore hide incomplete parsing. -10. Invalid integer filter tokens are skipped, and an empty aggregate filter - set is treated as no filter. Filtering all aggregate metrics can also exit - successfully with “No metrics to display.” -11. `--from-csv` plus input directories combines both sources despite help text - that says CSV is used “instead,” allowing unnoticed duplicate metrics. -12. Missing optional metadata can reduce report detail without violating the - current node-row assertion. - -## Stack-ranked gaps to close - -The ranking below considers expected frequency, consequence of silent error, -amount of code unique to the branch, existing lower-level coverage, and the -cost of adding a case. It is a value ranking, not a recommendation to create a -full cross product. - -1. **Default `worker-directories` plus direct I/O on SSH and Slurm.** This is - the shipped default and executes a large branch that the integration suite - currently bypasses: timed mkdir/write/read, derived file size, tree scan, - host rotation, and recursive cleanup. A small DIO case should assert exact - argv/metadata and both WRITE and READ report rows. -2. **Strengthen the existing result and report oracle.** Before multiplying - cases, require the current case to prove exact artifact names, snapshot - values, phase counters/states, expected metric count and values, BIO label, - both operations, workload metadata, and plot artifacts. This closes the - largest “tests run but can pass incorrectly” risk at modest runtime cost. -3. **Real failure followed by `--resume` on both substrates.** Long sweeps are - expensive, making partial-run recovery operationally critical. Exercise - stop-on-first-failure, preserved SUCCESS, stale or FAILED retry, lock - behavior, artifact replacement/deduplication, and final reporting. -4. **`--write-only` -> `--read-from` -> `--delete-only` lifecycle.** One chained - scenario can validate retained path publication, a real cache miss and hit, - read-only preservation, safe explicit deletion, and report annotations. - Root/outside-root rejection should be a mandatory negative check because - delete-only is destructive. -5. **`--write-no-read` and failure cleanup.** This verifies absence of READ, - distributed RMFILES evidence, no leaked data, and correct reporting of a - write-only metric without conflating the mode with retained write-only data. -6. **Single-big-file mode at one and two nodes.** Cover cooperative slicing, - `ELBENCHO_ALL_NODES_ACCESS_ALL_DATA=1`, custom basename/size, generated - cleanup, staged file read with inferred extent, and random/shared-layout - rejection. Bugs here can silently change how much of a shared file each - service accesses. -7. **A small multi-dimensional random/split-I/O matrix.** Vary at least two I/O - sizes (including independently random WRITE/READ), two threads, and two I/O - depths in one bounded run. Assert Cartesian numbering, phase argv, host - rotation, result association, report grouping, and filters. -8. **Weighted/multiple roots and active `-s` sizing.** Exercise the default - compatibility model where `FS_MAX_*` selects file counts, including one - bandwidth-limited and one IOPS-limited case. This is materially different - from exact shared-directory completion. -9. **Extended live CSV through reporting.** Generate real live files on SSH and - Slurm, then run aggregate live reporting and `--per-client-plots` with - nondefault limits. Live capture can become very large and is an important - diagnostic path, but its analysis library already has substantial unit - coverage. -10. **Slurm option variants with scheduling consequences.** Test - `SLURM_EXCLUSIVE_USER=1` and CPU derivation first, then an extra argument - containing spaces, include/exclude interaction, and a nonempty job prefix. - GPU GRES should follow when a suitable runner exists. -11. **Reporting persistence and output modes.** Add CLI round-trip coverage for - `--to-csv`/`--from-csv`, default terminal/report.txt output, - `--no-dual-y-axis`, exact plot sets, and multiple result directories. -12. **SSH parsing, selection, and shared homes.** Make separate and shared home - modes explicit CI cases or a sequential subcase, test host-file syntax, and - cover `ORDER_NODES=0`. This matters, but core SSH orchestration already has - a real two-worker happy path. -13. **CLI and validation error matrix.** Help, missing arguments, bad ranges, - conflicting modes, random single-file rejection, and invalid filters are - cheap and should be exhaustive. Most do not need the expensive kind - fixture and belong in entrypoint-level tests rather than the full workflow. -14. **Less common deployment variants.** Explicit account/reservation/module - settings, architecture mismatch, GPU client type, unusual paths, and - scheduler capacity failures are valuable environment-compatibility checks - but are less portable and lower value than the workload and recovery gaps - above. - -## Suggested coverage strategy - -A full Cartesian product would be expensive and redundant: - -1. Run one short real case for each workload/lifecycle branch on both - substrates, plus one multi-dimensional case for reification and grouping. -2. Put parsing, invalid combinations, ranges, and snapshot-schema contracts in - fast tests that stub only final dispatch. -3. Isolate process, service, retrieval, and scheduler failures in a focused - fault-injection suite. -4. Reuse real workload artifacts across reporting modes. Each case should - assert its branch, native command, phases, artifacts, snapshot, cleanup or - retention contract, and semantic report content. diff --git a/docs/research/single-host-integration-feasibility.md b/docs/research/single-host-integration-feasibility.md deleted file mode 100644 index 48ee557..0000000 --- a/docs/research/single-host-integration-feasibility.md +++ /dev/null @@ -1,322 +0,0 @@ - - -# Single-host integration environment feasibility and handoff - -## Outcome - -A functional Kubernetes, passwordless-SSH, shared-storage, and Slurm fixture is -feasible on one Linux server. The implemented harness uses rootful Docker and -kind to create three logical Kubernetes nodes: - -| Node | Label | Fixture roles | -|---|---|---| -| Control plane | `storage-scale-test/login=true` | Kubernetes control plane, negative scheduling control, Slinky LoginSet, MariaDB, and `slurmdbd` | -| Worker 1 | `storage-scale-test/target=true` | SSH worker and one `slurmd` | -| Worker 2 | `storage-scale-test/target=true` | SSH worker and one `slurmd` | - -The control-plane taint is removed so its lack of the target label is the -negative-selection test. A fourth node is unnecessary: the lightweight Slinky -LoginSet is schedulable on the control plane and is not a compute node. - -This fixture validates orchestration and substrate behavior. Logical nodes on -one server share physical resources, so it is not suitable for performance, -scaling, failure-domain, or network-isolation measurements. - -## Implemented lifecycle - -The executable driver is `integration-tests/bin/integration-test.py` and -supports five actions: - -- `setup` installs missing host dependencies, creates or reconciles the - environment, and validates every substrate. -- `start` is an exact synonym for `setup`. -- `stop` is disposable: it deletes only the marker-owned kind cluster and then - stops the NFS service only when the harness started it and no unrelated - exports exist. -- `teardown` performs stop and then removes all marker-owned fixture data and - host configuration while leaving installed packages and client tools. -- `test` requires an already-running setup and runs selected bounded filesystem - regression cases without reconciling the environment. - -Stop preserves installed packages and tools, downloaded charts, Docker image -caches, generated SSH and database credentials, rendered state, and external -NFS data. It intentionally discards Kubernetes objects, kind containers, and -MariaDB's node-local storage. A later start creates and validates a fresh -cluster, so it takes longer than setup against an already-running cluster. -The disposable stop and subsequent fresh start were exercised end to end, as -was a repeated stop with the cluster already absent. Full teardown was also -exercised after both filesystem substrate tests, its absence/disabled-state -postconditions were verified, and a second teardown succeeded with all fixture -state already absent. - -Deleting the cluster avoids relying on resumed kind container addresses. -Kubernetes Service DNS stabilizes application endpoints inside the cluster but -cannot make the node-container underlay stable, and the host cannot normally -route cluster-local DNS names. A fresh cluster is consequently simpler and more -reliable than restoring persisted CNI, kube-proxy, and host-network state. - -Lifecycle operations are serialized by a state-directory lock. The driver -uses a private kubeconfig on every kind, kubectl, and Helm command. A persistent -ownership marker prevents adoption or deletion of an unrelated same-named -cluster or export directory. - -## Pinned software - -The initial harness pins: - -- kind 0.33.0; -- Kubernetes node image and kubectl 1.37.0; -- Helm 3.22.0; -- NFS CSI chart 4.13.4 and its sidecar versions; -- Slinky charts 1.2.0; and -- a digest-pinned MariaDB 11.4 image. - -Client binaries are downloaded with their published checksums. The NFS CSI -chart is extracted from a versioned source archive after verifying both the -source archive and embedded chart SHA-256 values. CSI images use the production -Kubernetes registry with the official staging registry as a bounded fallback. - -The setup path currently targets Linux on x86-64 or ARM64, Python 3.12 or newer, -an accessible rootful Docker daemon, at least two CPUs, 8 GiB total memory, -6 GiB available memory, and 20 GiB free space. These are admission limits for -the fixture, not a benchmark sizing recommendation. - -## Shared storage - -The host runs a narrowly configured NFSv4.1 server with two server threads. -Its export is backed by a persistent 128 MiB sparse ext4 image. This gives the -project's mount validation a filesystem distinct from the host root while -keeping disk consumption bounded and retaining data across disposable stops. -Provisioning discovers the kind Docker network's IPv4 subnet and gateway; it -does not assume a fixed bridge address. The export: - -- is restricted to the discovered kind subnet; -- uses `all_squash` and maps requests to numeric UID/GID 2000; -- is installed through dedicated files in `/etc/exports.d` and - `/etc/nfs.conf.d` without replacing unrelated configuration; and -- opens TCP 2049 only for the kind subnet when UFW is already active. - -Numeric ownership is applied as `+2000:+2000` to avoid accidental name-service -resolution of numeric-looking account names. - -The upstream NFS CSI driver dynamically provisions two independent -`ReadWriteMany` claims: - -- `storage-test-rwx` is mounted at `/mnt/storage-test` by the SSH workers, - Slinky LoginSet, and Slurm workers. -- `ssh-home-rwx` optionally supplies the shared `/home/tester` mode. - -Both claims use deterministic namespace/PVC subdirectories and retain their -external NFS data when the disposable Kubernetes cluster is deleted. Requested -PVC sizes are metadata rather than NFS server quotas. MariaDB uses kind's local -`ReadWriteOnce` provisioner and is deliberately disposable. - -## SSH fixture - -The SSH substrate is a two-replica StatefulSet with required pod anti-affinity -and a target-node selector. Each pod uses host networking and listens on its -kind worker's bridge address, making both workers reachable from the Linux host -without a NodePort or LAN-facing host port. - -The image provides a dedicated `tester` user with UID/GID 2000. Setup generates -one Ed25519 fixture key, stores it in protected state, and creates a Kubernetes -Secret only when absent. An init container copies key material into the home -directory with strict ownership and modes. Password and root authentication are -disabled, while `/root` remains on local overlay storage. - -Two home modes are supported: - -- `separate` mounts a per-pod `emptyDir` at `/home/tester`. -- `shared` mounts the dedicated NFS RWX home claim at `/home/tester` in both - workers. - -The host discovers both live worker addresses, writes a strict known-hosts file -and an `ssh_hosts` list into protected state, and proves public-key access to -both. Validation also proves root rejection, bidirectional worker SSH, distinct -placement, the selected home visibility semantics, shared RWX visibility, and -local `/root` storage. - -## Slurm fixture - -Slinky installs CRDs, the operator, and the Slurm custom resources in that -order. The conservative profile disables cert-manager, monitoring, accelerator -support, container plug-ins, high availability, and external load balancers. It -uses: - -- one LoginSet on the login-labeled control plane; -- one DaemonSet-mode NodeSet selecting the two target workers; -- one partition containing that NodeSet; -- controller persistence disabled for the disposable cluster; and -- one directly managed MariaDB instance plus `slurmdbd` for accounting. - -The `slurmd` containers have a small CPU request but no CPU limit. A limit can -cause effective CPU discovery to reach zero on constrained hosts. Other -components retain modest requests and limits. SSH workers are temporarily -scaled to zero during the heaviest Slinky reconciliation and restored before -setup succeeds. - -Slinky 1.2.0 renders a new self-signed webhook CA during a same-values operator -upgrade when cert-manager is disabled. The driver therefore skips releases that -are already deployed at the pinned chart version. A real operator upgrade -restarts and waits for the webhook before applying Slurm custom resources. - -Slurm validation discovers pods by their actual workload container names rather -than relying on generic labels that the operator does not publish. It verifies: - -- the LoginSet runs on the control plane; -- exactly two `slurmd` pods run on the two target nodes; -- the LoginSet sees the NFS mount; -- a two-node `srun` reaches two distinct Slurm nodes; and -- a two-node `sbatch --wait` is reported by `sacct` as `COMPLETED|0:0`. - -Commands are submitted from the LoginSet, matching the intended login/submit -execution point. - -## Filesystem regression cases - -The `test` action accepts `all`, `filesystem`, `ssh`, and `slurm` selectors. -`all` and `filesystem` run both substrate cases; `ssh` and `slurm` can run -individually or together. Test execution deliberately refuses to run without a -matching saved setup and a healthy live three-node fixture. - -The harness first copies only tracked working-tree files to an isolated staging -tree and invokes `utils/build_tarball.sh` there. It validates the resulting -archive's paths, types, size, required files, and packaged benchmark executable -before extraction. Each case renders `env.sh` from the packaged user-facing -`env.sh.template`, injects only bounded fixture overrides, and executes -`validate_env.sh` before the benchmark. - -The SSH case runs both validation and `nv-elbencho-sweep.sh` on the test host; -the checked-in SSH implementation copies and launches its worker payloads in -the two pods. The Slurm case streams the same deployment archive to the -LoginSet, extracts it in the shared NFS filesystem, and runs both commands from -that extracted tree. The sweep uses one 4 KiB file, one thread, queue depth one, -buffered I/O, and one-node and two-node dimensions. Substrates run sequentially. -This covers deployment packaging, environment validation, sweep reification, -SSH and Slurm dispatch, service startup, write/read/delete phases, result -recording, and cleanup while writing only a few KiB per execution. - -The pinned elbencho release archive is architecture-selected, capped during -download, checksum-verified, and cached in protected state. Its verified native -binary is supplied to the isolated tree as the deployment builder's documented -local cache, so the integration test does not vendor a benchmark binary. A -minimal derived Slinky login image installs the standard `file` package required -by `validate_env.sh`; the test does not replace that prerequisite with a -test-specific implementation. - -Every run retains host-side logs beneath `test-runs/`. Success requires two -successful execution records, zero exit codes, workload manifests, environment -snapshots, nonempty CSV and text output, and no remaining generated benchmark -directory at the NFS root. Performance values are not asserted. - -## Idempotency and failure behavior - -Setup reuses generated credentials, existing claims, cached downloads, and -already-current Slinky releases. It uses declarative Kubernetes apply and Helm -upgrade/install for resources that require reconciliation. Re-running setup on -a healthy cluster repeats validation without rotating credentials. - -If setup finds marker-owned stopped or incomplete kind containers, it deletes -that disposable partial cluster and creates a fresh one. An unowned same-named -cluster or nonempty export directory is a hard error with remediation text. - -Every external command and readiness gate has a timeout. Downloads use bounded -retries and checksum verification. Secret-bearing commands are redacted from -logs. On setup failure, diagnostics include bounded host capacity, filesystem, -Docker, kind, NFS service/export, Kubernetes node/pod, and sorted event output; -Kubernetes Secrets are never dumped. - -Stop is idempotent. Repeated stop calls succeed when the cluster and owned NFS -service are already absent or inactive. It never stops Docker globally and -leaves a pre-existing or shared NFS service running. - -Teardown is also idempotent and is intended for CI workers and other disposable -fixtures. It first performs stop, then removes only marker-owned NFS data and -host configuration, stops and disables the dedicated NFS service, removes an -owned UFW rule, unmounts the verified loop device, and deletes generated state -and locally built image tags. It refuses cleanup if ownership, configuration, -mount-backing, or unrelated-export checks fail. Installed packages, client -tools, and reusable upstream image layers remain available. Image IDs are -recorded before and after local builds so teardown can remove only owned tags -and restore any prior tag target. - -## Provisioning sequence - -An implementation or future refactor should preserve this ordering: - -1. Acquire the lifecycle lock and initialize protected logging. -2. Validate the platform and conservative capacity floor. -3. Install the narrow host package set and verify rootful Docker. -4. Install checksum-verified kind, kubectl, and Helm clients as needed. -5. Validate cluster ownership, replacing only an owned partial cluster. -6. Create and validate the three-node kind topology and labels. -7. Create or mount the bounded sparse ext4 backing image, then configure and - probe the subnet-scoped NFSv4.1 export. -8. Preload pinned CSI images, install NFS CSI, and bind both RWX claims. -9. Build, preload, deploy, and validate the two SSH workers. -10. Scale SSH down, reconcile MariaDB and Slinky, and validate Slurm. -11. Restore and revalidate the SSH workers. -12. Persist a non-secret state summary and report success. - -For disposable stop: - -1. Acquire the lifecycle lock. -2. Discover the exact kind cluster and containers. -3. Require the matching ownership marker before deletion. -4. Delete the named kind cluster and verify its containers are gone. -5. Stop NFS only when it is harness-owned and has no unrelated exports. -6. Preserve all host packages, caches, keys, rendered state, and NFS data. - -For full teardown: - -1. Validate all ownership markers, installed configuration content, the ext4 - loop backing file, and the absence of unrelated NFS exports. -2. Perform the idempotent disposable stop. -3. Unexport the dedicated path and remove the two dedicated host config files. -4. Stop and disable NFS and remove only a UFW rule recorded as harness-created. -5. Unmount the export, detach its verified loop device, and remove the two - locally built fixture image tags. -6. Delete the dedicated export and state directories while retaining installed - packages, client tools, and reusable upstream container layers. - -## Checked-in implementation artifacts - -| Path | Purpose | -|---|---| -| `integration-tests/bin/integration-test.py` | Setup/start/stop/teardown driver, validation, logging, and diagnostics | -| `integration-tests/lib/filesystem_integration.py` | Filesystem test selection, staging, execution, and result assertions | -| `integration-tests/manifests/kind.yaml.tmpl` | Three-node kind topology and neutral role labels | -| `integration-tests/manifests/nfs-csi-values.yaml` | Low-footprint NFS CSI deployment values | -| `integration-tests/manifests/nfs-storage.yaml.tmpl` | StorageClass and two RWX claims | -| `integration-tests/ssh-image.Dockerfile` | Ubuntu/OpenSSH worker image with non-root fixture user | -| `integration-tests/slinky-login-image.Dockerfile` | Slinky login image with the standard `file` prerequisite | -| `integration-tests/manifests/ssh-workers.yaml.tmpl` | Two host-networked SSH workers and selectable home volume | -| `integration-tests/manifests/mariadb-accounting.yaml.tmpl` | Disposable local accounting database | -| `integration-tests/manifests/slinky-operator-values.yaml` | Low-footprint operator and webhook configuration | -| `integration-tests/manifests/slinky-slurm-values.yaml` | LoginSet, NodeSet, shared storage, partition, and accounting | - -The checked-in files are the authoritative associated artifact contents. They -should be changed together with this handoff whenever topology, versions, -resource policy, or lifecycle semantics change. - -## Remaining work - -The implemented first pass covers the basic filesystem sweep through SSH and -Slurm. Future cases can add metadata, write-only/read-from/resume, failure -recovery, shared-home mode, and Kubernetes dispatch coverage while preserving -the same bounded-data and no-performance-assertion policy. diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py index cc4e719..178da0c 100755 --- a/integration-tests/bin/integration-test.py +++ b/integration-tests/bin/integration-test.py @@ -101,6 +101,8 @@ SYSTEM_ADMIN_PATHS = ("/usr/local/sbin", "/usr/sbin", "/sbin") SBX_SHARED_DIRECTORY_MODE = 0o777 SSH_STORAGE_VISIBILITY_TIMEOUT_SECONDS = 30 +TEMPORARY_UNMOUNT_ATTEMPTS = 10 +TEMPORARY_UNMOUNT_RETRY_SECONDS = 1 DEFAULT_STATE_DIR = Path(__file__).resolve().parents[2] / "tmp" / "integration-state" DEFAULT_EXPORT_DIR = Path("/srv/storage-scale-test-integration") DEFAULT_SBX_SHARED_ROOT = ( @@ -1082,6 +1084,45 @@ def _export_mount_type(runner: Runner, config: Config) -> str: return result.stdout.strip() if result.returncode == 0 else "" +def _unmount_temporary_filesystem(runner: Runner, mountpoint: Path) -> None: + """Unmount a temporary filesystem and verify it is no longer mounted.""" + last_result: subprocess.CompletedProcess[str] | None = None + for attempt in range(1, TEMPORARY_UNMOUNT_ATTEMPTS + 1): + last_result = runner.run([*_sudo_prefix(), "umount", mountpoint], check=False) + mounted = runner.run( + [ + "findmnt", + "--noheadings", + "--mountpoint", + mountpoint, + ], + check=False, + ) + if mounted.returncode == 1: + return + if mounted.returncode != 0: + raise ProvisionError( + f"could not verify temporary mount was removed at {mountpoint}: " + f"findmnt exited {mounted.returncode}" + ) + if attempt < TEMPORARY_UNMOUNT_ATTEMPTS: + LOG.warning( + "Temporary mount %s remains active after unmount attempt %d/%d; " + "retrying", + mountpoint, + attempt, + TEMPORARY_UNMOUNT_ATTEMPTS, + ) + time.sleep(TEMPORARY_UNMOUNT_RETRY_SECONDS) + + detail = (last_result.stderr or last_result.stdout).strip() if last_result else "" + suffix = f": {detail}" if detail else "" + raise ProvisionError( + f"temporary mount remains active at {mountpoint} after " + f"{TEMPORARY_UNMOUNT_ATTEMPTS} unmount attempts{suffix}" + ) + + def _ensure_export_filesystem(runner: Runner, config: Config) -> None: """Mount a small persistent filesystem for realistic mount validation.""" _ensure_export_marker(runner, config) @@ -1133,7 +1174,7 @@ def _ensure_export_filesystem(runner: Runner, config: Config) -> None: ] ) finally: - runner.run([*_sudo_prefix(), "umount", migration_mount], check=False) + _unmount_temporary_filesystem(runner, migration_mount) runner.run([*_sudo_prefix(), "exportfs", "-u", config.export_dir], check=False) runner.run( [*_sudo_prefix(), "mount", "-o", "loop", config.nfs_image, config.export_dir] diff --git a/integration-tests/lib/deployment_cache.py b/integration-tests/lib/deployment_cache.py index ea5b8e0..18132a5 100644 --- a/integration-tests/lib/deployment_cache.py +++ b/integration-tests/lib/deployment_cache.py @@ -49,7 +49,6 @@ class DeploymentCacheRequest: binary: Path binary_name: str runtime: Path | None = None - build_options: tuple[str, ...] = ("--skip-object-tools",) recipe: int = DEFAULT_RECIPE build_timeout: int = 600 @@ -164,7 +163,7 @@ def _copy_tracked_snapshot( relative = _safe_relative(name) source = repo_root / relative if not source.exists() and not source.is_symlink(): - raise DeploymentCacheError(f"tracked path is absent: {source}") + continue target = destination / relative _copy_tracked_file(source, target) copied.append(target) @@ -239,7 +238,6 @@ def _identity_document( "schema": CACHE_SCHEMA, "recipe": request.recipe, "architecture": request.architecture, - "build_options": list(request.build_options), "source": source_manifest, "binary": binary_identity, "runtime": runtime_identity, @@ -292,18 +290,15 @@ def _valid_entry(cache_root: Path, key: str, identity: dict[str, object]) -> boo def _run_builder( runner: Any, request: DeploymentCacheRequest, staging: Path, snapshot: Path ) -> tuple[Path, str]: - """Invoke the repository's existing deployment builder from the snapshot.""" - builder = snapshot / "utils" / "build_tarball.sh" + """Invoke the existing builder from an exact disposable snapshot copy.""" + build_parent = staging / "builder" + build_tree = build_parent / "source" + shutil.copytree(snapshot, build_tree, symlinks=True) + builder = build_tree / "utils" / "build_tarball.sh" if not builder.is_file() or builder.is_symlink(): raise DeploymentCacheError(f"deployment builder is absent: {builder}") - arguments: list[str | Path] = [ - builder, - "--arch", - request.architecture, - *request.build_options, - ] - result = runner.run(arguments, cwd=snapshot, timeout=request.build_timeout) - archive = staging / ARCHIVE_NAME + result = runner.run([builder], cwd=build_tree, timeout=request.build_timeout) + archive = build_parent / ARCHIVE_NAME if not archive.is_file() or archive.is_symlink() or archive.stat().st_size <= 0: raise DeploymentCacheError(f"deployment builder did not create {archive}") return archive, result.stdout + result.stderr diff --git a/lib/reporting_common.py b/lib/reporting_common.py index d4c3008..da584c1 100644 --- a/lib/reporting_common.py +++ b/lib/reporting_common.py @@ -216,23 +216,27 @@ def histogram_axis_ranges( for x_values_iter, y_values_iter in series: x_values = list(x_values_iter) y_values = list(y_values_iter) - if x_values: - min_latency = min(min_latency, *x_values) - max_latency = max(max_latency, *x_values) + positive_latencies = [value for value in x_values if value > 0] + if positive_latencies: + min_latency = min(min_latency, *positive_latencies) + max_latency = max(max_latency, *positive_latencies) if y_values: nonzero_counts = [value for value in y_values if value > 0] if nonzero_counts: min_count = min(min_count, *nonzero_counts) max_count = max(max_count, *y_values) - min_latency = 0.1 if min_latency == float("inf") else min_latency + latency_data_found = min_latency != float("inf") + min_latency = 0.1 if not latency_data_found else min_latency max_latency = 1000.0 if max_latency == 0 else max_latency min_count = 1.0 if min_count == float("inf") else min_count max_count = 100.0 if max_count == 0 else max_count padding_factor = 0.1 return { - "min_latency": max(0.1, min_latency * (1 - padding_factor)), + "min_latency": ( + min_latency * (1 - padding_factor) if latency_data_found else min_latency + ), "max_latency": max_latency * (1 + padding_factor), "min_count": max(1.0, min_count * (1 - padding_factor)), "max_count": max_count * (1 + padding_factor), diff --git a/tests/test_check_license_headers.py b/tests/test_check_license_headers.py index 85b4380..3f5bb5b 100644 --- a/tests/test_check_license_headers.py +++ b/tests/test_check_license_headers.py @@ -76,6 +76,15 @@ def test_reports_all_text_failures_and_skips_binary(self) -> None: self.assertEqual([path for path, _error in violations], paths[:2]) + def test_skips_tracked_path_deleted_from_working_tree(self) -> None: + """A pending tracked-file deletion is not a header violation.""" + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + + violations = find_violations([Path("deleted.md")], root) + + self.assertEqual(violations, []) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_integration_deployment_cache.py b/tests/test_integration_deployment_cache.py index 8a6dd09..bc7f5bf 100644 --- a/tests/test_integration_deployment_cache.py +++ b/tests/test_integration_deployment_cache.py @@ -83,13 +83,8 @@ def _repository(tmp_path: Path) -> tuple[Path, Path]: repository / "utils" / "build_tarball.sh", """#!/usr/bin/env bash set -eu -while [[ $# -gt 0 ]]; do - case "$1" in - --arch) shift 2 ;; - --skip-object-tools|--fixture-option) shift ;; - *) exit 64 ;; - esac -done +[[ $# -eq 0 ]] +printf 'generated\n' > utils/generated-tool tar -czf ../storage-scale-test.tar.gz . """, 0o755, @@ -153,9 +148,29 @@ def test_snapshot_uses_current_tracked_content_and_excludes_untracked(tmp_path): payload = archive.extractfile("./payload.txt") assert payload is not None assert payload.read() == b"modified\n" + assert "./utils/generated-tool" in names assert "./untracked.txt" not in names +def test_snapshot_honors_tracked_working_tree_deletions(tmp_path): + """A tracked file deleted locally is absent from the archive and identity.""" + repository, binary = _repository(tmp_path) + runner = _Runner() + request = _request(tmp_path, repository, binary) + first = _CACHE.get_or_build_deployment(runner, request) + (repository / "payload.txt").unlink() + + second = _CACHE.get_or_build_deployment(runner, request) + + assert first.key != second.key + with tarfile.open(second.archive, "r:gz") as archive: + assert "./payload.txt" not in archive.getnames() + paths = [ + entry["path"] for entry in _manifest(second)["identity"]["source"]["entries"] + ] + assert "payload.txt" not in paths + + def test_file_mode_participates_in_cache_identity(tmp_path): """Changing a tracked mode invalidates an otherwise identical snapshot.""" repository, binary = _repository(tmp_path) @@ -176,12 +191,11 @@ def test_file_mode_participates_in_cache_identity(tmp_path): ("field", "value"), ( ("architecture", "aarch64"), - ("build_options", ("--skip-object-tools", "--fixture-option")), ("recipe", 2), ), ) -def test_build_identity_options_invalidate_cache(tmp_path, field, value): - """Architecture, builder options, and recipe are cache-key inputs.""" +def test_build_identity_inputs_invalidate_cache(tmp_path, field, value): + """Architecture and builder recipe are cache-key inputs.""" repository, binary = _repository(tmp_path) runner = _Runner() request = _request(tmp_path, repository, binary) @@ -266,10 +280,12 @@ class SnapshotRunner(_Runner): def __init__(self): super().__init__() self.builder_cwd = None + self.builder_arguments = None def run(self, arguments, *, cwd=None, timeout=None): if str(arguments[0]).endswith("build_tarball.sh"): self.builder_cwd = Path(cwd) + self.builder_arguments = [str(argument) for argument in arguments] assert self.builder_cwd != repository assert self.builder_cwd.name == "source" return super().run(arguments, cwd=cwd, timeout=timeout) @@ -279,6 +295,9 @@ def run(self, arguments, *, cwd=None, timeout=None): _CACHE.get_or_build_deployment(runner, _request(tmp_path, repository, binary)) assert runner.builder_cwd is not None + assert runner.builder_arguments == [ + str(runner.builder_cwd / "utils" / "build_tarball.sh") + ] def test_unsafe_tracked_symlink_is_rejected(tmp_path): diff --git a/tests/test_integration_driver_safety.py b/tests/test_integration_driver_safety.py index 88a42db..a4aef3d 100644 --- a/tests/test_integration_driver_safety.py +++ b/tests/test_integration_driver_safety.py @@ -53,6 +53,7 @@ _remove_nfs_configuration = getattr(_DRIVER, "_remove_nfs_configuration") _select_storage_backend = getattr(_DRIVER, "_select_storage_backend") _storage_backend_document = getattr(_DRIVER, "_storage_backend_document") +_unmount_temporary_filesystem = getattr(_DRIVER, "_unmount_temporary_filesystem") _validate_teardown_ownership = getattr(_DRIVER, "_validate_teardown_ownership") _validate_lifecycle_paths = getattr(_DRIVER, "_validate_lifecycle_paths") _validate_ssh_storage = getattr(_DRIVER, "_validate_ssh_storage") @@ -155,6 +156,64 @@ def test_standard_admin_path_resolves_nfs_tools(tmp_path, monkeypatch): assert _DRIVER.shutil.which("losetup") == str(losetup) +class _TemporaryUnmountRunner: + """Model a temporary mount that disappears after a chosen attempt.""" + + def __init__(self, unmounted_after): + self.unmounted_after = unmounted_after + self.unmount_attempts = 0 + + def run(self, arguments, **_kwargs): + """Return deterministic umount and findmnt outcomes.""" + command = [str(item) for item in arguments] + if "umount" in command: + self.unmount_attempts += 1 + return SimpleNamespace( + returncode=0 if self.unmount_attempts >= self.unmounted_after else 1, + stdout="", + stderr=( + "" + if self.unmount_attempts >= self.unmounted_after + else "target is busy" + ), + ) + assert command[0] == "findmnt" + is_mounted = self.unmount_attempts < self.unmounted_after + return SimpleNamespace( + returncode=0 if is_mounted else 1, + stdout="ext4\n" if is_mounted else "", + stderr="", + ) + + +def test_temporary_unmount_retries_until_mount_disappears(tmp_path, monkeypatch): + """A transient busy mount is retried and verified before cleanup continues.""" + runner = _TemporaryUnmountRunner(unmounted_after=3) + sleeps = [] + monkeypatch.setattr(_DRIVER.time, "sleep", sleeps.append) + + _unmount_temporary_filesystem(runner, tmp_path / "migration-mount") + + assert runner.unmount_attempts == 3 + assert sleeps == [ + _DRIVER.TEMPORARY_UNMOUNT_RETRY_SECONDS, + _DRIVER.TEMPORARY_UNMOUNT_RETRY_SECONDS, + ] + + +def test_temporary_unmount_rejects_persistent_mount(tmp_path, monkeypatch): + """Setup fails safely instead of deleting a still-mounted temporary path.""" + runner = _TemporaryUnmountRunner( + unmounted_after=_DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + 1 + ) + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + with pytest.raises(_DRIVER.ProvisionError, match="mount remains active"): + _unmount_temporary_filesystem(runner, tmp_path / "migration-mount") + + assert runner.unmount_attempts == _DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + + def test_default_state_bootstrap_creates_missing_tmp_parent(tmp_path, monkeypatch): """A clean checkout need not contain the ignored default tmp directory.""" checkout = tmp_path / "checkout" diff --git a/tests/test_reporting_common.py b/tests/test_reporting_common.py index 389c80f..745333f 100644 --- a/tests/test_reporting_common.py +++ b/tests/test_reporting_common.py @@ -92,3 +92,9 @@ def test_scale_filter_and_histogram_ranges(): "min_count": 1.0, "max_count": 110.00000000000001, } + assert histogram_axis_ranges([([0.0012, 0.091], [4080.0, 1.0])]) == { + "min_latency": 0.00108, + "max_latency": 0.10010000000000001, + "min_count": 1.0, + "max_count": 4488.0, + } diff --git a/utils/build_tarball.sh b/utils/build_tarball.sh index 934bc1f..7e6446c 100755 --- a/utils/build_tarball.sh +++ b/utils/build_tarball.sh @@ -288,21 +288,15 @@ check_s3test_binaries() { # Command-line argument processing usage() { cat <&2 - usage >&2 - exit 1 - fi - case "$2" in - x86_64|aarch64|all) - selected_arch="$2" - ;; - *) - echo "Unsupported architecture: $2" >&2 - usage >&2 - exit 1 - ;; - esac - shift 2 - ;; - --skip-object-tools) - skip_object_tools=true - shift - ;; -h|--help) usage exit 0 @@ -348,31 +320,24 @@ echo "Creating deployment tarball..." # Build list of elbencho binaries to download/extract. Format per entry: # arch|output_path|display_name -declare -a binaries_to_download=() -if [[ "${selected_arch}" == x86_64 || "${selected_arch}" == all ]]; then - binaries_to_download+=("x86_64|${UTILS_DIR}/elbencho|elbencho amd64") -fi -if [[ "${selected_arch}" == aarch64 || "${selected_arch}" == all ]]; then - binaries_to_download+=("aarch64|${UTILS_DIR}/elbencho.aarch64|elbencho arm64") -fi +declare -a binaries_to_download=( + "x86_64|${UTILS_DIR}/elbencho|elbencho amd64" + "aarch64|${UTILS_DIR}/elbencho.aarch64|elbencho arm64" +) # Refuse to produce a deployment tarball until the pinned upstream checksums have # been filled in. This prevents placeholder values from degrading into the # historical warn-and-continue download behavior below. validate_elbencho_checksum_configuration || exit 1 -if [[ "${skip_object_tools}" == false ]]; then - # Warp binaries are user-built from OSS source when object storage testing is needed. - # Warn if either architecture is missing, but keep building the tarball. - check_warp_binaries +# Warp binaries are user-built from OSS source when object storage testing is needed. +# Warn if either architecture is missing, but keep building the tarball. +check_warp_binaries - # Build s3test from in-tree source when the binaries are missing or stale. - # If one architecture cannot be built, keep creating the tarball but warn that - # object storage testing will not work for that architecture. - check_s3test_binaries ||: -else - echo "Skipping object-storage tool checks for filesystem-only packaging." -fi +# Build s3test from in-tree source when the binaries are missing or stale. +# If one architecture cannot be built, keep creating the tarball but warn that +# object storage testing will not work for that architecture. +check_s3test_binaries ||: # Download all selected files concurrently. echo "Downloading required files..." diff --git a/utils/check_license_headers.py b/utils/check_license_headers.py index 83c06ba..b3c0d3c 100755 --- a/utils/check_license_headers.py +++ b/utils/check_license_headers.py @@ -113,7 +113,7 @@ def find_violations( violations: list[tuple[Path, str]] = [] for path in paths: absolute_path = repo_root / path - if absolute_path.is_symlink(): + if not absolute_path.exists() or absolute_path.is_symlink(): continue data = absolute_path.read_bytes() if b"\0" in data: From 8ce7ac5fccf44cf92b1b7cd22e624216a39685df Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Sun, 20 Sep 2026 21:29:39 -0700 Subject: [PATCH 09/10] Harden NFS-backed Slurm integration recovery Raise the fixture NFS worker count to eight to prevent concurrent kind clients from starving service startup. Give the initial Slurm Elbencho services one bounded restart, preserve each failed service log, and keep the log in the result tree. Retain partial scenario logs and results before disposable cleanup and upload integration diagnostics before CI teardown. Replace temporary directory cleanup around NFS image migration with verified unmount and leaf removal so a busy mount cannot be recursively traversed. Restore and update the single-host feasibility handoff, refresh the repository context, and add regression coverage for NFS rendering, mount cleanup, service retry wiring, and retained failure diagnostics. --- .github/workflows/integration.yml | 10 ++ docs/CONTEXT.md | 7 +- .../single-host-integration-feasibility.md | 154 ++++++++++++++++++ integration-tests/bin/integration-test.py | 77 +++++---- .../lib/filesystem_integration.py | 73 +++++++++ .../fs/sbatch/_nv-elbencho-coordinator.sh | 46 ++++-- tests/test_elbencho_dispatch_shell.py | 6 +- tests/test_integration_driver_safety.py | 127 +++++++++++++++ 8 files changed, 452 insertions(+), 48 deletions(-) create mode 100644 docs/research/single-host-integration-feasibility.md diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 41c4768..ed01df0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -85,6 +85,16 @@ jobs: - name: Run all integration tests run: | "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs test + - name: Upload integration diagnostics + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: filesystem-integration-${{ matrix.architecture }} + path: | + tmp/integration-state/logs + tmp/integration-state/test-runs + if-no-files-found: warn + retention-days: 7 - name: Tear down the integration environment if: ${{ always() }} run: | diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index a4083ca..438d892 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -61,7 +61,8 @@ a shared RWX backend, two passwordless-SSH workers, and Slinky Slurm. Its `nfs` backend uses loop-backed NFSv4 and NFS CSI; Docker SBX instead shares a repository path across kind nodes through static RWX volumes. Both validate the same in-scope storage contract, and the selected backend persists until -teardown. The SBX profile pairs kind/Kubernetes 1.34 with kubectl 1.34. +teardown. The NFS server uses eight workers to avoid single-host client +starvation. The SBX profile pairs kind/Kubernetes 1.34 with kubectl 1.34. Lifecycle actions run as an ordinary user and keep state below `tmp/integration-state`; only narrow NFS host operations use `sudo`. Dedicated @@ -196,7 +197,9 @@ In Slurm mode, `dispatch_slurm_executions` allocates the largest node count needed by any non-successful cell and submits `storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh`. The coordinator starts elbencho services once across the allocation and processes cells sequentially, -using the first requested number of allocation hosts for each cell. +using the first requested number of allocation hosts for each cell. Initial +service health failure preserves its log and gets one bounded restart attempt; +phase-level checks can also restart unhealthy services. In SSH mode, `dispatch_ssh_executions` starts services once across the usable host pool. It selects a fresh host subset for each cell, runs cells sequentially, diff --git a/docs/research/single-host-integration-feasibility.md b/docs/research/single-host-integration-feasibility.md new file mode 100644 index 0000000..6c5b798 --- /dev/null +++ b/docs/research/single-host-integration-feasibility.md @@ -0,0 +1,154 @@ + + +# Single-host integration environment feasibility and handoff + +## Outcome + +A functional Kubernetes, passwordless-SSH, shared-storage, and Slurm fixture is +feasible on one Linux host. The implemented harness uses rootful Docker and +kind to provide three logical Kubernetes nodes: + +| Node | Label | Fixture roles | +|---|---|---| +| Control plane | `storage-scale-test/login=true` | Kubernetes control plane, Slinky LoginSet, MariaDB, and `slurmdbd` | +| Worker 1 | `storage-scale-test/target=true` | SSH worker and one `slurmd` | +| Worker 2 | `storage-scale-test/target=true` | SSH worker and one `slurmd` | + +This fixture validates orchestration, packaging, reporting, and execution +substrate behavior. Its logical nodes share one physical host, so it is not a +performance, scaling, network-isolation, or failure-domain test. + +## Lifecycle and safety model + +`integration-tests/bin/integration-test.py` supports `setup`, `start`, `stop`, +`teardown`, and `test`. Run it as an ordinary user. User-owned state defaults to +`tmp/integration-state`; the driver uses `sudo` only for narrow NFS, mount, +firewall, package, and service operations. Docker SBX uses no privileged NFS +path. + +Setup and start reconcile the fixture and validate both substrates. Stop +deletes the marker-owned kind cluster while retaining caches and persistent +fixture data. Teardown additionally removes marker-owned state and host NFS +configuration, but preserves pre-existing NFS services and unrelated exports. +Repeated setup, stop/start, teardown, and interrupted partial setup are expected +to be safe. + +Dedicated state and export leaves have ownership markers. The selected storage +backend, namespace, and export path cannot change while retained state exists. +Temporary migration mounts are retried and verified during unmount; an active +mountpoint is retained for recovery rather than traversed by recursive cleanup. + +## Pinned fixture profile + +The normal Linux profile uses kind 0.33 and Kubernetes/kubectl 1.37. The Docker +SBX compatibility profile uses kind 0.30 and Kubernetes/kubectl 1.34, with the +`/dev/null` to `/dev/kmsg` compatibility mapping only when the nested kubelet +requires it. Helm, NFS CSI, Slinky, MariaDB, and their images are version- or +digest-pinned in the driver. + +The Linux admission checks require rootful Docker, at least two CPUs, 8 GiB of +total memory, 6 GiB available memory, and 20 GiB free space. These are fixture +admission limits, not benchmark-sizing recommendations. + +## Shared storage backends + +The `nfs` backend creates a persistent sparse ext4 image and exports it through +a narrow NFSv4.1 configuration. The export is restricted to the discovered +kind subnet, uses `all_squash`, and maps clients to UID/GID 2000. Eight NFS +server workers prevent fixture clients from starving one another while sharing +the host kernel. The NFS CSI driver supplies the two RWX claims used for test +data and an optional shared SSH home. + +The `sbx-shared` backend mounts two repository-backed directories into every +kind node and binds static RWX volumes to them. The harness proves worker-to- +worker, login-to-worker, and agent-to-node visibility rather than relying on +the declared access mode. NFS and CSI provisioning are infrastructure choices, +not repository functionality; both backends exercise the same in-scope shared +filesystem contract. + +## SSH and Slurm fixture + +One two-pod SSH StatefulSet runs on the target workers with host networking and +passwordless access for the fixed `tester` UID/GID 2000 identity. Separate home +directories are canonical. The shared-home scenario atomically records a +transition, replaces only this StatefulSet with the RWX-home configuration, +then restores and revalidates separate-home mode. + +Slinky places one LoginSet on the control plane and one DaemonSet-mode NodeSet +across the target workers. Validation checks login placement, both `slurmd` +nodes, coordinator and scheduled-task identities, two-node `srun`, and completed +`sbatch` accounting. The same UID/GID 2000 workload identity matches the NFS +anonymous mapping, so pod-side commands never use the host operator identity. + +The Slurm filesystem coordinator starts one Elbencho service per allocated +node. Initial health checking is bounded. A failed first probe preserves and +prints the service log, stops the owner step, and makes one bounded restart and +probe attempt. Phase-level health checks retain their existing restart path. + +## Filesystem regression coverage + +The test CLI separates substrate selection from scenario selection: + +```text +integration-test.py test +integration-test.py test --substrate ssh +integration-test.py test --substrate slurm --scenario baseline +integration-test.py test --list-scenarios +``` + +No selectors run every applicable scenario. The real catalog covers baseline +and default direct I/O, failure/resume, retained write/read/delete lifecycles, +live capture, a Cartesian sweep, single-file and weighted-root behavior, shared +SSH homes, and Slurm scheduling. The full catalog runs on both amd64 and arm64 +in CI. Fast tests cover broader parsing, precedence, command construction, +failure contracts, workload arithmetic, path safety, and reporting boundaries. + +The harness builds the ordinary deployment archive from an immutable snapshot +of tracked source inputs. It caches the verified archive by source manifest, +architecture, build options, and seeded Elbencho/runtime identity, then extracts +an isolated workspace per scenario. The Elbencho binary remains an externally +downloaded, checksum-verified test input rather than vendored repository code. + +Each scenario validates `env.sh`, invokes the real sweep through SSH or Slurm, +copies immutable result snapshots to the user-owned run directory, and exercises +the supported reporting wrapper. Assertions enforce required coordinates, +state transitions, phases, dataset totals, and semantic report rows without +freezing timestamps, generated names, complete argument arrays, or optional +future artifacts. + +On failure, the harness copies partial logs and results out of the disposable +scenario workspace before cleanup. CI uploads the retained test-run and driver +logs before teardown, even when the scenario fails. + +## Local and CI validation + +On a capable Linux VM, validate the complete lifecycle with: + +```bash +integration-tests/bin/integration-test.py --storage-backend nfs setup +integration-tests/bin/integration-test.py --storage-backend nfs setup +integration-tests/bin/integration-test.py --storage-backend nfs stop +integration-tests/bin/integration-test.py --storage-backend nfs start +integration-tests/bin/integration-test.py --storage-backend nfs test +integration-tests/bin/integration-test.py --storage-backend nfs teardown +integration-tests/bin/integration-test.py --storage-backend nfs teardown +``` + +Docker SBX uses `--storage-backend sbx-shared` and exercises the same repository +behavior. CI explicitly selects `nfs` and runs the complete matrix concurrently +on `linux-amd64-cpu4` and `linux-arm64-cpu4`. diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py index 178da0c..7117565 100755 --- a/integration-tests/bin/integration-test.py +++ b/integration-tests/bin/integration-test.py @@ -96,6 +96,7 @@ NFS_UID = WORKLOAD_UID NFS_GID = WORKLOAD_GID NFS_IMAGE_BYTES = 128 * 1024 * 1024 +NFS_SERVER_THREADS = 8 GIB = 1024**3 STORAGE_BACKENDS = ("nfs", "sbx-shared") SYSTEM_ADMIN_PATHS = ("/usr/local/sbin", "/usr/sbin", "/sbin") @@ -1123,6 +1124,46 @@ def _unmount_temporary_filesystem(runner: Runner, mountpoint: Path) -> None: ) +def _migrate_export_data_to_image(runner: Runner, config: Config) -> None: + """Create the NFS image and safely migrate existing export contents.""" + LOG.info("Creating sparse %s-byte NFS backing filesystem", NFS_IMAGE_BYTES) + temporary_image = config.nfs_image.with_suffix(".ext4.new") + temporary_image.unlink(missing_ok=True) + runner.run(["truncate", "--size", str(NFS_IMAGE_BYTES), temporary_image]) + runner.run(["/usr/sbin/mkfs.ext4", "-F", "-q", "-m", "0", temporary_image]) + temporary_image.replace(config.nfs_image) + + migration_mount = Path( + tempfile.mkdtemp(prefix="nfs-migration-", dir=config.state_dir) + ) + try: + runner.run( + [ + *_sudo_prefix(), + "mount", + "-o", + "loop", + config.nfs_image, + migration_mount, + ] + ) + runner.run( + [ + *_sudo_prefix(), + "cp", + "-a", + f"{config.export_dir}/.", + f"{migration_mount}/", + ] + ) + finally: + # A failed mount command can still leave a live mount. Always perform + # bounded unmount verification, and deliberately retain the directory + # when verification fails so generic cleanup cannot traverse it. + _unmount_temporary_filesystem(runner, migration_mount) + migration_mount.rmdir() + + def _ensure_export_filesystem(runner: Runner, config: Config) -> None: """Mount a small persistent filesystem for realistic mount validation.""" _ensure_export_marker(runner, config) @@ -1145,36 +1186,7 @@ def _ensure_export_filesystem(runner: Runner, config: Config) -> None: return if not config.nfs_image.exists(): - LOG.info("Creating sparse %s-byte NFS backing filesystem", NFS_IMAGE_BYTES) - temporary_image = config.nfs_image.with_suffix(".ext4.new") - temporary_image.unlink(missing_ok=True) - runner.run(["truncate", "--size", str(NFS_IMAGE_BYTES), temporary_image]) - runner.run(["/usr/sbin/mkfs.ext4", "-F", "-q", "-m", "0", temporary_image]) - temporary_image.replace(config.nfs_image) - with tempfile.TemporaryDirectory(dir=config.state_dir) as directory: - migration_mount = Path(directory) - runner.run( - [ - *_sudo_prefix(), - "mount", - "-o", - "loop", - config.nfs_image, - migration_mount, - ] - ) - try: - runner.run( - [ - *_sudo_prefix(), - "cp", - "-a", - f"{config.export_dir}/.", - f"{migration_mount}/", - ] - ) - finally: - _unmount_temporary_filesystem(runner, migration_mount) + _migrate_export_data_to_image(runner, config) runner.run([*_sudo_prefix(), "exportfs", "-u", config.export_dir], check=False) runner.run( [*_sudo_prefix(), "mount", "-o", "loop", config.nfs_image, config.export_dir] @@ -1195,7 +1207,10 @@ def _configure_nfs(runner: Runner, config: Config, subnet: str, gateway: str) -> export_source = config.manifests_dir / "storage-scale-test.exports" nfs_source = config.manifests_dir / "storage-scale-test-nfs.conf" _write_text(export_source, export_line) - _write_text(nfs_source, "[nfsd]\nvers3 = n\nvers4 = y\nthreads = 2\n") + _write_text( + nfs_source, + f"[nfsd]\nvers3 = n\nvers4 = y\nthreads = {NFS_SERVER_THREADS}\n", + ) runner.run([*_sudo_prefix(), "install", "-d", "/etc/exports.d", "/etc/nfs.conf.d"]) runner.run( [ diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py index cbd5e8f..7f44f14 100644 --- a/integration-tests/lib/filesystem_integration.py +++ b/integration-tests/lib/filesystem_integration.py @@ -25,6 +25,7 @@ import re import shlex import shutil +import subprocess import tarfile import tempfile import time @@ -2231,6 +2232,60 @@ def _cleanup_ssh_remote_results(runner: Any, config: Any) -> None: ) +def _preserve_scenario_failure_diagnostics( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, + log_dir: Path, + error: Exception, +) -> None: + """Copy bounded scenario logs before its disposable workspace is removed.""" + destination = log_dir / "failure-diagnostics" + destination.mkdir(parents=True, exist_ok=True) + (destination / "failure.txt").write_text(f"{error!r}\n", encoding="utf-8") + failures: list[str] = [] + for name in ("logs", "results"): + source = f"{runtime.workspace}/{name}" + target = destination / name + try: + if runtime.selector == "ssh": + local_source = Path(source) + if local_source.exists(): + shutil.copytree(local_source, target, dirs_exist_ok=True) + else: + failures.append(f"missing local diagnostic path: {source}") + continue + result = runner.run( + [ + *_kubectl(config, "-n", config.namespace, "cp"), + "-c", + fixture.login_container, + f"{fixture.login_pod}:{source}", + target, + ], + check=False, + timeout=180, + ) + if result.returncode: + detail = (result.stderr or result.stdout).strip() + failures.append( + f"could not copy {source} (exit {result.returncode}): {detail}" + ) + except (OSError, subprocess.SubprocessError) as diagnostic_error: + failures.append(f"could not copy {source}: {diagnostic_error!r}") + if failures: + (destination / "collection-errors.txt").write_text( + "\n".join(failures) + "\n", encoding="utf-8" + ) + LOG.warning( + "Some %s/%s failure diagnostics could not be retained; see %s", + runtime.scenario.name, + runtime.selector, + destination / "collection-errors.txt", + ) + + def _cleanup_scenario_storage( runner: Any, config: Any, @@ -2830,6 +2885,24 @@ def run_filesystem_tests( report_workspace, scenario_logs, ) + except Exception as error: + try: + _preserve_scenario_failure_diagnostics( + runner, + config, + fixture, + runtime_state, + scenario_logs, + error, + ) + except (OSError, subprocess.SubprocessError) as diagnostic_error: + LOG.warning( + "Could not retain %s/%s failure diagnostics: %r", + scenario, + step.substrate.value, + diagnostic_error, + ) + raise finally: if step.substrate.value == "ssh": _cleanup_ssh_remote_results(runner, config) diff --git a/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh b/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh index 3cdfc2b..b8c1ef3 100755 --- a/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh +++ b/storage-tests/fs/sbatch/_nv-elbencho-coordinator.sh @@ -155,26 +155,44 @@ _elbencho_sweep_running_to_pending "$EXECUTIONS_DIR" || exit 1 # Start elbencho services on every allocation node, in background, ONCE. SRUN_ELBENCHO_PID="" SRUN_ELBENCHO_PID_FILE="" +SRUN_ELBENCHO_LOG="${OUTPUT_DIR}/elbencho-svc-j${SLURM_JOB_ID}-portdefault.log" ACTIVE_EXECUTION_ID="" if [[ "${SLURM_JOB_NUM_NODES:-1}" -gt 1 ]]; then stop_elbencho_services_srun "" || true # initial cleanup of stale processes - SRUN_ELBENCHO_PID=$(start_elbencho_services_srun "") - if [[ ! "$SRUN_ELBENCHO_PID" =~ ^[0-9]+$ ]]; then - echo "Error: unable to start elbencho service owner" >&2 - exit 1 - fi SRUN_ELBENCHO_PID_FILE="${EXECUTIONS_DIR}/.service-srun.pid" - if ! _atomic_write_sentinel "$SRUN_ELBENCHO_PID_FILE" "$SRUN_ELBENCHO_PID"; then - echo "Error: unable to record elbencho service owner PID" >&2 - stop_elbencho_services_srun "$SRUN_ELBENCHO_PID" || true - exit 1 - fi - if ! check_elbencho_services_srun ""; then - echo "Error: elbencho services failed initial health check" >&2 + for attempt in 1 2; do + SRUN_ELBENCHO_PID=$(start_elbencho_services_srun "" "$OUTPUT_DIR") + if [[ ! "$SRUN_ELBENCHO_PID" =~ ^[0-9]+$ ]]; then + echo "Error: unable to start elbencho service owner" >&2 + exit 1 + fi + if ! _atomic_write_sentinel \ + "$SRUN_ELBENCHO_PID_FILE" "$SRUN_ELBENCHO_PID"; then + echo "Error: unable to record elbencho service owner PID" >&2 + stop_elbencho_services_srun "$SRUN_ELBENCHO_PID" || true + exit 1 + fi + if check_elbencho_services_srun ""; then + break + fi + echo "Warning: elbencho services failed initial health check "\ + "(attempt $attempt/2)" >&2 + if [[ -f "$SRUN_ELBENCHO_LOG" ]]; then + echo "Service log from failed attempt $attempt:" >&2 + tail -n 200 -- "$SRUN_ELBENCHO_LOG" >&2 || true + cp -f -- "$SRUN_ELBENCHO_LOG" \ + "${SRUN_ELBENCHO_LOG}.attempt-${attempt}" || true + else + echo "Service log is missing: $SRUN_ELBENCHO_LOG" >&2 + fi stop_elbencho_services_srun "$SRUN_ELBENCHO_PID" || true rm -f -- "$SRUN_ELBENCHO_PID_FILE" - exit 1 - fi + if [[ "$attempt" -eq 2 ]]; then + echo "Error: elbencho services failed after one restart" >&2 + exit 1 + fi + echo "Retrying initial elbencho service startup once" >&2 + done fi # Refresh the coordinator's copy after a phase-level check restarts services diff --git a/tests/test_elbencho_dispatch_shell.py b/tests/test_elbencho_dispatch_shell.py index 5d292e7..e7acc9f 100644 --- a/tests/test_elbencho_dispatch_shell.py +++ b/tests/test_elbencho_dispatch_shell.py @@ -347,11 +347,15 @@ def test_phase_restart_persists_new_service_owner_pid(self) -> None: result = _run_bash(textwrap.dedent(script)) self.assertEqual(result.returncode, 0, result.stderr) - def test_coordinator_keeps_initial_check_without_post_execution_duplicate( + def test_coordinator_retries_initial_services_without_per_execution_probe( self, ) -> None: coordinator = _SLURM_COORDINATOR.read_text(encoding="utf-8") self.assertEqual(coordinator.count("check_elbencho_services_srun"), 1) + self.assertIn("for attempt in 1 2; do", coordinator) + self.assertIn('start_elbencho_services_srun "" "$OUTPUT_DIR"', coordinator) + self.assertIn('"${SRUN_ELBENCHO_LOG}.attempt-${attempt}"', coordinator) + self.assertIn("failed after one restart", coordinator) self.assertNotIn( 'maybe_restart_elbencho_services_slurm "execution-${ID}"', coordinator, diff --git a/tests/test_integration_driver_safety.py b/tests/test_integration_driver_safety.py index a4aef3d..8b87394 100644 --- a/tests/test_integration_driver_safety.py +++ b/tests/test_integration_driver_safety.py @@ -20,6 +20,7 @@ import importlib.util import json import os +import shutil import sys from dataclasses import replace from pathlib import Path, PurePosixPath @@ -39,6 +40,7 @@ _FILESYSTEM = sys.modules["filesystem_integration"] _bootstrap_state_dir = getattr(_DRIVER, "_bootstrap_state_dir") _collect_diagnostics = getattr(_DRIVER, "_collect_diagnostics") +_configure_nfs = getattr(_DRIVER, "_configure_nfs") _configure_user_tool_path = getattr(_DRIVER, "_configure_user_tool_path") _ensure_apt_packages = getattr(_DRIVER, "_ensure_apt_packages") _ensure_export_marker = getattr(_DRIVER, "_ensure_export_marker") @@ -47,6 +49,7 @@ _kind_clusters = getattr(_DRIVER, "_kind_clusters") _login_pod = getattr(_DRIVER, "_login_pod") _inspect_ssh_home_pool = getattr(_DRIVER, "_inspect_ssh_home_pool") +_migrate_export_data_to_image = getattr(_DRIVER, "_migrate_export_data_to_image") _prepare_host_dependencies = getattr(_DRIVER, "_prepare_host_dependencies") _prepare_sbx_shared = getattr(_DRIVER, "_prepare_sbx_shared") _driver_pods_with_container = getattr(_DRIVER, "_pods_with_container") @@ -62,6 +65,9 @@ _assert_ordered_workers = getattr(_FILESYSTEM, "_assert_ordered_workers") _ensure_elbencho = getattr(_FILESYSTEM, "_ensure_elbencho") _prepare_scenario_data = getattr(_FILESYSTEM, "_prepare_scenario_data") +_preserve_scenario_failure_diagnostics = getattr( + _FILESYSTEM, "_preserve_scenario_failure_diagnostics" +) _remote_staging_operations = getattr(_FILESYSTEM, "_remote_staging_operations") _reset_result_base = getattr(_FILESYSTEM, "_reset_result_base") _pods_with_container = getattr(_FILESYSTEM, "_pods_with_container") @@ -186,6 +192,20 @@ def run(self, arguments, **_kwargs): ) +class _MigrationRunner(_TemporaryUnmountRunner): + """Model image creation plus the temporary migration mount.""" + + def run(self, arguments, **kwargs): + """Create the sparse placeholder and model privileged operations.""" + command = [str(item) for item in arguments] + if command[0] == "truncate": + Path(command[-1]).touch() + return SimpleNamespace(returncode=0, stdout="", stderr="") + if "umount" in command or command[0] == "findmnt": + return super().run(arguments, **kwargs) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + def test_temporary_unmount_retries_until_mount_disappears(tmp_path, monkeypatch): """A transient busy mount is retried and verified before cleanup continues.""" runner = _TemporaryUnmountRunner(unmounted_after=3) @@ -214,6 +234,113 @@ def test_temporary_unmount_rejects_persistent_mount(tmp_path, monkeypatch): assert runner.unmount_attempts == _DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS +def test_nfs_migration_retains_verified_active_mountpoint(tmp_path, monkeypatch): + """Failed unmount verification cannot trigger recursive mount cleanup.""" + state_dir = tmp_path / "state" + export_dir = tmp_path / "export" + state_dir.mkdir() + export_dir.mkdir() + mountpoint = state_dir / "known-migration-mount" + + def _make_mountpoint(**_kwargs): + mountpoint.mkdir() + return str(mountpoint) + + runner = _MigrationRunner(unmounted_after=_DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + 1) + monkeypatch.setattr(_DRIVER.tempfile, "mkdtemp", _make_mountpoint) + monkeypatch.setattr(_DRIVER.time, "sleep", lambda _seconds: None) + + with pytest.raises(_DRIVER.ProvisionError, match="mount remains active"): + _migrate_export_data_to_image(runner, _config(state_dir, export_dir)) + + assert mountpoint.is_dir() + assert runner.unmount_attempts == _DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS + + +def test_nfs_migration_removes_mountpoint_only_after_verified_unmount( + tmp_path, monkeypatch +): + """Successful migration removes its now-unmounted temporary leaf.""" + state_dir = tmp_path / "state" + export_dir = tmp_path / "export" + state_dir.mkdir() + export_dir.mkdir() + mountpoint = state_dir / "known-migration-mount" + + def _make_mountpoint(**_kwargs): + mountpoint.mkdir() + return str(mountpoint) + + runner = _MigrationRunner(unmounted_after=1) + monkeypatch.setattr(_DRIVER.tempfile, "mkdtemp", _make_mountpoint) + + _migrate_export_data_to_image(runner, _config(state_dir, export_dir)) + + assert not mountpoint.exists() + assert runner.unmount_attempts == 1 + + +def test_nfs_server_has_capacity_for_concurrent_fixture_clients(tmp_path, monkeypatch): + """Rendered NFS state avoids starving the fixture's concurrent clients.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + runner = _RecordingRunner() + for name in ( + "_record_nfs_service_state", + "_ensure_export_filesystem", + "_ensure_export_marker", + "_ensure_nfs_firewall", + ): + monkeypatch.setattr(_DRIVER, name, lambda *_args: None) + + _configure_nfs(runner, config, "172.18.0.0/16", "172.18.0.1") + + daemon_config = (config.manifests_dir / "storage-scale-test-nfs.conf").read_text( + encoding="utf-8" + ) + assert f"threads = {_DRIVER.NFS_SERVER_THREADS}\n" in daemon_config + assert _DRIVER.NFS_SERVER_THREADS == 8 + + +def test_local_scenario_failure_diagnostics_survive_workspace_cleanup(tmp_path): + """SSH logs and partial results are copied outside disposable workspace.""" + workspace = tmp_path / "workspace" + (workspace / "logs").mkdir(parents=True) + (workspace / "results").mkdir() + (workspace / "logs" / "service.log").write_text("service failed\n") + (workspace / "results" / "coordinator.log").write_text("probe failed\n") + scenario = SimpleNamespace(name="example") + runtime = _FILESYSTEM.ScenarioRuntime( + scenario=scenario, + selector="ssh", + workspace=str(workspace), + local_workspace=tmp_path, + artifact_root=tmp_path / "artifacts", + data_root="/mnt/storage-test/example", + values={}, + copied_results=[], + ) + log_dir = tmp_path / "retained" + log_dir.mkdir() + + _preserve_scenario_failure_diagnostics( + _RecordingRunner(), + SimpleNamespace(), + SimpleNamespace(), + runtime, + log_dir, + RuntimeError("scenario failed"), + ) + shutil.rmtree(workspace) + + diagnostics = log_dir / "failure-diagnostics" + assert (diagnostics / "logs" / "service.log").read_text() == "service failed\n" + assert (diagnostics / "results" / "coordinator.log").read_text() == "probe failed\n" + assert "scenario failed" in (diagnostics / "failure.txt").read_text() + + def test_default_state_bootstrap_creates_missing_tmp_parent(tmp_path, monkeypatch): """A clean checkout need not contain the ignored default tmp directory.""" checkout = tmp_path / "checkout" From 83a170819162e0f0471fa8b8046eda17c76c166f Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Mon, 21 Sep 2026 08:50:59 -0700 Subject: [PATCH 10/10] Fix NFS integration capacity and live workers Grow the sparse NFS backing filesystem to 4 GiB so the advertised RWX claims, staged deployments, results, and bounded workloads fit. Expand retained images and ext4 filesystems online during repeated setup. Record the worker count for a pre-existing NFS service, reconcile the live kernel count to eight without restarting it, and restore the prior count during teardown. Add capacity, reconciliation, and cleanup tests and document the lifecycle. Move diagnostic artifact upload to the Node.js 24 action release used by the current GitHub runner environment. Harden integration fixture lifecycle invariants Unify fixture capacity declarations, sparse NFS sizing, deployment bounds, and runtime free-space validation. Grow retained filesystems and publish newly migrated images only after verified copy, unmount, and filesystem checks. Track NFS service existence, active state, boot policy, and live worker count independently so teardown restores only harness-owned changes. Check free space on the configured fixture and Docker filesystems, and replace retained kind clusters whose observed kubelet version no longer matches the selected profile. Treat scenario cleanup failures as test failures without masking an earlier workload error. Derive Slurm allocation limits from each scenario deadline and add focused lifecycle regression coverage. --- .github/workflows/integration.yml | 2 +- docs/CONTEXT.md | 29 +- .../single-host-integration-feasibility.md | 12 +- integration-tests/bin/integration-test.py | 441 ++++++++++++++--- .../lib/filesystem_integration.py | 103 +++- integration-tests/lib/fixture_capacity.py | 37 ++ .../manifests/nfs-storage.yaml.tmpl | 4 +- .../manifests/sbx-storage.yaml.tmpl | 8 +- tests/test_integration_driver_safety.py | 446 +++++++++++++++++- 9 files changed, 960 insertions(+), 122 deletions(-) create mode 100644 integration-tests/lib/fixture_capacity.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ed01df0..416ed66 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -87,7 +87,7 @@ jobs: "$(command -v python)" integration-tests/bin/integration-test.py --storage-backend nfs test - name: Upload integration diagnostics if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: filesystem-integration-${{ matrix.architecture }} path: | diff --git a/docs/CONTEXT.md b/docs/CONTEXT.md index 438d892..8aab34d 100644 --- a/docs/CONTEXT.md +++ b/docs/CONTEXT.md @@ -57,24 +57,27 @@ and are responsible for every binary they place in it. Slurm is the default execution substrate. Setting `SSH_HOST_LIST` selects passwordless SSH instead. Kubernetes execution is not implemented. The separate `integration-tests/` fixture provisions a three-node kind cluster, -a shared RWX backend, two passwordless-SSH workers, and Slinky Slurm. Its `nfs` -backend uses loop-backed NFSv4 and NFS CSI; Docker SBX instead shares a -repository path across kind nodes through static RWX volumes. Both validate the -same in-scope storage contract, and the selected backend persists until -teardown. The NFS server uses eight workers to avoid single-host client -starvation. The SBX profile pairs kind/Kubernetes 1.34 with kubectl 1.34. +a shared RWX backend, two passwordless-SSH workers, and Slinky Slurm. The `nfs` +backend uses loop-backed NFSv4 and NFS CSI; Docker SBX uses static volumes over +a repository-shared path. Both validate the same storage contract. + +One budget drives PVC capacity and the growable 4 GiB sparse NFS image. Setup +publishes new images transactionally, grows retained images, validates mounted +free space, and reconciles eight NFS workers. Teardown restores the service's +recorded active, enabled, and worker-count states. The SBX profile uses +kind/Kubernetes and kubectl 1.34; setup replaces retained clusters with a +different observed kubelet version. Lifecycle actions run as an ordinary user and keep state below `tmp/integration-state`; only narrow NFS host operations use `sudo`. Dedicated -state and export leaves require exact ownership markers, and setup locks its -namespace and export path until teardown. Cleanup removes only harness-owned -resources, preserves pre-existing NFS services and exports, tolerates partial -bootstrap, and verifies temporary and final unmounts before deleting paths. +state and export leaves require ownership markers. Cleanup removes only owned +resources, preserves pre-existing NFS state, tolerates partial bootstrap, and +verifies unmounts before deletion. Capacity checks use the fixture and Docker +backing filesystems. The test CLI selects substrates and named scenarios independently. Its planner -uses substrate-aware preflight and batches the shared-home SSH scenario behind -a crash-recoverable transition, with separate homes as the canonical state. -All pod-side work uses the fixed `tester` UID/GID 2000 identity, matching the +batches shared-home SSH work behind a crash-recoverable transition; separate +homes are canonical. Pod-side work uses `tester` UID/GID 2000, matching the all-squashed NFS export and Slurm account. The harness builds the ordinary deployment archive from an immutable tracked diff --git a/docs/research/single-host-integration-feasibility.md b/docs/research/single-host-integration-feasibility.md index 6c5b798..c88d260 100644 --- a/docs/research/single-host-integration-feasibility.md +++ b/docs/research/single-host-integration-feasibility.md @@ -67,12 +67,14 @@ admission limits, not benchmark-sizing recommendations. ## Shared storage backends -The `nfs` backend creates a persistent sparse ext4 image and exports it through -a narrow NFSv4.1 configuration. The export is restricted to the discovered -kind subnet, uses `all_squash`, and maps clients to UID/GID 2000. Eight NFS +The `nfs` backend creates a persistent 4 GiB sparse ext4 image and exports it +through a narrow NFSv4.1 configuration. Setup grows older retained images and +their filesystems in place. The export is restricted to the discovered kind +subnet, uses `all_squash`, and maps clients to UID/GID 2000. Eight live NFS server workers prevent fixture clients from starving one another while sharing -the host kernel. The NFS CSI driver supplies the two RWX claims used for test -data and an optional shared SSH home. +the host kernel. If the service predates the fixture, teardown restores its +recorded worker count. The NFS CSI driver supplies the two RWX claims used for +test data and an optional shared SSH home. The `sbx-shared` backend mounts two repository-backed directories into every kind node and binds static RWX volumes to them. The harness proves worker-to- diff --git a/integration-tests/bin/integration-test.py b/integration-tests/bin/integration-test.py index 7117565..e4e0079 100755 --- a/integration-tests/bin/integration-test.py +++ b/integration-tests/bin/integration-test.py @@ -47,6 +47,13 @@ IntegrationTestError, run_filesystem_tests, ) +from fixture_capacity import ( # pylint: disable=wrong-import-position + GIB, + NFS_BUDGET_BYTES, + NFS_IMAGE_BYTES, + SSH_HOME_CAPACITY, + STORAGE_TEST_CAPACITY, +) from scenario_planner import ( # pylint: disable=wrong-import-position SUBSTRATES, format_scenario_listing, @@ -95,9 +102,9 @@ WORKLOAD_ACCOUNT = "storage-test" NFS_UID = WORKLOAD_UID NFS_GID = WORKLOAD_GID -NFS_IMAGE_BYTES = 128 * 1024 * 1024 NFS_SERVER_THREADS = 8 -GIB = 1024**3 +NFS_THREADS_PATH = Path("/proc/fs/nfsd/threads") +NFS_SERVICE_STATE_SCHEMA = 2 STORAGE_BACKENDS = ("nfs", "sbx-shared") SYSTEM_ADMIN_PATHS = ("/usr/local/sbin", "/usr/sbin", "/sbin") SBX_SHARED_DIRECTORY_MODE = 0o777 @@ -366,11 +373,34 @@ def _require_python() -> None: raise ProvisionError("integration-test.py requires Python 3.12 or newer") +def _nearest_existing_parent(path: Path) -> Path: + """Return the path itself or its nearest existing parent.""" + candidate = path.resolve(strict=False) + while not candidate.exists() and candidate != candidate.parent: + candidate = candidate.parent + return candidate + + +def _check_disk_capacity(disk_path: Path, label: str) -> None: + """Require free space on the filesystem backing one concrete path.""" + disk = shutil.disk_usage(disk_path) + if disk.free < 20 * GIB: + raise ProvisionError( + f"host capacity check failed: need at least 20 GiB free on " + f"{label} ({disk_path})" + ) + LOG.info( + "Host capacity accepted: %.1f GiB free on %s (%s)", + disk.free / GIB, + label, + disk_path, + ) + + def _check_host_capacity(disk_path: Path) -> None: """Fail before provisioning an undersized host.""" cpu_count = os.cpu_count() or 0 memory = _meminfo() - disk = shutil.disk_usage(disk_path) failures: list[str] = [] if cpu_count < 2: failures.append(f"need at least 2 CPUs; found {cpu_count}") @@ -378,18 +408,14 @@ def _check_host_capacity(disk_path: Path) -> None: failures.append("need at least 8 GiB total memory") if memory.get("MemAvailable", 0) < 6 * GIB: failures.append("need at least 6 GiB available memory") - if disk.free < 20 * GIB: - failures.append(f"need at least 20 GiB free on {disk_path}") if failures: raise ProvisionError("host capacity check failed: " + "; ".join(failures)) LOG.info( - "Host capacity accepted: %s CPUs, %.1f GiB available RAM, %.1f GiB " - "free on %s", + "Host capacity accepted: %s CPUs, %.1f GiB available RAM", cpu_count, memory["MemAvailable"] / GIB, - disk.free / GIB, - disk_path, ) + _check_disk_capacity(_nearest_existing_parent(disk_path), "fixture storage") def _nfs_capability_failures() -> list[str]: @@ -580,6 +606,22 @@ def _ensure_docker(runner: Runner) -> None: LOG.info("Rootful Docker is available") +def _check_docker_capacity(runner: Runner) -> None: + """Check Docker's data filesystem when it is visible to this process.""" + result = runner.run( + ["docker", "info", "--format", "{{.DockerRootDir}}"], timeout=60 + ) + docker_root = Path(result.stdout.strip()) + if not result.stdout.strip() or not docker_root.exists(): + LOG.info( + "Docker data root %s is not visible locally; skipping host-side " + "free-space validation for it", + docker_root if result.stdout.strip() else "", + ) + return + _check_disk_capacity(docker_root, "Docker data root") + + def _command_version(runner: Runner, command: str) -> str: """Return normalized version output, or an empty string if unavailable.""" if not shutil.which(command): @@ -979,6 +1021,46 @@ def _wait_for_cluster(runner: Runner, config: Config) -> None: _validate_node_labels(nodes) +def _expected_kubernetes_version(backend: str) -> str: + """Return the kubelet version selected by one fixture profile.""" + return SBX_KUBECTL_VERSION if backend == "sbx-shared" else KUBECTL_VERSION + + +def _observed_kubernetes_version(runner: Runner, config: Config) -> str: + """Return the one kubelet version observed on every fixture node.""" + nodes = json.loads( + runner.run(_kubectl(config, "get", "nodes", "-o", "json")).stdout + ).get("items", []) + versions = { + node.get("status", {}).get("nodeInfo", {}).get("kubeletVersion") + for node in nodes + } + versions.discard(None) + if len(nodes) != 3 or len(versions) != 1: + raise ProvisionError( + "retained cluster does not have three nodes at one kubelet version: " + f"nodes={len(nodes)}, versions={sorted(versions)}" + ) + return str(versions.pop()) + + +def _retained_cluster_matches_profile( + runner: Runner, config: Config, backend: str +) -> bool: + """Return whether a running retained cluster matches the selected profile.""" + _wait_for_kube_api(runner, config) + observed = _observed_kubernetes_version(runner, config) + expected = _expected_kubernetes_version(backend) + if observed == expected: + return True + LOG.warning( + "Replacing retained kind cluster with kubelet %s; profile requires %s", + observed, + expected, + ) + return False + + def _wait_for_kube_api(runner: Runner, config: Config) -> None: """Wait for the Kubernetes API to become usable.""" deadline = time.monotonic() + 90 @@ -1124,48 +1206,135 @@ def _unmount_temporary_filesystem(runner: Runner, mountpoint: Path) -> None: ) +def _loop_devices_for_file(runner: Runner, path: Path) -> list[str]: + """Return loop devices associated with one exact backing file.""" + if not path.exists(): + return [] + result = runner.run([*_sudo_prefix(), "losetup", "--associated", path], check=False) + return [line.split(":", maxsplit=1)[0] for line in result.stdout.splitlines()] + + def _migrate_export_data_to_image(runner: Runner, config: Config) -> None: - """Create the NFS image and safely migrate existing export contents.""" + """Transactionally create an NFS image containing existing export data.""" LOG.info("Creating sparse %s-byte NFS backing filesystem", NFS_IMAGE_BYTES) temporary_image = config.nfs_image.with_suffix(".ext4.new") + stale_loops = _loop_devices_for_file(runner, temporary_image) + if stale_loops: + raise ProvisionError( + f"temporary NFS image remains attached to {', '.join(stale_loops)}; " + "unmount it before retrying setup" + ) temporary_image.unlink(missing_ok=True) runner.run(["truncate", "--size", str(NFS_IMAGE_BYTES), temporary_image]) runner.run(["/usr/sbin/mkfs.ext4", "-F", "-q", "-m", "0", temporary_image]) - temporary_image.replace(config.nfs_image) migration_mount = Path( tempfile.mkdtemp(prefix="nfs-migration-", dir=config.state_dir) ) try: - runner.run( - [ - *_sudo_prefix(), - "mount", - "-o", - "loop", - config.nfs_image, - migration_mount, - ] + try: + runner.run( + [ + *_sudo_prefix(), + "mount", + "-o", + "loop", + temporary_image, + migration_mount, + ] + ) + runner.run( + [ + *_sudo_prefix(), + "cp", + "-a", + f"{config.export_dir}/.", + f"{migration_mount}/", + ] + ) + finally: + # A failed mount command can still leave a live mount. Always + # verify unmount before the path or temporary image can be removed. + _unmount_temporary_filesystem(runner, migration_mount) + check = runner.run( + [*_sudo_prefix(), "e2fsck", "-pf", temporary_image], + check=False, + timeout=300, ) - runner.run( - [ - *_sudo_prefix(), - "cp", - "-a", - f"{config.export_dir}/.", - f"{migration_mount}/", - ] + if check.returncode not in {0, 1}: + raise ProvisionError( + "temporary NFS filesystem validation failed with exit " + f"{check.returncode}: {(check.stderr or check.stdout).strip()}" + ) + temporary_image.replace(config.nfs_image) + except Exception: + mounted = runner.run( + ["findmnt", "--noheadings", "--mountpoint", migration_mount], + check=False, ) - finally: - # A failed mount command can still leave a live mount. Always perform - # bounded unmount verification, and deliberately retain the directory - # when verification fails so generic cleanup cannot traverse it. - _unmount_temporary_filesystem(runner, migration_mount) + if mounted.returncode == 1 and not _loop_devices_for_file( + runner, temporary_image + ): + temporary_image.unlink(missing_ok=True) + migration_mount.rmdir() + raise migration_mount.rmdir() +def _ensure_nfs_image_capacity( + runner: Runner, config: Config, loop_device: str | None = None +) -> None: + """Grow the sparse NFS image and its ext4 filesystem when required.""" + current_size = config.nfs_image.stat().st_size + if current_size < NFS_IMAGE_BYTES: + LOG.info( + "Growing sparse NFS backing image from %s to %s bytes", + current_size, + NFS_IMAGE_BYTES, + ) + runner.run(["truncate", "--size", str(NFS_IMAGE_BYTES), config.nfs_image]) + if loop_device: + runner.run( + [*_sudo_prefix(), "losetup", "--set-capacity", loop_device], + timeout=60, + ) + runner.run( + [*_sudo_prefix(), "resize2fs", loop_device or config.nfs_image], + timeout=300, + ) + + +def _validate_nfs_filesystem_capacity(runner: Runner, config: Config) -> None: + """Require the mounted export to satisfy the shared fixture budget.""" + result = runner.run( + [ + *_sudo_prefix(), + "stat", + "-f", + "-c", + "%b %S %a", + config.export_dir, + ], + timeout=30, + ) + fields = result.stdout.split() + if len(fields) != 3 or not all(field.isdigit() for field in fields): + raise ProvisionError( + f"could not parse NFS filesystem capacity: {result.stdout!r}" + ) + blocks, fragment_size, available_blocks = map(int, fields) + total_bytes = blocks * fragment_size + available_bytes = available_blocks * fragment_size + if total_bytes < NFS_BUDGET_BYTES or available_bytes < NFS_BUDGET_BYTES: + raise ProvisionError( + "NFS backing filesystem is below the fixture capacity budget: " + f"total={total_bytes}, available={available_bytes}, " + f"required_available={NFS_BUDGET_BYTES}" + ) + + def _ensure_export_filesystem(runner: Runner, config: Config) -> None: - """Mount a small persistent filesystem for realistic mount validation.""" + """Mount a sized persistent filesystem for realistic mount validation.""" _ensure_export_marker(runner, config) mounted_type = _export_mount_type(runner, config) if mounted_type: @@ -1174,24 +1343,20 @@ def _ensure_export_filesystem(runner: Runner, config: Config) -> None: f"refusing non-ext4 mount at dedicated export {config.export_dir}: " f"{mounted_type}" ) - loops = runner.run( - [*_sudo_prefix(), "losetup", "--associated", config.nfs_image], - check=False, - ).stdout - if not loops.strip(): - raise ProvisionError( - f"mounted export {config.export_dir} is not backed by " - f"{config.nfs_image}" - ) + loop_device = _verified_export_loop(runner, config) + _ensure_nfs_image_capacity(runner, config, loop_device) + _validate_nfs_filesystem_capacity(runner, config) return if not config.nfs_image.exists(): _migrate_export_data_to_image(runner, config) + _ensure_nfs_image_capacity(runner, config) runner.run([*_sudo_prefix(), "exportfs", "-u", config.export_dir], check=False) runner.run( [*_sudo_prefix(), "mount", "-o", "loop", config.nfs_image, config.export_dir] ) _ensure_export_marker(runner, config) + _validate_nfs_filesystem_capacity(runner, config) def _configure_nfs(runner: Runner, config: Config, subnet: str, gateway: str) -> None: @@ -1235,22 +1400,96 @@ def _configure_nfs(runner: Runner, config: Config, subnet: str, gateway: str) -> _ensure_nfs_firewall(runner, config, subnet) runner.run([*_sudo_prefix(), "exportfs", "-rav"]) runner.run([*_sudo_prefix(), "systemctl", "enable", "--now", "nfs-server"]) + _set_live_nfs_threads(runner, NFS_SERVER_THREADS) state = {"subnet": subnet, "gateway": gateway} _write_text(config.state_dir / "network.json", json.dumps(state, indent=2) + "\n") -def _record_nfs_service_state(runner: Runner, config: Config) -> None: - """Remember whether this harness was responsible for starting NFS.""" - path = config.state_dir / "nfs-service.json" - if path.exists(): +def _live_nfs_threads(runner: Runner) -> int | None: + """Return the active kernel NFS worker count when it is available.""" + result = runner.run( + [*_sudo_prefix(), "cat", NFS_THREADS_PATH], + check=False, + timeout=30, + ) + value = result.stdout.strip() + if result.returncode or not value.isdigit() or int(value) < 1: + return None + return int(value) + + +def _set_live_nfs_threads(runner: Runner, count: int) -> None: + """Apply and verify a kernel NFS worker count without restarting NFS.""" + current = _live_nfs_threads(runner) + if current == count: return + rpc_nfsd = shutil.which("rpc.nfsd") + if not rpc_nfsd: + raise ProvisionError("rpc.nfsd is unavailable; cannot set NFS worker count") + runner.run([*_sudo_prefix(), rpc_nfsd, str(count)], timeout=60) + actual = _live_nfs_threads(runner) + if actual != count: + raise ProvisionError( + f"NFS worker reconciliation requested {count}, found {actual!r}" + ) + + +def _systemd_unit_exists(runner: Runner, unit: str) -> bool: + """Return whether systemd currently knows about one unit.""" + result = runner.run( + [*_sudo_prefix(), "systemctl", "show", "--property=LoadState", "--value", unit], + check=False, + ) + return result.returncode == 0 and result.stdout.strip() not in {"", "not-found"} + + +def _systemd_unit_state(runner: Runner, unit: str) -> tuple[bool, bool, bool]: + """Return unit existence plus independent active and enabled states.""" + exists = _systemd_unit_exists(runner, unit) + if not exists: + return False, False, False active = ( runner.run( - [*_sudo_prefix(), "systemctl", "is-active", "nfs-server"], check=False + [*_sudo_prefix(), "systemctl", "is-active", unit], check=False + ).returncode + == 0 + ) + enabled = ( + runner.run( + [*_sudo_prefix(), "systemctl", "is-enabled", unit], check=False ).returncode == 0 ) - _write_text(path, json.dumps({"started_by_harness": not active}) + "\n") + return True, active, enabled + + +def _record_nfs_service_state(runner: Runner, config: Config) -> None: + """Remember independent NFS installation, runtime, and boot states.""" + path = config.state_dir / "nfs-service.json" + if path.exists(): + state = json.loads(path.read_text(encoding="utf-8")) + if state.get("schema") != NFS_SERVICE_STATE_SCHEMA: + raise ProvisionError( + "retained NFS service state predates independent active/enabled " + "tracking; run teardown before setup" + ) + return + existed, active, enabled = _systemd_unit_state(runner, "nfs-server") + state: dict[str, object] = { + "schema": NFS_SERVICE_STATE_SCHEMA, + "service_existed": existed, + "was_active": active, + "was_enabled": enabled, + "previous_threads": None, + } + if active: + previous_threads = _live_nfs_threads(runner) + if previous_threads is None: + raise ProvisionError( + "cannot record the pre-existing NFS worker count before setup" + ) + state["previous_threads"] = previous_threads + _write_text(path, json.dumps(state, sort_keys=True) + "\n") def _ensure_nfs_firewall(runner: Runner, config: Config, subnet: str) -> None: @@ -1460,7 +1699,12 @@ def _install_nfs_csi(runner: Runner, config: Config, gateway: str) -> None: storage = _render_resource( config, "manifests/nfs-storage.yaml.tmpl", - {"NAMESPACE": config.namespace, "NFS_SERVER": gateway}, + { + "NAMESPACE": config.namespace, + "NFS_SERVER": gateway, + "STORAGE_TEST_CAPACITY": STORAGE_TEST_CAPACITY, + "SSH_HOME_CAPACITY": SSH_HOME_CAPACITY, + }, ) runner.run(_kubectl(config, "apply", "-f", storage)) runner.run( @@ -1484,7 +1728,11 @@ def _install_sbx_shared_storage(runner: Runner, config: Config) -> None: storage = _render_resource( config, "manifests/sbx-storage.yaml.tmpl", - {"NAMESPACE": config.namespace}, + { + "NAMESPACE": config.namespace, + "STORAGE_TEST_CAPACITY": STORAGE_TEST_CAPACITY, + "SSH_HOME_CAPACITY": SSH_HOME_CAPACITY, + }, ) runner.run(_kubectl(config, "apply", "-f", storage)) runner.run( @@ -2537,7 +2785,11 @@ def _validate_slurm(runner: Runner, config: Config) -> None: def _write_state_summary( - config: Config, backend: str, subnet: str = "", gateway: str = "" + config: Config, + backend: str, + kubernetes_version: str, + subnet: str = "", + gateway: str = "", ) -> None: """Persist non-secret desired state for later diagnostics.""" state = { @@ -2558,9 +2810,7 @@ def _write_state_summary( "kubectl_version": ( SBX_KUBECTL_VERSION if backend == "sbx-shared" else KUBECTL_VERSION ), - "kubernetes_version": ( - SBX_KUBECTL_VERSION if backend == "sbx-shared" else KUBECTL_VERSION - ), + "kubernetes_version": kubernetes_version, "nfs_csi_version": NFS_CSI_VERSION if backend == "nfs" else None, "slinky_version": SLINKY_VERSION, "kind_subnet": subnet, @@ -2573,10 +2823,13 @@ def setup_environment(runner: Runner, config: Config) -> None: """Idempotently provision and validate the complete fixture.""" architecture = _check_platform() backend = _select_storage_backend(config) - capacity_path = _repository_root() if backend == "sbx-shared" else Path("/") + capacity_path = ( + config.sbx_shared_root if backend == "sbx-shared" else config.state_dir + ) _check_host_capacity(capacity_path) _prepare_host_dependencies(runner, config, backend) _ensure_docker(runner) + _check_docker_capacity(runner) _ensure_client_tools(runner, config, architecture, backend) running_clusters = _kind_clusters(runner) containers = _kind_containers(runner, config, running_only=False) @@ -2598,6 +2851,9 @@ def setup_environment(runner: Runner, config: Config) -> None: _prepare_sbx_shared(runner, config) if config.cluster_name in running_clusters and len(running_containers) == 3: _export_kubeconfig(runner, config) + if not _retained_cluster_matches_profile(runner, config, backend): + _delete_cluster(runner, config) + _create_cluster(runner, config, backend) elif cluster_exists: LOG.warning("Replacing incomplete or stopped disposable kind cluster") _delete_cluster(runner, config) @@ -2607,6 +2863,7 @@ def setup_environment(runner: Runner, config: Config) -> None: if backend == "sbx-shared": _configure_sbx_node_trust(runner, config) _wait_for_cluster(runner, config) + kubernetes_version = _observed_kubernetes_version(runner, config) subnet = "" gateway = "" if backend == "nfs": @@ -2623,7 +2880,7 @@ def setup_environment(runner: Runner, config: Config) -> None: _wait_for_ssh(runner, config) private_key = config.keys_dir / "id_ed25519" _validate_ssh_workers(runner, config, private_key, "separate") - _write_state_summary(config, backend, subnet, gateway) + _write_state_summary(config, backend, kubernetes_version, subnet, gateway) LOG.info("Integration environment is provisioned and running") @@ -3052,12 +3309,7 @@ def _verified_export_loop(runner: Runner, config: Config) -> str: def _associated_loop_devices(runner: Runner, config: Config) -> list[str]: """Return loop devices associated with the exact NFS backing image.""" - if not config.nfs_image.exists(): - return [] - result = runner.run( - [*_sudo_prefix(), "losetup", "--associated", config.nfs_image], check=False - ) - return [line.split(":", maxsplit=1)[0] for line in result.stdout.splitlines()] + return _loop_devices_for_file(runner, config.nfs_image) def _validate_loop_associations(runner: Runner, config: Config) -> None: @@ -3122,8 +3374,7 @@ def _remove_nfs_configuration(runner: Runner, config: Config) -> None: LOG.info("exportfs is unavailable; skipping partial-bootstrap reload") if str(config.export_dir) in _export_paths(runner): raise ProvisionError(f"NFS export is still active: {config.export_dir}") - if _nfs_service_started_by_harness(config) and not _export_paths(runner): - runner.run([*_sudo_prefix(), "systemctl", "disable", "nfs-server"]) + _restore_nfs_service_state(runner, config) firewall_state = config.state_dir / "ufw-rule.json" if firewall_state.exists(): state = json.loads(firewall_state.read_text(encoding="utf-8")) @@ -3189,20 +3440,13 @@ def _remove_owned_directories( def _stop_owned_nfs(runner: Runner, config: Config) -> None: - """Stop NFS only when this harness started the otherwise-dedicated service.""" + """Restore an initially inactive NFS service to its runtime state.""" if not (config.state_dir / "nfs-service.json").exists(): LOG.info("NFS ownership state is absent; leaving nfs-server unchanged") return if not _nfs_service_started_by_harness(config): LOG.info("nfs-server predated this fixture; leaving it running") return - export_paths = _export_paths(runner) - unrelated = [path for path in export_paths if path != str(config.export_dir)] - if unrelated: - LOG.warning( - "Leaving nfs-server running because unrelated exports exist: %s", unrelated - ) - return service = runner.run( [*_sudo_prefix(), "systemctl", "is-active", "nfs-server"], check=False ) @@ -3218,11 +3462,54 @@ def _nfs_service_started_by_harness(config: Config) -> bool: state_path = config.state_dir / "nfs-service.json" if not state_path.exists(): return False - return bool( - json.loads(state_path.read_text(encoding="utf-8")).get( - "started_by_harness", False - ) + state = json.loads(state_path.read_text(encoding="utf-8")) + if state.get("schema") == NFS_SERVICE_STATE_SCHEMA: + return not bool(state.get("was_active")) + return bool(state.get("started_by_harness", False)) + + +def _restore_preexisting_nfs_threads(runner: Runner, config: Config) -> None: + """Restore the worker count of an NFS service that predates the fixture.""" + state_path = config.state_dir / "nfs-service.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text(encoding="utf-8")) + previous = state.get("previous_threads") + was_active = ( + bool(state.get("was_active")) + if state.get("schema") == NFS_SERVICE_STATE_SCHEMA + else not bool(state.get("started_by_harness")) + ) + if not was_active or not isinstance(previous, int): + return + active = runner.run( + [*_sudo_prefix(), "systemctl", "is-active", "nfs-server"], + check=False, ) + if active.returncode: + LOG.info("Pre-existing nfs-server is inactive; worker restoration skipped") + return + _set_live_nfs_threads(runner, previous) + LOG.info("Restored pre-existing NFS worker count to %d", previous) + + +def _restore_nfs_service_state(runner: Runner, config: Config) -> None: + """Restore the pre-setup NFS runtime and boot-policy state.""" + state_path = config.state_dir / "nfs-service.json" + if not state_path.exists(): + return + state = json.loads(state_path.read_text(encoding="utf-8")) + _restore_preexisting_nfs_threads(runner, config) + if not _systemd_unit_exists(runner, "nfs-server"): + LOG.info("nfs-server is unavailable; no boot policy needs restoration") + return + if state.get("schema") != NFS_SERVICE_STATE_SCHEMA: + if state.get("started_by_harness") and not _export_paths(runner): + runner.run([*_sudo_prefix(), "systemctl", "disable", "nfs-server"]) + return + was_enabled = bool(state.get("was_enabled")) + action = "enable" if was_enabled else "disable" + runner.run([*_sudo_prefix(), "systemctl", action, "nfs-server"]) def _diagnostic_backend(config: Config) -> str | None: diff --git a/integration-tests/lib/filesystem_integration.py b/integration-tests/lib/filesystem_integration.py index 7f44f14..ca2eca0 100644 --- a/integration-tests/lib/filesystem_integration.py +++ b/integration-tests/lib/filesystem_integration.py @@ -46,6 +46,10 @@ build_ssh_failure_injection_plan, staged_failure_injection, ) +from fixture_capacity import ( + MAX_DEPLOYMENT_CONTENT_BYTES, + MAX_LIVE_CAPTURE_DATASET_BYTES, +) from filesystem_scenario_specs import ( SCENARIO_SPECS_BY_NAME, CommandKind, @@ -76,7 +80,7 @@ MAX_ARCHIVE_BYTES = 32 * 1024 * 1024 MAX_DEPLOYMENT_ARCHIVE_BYTES = 128 * 1024 * 1024 MAX_DEPLOYMENT_FILES = 10_000 -MAX_DEPLOYMENT_CONTENT_BYTES = 512 * 1024 * 1024 +SLURM_CLEANUP_MARGIN_SECONDS = 120 ELBENCHO_ARCHIVES = { "x86_64": ( "elbencho-static-x86_64.tar.gz", @@ -739,6 +743,14 @@ def _shell(value: str | Path) -> str: return shlex.quote(str(value)) +def _slurm_run_time(timeout_seconds: int) -> str: + """Return an allocation limit that outlives the harness deadline.""" + total = timeout_seconds + SLURM_CLEANUP_MARGIN_SECONDS + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + + def _override_block( selector: str, remote_root: str, @@ -747,6 +759,7 @@ def _override_block( results_dir: str | None = None, logs_dir: str | None = None, extra_env: str = "", + timeout_seconds: int = 300, ) -> tuple[str, dict[str, str]]: """Return template overrides and small support-file contents.""" data = "/mnt/storage-test" @@ -799,7 +812,7 @@ def _override_block( 'account="storage-test"', 'reservation=""', 'partition="all"', - 'run_time="00:05:00"', + f"run_time={_shell(_slurm_run_time(timeout_seconds))}", "SLURM_EXCLUSIVE_USER=0", f"export SLURM_NODE_INCLUDES={_shell(include_file)}", f"export SLURM_NODE_IGNORES={_shell(ignore_file)}", @@ -821,6 +834,7 @@ def _render_env( results_dir: str | None = None, logs_dir: str | None = None, extra_env: str = "", + timeout_seconds: int = 300, ) -> tuple[str, dict[str, str]]: """Render one runtime env from the repository's real user template.""" text = template.read_text(encoding="utf-8") @@ -836,6 +850,7 @@ def _render_env( results_dir=results_dir, logs_dir=logs_dir, extra_env=extra_env, + timeout_seconds=timeout_seconds, ) rendered = text.replace(anchor, overrides + "\n" + anchor) return rendered, support @@ -982,6 +997,7 @@ def _write_runtime_files( logs_dir: str | None = None, extra_env: str = "", extra_support: dict[str, str] | None = None, + timeout_seconds: int = 300, ) -> None: """Add the generated environment and support files to a deployment.""" rendered, support = _render_env( @@ -992,6 +1008,7 @@ def _write_runtime_files( results_dir=results_dir, logs_dir=logs_dir, extra_env=extra_env, + timeout_seconds=timeout_seconds, ) (workspace / "env.sh").write_text(rendered, encoding="utf-8") (workspace / "env.sh").chmod(0o640) @@ -1468,6 +1485,7 @@ def _sync_step_runtime( logs_dir=f"{runtime.workspace}/logs/{step.name}", extra_env=extra_env, extra_support=extra_support, + timeout_seconds=step.timeout_seconds, ) return stage = runtime.local_workspace / f"runtime-{step.name}" @@ -1484,6 +1502,7 @@ def _sync_step_runtime( logs_dir=f"{runtime.workspace}/logs/{step.name}", extra_env=extra_env, extra_support=extra_support, + timeout_seconds=step.timeout_seconds, ) archive_path = runtime.local_workspace / f"runtime-{step.name}.tar" with tarfile.open(archive_path, "w") as archive: @@ -1627,7 +1646,7 @@ def _expected_dataset_totals( if scenario in {"baseline", "ssh-shared-home", "failure-resume"}: return nodes, nodes * 16 * 1024 * 1024 if scenario == "live-capture": - return nodes * 2, nodes * 2 * 16 * 1024 * 1024 + return nodes * 2, nodes * (MAX_LIVE_CAPTURE_DATASET_BYTES // 2) if scenario == "slurm-cartesian": return nodes * 2, nodes * 2 * 1024 * 1024 if scenario == "retained-lifecycle" and step.kind is not CommandKind.DELETE: @@ -2206,8 +2225,9 @@ def _run_regular_scenario( def _cleanup_ssh_remote_results(runner: Any, config: Any) -> None: """Remove retrieved per-scenario output trees from bounded SSH homes.""" + failures = [] for pod in _pods_with_container(_pod_inventory(runner, config), "sshd"): - runner.run( + result = runner.run( [ *_kubectl( config, @@ -2230,6 +2250,15 @@ def _cleanup_ssh_remote_results(runner: Any, config: Any) -> None: check=False, timeout=60, ) + if result.returncode: + failures.append( + f"{pod['metadata']['name']}: " + f"{(result.stderr or result.stdout).strip()}" + ) + if failures: + raise IntegrationTestError( + "could not clean SSH scenario results: " + "; ".join(failures) + ) def _preserve_scenario_failure_diagnostics( @@ -2322,10 +2351,48 @@ def _cleanup_scenario_storage( timeout=135, ) if result.returncode: - LOG.warning( - "Could not clean scenario-owned remote storage for %s/%s", - runtime.scenario.name, - runtime.selector, + raise IntegrationTestError( + "could not clean scenario-owned remote storage for " + f"{runtime.scenario.name}/{runtime.selector}: " + f"{(result.stderr or result.stdout).strip()}" + ) + + +def _cleanup_scenario_resources( + runner: Any, + config: Any, + fixture: Fixture, + runtime: ScenarioRuntime, +) -> None: + """Attempt every scenario cleanup and report all failures together.""" + failures = [] + if runtime.selector == "ssh": + try: + _cleanup_ssh_remote_results(runner, config) + except IntegrationTestError as error: + failures.append(str(error)) + try: + _cleanup_scenario_storage(runner, config, fixture, runtime) + except IntegrationTestError as error: + failures.append(str(error)) + if failures: + raise IntegrationTestError("; ".join(failures)) + + +def _record_secondary_cleanup_failure(log_dir: Path, error: Exception) -> None: + """Retain cleanup failure without replacing the scenario's primary error.""" + try: + destination = log_dir / "failure-diagnostics" + destination.mkdir(parents=True, exist_ok=True) + path = destination / "cleanup-error.txt" + path.write_text(f"{type(error).__name__}: {error}\n", encoding="utf-8") + LOG.error("Scenario cleanup also failed; retained details in %s", path) + except OSError as diagnostic_error: + LOG.error( + "Scenario cleanup also failed (%r), and its diagnostic could not " + "be retained: %r", + error, + diagnostic_error, ) @@ -2861,6 +2928,7 @@ def run_filesystem_tests( scenario_logs, run_id, ) + primary_error: Exception | None = None try: if scenario == "failure-resume": _run_failure_resume( @@ -2886,6 +2954,7 @@ def run_filesystem_tests( scenario_logs, ) except Exception as error: + primary_error = error try: _preserve_scenario_failure_diagnostics( runner, @@ -2904,14 +2973,16 @@ def run_filesystem_tests( ) raise finally: - if step.substrate.value == "ssh": - _cleanup_ssh_remote_results(runner, config) - _cleanup_scenario_storage( - runner, - config, - fixture, - runtime_state, - ) + try: + _cleanup_scenario_resources( + runner, config, fixture, runtime_state + ) + # Cleanup must never replace an active scenario exception. + # pylint: disable-next=broad-exception-caught + except Exception as cleanup_error: + if primary_error is None: + raise + _record_secondary_cleanup_failure(scenario_logs, cleanup_error) finally: if home_mode == "shared" and transition_ssh_home is not None: transition_ssh_home("separate", "ssh-shared-home-restore") diff --git a/integration-tests/lib/fixture_capacity.py b/integration-tests/lib/fixture_capacity.py new file mode 100644 index 0000000..4db92aa --- /dev/null +++ b/integration-tests/lib/fixture_capacity.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One capacity budget for the single-host integration fixture.""" + +MIB = 1024**2 +GIB = 1024**3 + +STORAGE_TEST_CAPACITY_BYTES = 2 * GIB +SSH_HOME_CAPACITY_BYTES = 64 * MIB +MAX_DEPLOYMENT_CONTENT_BYTES = 512 * MIB +MAX_LIVE_CAPTURE_DATASET_BYTES = 64 * MIB +RESULT_AND_REPORT_HEADROOM_BYTES = 448 * MIB + +NFS_BUDGET_BYTES = ( + STORAGE_TEST_CAPACITY_BYTES + + SSH_HOME_CAPACITY_BYTES + + MAX_DEPLOYMENT_CONTENT_BYTES + + MAX_LIVE_CAPTURE_DATASET_BYTES + + RESULT_AND_REPORT_HEADROOM_BYTES +) +NFS_IMAGE_BYTES = ((NFS_BUDGET_BYTES + GIB - 1) // GIB) * GIB + +STORAGE_TEST_CAPACITY = f"{STORAGE_TEST_CAPACITY_BYTES // GIB}Gi" +SSH_HOME_CAPACITY = f"{SSH_HOME_CAPACITY_BYTES // MIB}Mi" diff --git a/integration-tests/manifests/nfs-storage.yaml.tmpl b/integration-tests/manifests/nfs-storage.yaml.tmpl index dc443e1..385052d 100644 --- a/integration-tests/manifests/nfs-storage.yaml.tmpl +++ b/integration-tests/manifests/nfs-storage.yaml.tmpl @@ -44,7 +44,7 @@ spec: storageClassName: storage-scale-nfs-csi resources: requests: - storage: 2Gi + storage: @@STORAGE_TEST_CAPACITY@@ --- apiVersion: v1 kind: PersistentVolumeClaim @@ -57,4 +57,4 @@ spec: storageClassName: storage-scale-nfs-csi resources: requests: - storage: 64Mi + storage: @@SSH_HOME_CAPACITY@@ diff --git a/integration-tests/manifests/sbx-storage.yaml.tmpl b/integration-tests/manifests/sbx-storage.yaml.tmpl index 590a5c0..48cd3ba 100644 --- a/integration-tests/manifests/sbx-storage.yaml.tmpl +++ b/integration-tests/manifests/sbx-storage.yaml.tmpl @@ -26,7 +26,7 @@ metadata: name: storage-scale-sbx-data spec: capacity: - storage: 2Gi + storage: @@STORAGE_TEST_CAPACITY@@ accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain @@ -50,7 +50,7 @@ spec: volumeName: storage-scale-sbx-data resources: requests: - storage: 2Gi + storage: @@STORAGE_TEST_CAPACITY@@ --- apiVersion: v1 kind: PersistentVolume @@ -58,7 +58,7 @@ metadata: name: storage-scale-sbx-home spec: capacity: - storage: 64Mi + storage: @@SSH_HOME_CAPACITY@@ accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain @@ -82,4 +82,4 @@ spec: volumeName: storage-scale-sbx-home resources: requests: - storage: 64Mi + storage: @@SSH_HOME_CAPACITY@@ diff --git a/tests/test_integration_driver_safety.py b/tests/test_integration_driver_safety.py index 8b87394..a82a76f 100644 --- a/tests/test_integration_driver_safety.py +++ b/tests/test_integration_driver_safety.py @@ -39,11 +39,15 @@ _SPEC.loader.exec_module(_DRIVER) _FILESYSTEM = sys.modules["filesystem_integration"] _bootstrap_state_dir = getattr(_DRIVER, "_bootstrap_state_dir") +_check_docker_capacity = getattr(_DRIVER, "_check_docker_capacity") +_check_host_capacity = getattr(_DRIVER, "_check_host_capacity") _collect_diagnostics = getattr(_DRIVER, "_collect_diagnostics") _configure_nfs = getattr(_DRIVER, "_configure_nfs") _configure_user_tool_path = getattr(_DRIVER, "_configure_user_tool_path") _ensure_apt_packages = getattr(_DRIVER, "_ensure_apt_packages") _ensure_export_marker = getattr(_DRIVER, "_ensure_export_marker") +_ensure_nfs_image_capacity = getattr(_DRIVER, "_ensure_nfs_image_capacity") +_observed_kubernetes_version = getattr(_DRIVER, "_observed_kubernetes_version") _ensure_slurm_workload_account = getattr(_DRIVER, "_ensure_slurm_workload_account") _export_paths = getattr(_DRIVER, "_export_paths") _kind_clusters = getattr(_DRIVER, "_kind_clusters") @@ -52,11 +56,22 @@ _migrate_export_data_to_image = getattr(_DRIVER, "_migrate_export_data_to_image") _prepare_host_dependencies = getattr(_DRIVER, "_prepare_host_dependencies") _prepare_sbx_shared = getattr(_DRIVER, "_prepare_sbx_shared") +_record_nfs_service_state = getattr(_DRIVER, "_record_nfs_service_state") +_retained_cluster_matches_profile = getattr( + _DRIVER, "_retained_cluster_matches_profile" +) +_stop_owned_nfs = getattr(_DRIVER, "_stop_owned_nfs") _driver_pods_with_container = getattr(_DRIVER, "_pods_with_container") _remove_nfs_configuration = getattr(_DRIVER, "_remove_nfs_configuration") +_restore_nfs_service_state = getattr(_DRIVER, "_restore_nfs_service_state") +_restore_preexisting_nfs_threads = getattr(_DRIVER, "_restore_preexisting_nfs_threads") _select_storage_backend = getattr(_DRIVER, "_select_storage_backend") +_set_live_nfs_threads = getattr(_DRIVER, "_set_live_nfs_threads") _storage_backend_document = getattr(_DRIVER, "_storage_backend_document") _unmount_temporary_filesystem = getattr(_DRIVER, "_unmount_temporary_filesystem") +_validate_nfs_filesystem_capacity = getattr( + _DRIVER, "_validate_nfs_filesystem_capacity" +) _validate_teardown_ownership = getattr(_DRIVER, "_validate_teardown_ownership") _validate_lifecycle_paths = getattr(_DRIVER, "_validate_lifecycle_paths") _validate_ssh_storage = getattr(_DRIVER, "_validate_ssh_storage") @@ -65,6 +80,12 @@ _assert_ordered_workers = getattr(_FILESYSTEM, "_assert_ordered_workers") _ensure_elbencho = getattr(_FILESYSTEM, "_ensure_elbencho") _prepare_scenario_data = getattr(_FILESYSTEM, "_prepare_scenario_data") +_cleanup_scenario_storage = getattr(_FILESYSTEM, "_cleanup_scenario_storage") +_cleanup_ssh_remote_results = getattr(_FILESYSTEM, "_cleanup_ssh_remote_results") +_record_secondary_cleanup_failure = getattr( + _FILESYSTEM, "_record_secondary_cleanup_failure" +) +_slurm_run_time = getattr(_FILESYSTEM, "_slurm_run_time") _preserve_scenario_failure_diagnostics = getattr( _FILESYSTEM, "_preserve_scenario_failure_diagnostics" ) @@ -162,6 +183,52 @@ def test_standard_admin_path_resolves_nfs_tools(tmp_path, monkeypatch): assert _DRIVER.shutil.which("losetup") == str(losetup) +def test_capacity_check_uses_fixture_paths_nearest_existing_parent( + tmp_path, monkeypatch +): + """A configurable state path is checked on its actual backing filesystem.""" + existing = tmp_path / "existing" + existing.mkdir() + requested = existing / "not-yet-created" / "state" + observed = [] + monkeypatch.setattr(_DRIVER.os, "cpu_count", lambda: 4) + monkeypatch.setattr( + _DRIVER, + "_meminfo", + lambda: {"MemTotal": 16 * _DRIVER.GIB, "MemAvailable": 12 * _DRIVER.GIB}, + ) + monkeypatch.setattr( + _DRIVER.shutil, + "disk_usage", + lambda path: observed.append(Path(path)) + or SimpleNamespace(free=30 * _DRIVER.GIB), + ) + + _check_host_capacity(requested) + + assert observed == [existing] + + +def test_docker_capacity_checks_visible_data_root(tmp_path, monkeypatch): + """Docker storage is checked separately when its data root is host-visible.""" + observed = [] + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, stdout=f"{tmp_path}\n", stderr="" + ) + ) + monkeypatch.setattr( + _DRIVER.shutil, + "disk_usage", + lambda path: observed.append(Path(path)) + or SimpleNamespace(free=30 * _DRIVER.GIB), + ) + + _check_docker_capacity(runner) + + assert observed == [tmp_path] + + class _TemporaryUnmountRunner: """Model a temporary mount that disappears after a chosen attempt.""" @@ -254,6 +321,8 @@ def _make_mountpoint(**_kwargs): _migrate_export_data_to_image(runner, _config(state_dir, export_dir)) assert mountpoint.is_dir() + assert not (state_dir / "nfs-export.ext4").exists() + assert (state_dir / "nfs-export.ext4.new").exists() assert runner.unmount_attempts == _DRIVER.TEMPORARY_UNMOUNT_ATTEMPTS @@ -277,11 +346,48 @@ def _make_mountpoint(**_kwargs): _migrate_export_data_to_image(runner, _config(state_dir, export_dir)) assert not mountpoint.exists() + assert (state_dir / "nfs-export.ext4").exists() + assert not (state_dir / "nfs-export.ext4.new").exists() assert runner.unmount_attempts == 1 -def test_nfs_server_has_capacity_for_concurrent_fixture_clients(tmp_path, monkeypatch): - """Rendered NFS state avoids starving the fixture's concurrent clients.""" +class _CopyFailureMigrationRunner(_MigrationRunner): + """Fail while copying retained export data into the new image.""" + + def run(self, arguments, **kwargs): + command = [str(item) for item in arguments] + if "cp" in command: + raise _DRIVER.ProvisionError("copy failed") + return super().run(arguments, **kwargs) + + +def test_nfs_migration_does_not_publish_a_failed_image(tmp_path, monkeypatch): + """A migration failure cannot leave an incomplete image at the final path.""" + state_dir = tmp_path / "state" + export_dir = tmp_path / "export" + state_dir.mkdir() + export_dir.mkdir() + mountpoint = state_dir / "known-migration-mount" + + def _make_mountpoint(**_kwargs): + mountpoint.mkdir() + return str(mountpoint) + + monkeypatch.setattr(_DRIVER.tempfile, "mkdtemp", _make_mountpoint) + + with pytest.raises(_DRIVER.ProvisionError, match="copy failed"): + _migrate_export_data_to_image( + _CopyFailureMigrationRunner(unmounted_after=1), + _config(state_dir, export_dir), + ) + + assert not (state_dir / "nfs-export.ext4").exists() + assert not (state_dir / "nfs-export.ext4.new").exists() + assert not mountpoint.exists() + + +def test_nfs_server_configures_eight_workers(tmp_path, monkeypatch): + """Rendered NFS state requests enough workers for concurrent clients.""" config = replace( _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" ) @@ -292,6 +398,7 @@ def test_nfs_server_has_capacity_for_concurrent_fixture_clients(tmp_path, monkey "_ensure_export_filesystem", "_ensure_export_marker", "_ensure_nfs_firewall", + "_set_live_nfs_threads", ): monkeypatch.setattr(_DRIVER, name, lambda *_args: None) @@ -304,6 +411,224 @@ def test_nfs_server_has_capacity_for_concurrent_fixture_clients(tmp_path, monkey assert _DRIVER.NFS_SERVER_THREADS == 8 +class _ImageCapacityRunner: + """Apply sparse truncation while recording filesystem growth commands.""" + + def __init__(self): + self.commands = [] + + def run(self, arguments, **_kwargs): + """Record a command and enact only the harmless sparse resize.""" + command = [str(item) for item in arguments] + self.commands.append(command) + if command[0] == "truncate": + os.truncate(command[-1], int(command[-2])) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_existing_nfs_image_grows_online_to_advertised_capacity(tmp_path): + """Repeated setup expands both a retained sparse image and loop device.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + config.nfs_image.touch() + os.truncate(config.nfs_image, 128 * 1024 * 1024) + runner = _ImageCapacityRunner() + + _ensure_nfs_image_capacity(runner, config, "/dev/loop9") + + assert config.nfs_image.stat().st_size == _DRIVER.NFS_IMAGE_BYTES + assert _DRIVER.NFS_IMAGE_BYTES == 4 * _DRIVER.GIB + rendered = [" ".join(command) for command in runner.commands] + assert any("losetup --set-capacity /dev/loop9" in line for line in rendered) + assert any("resize2fs /dev/loop9" in line for line in rendered) + + +def test_fixture_capacity_budget_drives_storage_declarations(): + """The backing image and manifests share one explicit capacity budget.""" + assert _DRIVER.NFS_IMAGE_BYTES >= _DRIVER.NFS_BUDGET_BYTES + assert _DRIVER.NFS_BUDGET_BYTES > 2 * _DRIVER.GIB + for name in ("nfs-storage.yaml.tmpl", "sbx-storage.yaml.tmpl"): + text = (_REPO_ROOT / "integration-tests" / "manifests" / name).read_text( + encoding="utf-8" + ) + assert "@@STORAGE_TEST_CAPACITY@@" in text + assert "@@SSH_HOME_CAPACITY@@" in text + + +def test_mounted_nfs_capacity_must_satisfy_the_runtime_budget(tmp_path): + """A retained image cannot pass based on sparse-file length alone.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=0, + stdout=f"{_DRIVER.NFS_BUDGET_BYTES // 4096} 4096 1\n", + stderr="", + ) + ) + + with pytest.raises(_DRIVER.ProvisionError, match="capacity budget"): + _validate_nfs_filesystem_capacity(runner, config) + + +class _NfsThreadRunner: + """Model a live pre-existing NFS service and its mutable worker count.""" + + def __init__(self, threads, *, active=True, enabled=False): + self.threads = threads + self.active = active + self.enabled = enabled + self.commands = [] + + def run(self, arguments, **_kwargs): + """Return and update the modeled live NFS worker count.""" + command = [str(item) for item in arguments] + self.commands.append(command) + if "show" in command: + return SimpleNamespace(returncode=0, stdout="loaded\n", stderr="") + if "is-active" in command: + return SimpleNamespace( + returncode=0 if self.active else 3, + stdout="active\n" if self.active else "inactive\n", + stderr="", + ) + if "is-enabled" in command: + return SimpleNamespace( + returncode=0 if self.enabled else 1, + stdout="enabled\n" if self.enabled else "disabled\n", + stderr="", + ) + if str(_DRIVER.NFS_THREADS_PATH) in command: + return SimpleNamespace(returncode=0, stdout=f"{self.threads}\n", stderr="") + if any(Path(item).name == "rpc.nfsd" for item in command): + self.threads = int(command[-1]) + if "systemctl" in command and "stop" in command: + self.active = False + if "systemctl" in command and "enable" in command: + self.enabled = True + if "systemctl" in command and "disable" in command: + self.enabled = False + return SimpleNamespace(returncode=0, stdout="", stderr="") + + +def test_preexisting_nfs_worker_count_is_reconciled_and_restored(tmp_path, monkeypatch): + """Setup applies eight workers and teardown restores the host's prior count.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + runner = _NfsThreadRunner(threads=2) + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _name: "/usr/sbin/rpc.nfsd") + + _record_nfs_service_state(runner, config) + _set_live_nfs_threads(runner, _DRIVER.NFS_SERVER_THREADS) + _record_nfs_service_state(runner, config) + + state = json.loads( + (config.state_dir / "nfs-service.json").read_text(encoding="utf-8") + ) + assert state == { + "previous_threads": 2, + "schema": _DRIVER.NFS_SERVICE_STATE_SCHEMA, + "service_existed": True, + "was_active": True, + "was_enabled": False, + } + assert runner.threads == 8 + + _restore_nfs_service_state(runner, config) + + assert runner.threads == 2 + assert any("disable" in command for command in runner.commands) + + +def test_nfs_service_records_active_and_enabled_states_independently( + tmp_path, monkeypatch +): + """Runtime ownership cannot be inferred from systemd boot policy.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + runner = _NfsThreadRunner(threads=3, enabled=True) + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _name: "/usr/sbin/rpc.nfsd") + + _record_nfs_service_state(runner, config) + + state = json.loads( + (config.state_dir / "nfs-service.json").read_text(encoding="utf-8") + ) + assert state["service_existed"] is True + assert state["was_active"] is True + assert state["was_enabled"] is True + assert state["previous_threads"] == 3 + + +def test_enabled_inactive_nfs_service_is_stopped_but_remains_enabled( + tmp_path, monkeypatch +): + """Teardown restores runtime and boot states without conflating them.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.state_dir.mkdir() + runner = _NfsThreadRunner(threads=0, active=False, enabled=True) + monkeypatch.setattr(_DRIVER.shutil, "which", lambda _name: "/usr/sbin/rpc.nfsd") + _record_nfs_service_state(runner, config) + runner.active = True + + _stop_owned_nfs(runner, config) + _restore_nfs_service_state(runner, config) + + assert runner.active is False + assert runner.enabled is True + + +def test_observed_kubernetes_version_requires_three_matching_nodes(tmp_path): + """Retained cluster identity comes from kubelets rather than configured pins.""" + config = _config(tmp_path / "state", tmp_path / "export") + + class _NodeRunner: + def __init__(self, versions): + self.versions = versions + + def run(self, _arguments, **_kwargs): + items = [ + {"status": {"nodeInfo": {"kubeletVersion": version}}} + for version in self.versions + ] + return SimpleNamespace( + returncode=0, stdout=json.dumps({"items": items}), stderr="" + ) + + assert ( + _observed_kubernetes_version( + _NodeRunner(["v1.34.0", "v1.34.0", "v1.34.0"]), config + ) + == "v1.34.0" + ) + with pytest.raises(_DRIVER.ProvisionError, match="one kubelet version"): + _observed_kubernetes_version( + _NodeRunner(["v1.34.0", "v1.34.0", "v1.33.0"]), config + ) + + +def test_retained_cluster_profile_uses_observed_kubelet_version(tmp_path, monkeypatch): + """A pin change marks an otherwise-running disposable cluster stale.""" + config = _config(tmp_path / "state", tmp_path / "export") + monkeypatch.setattr(_DRIVER, "_wait_for_kube_api", lambda *_args: None) + monkeypatch.setattr( + _DRIVER, "_observed_kubernetes_version", lambda *_args: "v1.33.0" + ) + + assert not _retained_cluster_matches_profile( + SimpleNamespace(), config, "sbx-shared" + ) + + def test_local_scenario_failure_diagnostics_survive_workspace_cleanup(tmp_path): """SSH logs and partial results are copied outside disposable workspace.""" workspace = tmp_path / "workspace" @@ -341,6 +666,82 @@ def test_local_scenario_failure_diagnostics_survive_workspace_cleanup(tmp_path): assert "scenario failed" in (diagnostics / "failure.txt").read_text() +def _cleanup_runtime(tmp_path): + """Return one marker-safe scenario runtime for cleanup tests.""" + return _FILESYSTEM.ScenarioRuntime( + scenario=SimpleNamespace(name="example"), + selector="ssh", + workspace=str(tmp_path / "workspace"), + local_workspace=tmp_path, + artifact_root=tmp_path / "artifacts", + data_root=f"{_FILESYSTEM.REMOTE_BASE}/test-data/example", + values={}, + copied_results=[], + ) + + +def test_scenario_storage_cleanup_failure_is_fatal(tmp_path): + """A successful workload cannot pass while isolated storage remains.""" + runner = SimpleNamespace( + run=lambda *_args, **_kwargs: SimpleNamespace( + returncode=1, stdout="", stderr="permission denied" + ) + ) + fixture = SimpleNamespace(login_pod="login", login_container="login") + + with pytest.raises(_FILESYSTEM.IntegrationTestError, match="permission denied"): + _cleanup_scenario_storage( + runner, + _config(tmp_path / "state", tmp_path / "export"), + fixture, + _cleanup_runtime(tmp_path), + ) + + +def test_ssh_home_cleanup_failure_is_fatal(tmp_path): + """Failed worker-home cleanup cannot silently contaminate later scenarios.""" + pod = { + "metadata": {"name": "ssh-worker-0"}, + "spec": {"containers": [{"name": "sshd"}]}, + "status": { + "phase": "Running", + "containerStatuses": [{"name": "sshd", "ready": True}], + }, + } + + class _SshCleanupRunner: + def run(self, arguments, **_kwargs): + command = [str(item) for item in arguments] + if "get" in command and "pods" in command: + return SimpleNamespace( + returncode=0, + stdout=json.dumps({"items": [pod]}), + stderr="", + ) + return SimpleNamespace( + returncode=1, stdout="", stderr="remote cleanup failed" + ) + + with pytest.raises(_FILESYSTEM.IntegrationTestError, match="remote cleanup"): + _cleanup_ssh_remote_results( + _SshCleanupRunner(), + _config(tmp_path / "state", tmp_path / "export"), + ) + + +def test_secondary_cleanup_failure_is_retained_without_replacing_primary(tmp_path): + """Cleanup diagnostics survive beside an already-recorded workload error.""" + _record_secondary_cleanup_failure(tmp_path, RuntimeError("cleanup failed")) + + error = tmp_path / "failure-diagnostics" / "cleanup-error.txt" + assert error.read_text(encoding="utf-8") == "RuntimeError: cleanup failed\n" + + +def test_slurm_allocation_outlives_slowest_scenario(): + """The scheduler cannot expire before a declared harness deadline.""" + assert _slurm_run_time(600) == "00:12:00" + + def test_default_state_bootstrap_creates_missing_tmp_parent(tmp_path, monkeypatch): """A clean checkout need not contain the ignored default tmp directory.""" checkout = tmp_path / "checkout" @@ -972,13 +1373,20 @@ def test_teardown_validation_allows_unrelated_nfs_exports(tmp_path, monkeypatch) class _NfsRemovalRunner: """Record NFS cleanup while reporting one unrelated active export.""" - def __init__(self): + def __init__(self, threads=None): self.commands = [] + self.threads = threads def run(self, arguments, **_kwargs): """Record one cleanup command and return stable export state.""" command = [str(item) for item in arguments] self.commands.append(command) + if "is-active" in command: + return SimpleNamespace(returncode=0, stdout="active\n", stderr="") + if str(_DRIVER.NFS_THREADS_PATH) in command: + return SimpleNamespace(returncode=0, stdout=f"{self.threads}\n", stderr="") + if any(Path(item).name == "rpc.nfsd" for item in command): + self.threads = int(command[-1]) stdout = ( "/srv/unrelated-export 10.0.0.0/24(options)\n" if "-v" in command else "" ) @@ -1006,13 +1414,43 @@ def test_nfs_cleanup_preserves_preexisting_service(tmp_path, monkeypatch): _remove_nfs_configuration(runner, config) - assert not any("systemctl" in command for command in runner.commands) + assert not any( + "systemctl" in command + and any(action in command for action in ("enable", "disable", "stop")) + for command in runner.commands + ) assert any( any(Path(item).name == "exportfs" for item in command) and "-u" in command for command in runner.commands ) +def test_nfs_cleanup_restores_preexisting_worker_count(tmp_path, monkeypatch): + """Teardown restores the live count after removing fixture configuration.""" + config = replace( + _config(tmp_path / "state", tmp_path / "export"), storage_backend="nfs" + ) + config.manifests_dir.mkdir(parents=True) + (config.state_dir / "nfs-service.json").write_text( + json.dumps({"started_by_harness": False, "previous_threads": 2}), + encoding="utf-8", + ) + runner = _NfsRemovalRunner(threads=8) + monkeypatch.setattr( + _DRIVER.shutil, + "which", + lambda command: f"/usr/sbin/{command}", + ) + + _remove_nfs_configuration(runner, config) + + assert runner.threads == 2 + assert any( + any(Path(item).name == "rpc.nfsd" for item in command) and command[-1] == "2" + for command in runner.commands + ) + + def test_nfs_cleanup_survives_missing_exportfs(tmp_path, monkeypatch): """Interrupted package bootstrap leaves teardown able to remove state.""" config = replace(