From de2dde1393c3a71a9be935585c9cfe014302d526 Mon Sep 17 00:00:00 2001 From: "PD User (shared)" Date: Wed, 15 Jul 2026 05:52:30 +0000 Subject: [PATCH] fix(pd-lm): SIGPIPE-proof the driver-version detection in the node-setup script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nvidia-smi --query-gpu writes one line per GPU; '| head -n 1' closes the pipe after the first, and under 'set -o pipefail' a late write from nvidia-smi turns into SIGPIPE -> exit 141, killing the whole srun step before the workspace even builds. Racy per node: observed three times across 1- and 4-node submissions (jobs 1050555, 1050877 requeue, 1090526). Query GPU 0 alone instead — 'cut' consumes the full stream, no pipe truncation, same DRIVER_MAJOR. Crew-Address: task/scan-read-activations-taps --- param_decomp_lab/experiments/lm/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/param_decomp_lab/experiments/lm/launch.py b/param_decomp_lab/experiments/lm/launch.py index ea9d285cd..41c3af383 100644 --- a/param_decomp_lab/experiments/lm/launch.py +++ b/param_decomp_lab/experiments/lm/launch.py @@ -100,7 +100,7 @@ def _node_workspace_setup(run_id: str, snapshot_ref: str, source_repo: Path, run git checkout --quiet FETCH_HEAD cp "{run_dir}/.env" .env unset VIRTUAL_ENV -DRIVER_MAJOR=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1 | cut -d. -f1) +DRIVER_MAJOR=$(nvidia-smi --id=0 --query-gpu=driver_version --format=csv,noheader | cut -d. -f1) if [ "$DRIVER_MAJOR" -ge 580 ]; then CUDA_EXTRA=cuda13; else CUDA_EXTRA=cuda; fi echo "cuda extra: $CUDA_EXTRA (driver major $DRIVER_MAJOR)" uv sync --all-packages --no-dev --extra "$CUDA_EXTRA" --link-mode copy -q