From b7ad3062d5726bf1addcbbd57f1f675955457aff Mon Sep 17 00:00:00 2001 From: "PD User (shared)" Date: Tue, 14 Jul 2026 00:06:46 +0000 Subject: [PATCH] fix(pd-lm): SIGPIPE race in node-setup driver check kills 16-node jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-node workspace setup runs under `set -euo pipefail` and gated the CUDA extra on `nvidia-smi --query-gpu=driver_version ... | head -n 1`. On an 8-GPU node nvidia-smi prints 8 lines; head exits after the first, and when nvidia-smi loses the race it dies of SIGPIPE (exit 141), pipefail fails the pipeline, and set -e kills the whole srun task — cancelling all 16 nodes seconds into the job. Observed on jobs 172348, 172349 (Jul 8) and 174502, 174513 (Jul 13); roughly a coin flip per 16-node launch. Query only GPU 0 instead: single output line, nothing truncates the pipe. Verified on job 174520 (16 nodes, all through setup, run completed). Crew-Address: task/profile-llama-32b-cw-east-artifacts Co-Authored-By: Claude Fable 5 --- param_decomp_lab/experiments/lm/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/param_decomp_lab/experiments/lm/launch.py b/param_decomp_lab/experiments/lm/launch.py index ea9d285cd..41c3af383 100644 --- a/param_decomp_lab/experiments/lm/launch.py +++ b/param_decomp_lab/experiments/lm/launch.py @@ -100,7 +100,7 @@ def _node_workspace_setup(run_id: str, snapshot_ref: str, source_repo: Path, run git checkout --quiet FETCH_HEAD cp "{run_dir}/.env" .env unset VIRTUAL_ENV -DRIVER_MAJOR=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1 | cut -d. -f1) +DRIVER_MAJOR=$(nvidia-smi --id=0 --query-gpu=driver_version --format=csv,noheader | cut -d. -f1) if [ "$DRIVER_MAJOR" -ge 580 ]; then CUDA_EXTRA=cuda13; else CUDA_EXTRA=cuda; fi echo "cuda extra: $CUDA_EXTRA (driver major $DRIVER_MAJOR)" uv sync --all-packages --no-dev --extra "$CUDA_EXTRA" --link-mode copy -q