diff --git a/.github/workflows/gpu-ci.yml b/.github/workflows/gpu-ci.yml index 46caf703..2b42a2d2 100644 --- a/.github/workflows/gpu-ci.yml +++ b/.github/workflows/gpu-ci.yml @@ -15,67 +15,39 @@ on: - '.github/workflows/gpu-ci.yml' types: [ opened, synchronize, reopened, labeled ] -concurrency: - group: gpu-ci-serial - cancel-in-progress: false - +# The orchestrator is checked out from the base branch, then run against the PR +# revision in an isolated temporary worktree. Do not grant repository-write or +# cloud credentials to this job: it executes untrusted PR test code. +permissions: + contents: read jobs: gpu-tests: if: contains(github.event.pull_request.labels.*.name, 'needs-gpu-ci') - runs-on: ubuntu-latest - timeout-minutes: 60 - - strategy: - fail-fast: false - matrix: - include: - - { gpu_id: "NVIDIA RTX A4000", target_sm: "8.6" } # SM86 - - { gpu_id: "NVIDIA H100 80GB HBM3", target_sm: "9.0", force_sm90: "1" } # SM90 + # This label must be attached to the local eight-H100-SXM runner fleet. + # Multiple runner agents may share one host; ci/run_gpu_ci.sh arbitrates + # the four physical two-GPU pairs on that host. + runs-on: [self-hosted, linux, x64, h100-sxm-8x] + timeout-minutes: 180 steps: - - name: Checkout secure orchestrator script from base branch + - name: Checkout trusted GPU CI orchestrator from base branch uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 1 + persist-credentials: false - - name: Install runpodctl - run: | - wget -qO runpodctl https://github.com/runpod/runpodctl/releases/latest/download/runpodctl-linux-amd64 - chmod +x runpodctl - sudo mv runpodctl /usr/local/bin/runpodctl - runpodctl version - - - name: Configure runpodctl - run: runpodctl config --apiKey "${{ secrets.RUNPOD_API_KEY }}" - - - name: Setup SSH key - run: | - mkdir -p ~/.ssh && chmod 700 ~/.ssh - printf '%s\n' "${{ secrets.RUNPOD_SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - ssh-keygen -y -f ~/.ssh/id_ed25519 > /dev/null && echo "key OK" || echo "key BROKEN" - - # Follow-up (#191): to test multiple architectures, add a matrix and pass - # GPU_ID + TARGET_SM (+ KERNEL_ALIGN_FORCE_SM90 for Hopper) through to the script. - # run_gpu_ci.sh reads all three, normalizes TARGET_SM, asserts the pod matches it, - # and forwards KERNEL_ALIGN_FORCE_SM90 into the remote build: - # strategy: - # matrix: - # include: - # - { gpu_id: "NVIDIA RTX A4000", target_sm: "8.6" } # Ampere - # - { gpu_id: "NVIDIA A100 80GB PCIe", target_sm: "8.0" } - # - { gpu_id: "NVIDIA H100 PCIe", target_sm: "9.0", force_sm90: "1" } # build Hopper TMA/WGMMA kernels - # - { gpu_id: "NVIDIA B200", target_sm: "10.0" } - # Per-arch jobs must NOT fall back to a different-capability GPU: the script - # fails fast when the pod arch != requested TARGET_SM, so keep fallback within - # the same compute capability (or unset it for these jobs). - - name: Run GPU tests on RunPod + - name: Run GPU tests on the local H100 SXM server env: - RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} + # These directories must be shared by every Actions runner process + # on the same physical server and writable by the runner account. + GPU_CI_LOCK_DIR: /var/tmp/rl-kernel-gpu-ci/locks + GPU_CI_WORK_ROOT: /var/tmp/rl-kernel-gpu-ci/workspaces PR_REPO_URL: ${{ github.event.pull_request.head.repo.clone_url }} PR_SHA: ${{ github.event.pull_request.head.sha }} - GPU_ID: ${{ matrix.gpu_id }} - TARGET_SM: ${{ matrix.target_sm }} - KERNEL_ALIGN_FORCE_SM90: ${{ matrix.force_sm90 }} + EXPECTED_GPU_NAME: H100 + EXPECTED_GPU_COUNT: "8" + TARGET_SM: "9.0" + KERNEL_ALIGN_FORCE_SM90: "1" run: bash ci/run_gpu_ci.sh diff --git a/ci/run_gpu_ci.sh b/ci/run_gpu_ci.sh index 5a757464..1e87d538 100644 --- a/ci/run_gpu_ci.sh +++ b/ci/run_gpu_ci.sh @@ -1,202 +1,268 @@ #!/usr/bin/env bash -set -uo pipefail - -# TP=2 (override via env; GPU_ID/GPU_COUNT accepted as matrix-friendly aliases) -PRIMARY_GPU_ID="${PRIMARY_GPU_ID:-${GPU_ID:-NVIDIA RTX A4000}}" -PRIMARY_GPU_COUNT="${PRIMARY_GPU_COUNT:-${GPU_COUNT:-2}}" - -# TP=1 -FALLBACK_GPU_ID="${FALLBACK_GPU_ID:-NVIDIA A40}" -FALLBACK_GPU_COUNT="${FALLBACK_GPU_COUNT:-1}" +# Run one GPU CI job on a shared, local 8x H100 SXM host. +# +# Lock files are deliberately persistent. Removing a lock file while another +# process still holds its inode can split the lock domain. A GPU pair is +# released by closing its flock file descriptor; the optional owner file is +# only diagnostic metadata and is removed during cleanup. +set -Eeuo pipefail + +readonly GPU_PAIRS=("0,1" "2,3" "4,5" "6,7") +# Fixed descriptors keep this script compatible with Bash versions that do not +# support `exec {var}>file`. fd 9 serializes allocation; fds 8..5 are the +# four pair-lock descriptors in the same order as GPU_PAIRS. +readonly SCHEDULER_FD=9 +readonly GPU_PAIR_FDS=(8 7 6 5) +readonly GPU_CI_LOCK_DIR="${GPU_CI_LOCK_DIR:-/var/tmp/rl-kernel-gpu-ci/locks}" +readonly GPU_CI_WORK_ROOT="${GPU_CI_WORK_ROOT:-/var/tmp/rl-kernel-gpu-ci/workspaces}" +readonly GPU_CI_WAIT_SECONDS="${GPU_CI_WAIT_SECONDS:-5}" +readonly EXPECTED_GPU_NAME="${EXPECTED_GPU_NAME:-H100}" +readonly EXPECTED_GPU_COUNT="${EXPECTED_GPU_COUNT:-8}" +readonly TARGET_SM="${TARGET_SM:-9.0}" +readonly KERNEL_ALIGN_FORCE_SM90="${KERNEL_ALIGN_FORCE_SM90:-1}" + +GPU_PAIR_FD="" +GPU_PAIR="" +GPU_PAIR_LABEL="" +GPU_PAIR_OWNER_FILE="" +WORK_DIR="" +SOURCE_DIR="" +PYTHON="" +TEST_PID="" + +die() { + echo "[gpu-ci] FATAL: $*" >&2 + exit 1 +} -# Optional arch override; asserted against the pod's real cap in the remote build so a -# cross-arch resource fallback cannot build mismatched SASS. -TARGET_SM="${TARGET_SM:-}" +release_scheduler_lock() { + flock -u "$SCHEDULER_FD" || true +} -# Forwarded to the remote build; setup.py compiles the Hopper (sm90) kernels only when "1". -KERNEL_ALIGN_FORCE_SM90="${KERNEL_ALIGN_FORCE_SM90:-}" +release_gpu_pair() { + if [[ -n "$GPU_PAIR_FD" ]]; then + rm -f -- "$GPU_PAIR_OWNER_FILE" || true + flock -u "$GPU_PAIR_FD" || true + echo "[gpu-ci] Released GPU pair ${GPU_PAIR}." + GPU_PAIR_FD="" + GPU_PAIR="" + fi +} -CI_IMAGE="${CI_IMAGE:-runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04}" -DISK_GB=40 -PR_SHA="${PR_SHA:-$(date +%s)}" -POD_NAME="rl-kernel-ci-${PR_SHA:0:7}" -READY_RETRIES=60 +close_lock_files() { + exec 5>&- || true + exec 6>&- || true + exec 7>&- || true + exec 8>&- || true + exec 9>&- || true +} -POD_ID="" +stop_test_process_group() { + if [[ -z "$TEST_PID" ]]; then + return + fi -cleanup() { - trap - EXIT INT TERM - - if [ -n "$POD_ID" ]; then - echo "" - echo "[ci] ========================================================" - echo "[ci] === AUTOMATIC CLEANUP: Removing pod $POD_ID ===" - echo "[ci] ========================================================" - - REMOVE_OUT=$(runpodctl pod remove "$POD_ID" 2>&1) - if echo "$REMOVE_OUT" | grep -qi "not found"; then - echo "[ci] Pod $POD_ID was already cleared from the cloud. Safe to exit." - else - echo "$REMOVE_OUT" - fi + if kill -0 "$TEST_PID" 2>/dev/null; then + echo "[gpu-ci] Stopping test process group ${TEST_PID}." + kill -TERM -- "-${TEST_PID}" 2>/dev/null || true + for _ in $(seq 1 10); do + kill -0 "$TEST_PID" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${TEST_PID}" 2>/dev/null || true fi + wait "$TEST_PID" 2>/dev/null || true + TEST_PID="" } -trap cleanup EXIT INT TERM - -GPU_ID=$PRIMARY_GPU_ID -GPU_COUNT=$PRIMARY_GPU_COUNT - -echo "[ci] Attempt 1: create pod: ${GPU_COUNT}x ${GPU_ID}" -CREATE_OUT=$(runpodctl pod create \ - --name "$POD_NAME" \ - --gpu-id "$GPU_ID" \ - --gpu-count "$GPU_COUNT" \ - --image "$CI_IMAGE" \ - --container-disk-in-gb "$DISK_GB" \ - --cloud-type SECURE \ - --ports "22/tcp" 2>&1) - -# Fallback 触发 -if echo "$CREATE_OUT" | grep -qi "no longer any instances available"; then - echo "[ci] WARN: ${GPU_COUNT}x ${GPU_ID} sold out! Triggering elastic Fallback..." - - GPU_ID=$FALLBACK_GPU_ID - GPU_COUNT=$FALLBACK_GPU_COUNT - - echo "[ci] Attempt 2 (Fallback): create pod: ${GPU_COUNT}x ${GPU_ID}" - CREATE_OUT=$(runpodctl pod create \ - --name "$POD_NAME" \ - --gpu-id "$GPU_ID" \ - --gpu-count "$GPU_COUNT" \ - --image "$CI_IMAGE" \ - --container-disk-in-gb "$DISK_GB" \ - --cloud-type SECURE \ - --ports "22/tcp" 2>&1) - - if echo "$CREATE_OUT" | grep -qi "no longer any instances available"; then - echo "[ci] FATAL: Alternatives (${GPU_COUNT}x ${GPU_ID}) have also been exhausted. Please try CI again later." - exit 1 - fi -fi -POD_ID=$(echo "$CREATE_OUT" | grep -oE '"id":\s*"[a-z0-9]{8,}"' | cut -d '"' -f4 | head -1) -if [ -z "$POD_ID" ]; then - POD_ID=$(echo "$CREATE_OUT" | grep -oE '"[a-z0-9]{8,}"' | tr -d '"' | head -1) -fi +cleanup() { + local status=$? + trap - EXIT INT TERM HUP -if [ -z "$POD_ID" ]; then - echo "[ci] ERROR: Unable to resolve pod id. Output: $CREATE_OUT" - exit 1 -fi -echo "[ci] Successfully rented pod: $POD_ID" + stop_test_process_group + release_gpu_pair + release_scheduler_lock + close_lock_files -echo "[ci] Waiting for pod network infrastructure to be fully ready..." -SSH_IP="" -SSH_PORT="" + if [[ -n "$WORK_DIR" && -d "$WORK_DIR" ]]; then + rm -rf -- "$WORK_DIR" || true + fi + exit "$status" +} -for i in $(seq 1 "$READY_RETRIES"); do - POD_INFO=$(runpodctl pod get "$POD_ID" -o json) +on_signal() { + local signal="$1" + local status="$2" + echo "[gpu-ci] Received ${signal}; cleaning up GPU lease." + exit "$status" +} - SSH_IP=$(echo "$POD_INFO" | grep -iE '"ip"|"publicIp"|"address"' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | head -1 || true) - SSH_PORT=$(echo "$POD_INFO" | grep -iE '"port"|"externalPort"|"publicPort"' | grep -oE '[0-9]+' | grep -v '^22$' | head -1 || true) +trap cleanup EXIT +trap 'on_signal INT 130' INT +trap 'on_signal TERM 143' TERM +trap 'on_signal HUP 129' HUP + +require_host_prerequisites() { + command -v flock >/dev/null || die "flock is required for GPU-pair scheduling." + command -v git >/dev/null || die "git is required to fetch the PR revision." + command -v setsid >/dev/null || die "setsid is required to terminate distributed tests safely." + command -v nvidia-smi >/dev/null || die "nvidia-smi is required on the local GPU runner." + command -v python3 >/dev/null || die "python3 is required on the local GPU runner." + + local detected_count + detected_count=$(nvidia-smi --query-gpu=index --format=csv,noheader | wc -l | tr -d '[:space:]') + [[ "$detected_count" == "$EXPECTED_GPU_COUNT" ]] || die \ + "expected ${EXPECTED_GPU_COUNT} GPUs, found ${detected_count}. Refusing to allocate an unknown host." + + local gpu_index gpu_name gpu_sm + for gpu_index in $(seq 0 7); do + gpu_name=$(nvidia-smi --id="$gpu_index" --query-gpu=name --format=csv,noheader | xargs) + gpu_sm=$(nvidia-smi --id="$gpu_index" --query-gpu=compute_cap --format=csv,noheader | xargs) + [[ "$gpu_name" == *"$EXPECTED_GPU_NAME"* ]] || die \ + "GPU ${gpu_index} is '${gpu_name}', expected a ${EXPECTED_GPU_NAME}." + [[ "$gpu_sm" == "$TARGET_SM" ]] || die \ + "GPU ${gpu_index} is sm_${gpu_sm}, expected sm_${TARGET_SM}." + done +} - if [ -n "$SSH_IP" ] && [ -n "$SSH_PORT" ] && ! echo "$POD_INFO" | grep -qi "not ready"; then - echo "[ci] Pod infrastructure is 100% READY!" - break - fi +acquire_gpu_pair() { + local scheduler_lock_file="${GPU_CI_LOCK_DIR}/scheduler.lock" + local pair pair_index candidate_fd + + mkdir -p "$GPU_CI_LOCK_DIR" + [[ -d "$GPU_CI_LOCK_DIR" && -w "$GPU_CI_LOCK_DIR" ]] || die \ + "GPU_CI_LOCK_DIR is not writable: ${GPU_CI_LOCK_DIR}" + + # Serialize only the probe-and-claim transaction. The selected pair lock is + # retained while tests run, so unrelated jobs can immediately claim another pair. + exec 9>>"$scheduler_lock_file" + exec 8>>"${GPU_CI_LOCK_DIR}/pair-0-1.lock" + exec 7>>"${GPU_CI_LOCK_DIR}/pair-2-3.lock" + exec 6>>"${GPU_CI_LOCK_DIR}/pair-4-5.lock" + exec 5>>"${GPU_CI_LOCK_DIR}/pair-6-7.lock" + + while true; do + flock "$SCHEDULER_FD" + + for pair_index in "${!GPU_PAIRS[@]}"; do + pair="${GPU_PAIRS[$pair_index]}" + candidate_fd="${GPU_PAIR_FDS[$pair_index]}" + if flock -n "$candidate_fd"; then + GPU_PAIR="$pair" + GPU_PAIR_LABEL="${pair/,/-}" + GPU_PAIR_FD="$candidate_fd" + GPU_PAIR_OWNER_FILE="${GPU_CI_LOCK_DIR}/pair-${GPU_PAIR_LABEL}.owner" + printf 'pid=%s\nrun_id=%s\nrepository=%s\nacquired_at=%s\n' \ + "$$" "${GITHUB_RUN_ID:-local}" "${GITHUB_REPOSITORY:-local}" \ + "$(date -u +%FT%TZ)" > "$GPU_PAIR_OWNER_FILE" + release_scheduler_lock + echo "[gpu-ci] Acquired GPU pair ${GPU_PAIR}." + return + fi + done + + release_scheduler_lock + echo "[gpu-ci] All GPU pairs are busy; waiting ${GPU_CI_WAIT_SECONDS}s before retrying." + sleep "$GPU_CI_WAIT_SECONDS" + done +} - if [ "$i" -eq "$READY_RETRIES" ]; then - echo "[ci] ERROR: Pod network/SSH infrastructure initialization timed out." - exit 1 +prepare_pr_worktree() { + mkdir -p "$GPU_CI_WORK_ROOT" + [[ -d "$GPU_CI_WORK_ROOT" && -w "$GPU_CI_WORK_ROOT" ]] || die \ + "GPU_CI_WORK_ROOT is not writable: ${GPU_CI_WORK_ROOT}" + + WORK_DIR=$(mktemp -d "${GPU_CI_WORK_ROOT%/}/rl-kernel-gpu-ci.XXXXXX") + SOURCE_DIR="${WORK_DIR}/repo" + + if [[ -n "${PR_REPO_URL:-}" && -n "${PR_SHA:-}" ]]; then + echo "[gpu-ci] Fetching PR revision ${PR_SHA} into an isolated worktree." + git clone --no-checkout "$PR_REPO_URL" "$SOURCE_DIR" + git -C "$SOURCE_DIR" fetch --depth=1 origin "$PR_SHA" + git -C "$SOURCE_DIR" checkout --detach FETCH_HEAD + else + # This branch is useful for local reproduction only. GitHub Actions always + # supplies PR_REPO_URL and PR_SHA, so production CI never tests the base checkout. + [[ -n "${GITHUB_WORKSPACE:-}" ]] || die "PR_REPO_URL/PR_SHA or GITHUB_WORKSPACE is required." + SOURCE_DIR="$GITHUB_WORKSPACE" fi +} - echo "[ci] Pod layer status: RUNNING, but network routing is initializing... waiting 10s (Attempt $i/$READY_RETRIES)" - sleep 10 -done +create_job_venv() { + local base_python="${PYTHON_BIN:-python3}" + local venv_dir="${WORK_DIR}/venv" -echo "[ci] Target Establish -> root@$SSH_IP:$SSH_PORT" + "$base_python" -m venv --system-site-packages "$venv_dir" + PYTHON="${venv_dir}/bin/python" + "$PYTHON" -c 'import torch; assert torch.cuda.is_available(), "CUDA-enabled torch is required"' -SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -p $SSH_PORT" + # Keep package installation job-local; concurrent CI jobs must not mutate the + # self-hosted runner's global Python environment. + "$PYTHON" -m pip install --no-build-isolation --no-deps -e "$SOURCE_DIR" + "$PYTHON" -m pip install --no-cache-dir numpy tabulate accelerate "transformers==5.13.1" pytest +} -if [ "${GPU_COUNT}" -gt 1 ]; then - TEST_CMD='"$PY" -m torch.distributed.run --nproc_per_node='"${GPU_COUNT}"' -m pytest tests/ -v' -else - TEST_CMD='"$PY" -m pytest tests/ -v' -fi +configure_gpu_isolation() { + export CUDA_DEVICE_ORDER=PCI_BUS_ID + export CUDA_VISIBLE_DEVICES="$GPU_PAIR" + export FORCE_CUDA=1 + export MAX_JOBS="${MAX_JOBS:-8}" + export KERNEL_ALIGN_FORCE_SM90 + export TORCH_CUDA_ARCH_LIST="${TARGET_SM}+PTX" + export RL_KERNEL_REQUIRE_EXT=1 -REMOTE_CMD='set -e -PY=$(command -v python3.11 || command -v python3) -if [ -z "$PY" ]; then echo "[remote] FATAL: python not found in PATH"; exit 127; fi -if ! "$PY" -c "import torch" >/dev/null 2>&1; then - for cand in python3.11 python3.10 python3; do - p=$(command -v "$cand" 2>/dev/null) || continue - if "$p" -c "import torch" >/dev/null 2>&1; then PY="$p"; break; fi - done -fi -echo "[remote] Using interpreter: $PY" -export FORCE_CUDA=1 -export MAX_JOBS=8 -export KERNEL_ALIGN_FORCE_SM90="'"${KERNEL_ALIGN_FORCE_SM90}"'" - -# normalize_sm: compact (90) or dotted (9.0) compute cap -> torch dotted form, keeping +PTX. -normalize_sm() { - sm_in="$1"; sm_ptx="" - case "$sm_in" in *+PTX) sm_ptx="+PTX"; sm_in="${sm_in%+PTX}";; esac - case "$sm_in" in - *.*) : ;; - [0-9][0-9]|[0-9][0-9][0-9]) sm_major="${sm_in%?}"; sm_in="${sm_major}.${sm_in#$sm_major}" ;; - *) return 1 ;; - esac - case "$sm_in" in [0-9]*.[0-9]|[0-9]*.[0-9][0-9]) echo "${sm_in}${sm_ptx}" ;; *) return 1 ;; esac + echo "[gpu-ci] CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" } -ACTUAL_SM=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d "[:space:]") -[ -z "$ACTUAL_SM" ] && ACTUAL_SM=$("$PY" -c "import torch;a,b=torch.cuda.get_device_capability();print(f\"{a}.{b}\")" 2>/dev/null || true) -[ -z "$ACTUAL_SM" ] && { echo "[remote] FATAL: cannot determine GPU compute capability"; exit 3; } - -REQUESTED_SM="'"${TARGET_SM}"'" -if [ -n "$REQUESTED_SM" ]; then - NORM_REQ=$(normalize_sm "$REQUESTED_SM") || { echo "[remote] FATAL: unsupported TARGET_SM=$REQUESTED_SM"; exit 3; } - NORM_REQ_BASE="${NORM_REQ%+PTX}" - if [ "$NORM_REQ_BASE" != "$ACTUAL_SM" ]; then - echo "[remote] FATAL: requested TARGET_SM=$REQUESTED_SM (sm_$NORM_REQ_BASE) but provisioned GPU is sm_$ACTUAL_SM." - echo "[remote] Refusing to build mismatched kernels (likely a cross-arch resource fallback)." - exit 3 - fi - BUILD_SM="$NORM_REQ_BASE" -else - BUILD_SM=$(normalize_sm "$ACTUAL_SM") || { echo "[remote] FATAL: unsupported detected arch $ACTUAL_SM"; exit 3; } -fi -# BUILD_SM is always bare here (both paths strip +PTX); +PTX gives forward-compat JIT. -export TORCH_CUDA_ARCH_LIST="${BUILD_SM}+PTX" -echo "[remote] Detected GPU sm_$ACTUAL_SM; building _C for TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST" - -cd /workspace -git clone '"${PR_REPO_URL:-https://github.com/RL-Align/RL-Kernel.git}"' repo -cd repo -git fetch origin '"${PR_SHA}"' -git checkout --detach '"${PR_SHA}"' -"$PY" -c "import torch;print(f\"[remote] image torch {torch.__version__} cuda {torch.version.cuda}\")" -# Pin torch (cu124, matching the CI image) so the extension is built against the exact -# runtime torch, not a non-deterministic bare-install upgrade of the 2.4.0 in the image. -TORCH_SPEC="${TORCH_SPEC:-torch==2.4.1}" -TORCH_INDEX_URL="${TORCH_INDEX_URL:-https://download.pytorch.org/whl/cu124}" -"$PY" -m pip install --no-cache-dir "$TORCH_SPEC" --index-url "$TORCH_INDEX_URL" -"$PY" -c "import torch;print(f\"[remote] pinned torch {torch.__version__} cuda {torch.version.cuda}\")" -# --no-build-isolation: torch must be visible to setup.py, else the extension is silently skipped. -# --no-deps: keep the pinned torch; do not let the editable install re-resolve it. -"$PY" -m pip install --no-build-isolation --no-deps -e . -"$PY" -m pip install --no-cache-dir numpy tabulate accelerate "transformers==5.13.1" pytest -nvidia-smi -# Fail fast if _C did not build or cannot launch, instead of silently using native fallbacks. -"$PY" scripts/ci_smoke.py -# Enforce _C in the pytest suite too (test_extension_smoke.py skips unless this is set). -export RL_KERNEL_REQUIRE_EXT=1 -'"${TEST_CMD}" - -echo "[ci] Launching remote test suite on GPU pod (Distributed Execution Mode: TP=${GPU_COUNT})..." -ssh $SSH_OPTIONS root@"$SSH_IP" "bash -lc '$REMOTE_CMD'" -TEST_EXIT=$? - -echo "[ci] Remote execution finished with exit code = $TEST_EXIT" -exit $TEST_EXIT +run_tests() { + "$PYTHON" - <<'PY' +import os +import torch + +assert torch.cuda.is_available(), "CUDA is not available after GPU allocation" +assert torch.cuda.device_count() == 2, ( + f"expected exactly 2 visible GPUs, got {torch.cuda.device_count()} " + f"for CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')}" +) +for local_index in range(2): + name = torch.cuda.get_device_name(local_index) + print(f"[gpu-ci] local cuda:{local_index} -> {name}") + assert "H100" in name, f"unexpected visible device: {name}" +PY + + ( + cd "$SOURCE_DIR" + "$PYTHON" scripts/ci_smoke.py + ) + + echo "[gpu-ci] Starting TP=2 test suite on physical GPUs ${GPU_PAIR}." + # setsid gives the signal handler an isolated process group to terminate on + # workflow cancellation before the GPU-pair flock is released. + setsid "$PYTHON" -m torch.distributed.run \ + --standalone \ + --nnodes=1 \ + --nproc_per_node=2 \ + -m pytest "$SOURCE_DIR/tests" -v & + TEST_PID=$! + + local test_status + set +e + wait "$TEST_PID" + test_status=$? + set -e + TEST_PID="" + + echo "[gpu-ci] Test suite exited with code ${test_status}." + return "$test_status" +} + +main() { + require_host_prerequisites + acquire_gpu_pair + configure_gpu_isolation + prepare_pr_worktree + create_job_venv + run_tests +} + +main "$@"