[ws1]: complete C1-C5/C8 gtest validation and GPU CI gates

.github/workflows/ci.yml

@@ -67,6 +67,7 @@ jobs:
         run: |
           python -m pytest rl_engine/tests/test_dispatch.py -v
           PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest tests/test_attention_correctness.py -q -rs
+          python -m pytest tests/test_forward_invariance.py tests/test_tolerance_contract.py tests/test_ws1_workload.py tests/test_gradient_invariance.py tests/test_elementwise_inventory.py tests/test_four_judgment_matrix.py tests/test_op_checks.py tests/test_operator_inputs.py tests/test_profiler.py -q
 
       - name: Run Attention Ground-Truth Tests (CPU-safe)
         run: |

.github/workflows/ws1-gtest-gpu.yml

@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: Apache-2.0
+# WS1 single-op gtest + C8 four-judgment GPU gate (CUDA BF16 and Triton-on-CUDA BF16).
+# Uses the same RunPod orchestrator as gpu-ci.yml. Fails closed on C8 red cells
+# and on C3/C4 silent fallback. Hopper must have zero pending_hopper.
+#
+# Security: do not use pull_request_target. Fork PRs never see RunPod secrets.
+# Same-repo PRs, pushes to main, and maintainer workflow_dispatch are allowed.
+
+name: WS1-gtest-GPU
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - "rl_engine/kernels/gtest/**"
+      - "rl_engine/kernels/ops/**"
+      - "rl_engine/testing/**"
+      - "scripts/sweep_ws1_four_judgments.py"
+      - "scripts/check_forward_invariance.py"
+      - "scripts/check_gradient_invariance.py"
+      - "scripts/ws1_candidate_evidence.py"
+      - "tests/test_ws1_*.py"
+      - "tests/test_forward_invariance.py"
+      - "tests/test_gradient_invariance.py"
+      - "tests/test_four_judgment_matrix.py"
+      - "tests/test_triton_batch_invariant_attention.py"
+      - "ci/run_ws1_gtest.sh"
+      - "ci/run_gpu_ci.sh"
+      - ".github/workflows/ws1-gtest-gpu.yml"
+  push:
+    branches: [ main ]
+    paths:
+      - "rl_engine/kernels/gtest/**"
+      - "rl_engine/kernels/ops/**"
+      - "rl_engine/testing/**"
+      - "scripts/sweep_ws1_four_judgments.py"
+      - "scripts/check_forward_invariance.py"
+      - "scripts/check_gradient_invariance.py"
+      - "scripts/ws1_candidate_evidence.py"
+      - "tests/test_ws1_*.py"
+      - "tests/test_forward_invariance.py"
+      - "tests/test_gradient_invariance.py"
+      - "tests/test_four_judgment_matrix.py"
+      - "tests/test_triton_batch_invariant_attention.py"
+      - "ci/run_ws1_gtest.sh"
+      - "ci/run_gpu_ci.sh"
+      - ".github/workflows/ws1-gtest-gpu.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: ws1-gtest-gpu-${{ github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  ws1-gtest:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { gpu_id: "NVIDIA RTX A4000", target_sm: "8.6", name: "sm86-cuda-triton" }
+          - { gpu_id: "NVIDIA H100 80GB HBM3", target_sm: "9.0", force_sm90: "1", name: "sm90-c8-execute" }
+    steps:
+      - name: Checkout the commit under test
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+
+      - name: Install runpodctl
+        run: |
+          wget -qO runpodctl https://github.com/runpod/runpodctl/releases/latest/download/runpodctl-linux-amd64
+          chmod +x runpodctl
+          sudo mv runpodctl /usr/local/bin/runpodctl
+
+      - name: Configure runpodctl
+        run: runpodctl config --apiKey "${{ secrets.RUNPOD_API_KEY }}"
+
+      - name: Setup SSH key
+        run: |
+          mkdir -p ~/.ssh && chmod 700 ~/.ssh
+          printf '%s\n' "${{ secrets.RUNPOD_SSH_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+
+      - name: Run WS1 CUDA/Triton gtest + C8 on RunPod
+        env:
+          RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }}
+          PR_REPO_URL: ${{ github.event.pull_request.head.repo.clone_url || github.event.repository.clone_url }}
+          PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          GPU_ID: ${{ matrix.gpu_id }}
+          GPU_COUNT: "1"
+          TARGET_SM: ${{ matrix.target_sm }}
+          KERNEL_ALIGN_FORCE_SM90: ${{ matrix.force_sm90 }}
+          TEST_SUITE: ws1-gtest
+        run: bash ci/run_gpu_ci.sh
+
+      - name: Upload C8 execute JSON
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ws1-c8-execute-${{ matrix.name }}
+          path: artifacts/ws1-c8-ci.json
+          if-no-files-found: error

.gitignore

@@ -209,3 +209,6 @@ __marimo__/
 
 # Local dev notes (not for upstream)
 _dev_notes/
+
+# Local C8 execute dumps; default output is under TMPDIR.
+ws1-c8-ci.json

benchmarks/benchmark_sampling.py

@@ -5,7 +5,6 @@
 import time
 
 import torch
-from tabulate import tabulate
 
 from rl_engine.kernels.sampling import SamplerBackend as RL_Sampler
 from rl_engine.platforms.device import device_ctx
@@ -91,6 +90,8 @@ def run_benchmark(args, return_data: bool = False):
     if return_data:
         return raw_metrics
 
+    from tabulate import tabulate
+
     headers = ["Batch Size (G)", "Native Latency", "RL-Kernel", "Speedup"]
     print("\n" + "=" * 80)
     print(f"RL-KERNEL SAMPLING BENCHMARK REPORT (TopK={args.top_k}, TopP={args.top_p})")

ci/run_gpu_ci.sh

@@ -118,7 +118,10 @@ echo "[ci] Target Establish -> root@$SSH_IP:$SSH_PORT"
 
 SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -p $SSH_PORT"
 
-if [ "${GPU_COUNT}" -gt 1 ]; then
+TEST_SUITE="${TEST_SUITE:-full}"
+if [ "$TEST_SUITE" = "ws1-gtest" ]; then
+  TEST_CMD='bash ci/run_ws1_gtest.sh'
+elif [ "${GPU_COUNT}" -gt 1 ]; then
   TEST_CMD='"$PY" -m torch.distributed.run --nproc_per_node='"${GPU_COUNT}"' -m pytest tests/ -v'
 else
   TEST_CMD='"$PY" -m pytest tests/ -v'
@@ -186,17 +189,29 @@ TORCH_INDEX_URL="${TORCH_INDEX_URL:-https://download.pytorch.org/whl/cu124}"
 # --no-build-isolation: torch must be visible to setup.py, else the extension is silently skipped.
 # --no-deps: keep the pinned torch; do not let the editable install re-resolve it.
 "$PY" -m pip install --no-build-isolation --no-deps -e .
-"$PY" -m pip install --no-cache-dir numpy tabulate accelerate "transformers==5.13.1" pytest
+"$PY" -m pip install --no-cache-dir numpy tabulate accelerate "transformers==5.13.1" pytest triton
 nvidia-smi
 # Fail fast if _C did not build or cannot launch, instead of silently using native fallbacks.
 "$PY" scripts/ci_smoke.py
 # Enforce _C in the pytest suite too (test_extension_smoke.py skips unless this is set).
 export RL_KERNEL_REQUIRE_EXT=1
+export WS1_C8_JSON=/tmp/ws1-c8-ci.json
 '"${TEST_CMD}"
 
-echo "[ci] Launching remote test suite on GPU pod (Distributed Execution Mode: TP=${GPU_COUNT})..."
+echo "[ci] Launching remote test suite on GPU pod (Distributed Execution Mode: TP=${GPU_COUNT}, suite=${TEST_SUITE})..."
 ssh $SSH_OPTIONS root@"$SSH_IP" "bash -lc '$REMOTE_CMD'"
 TEST_EXIT=$?
 
+if [ "$TEST_SUITE" = "ws1-gtest" ]; then
+  if [ "$TEST_EXIT" -ne 0 ]; then
+    echo "[ci] Remote WS1 gtest failed with exit code = $TEST_EXIT"
+    exit "$TEST_EXIT"
+  fi
+  echo "[ci] Fetching C8 execute artifact from the pod"
+  mkdir -p artifacts
+  scp $SSH_OPTIONS root@"$SSH_IP":/tmp/ws1-c8-ci.json artifacts/ws1-c8-ci.json
+  test -s artifacts/ws1-c8-ci.json
+fi
+
 echo "[ci] Remote execution finished with exit code = $TEST_EXIT"
 exit $TEST_EXIT

ci/run_ws1_gtest.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# WS1 single-op gtest + C8 four-judgment GPU gate.
+# Assumes an editable install and a CUDA device. Fails closed on red cells
+# and on silent fallback (C3/C4 already reject those).
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+
+PY="${PY:-python3}"
+# Keep the artifact outside the repo so `_git_identity()` is not dirtied by
+# the file we are in the process of writing.
+OUT="${WS1_C8_JSON:-${TMPDIR:-/tmp}/ws1-c8-ci.json}"
+export RL_KERNEL_REQUIRE_EXT="${RL_KERNEL_REQUIRE_EXT:-1}"
+
+echo "[ws1-gtest] interpreter=$PY out=$OUT"
+
+"$PY" -m pytest -q \
+  tests/test_ws1_gtest_gpu.py \
+  tests/test_triton_batch_invariant_attention.py \
+  tests/test_four_judgment_matrix.py \
+  tests/test_ws1_candidate_evidence.py \
+  tests/test_op_checks.py \
+  tests/test_elementwise_inventory.py
+
+echo "[ws1-gtest] C3/C4 CUDA + Triton smoke (silu)"
+"$PY" scripts/check_forward_invariance.py \
+  --op silu --candidate cuda --backend-profile cuda_bf16
+"$PY" scripts/check_gradient_invariance.py \
+  --op silu --candidate cuda --backend-profile cuda_bf16
+"$PY" scripts/check_forward_invariance.py \
+  --op silu --candidate triton --backend-profile triton_cuda_bf16
+"$PY" scripts/check_gradient_invariance.py \
+  --op silu --candidate triton --backend-profile triton_cuda_bf16
+
+HOPPER=0
+if "$PY" -c "import torch,sys; sys.exit(0 if torch.cuda.is_available() and torch.cuda.get_device_capability(0)[0]==9 else 1)"; then
+  HOPPER=1
+fi
+
+echo "[ws1-gtest] C8 --execute hopper=$HOPPER"
+if [ "$HOPPER" = 1 ]; then
+  "$PY" scripts/sweep_ws1_four_judgments.py --execute --json > "$OUT"
+else
+  "$PY" scripts/sweep_ws1_four_judgments.py --execute --json --allow-pending-hopper > "$OUT"
+fi
+
+"$PY" - "$OUT" <<'PY'
+import json
+import sys
+
+path = sys.argv[1]
+payload = json.load(open(path, encoding="utf-8"))
+counts = payload.get("counts") or {}
+red = int(counts.get("red", 0))
+print(f"[ws1-gtest] C8 counts={counts} source={payload.get('git')}")
+if red:
+    raise SystemExit(f"C8 has {red} red cells")
+cells = payload.get("cells") or []
+if not cells:
+    raise SystemExit("C8 artifact contains no cells")
+if int(counts.get("green", 0)) == 0:
+    raise SystemExit("C8 artifact has no green cells")
+required = [c for c in cells if c.get("op_name") != "pack" and c.get("status") == "green"]
+if not required:
+    raise SystemExit("C8 artifact has no green required cells")
+for cell in required:
+    if not cell.get("judgment", "").endswith("invariance"):
+        continue
+    if not cell.get("actual_backend_id") or not cell.get("actual_kernel_config_id"):
+        raise SystemExit(
+            f"invariance cell missing provenance: {cell.get('profile')} {cell.get('op_name')}"
+        )
+print("[ws1-gtest] C8 gate passed")
+PY