diff --git a/.github/workflows/local-demo-benchmarks.yml b/.github/workflows/local-demo-benchmarks.yml new file mode 100644 index 00000000..3944f2be --- /dev/null +++ b/.github/workflows/local-demo-benchmarks.yml @@ -0,0 +1,201 @@ +name: Demo Benchmarks + +on: + push: + branches: + - main + - "pull-request/[0-9]+" + paths: + - ".github/workflows/local-demo-benchmarks.yml" + - "configs/deterministic_quality_benchmarks.json" + - "flashdreams/flashdreams/**" + - "flashdreams/pyproject.toml" + - "flashdreams/tools/benchmarks/**" + - "integrations/omnidreams/**" + - "pyproject.toml" + - "uv.lock" + merge_group: + branches: [main] + workflow_dispatch: + inputs: + cuda_group: + description: CUDA dependency group to install. + required: true + default: cuda13 + type: choice + options: + - cuda13 + - cuda12 + +permissions: + contents: read + +jobs: + benchmark: + name: Demo Benchmarks + runs-on: linux-amd64-gpu-rtxpro6000-latest-2 + timeout-minutes: 180 + continue-on-error: true + defaults: + run: + shell: bash + container: + image: nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 + options: --gpus all + env: + UV_PROJECT_ENVIRONMENT: /tmp/flashdreams-local-demo-benchmark-venv + UV_LINK_MODE: copy + UV_PYTHON: "3.10" + MAX_JOBS: 8 + CUDA_GROUP: ${{ github.event.inputs.cuda_group || 'cuda13' }} + SCENARIO_FILE: configs/deterministic_quality_benchmarks.json + BENCHMARK_BASELINE_DIR: artifacts/benchmarks/local-demo-canary-baseline + BENCHMARK_CANDIDATE_DIR: artifacts/benchmarks/local-demo-canary-candidate + steps: + - name: Detect GPU architecture + id: gpu-arch + run: | + nvidia-smi + compute_cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '[:space:]') + arch=$(echo "${compute_cap}" | tr -d '.') + echo "arch=${arch}" >> "$GITHUB_OUTPUT" + echo "Detected GPU compute capability: ${compute_cap} -> sm_${arch}" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \ + python3 python3-dev python3-venv \ + ffmpeg \ + gcc g++ ninja-build \ + libnccl-dev \ + curl git ca-certificates unzip jq + rm -rf /var/lib/apt/lists/* + + - name: Setup proxy cache + uses: nv-gha-runners/setup-proxy-cache@main + + - name: Setup uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-suffix: "local-demo-benchmark-canary-${{ github.event.inputs.cuda_group || 'cuda13' }}-sm${{ steps.gpu-arch.outputs.arch }}" + prune-cache: false + + - name: Check benchmark baseline host reachability + run: | + set -ux + host=2u2g-gen-0801.ipp3a2.colossus.nvidia.com + getent hosts "${host}" || true + timeout 10 bash -c "= 100 then ((. * 10 | round) / 10 | tostring) + else ((. * 10000 | round) / 10000 | tostring) + end + else tostring + end; + def duration: + if . == null then "n/a" + elif . >= 60 then (((. / 60 | floor) | tostring) + "m " + ((. % 60 | round) | tostring) + "s") + else (((. * 10 | round) / 10 | tostring) + "s") + end; + .scenarios[] + | "| \(.id) | \(.status) | \(.wall_time_s | duration) | \(metric("gen_fps") | fmt) | \(metric("quality_score") | fmt) | \(metric("quality_similarity_score") | fmt) | \(metric("quality_psnr_db") | fmt) | \(metric("quality_rmse") | fmt) |" + ' "${manifest}" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Trim uv cache for upload + if: always() + run: | + cache_dir="${UV_CACHE_DIR:-/github/home/.cache/uv}" + echo "=== Cache size before trim ===" + du -sh "${cache_dir}" || true + du -sh "${cache_dir}"/*/ 2>/dev/null || true + + # The proxy cache can re-download wheels faster than Actions can + # upload and restore a multi-GB uv cache. + rm -rf "${cache_dir}/wheels-v6" + + # uv can re-extract unzipped wheel archives on demand. + rm -rf "${cache_dir}/archive-v0" + + # Build artifacts from git checkouts bloat the saved cache but are + # not needed for reuse. + find "${cache_dir}/git-v0/checkouts" \ + \( -name "build" -o -name "*.egg-info" -o -name "__pycache__" \) \ + -type d -exec rm -rf {} + 2>/dev/null || true + + # Workspace editable installs rebuild quickly from source. + rm -rf "${cache_dir}/sdists-v9/editable" + + echo "" + echo "=== Cache size after trim ===" + du -sh "${cache_dir}" || true + du -sh "${cache_dir}"/*/ 2>/dev/null || true + + - name: Upload benchmark artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: local-demo-benchmarks-${{ github.run_id }} + path: artifacts/benchmarks + if-no-files-found: warn