diff --git a/.buildkite/common/scripts/upload_pipeline.py b/.buildkite/common/scripts/upload_pipeline.py index 805ed12e..0eb84106 100644 --- a/.buildkite/common/scripts/upload_pipeline.py +++ b/.buildkite/common/scripts/upload_pipeline.py @@ -6,10 +6,10 @@ - Hook uploads the entry YAML (``pipeline.yml``) with one step that runs ``upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml``. - Injects ``if`` by step ``key`` from skip-ci and uploads child steps - (image build, L2/L3). + (image build, L2/L3/weekly). - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff. - When only CI level YAML changes, enable **L2/L3** upload steps for - affected levels only. + affected levels only. Weekly uses ``WEEKLY`` or the ``weekly-test`` PR label. Test pipeline mode (e.g. test-merge.yml): - Drop steps whose ``source_file_dependencies`` do not match changed files. @@ -54,10 +54,18 @@ BOOTSTRAP_UPLOAD_IF_KEYS = { "upload-ready-pipeline": "ready", "upload-merge-pipeline": "merge", + "upload-weekly-pipeline": "weekly", } E2E_GROUP_MARKER = "E2E Test" CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml" +# Weekly schedule (aligned with vllm-omni): main + WEEKLY, or PR label. +WEEKLY_MAIN_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"' +WEEKLY_LABEL_IF = ( + f"({WEEKLY_MAIN_IF}) || " + '(build.branch != "main" && build.pull_request.labels includes "weekly-test")' +) + # --- Logging --- @@ -93,40 +101,48 @@ def _format_bootstrap_if(expr: str) -> str: def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]: disabled = "false" ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"' - merge_main = 'build.branch == "main"' + # Skip regular main merge when a weekly schedule is running. + merge_main = 'build.branch == "main" && build.env("WEEKLY") != "1"' merge_pr = ( 'build.branch != "main" && build.pull_request.labels includes "merge-test"' ) ready_base = ready_pr merge_base = f"({merge_main}) || ({merge_pr})" + weekly_base = WEEKLY_LABEL_IF if platform == "cuda" else disabled if decision.skip_all: - # Docs / skip-mark only: suppress image + ready/merge uploads. - image_expr = disabled + # Docs / skip-mark only: suppress ready/merge; weekly schedule still runs. + image_expr = weekly_base if platform == "cuda" else disabled ready_expr = disabled merge_expr = disabled + weekly_expr = weekly_base elif decision.skip_l2_l3: l2_enabled = decision.is_run(platform, "l2") l3_enabled = decision.is_run(platform, "l3") ready_expr = ready_base if l2_enabled else disabled merge_expr = merge_base if l3_enabled else disabled + weekly_expr = weekly_base image_parts: list[str] = [] if l2_enabled: image_parts.append(ready_base) if l3_enabled: image_parts.append(merge_base) + if platform == "cuda": + image_parts.append(f"({weekly_base})") image_expr = " || ".join(image_parts) if image_parts else disabled else: image_expr = "true" ready_expr = ready_base merge_expr = merge_base + weekly_expr = weekly_base return { "image": _format_bootstrap_if(image_expr), "ready": _format_bootstrap_if(ready_expr), "merge": _format_bootstrap_if(merge_expr), + "weekly": _format_bootstrap_if(weekly_expr), } diff --git a/.buildkite/cuda/bootstrap-upload-steps.yml b/.buildkite/cuda/bootstrap-upload-steps.yml index 2ae71fc4..b5755857 100644 --- a/.buildkite/cuda/bootstrap-upload-steps.yml +++ b/.buildkite/cuda/bootstrap-upload-steps.yml @@ -1,6 +1,6 @@ # Uploaded by upload_pipeline.py --upload (not the Buildkite hook). # ``if`` is injected by step ``key`` from skip-ci at upload time. -# CUDA ready (L2) + merge (L3). NPU will use `.buildkite/npu/` when added. +# CUDA ready (L2) + merge (L3) + weekly. NPU will use `.buildkite/npu/` when added. steps: - label: ":docker: Build image" key: image-build @@ -26,3 +26,12 @@ steps: - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" + + # Weekly — main+WEEKLY=1 (scheduled), or PR label weekly-test + - label: "Upload Weekly Pipeline" + depends_on: image-build + key: upload-weekly-pipeline + commands: + - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml + agents: + queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml new file mode 100644 index 00000000..db3e9b0a --- /dev/null +++ b/.buildkite/cuda/test-weekly.yml @@ -0,0 +1,13 @@ +# Weekly — Simple Test + full GSM8K E2E. Uploaded by cuda/bootstrap-upload-steps.yml. +# Trigger: main + WEEKLY=1, or PR label weekly-test. +# Job layout mirrors test-ready.yml; E2E uses AFD_GSM8K_LIMIT=all (see e2e_testing.md). +# debug for test +steps: + - label: "DeepSeekV2-Lite Test" + timeout_in_minutes: 120 + commands: + - "pip install 'lm_eval[api]==0.4.12'" + - "export AFD_E2E_BACKEND=gpu" + - "export AFD_GSM8K_LIMIT=all" + - 'pytest -sv tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-graph-dbo-2a2f]' + mirror_hardwares: h100_4