From 41c48856c2f25c6bc8f7556fd2e7ee87c7ab0a8f Mon Sep 17 00:00:00 2001 From: wangyu <410167048@qq.com> Date: Tue, 25 Aug 2026 14:34:19 +0800 Subject: [PATCH 1/2] Enhance upload_pipeline.py and bootstrap-upload-steps.yml for weekly pipeline support - Updated upload_pipeline.py to include logic for handling weekly uploads based on the WEEKLY environment variable and the 'weekly-test' PR label. - Modified bootstrap-upload-steps.yml to add a new step for uploading the weekly pipeline. - Introduced a new test-weekly.yml file to define the steps for the weekly test execution. This change allows for better scheduling and management of weekly test pipelines in the CI/CD process. Signed-off-by: wangyu <410167048@qq.com> --- .buildkite/common/scripts/upload_pipeline.py | 26 ++++++++++++++++---- .buildkite/cuda/bootstrap-upload-steps.yml | 11 ++++++++- .buildkite/cuda/test-weekly.yml | 12 +++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 .buildkite/cuda/test-weekly.yml diff --git a/.buildkite/common/scripts/upload_pipeline.py b/.buildkite/common/scripts/upload_pipeline.py index 805ed12e..0eb84106 100644 --- a/.buildkite/common/scripts/upload_pipeline.py +++ b/.buildkite/common/scripts/upload_pipeline.py @@ -6,10 +6,10 @@ - Hook uploads the entry YAML (``pipeline.yml``) with one step that runs ``upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml``. - Injects ``if`` by step ``key`` from skip-ci and uploads child steps - (image build, L2/L3). + (image build, L2/L3/weekly). - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff. - When only CI level YAML changes, enable **L2/L3** upload steps for - affected levels only. + affected levels only. Weekly uses ``WEEKLY`` or the ``weekly-test`` PR label. Test pipeline mode (e.g. test-merge.yml): - Drop steps whose ``source_file_dependencies`` do not match changed files. @@ -54,10 +54,18 @@ BOOTSTRAP_UPLOAD_IF_KEYS = { "upload-ready-pipeline": "ready", "upload-merge-pipeline": "merge", + "upload-weekly-pipeline": "weekly", } E2E_GROUP_MARKER = "E2E Test" CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml" +# Weekly schedule (aligned with vllm-omni): main + WEEKLY, or PR label. +WEEKLY_MAIN_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"' +WEEKLY_LABEL_IF = ( + f"({WEEKLY_MAIN_IF}) || " + '(build.branch != "main" && build.pull_request.labels includes "weekly-test")' +) + # --- Logging --- @@ -93,40 +101,48 @@ def _format_bootstrap_if(expr: str) -> str: def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]: disabled = "false" ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"' - merge_main = 'build.branch == "main"' + # Skip regular main merge when a weekly schedule is running. + merge_main = 'build.branch == "main" && build.env("WEEKLY") != "1"' merge_pr = ( 'build.branch != "main" && build.pull_request.labels includes "merge-test"' ) ready_base = ready_pr merge_base = f"({merge_main}) || ({merge_pr})" + weekly_base = WEEKLY_LABEL_IF if platform == "cuda" else disabled if decision.skip_all: - # Docs / skip-mark only: suppress image + ready/merge uploads. - image_expr = disabled + # Docs / skip-mark only: suppress ready/merge; weekly schedule still runs. + image_expr = weekly_base if platform == "cuda" else disabled ready_expr = disabled merge_expr = disabled + weekly_expr = weekly_base elif decision.skip_l2_l3: l2_enabled = decision.is_run(platform, "l2") l3_enabled = decision.is_run(platform, "l3") ready_expr = ready_base if l2_enabled else disabled merge_expr = merge_base if l3_enabled else disabled + weekly_expr = weekly_base image_parts: list[str] = [] if l2_enabled: image_parts.append(ready_base) if l3_enabled: image_parts.append(merge_base) + if platform == "cuda": + image_parts.append(f"({weekly_base})") image_expr = " || ".join(image_parts) if image_parts else disabled else: image_expr = "true" ready_expr = ready_base merge_expr = merge_base + weekly_expr = weekly_base return { "image": _format_bootstrap_if(image_expr), "ready": _format_bootstrap_if(ready_expr), "merge": _format_bootstrap_if(merge_expr), + "weekly": _format_bootstrap_if(weekly_expr), } diff --git a/.buildkite/cuda/bootstrap-upload-steps.yml b/.buildkite/cuda/bootstrap-upload-steps.yml index 2ae71fc4..b5755857 100644 --- a/.buildkite/cuda/bootstrap-upload-steps.yml +++ b/.buildkite/cuda/bootstrap-upload-steps.yml @@ -1,6 +1,6 @@ # Uploaded by upload_pipeline.py --upload (not the Buildkite hook). # ``if`` is injected by step ``key`` from skip-ci at upload time. -# CUDA ready (L2) + merge (L3). NPU will use `.buildkite/npu/` when added. +# CUDA ready (L2) + merge (L3) + weekly. NPU will use `.buildkite/npu/` when added. steps: - label: ":docker: Build image" key: image-build @@ -26,3 +26,12 @@ steps: - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-merge.yml agents: queue: "cpu_queue_premerge" + + # Weekly — main+WEEKLY=1 (scheduled), or PR label weekly-test + - label: "Upload Weekly Pipeline" + depends_on: image-build + key: upload-weekly-pipeline + commands: + - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml + agents: + queue: "cpu_queue_premerge" diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml new file mode 100644 index 00000000..152944a6 --- /dev/null +++ b/.buildkite/cuda/test-weekly.yml @@ -0,0 +1,12 @@ +# Weekly — Simple Test + full GSM8K E2E. Uploaded by cuda/bootstrap-upload-steps.yml. +# Trigger: main + WEEKLY=1, or PR label weekly-test. +# Job layout mirrors test-ready.yml; E2E uses AFD_GSM8K_LIMIT=all (see e2e_testing.md). +steps: + - label: "DeepSeekV2-Lite Test" + timeout_in_minutes: 120 + commands: + - "pip install 'lm_eval[api]==0.4.12'" + - "export AFD_E2E_BACKEND=gpu" + - "export AFD_GSM8K_LIMIT=all" + - 'pytest -sv tests/e2e/models/deepseek_v2_lite/test_deepseek_v2_lite.py::test_deepseek_v2_lite[afd-graph-dbo-2a2f]' + mirror_hardwares: h100_4 From da3d05da38372311ef175ee4f0a0721132dd0293 Mon Sep 17 00:00:00 2001 From: wangyu <410167048@qq.com> Date: Tue, 25 Aug 2026 15:04:37 +0800 Subject: [PATCH 2/2] debug for test Signed-off-by: wangyu <410167048@qq.com> --- .buildkite/cuda/test-weekly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 152944a6..db3e9b0a 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -1,6 +1,7 @@ # Weekly — Simple Test + full GSM8K E2E. Uploaded by cuda/bootstrap-upload-steps.yml. # Trigger: main + WEEKLY=1, or PR label weekly-test. # Job layout mirrors test-ready.yml; E2E uses AFD_GSM8K_LIMIT=all (see e2e_testing.md). +# debug for test steps: - label: "DeepSeekV2-Lite Test" timeout_in_minutes: 120