From 86047d4901e141734a93b0081cc804a4a4a4edfb Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 26 Jul 2026 19:52:37 +0800 Subject: [PATCH 01/15] ci: split coverage into parallel producers --- .github/workflows/coverage-merge.yaml | 163 ++++++++++++++++++++++++++ .github/workflows/coverage-ut.yaml | 127 ++++++++++++++++++++ .github/workflows/e2e-compose.yaml | 55 +++++++-- .github/workflows/e2e-standalone.yaml | 61 +++++++++- .github/workflows/utils.yaml | 7 +- 5 files changed, 398 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/coverage-merge.yaml create mode 100644 .github/workflows/coverage-ut.yaml diff --git a/.github/workflows/coverage-merge.yaml b/.github/workflows/coverage-merge.yaml new file mode 100644 index 0000000..3336806 --- /dev/null +++ b/.github/workflows/coverage-merge.yaml @@ -0,0 +1,163 @@ +name: MatrixOne Coverage Merge + +on: + workflow_call: + secrets: + TOKEN_ACTION: + description: "Token for checkout and pull-request metadata" + required: false + +jobs: + check_organization_user: + name: Check coverage eligibility + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + outputs: + in_org: ${{ steps.check_in_org.outputs.in_org }} + safe_label: ${{ steps.check_safe_label.outputs.safe_label }} + steps: + - id: check_in_org + name: CHECK ORGANIZATION USER + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + set -euo pipefail + page=1 + user_in_org=0 + while :; do + users=$(curl -fsSL \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/orgs/matrixorigin/members?page=${page}&per_page=100") + if jq -e --arg user "${PR_AUTHOR}" '.[] | select(.login == $user)' <<<"${users}" >/dev/null; then + user_in_org=1 + break + fi + [ "$(jq length <<<"${users}")" -eq 100 ] || break + page=$((page + 1)) + done + echo "in_org=${user_in_org}" >> "$GITHUB_OUTPUT" + - id: check_safe_label + name: CHECK PULL REQUEST LABEL + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + PR_URL: ${{ github.event.pull_request.url }} + run: | + set -euo pipefail + labels=$(curl -fsSL \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "${PR_URL}" | jq -r '.labels[].name') + if grep -Fxq 'safe-to-test' <<<"${labels}"; then + echo 'safe_label=1' >> "$GITHUB_OUTPUT" + else + echo 'safe_label=0' >> "$GITHUB_OUTPUT" + fi + + coverage_merge: + name: Coverage + needs: [check_organization_user] + if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }} + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "1" + path: ./matrixone + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + - name: Checkout coverage parser + uses: actions/checkout@v6 + with: + token: ${{ secrets.TOKEN_ACTION }} + repository: matrixorigin/CI + fetch-depth: "1" + path: ./CI + - name: Set up Go + uses: matrixorigin/CI/actions/setup-env@main + with: + setup-java: false + go-version-file: "${{ github.workspace }}/matrixone/go.mod" + - name: Generate diff.patch + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + IS_PUB_REPO: ${{ vars.IS_PUB_REPO }} + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE/matrixone" + pr_repo='${{ github.event.pull_request.base.repo.full_name }}' + pr_number='${{ github.event.pull_request.number }}' + if [ -n "${IS_PUB_REPO:-}" ] && [ "${IS_PUB_REPO}" != "0" ] && [ "${IS_PUB_REPO}" != "false" ]; then + curl -fL --retry 5 --retry-delay 3 "https://github.com/${pr_repo}/pull/${pr_number}.diff" -o diff.patch + else + curl -fL --retry 5 --retry-delay 3 \ + -H 'Accept: application/vnd.github.v3.diff' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + "https://api.github.com/repos/${pr_repo}/pulls/${pr_number}" -o diff.patch + fi + - name: Download test coverage artifacts + uses: actions/download-artifact@v7 + with: + pattern: '*-coverage' + merge-multiple: true + path: ${{ github.workspace }}/coverage-artifacts + - name: Merge UT and BVT coverage + run: | + set -euo pipefail + artifact_dir="$GITHUB_WORKSPACE/coverage-artifacts" + processed_dir="$GITHUB_WORKSPACE/coverage-processed" + mkdir -p "${processed_dir}" + mapfile -t ut_profiles < <(find "${artifact_dir}" -type f -name 'ut-coverage.out' | sort) + mapfile -t bvt_profiles < <(find "${artifact_dir}" -type f -name 'bvt-*.out' | sort) + if [ "${#ut_profiles[@]}" -ne 1 ] || [ "${#bvt_profiles[@]}" -ne 2 ]; then + echo '::error::expected one UT profile and two BVT profiles' + find "${artifact_dir}" -type f -print || true + exit 1 + fi + coverage_files=() + for profile in "${ut_profiles[@]}"; do + output="${processed_dir}/$(basename "${profile}")" + grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${profile}" > "${output}" || true + test -s "${output}" + coverage_files+=("${output}") + done + for profile in "${bvt_profiles[@]}"; do + output="${processed_dir}/$(basename "${profile}")" + grep -Ev 'pkg/pb|yaccpar' "${profile}" > "${output}" || true + test -s "${output}" + coverage_files+=("${output}") + done + cd "$GITHUB_WORKSPACE" + cp "$GITHUB_WORKSPACE/CI/scripts/.ignore" "$GITHUB_WORKSPACE/.ignore" + python CI/scripts/parse_coverage.py \ + -coverage_files "${coverage_files[@]}" \ + -diff_path "$GITHUB_WORKSPACE/matrixone/diff.patch" + go tool cover -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" -html=merged_coverage.out + - name: Upload coverage result + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: final-result-files + path: | + ${{ github.workspace }}/matrixone/pr_coverage.html + ${{ github.workspace }}/pr_coverage.out + ${{ github.workspace }}/merged_coverage.out + if-no-files-found: warn + retention-days: 7 + - name: Upload raw coverage inputs for failed merge + if: ${{ failure() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: coverage-merge-debug + path: | + ${{ github.workspace }}/coverage-artifacts + ${{ github.workspace }}/coverage-processed + ${{ github.workspace }}/matrixone/diff.patch + if-no-files-found: warn + retention-days: 1 diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml new file mode 100644 index 0000000..f6348e7 --- /dev/null +++ b/.github/workflows/coverage-ut.yaml @@ -0,0 +1,127 @@ +name: MatrixOne UT Coverage + +on: + workflow_call: + outputs: + coverage_ready: + description: "Whether the UT coverage profile was eligible to run" + value: ${{ jobs.ut-coverage-linux-x86.outputs.coverage_ready }} + secrets: + TOKEN_ACTION: + description: "Token for checkout (e.g. pull from fork/private)" + required: false + TEST_S3FS_ALIYUN: + description: "Object storage configuration for UT coverage" + required: false + TEST_S3FS_QCLOUD: + description: "Object storage configuration for UT coverage" + required: false + S3ENDPOINT: + description: "S3ENDPOINT For Test" + required: true + S3REGION: + description: "S3REGION For Test" + required: true + S3APIKEY: + description: "S3APIKEY For Test" + required: true + S3APISECRET: + description: "S3APISECRET For Test" + required: true + S3BUCKET: + description: "S3BUCKET For Test" + required: true + +jobs: + ut-coverage-linux-x86: + name: UT Coverage on Ubuntu/x86 + environment: ci + # Match the existing PR BVT runner policy: use the configured runner label + # when present, otherwise use GitHub-hosted Ubuntu. + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + timeout-minutes: 60 + outputs: + coverage_ready: ${{ steps.coverage_secrets.outputs.ready }} + steps: + - id: coverage_secrets + name: Check coverage secrets + env: + TEST_S3FS_ALIYUN: ${{ secrets.TEST_S3FS_ALIYUN }} + TEST_S3FS_QCLOUD: ${{ secrets.TEST_S3FS_QCLOUD }} + run: | + set -euo pipefail + if [ -z "${TEST_S3FS_ALIYUN:-}" ] || [ -z "${TEST_S3FS_QCLOUD:-}" ]; then + echo 'ready=false' >> "$GITHUB_OUTPUT" + echo '::notice::UT coverage skipped because TEST_S3FS_ALIYUN or TEST_S3FS_QCLOUD is unavailable' + else + echo 'ready=true' >> "$GITHUB_OUTPUT" + fi + - name: Free hosted-runner disk space + if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} + run: | + set -euo pipefail + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /usr/share/swift \ + /usr/local/share/powershell + sudo docker system prune -af || true + if [ -d /mnt ]; then + sudo mkdir -p /mnt/runner-ut-workspace + sudo chmod 777 /mnt/runner-ut-workspace + sudo rm -rf /tmp/runner-ut-workspace + sudo ln -sfn /mnt/runner-ut-workspace /tmp/runner-ut-workspace + fi + df -h / /mnt || true + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + - name: Set up Go + if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} + uses: matrixorigin/CI/actions/setup-env@main + with: + setup-java: true + go-version-file: "${{ github.workspace }}/go.mod" + - name: Run coverage unit tests + if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE" + export endpoint='${{ secrets.S3ENDPOINT }}' + export region='${{ secrets.S3REGION }}' + export apikey='${{ secrets.S3APIKEY }}' + export apisecret='${{ secrets.S3APISECRET }}' + export bucket='${{ secrets.S3BUCKET }}' + export TEST_S3FS_ALIYUN='${{ secrets.TEST_S3FS_ALIYUN }}' + export TEST_S3FS_QCLOUD='${{ secrets.TEST_S3FS_QCLOUD }}' + + test_scope=$(go list -mod=readonly ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog') + cover_pkgs=$(printf '%s\n' "${test_scope}" | paste -sd, -) + test -n "${cover_pkgs}" + + make clean && make config && make cgo && make thirdparties + thirdparties_install_dir="$GITHUB_WORKSPACE/thirdparties/install" + cgo_cflags="-I${GITHUB_WORKSPACE}/cgo -I${thirdparties_install_dir}/include" + cgo_ldflags="-L${GITHUB_WORKSPACE}/cgo -lmo -L${thirdparties_install_dir}/lib -lusearch_c -Wl,-rpath,${thirdparties_install_dir}/lib -lm" + coverage_profile="${RUNNER_TEMP}/ut-coverage.out" + + CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" \ + go test -mod=readonly -json -short -v -tags matrixone_test -p 6 \ + -covermode=set -coverprofile="${coverage_profile}" -coverpkg="${cover_pkgs}" \ + ${test_scope} | tee "${RUNNER_TEMP}/ut-coverage-report.json" + test -s "${coverage_profile}" + grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true + test -s "${coverage_profile}.filtered" + mv "${coverage_profile}.filtered" "${coverage_profile}" + - name: Upload UT coverage + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' }} + uses: actions/upload-artifact@v7 + with: + name: ut-coverage + path: | + ${{ runner.temp }}/ut-coverage.out + ${{ runner.temp }}/ut-coverage-report.json + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/e2e-compose.yaml b/.github/workflows/e2e-compose.yaml index 4ffc3ab..10a00a4 100644 --- a/.github/workflows/e2e-compose.yaml +++ b/.github/workflows/e2e-compose.yaml @@ -180,11 +180,12 @@ jobs: ${{ github.workspace }}/docker-compose-log retention-days: 7 - multi-CN-bvt-docker-compose-pessimistic: + multi-CN-bvt-docker-compose-proxy: if: ${{ !github.event.pull_request.draft }} environment: ci + # Match the existing PR BVT runner policy. runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - name: multi cn e2e bvt test docker compose(PESSIMISTIC) + name: multi cn e2e bvt test docker compose(PROXY) timeout-minutes: 75 steps: - name: Pre Clean Some Unneeded Images @@ -233,8 +234,11 @@ jobs: # Handle various docker-compose volume formats: /test, /test:options, /test followed by space sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml - # build matrixone docker image first - docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + # Build once for this single Compose job. Splitting this image build + # into an artifact would add transfer time without reducing latency. + mkdir -p "$GITHUB_WORKSPACE/coverage" + docker build --build-arg GOBUILD_OPT=-cover \ + -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile mkdir -p ${{ github.workspace }}/docker-compose-log docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d @@ -292,7 +296,20 @@ jobs: cat mo.yml echo "=============================" - ./run.sh -n -g -o -p $GITHUB_WORKSPACE/test/distributed/cases -e optimistic 2>&1 + run_id="${{ github.run_id }}" + run_attempt="${{ github.run_attempt }}" + bvt_group=$(( (run_id + run_attempt) % 2 )) + echo "Compose + Proxy BVT: run ${run_id}, attempt ${run_attempt}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- Deployment: Compose Multi-CN + Proxy" + echo "- Group: ${bvt_group}" + } >> "$GITHUB_STEP_SUMMARY" + cd "$GITHUB_WORKSPACE" + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/test/distributed/cases" \ + "${bvt_group}" 2>&1 - name: Print Docker Info Before Container Shutdown if: ${{ always() }} @@ -330,13 +347,37 @@ jobs: run: | docker ps docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans - docker volume rm launch-tae-compose_minio_storage + docker volume rm launch-tae-compose_minio_storage || true + + - name: Generate Compose BVT coverage profile + if: ${{ always() && !cancelled() }} + run: | + set -uo pipefail + coverage_dir="$GITHUB_WORKSPACE/coverage" + coverage_profile="$RUNNER_TEMP/bvt-compose.out" + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo '::error::failed to generate Compose BVT coverage after a successful BVT' + exit 1 + else + echo '::warning::Compose BVT coverage unavailable because BVT did not finish successfully' + fi + + - name: Upload Compose BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-compose + path: ${{ runner.temp }}/bvt-compose.out + if-no-files-found: warn + retention-days: 7 - uses: actions/upload-artifact@v7 if: ${{ failure() || cancelled()}} continue-on-error: true with: - name: Compose-multi-cn-e2e-bvt-test-docker-log(PESSIMISTIC) + name: Compose-multi-cn-e2e-bvt-test-docker-log(PROXY) path: | ${{ github.workspace }}/docker-compose-log retention-days: 7 diff --git a/.github/workflows/e2e-standalone.yaml b/.github/workflows/e2e-standalone.yaml index 1ecbb25..01c41c2 100644 --- a/.github/workflows/e2e-standalone.yaml +++ b/.github/workflows/e2e-standalone.yaml @@ -145,6 +145,9 @@ jobs: retention-days: 7 multi-cn-proxy-bvt-linux-x86: + # Compose + Proxy is the active proxy coverage deployment. Keep the + # native proxy launcher out of the PR matrix until it has a distinct role. + if: false runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} name: Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) timeout-minutes: 75 @@ -234,6 +237,7 @@ jobs: retention-days: 7 pessimistic-bvt-linux-x86: + # Match the existing PR BVT runner policy. runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} name: e2e BVT Test on Linux/x64(LAUNCH, PESSIMISTIC) timeout-minutes: 75 @@ -256,7 +260,7 @@ jobs: - name: Build MatrixOne run: | - cd $GITHUB_WORKSPACE/head && make clean && make build + cd $GITHUB_WORKSPACE/head && make clean && make GOBUILD_OPT=-cover build git rev-parse --short HEAD - name: Add cn.txn @@ -275,6 +279,9 @@ jobs: - name: Start MO run: | cd $GITHUB_WORKSPACE/head + export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-pessimistic" + rm -rf "${GOCOVERDIR}" + mkdir -p "${GOCOVERDIR}" ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch - name: Clone test-tool repository @@ -292,12 +299,25 @@ jobs: run: | export LC_ALL="C.UTF-8" locale - cd $GITHUB_WORKSPACE/mo-tester + cd "$GITHUB_WORKSPACE/mo-tester" sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml cat mo.yml echo "=============================" - ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -s $GITHUB_WORKSPACE/head/test/distributed/resources -e optimistic 2>&1 + run_id="${{ github.run_id }}" + run_attempt="${{ github.run_attempt }}" + bvt_group=$(( 1 - ((run_id + run_attempt) % 2) )) + echo "Launch + Pessimistic BVT: run ${run_id}, attempt ${run_attempt}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- Deployment: native launch (PESSIMISTIC)" + echo "- Group: ${bvt_group}" + } >> "$GITHUB_STEP_SUMMARY" + cd "$GITHUB_WORKSPACE/head" + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/head/test/distributed/cases" \ + "${bvt_group}" 2>&1 - name: Dump mo-service goroutines if: ${{ always() && !cancelled() }} @@ -311,10 +331,41 @@ jobs: # 4 nodes in one Process ./optools/check_log_count.sh 4000 60 # {count threshold} {metric collected interval} - - name: Check mo-service Status + - name: Stop service and generate BVT coverage profile if: ${{ always() && !cancelled() }} run: | - if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then pkill -9 mo-service; else echo 'current mo-service has already crashed'; exit 1; fi + set -uo pipefail + coverage_dir="${RUNNER_TEMP}/bvt-coverage-pessimistic" + coverage_profile="${RUNNER_TEMP}/bvt-pessimistic.out" + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -TERM ${pids} || true + fi + for _ in $(seq 1 30); do + pgrep -f 'mo-service' >/dev/null || break + sleep 1 + done + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -KILL ${pids} || true + fi + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo '::error::failed to generate Launch + Pessimistic BVT coverage after a successful BVT' + exit 1 + else + echo '::warning::Launch + Pessimistic BVT coverage unavailable because BVT did not finish successfully' + fi + + - name: Upload Launch + Pessimistic BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-pessimistic + path: ${{ runner.temp }}/bvt-pessimistic.out + if-no-files-found: warn + retention-days: 7 - name: generate upload files if: ${{ always() }} diff --git a/.github/workflows/utils.yaml b/.github/workflows/utils.yaml index 4656f11..37a8478 100644 --- a/.github/workflows/utils.yaml +++ b/.github/workflows/utils.yaml @@ -111,9 +111,10 @@ jobs: fi pr_coverage: - # Secrets cannot be referenced directly in job-level conditions; preflight prevents allocating the large ARC runner. - if: ${{ (needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1') && needs.check_organization_user.outputs.coverage_secrets_available == 'true' }} - name: Coverage + # Replaced by coverage-ut.yaml plus coverage-merge.yaml. Keep this job + # disabled during the rollout so the legacy serial UT+BVT path cannot run. + if: ${{ false }} + name: Legacy serial coverage (disabled) needs: [check_organization_user] environment: ci # If the runner/workspace layout changes, also update setup-env's go-version-file below. From 3ebd472e8618e6c259ac10ca93104f86bf75bb98 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 26 Jul 2026 21:35:57 +0800 Subject: [PATCH 02/15] ci: report coverage UT failures --- .github/workflows/coverage-ut.yaml | 38 ++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index f6348e7..894bbb3 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -84,8 +84,10 @@ jobs: with: setup-java: true go-version-file: "${{ github.workspace }}/go.mod" - - name: Run coverage unit tests + - id: coverage_ut + name: Run coverage unit tests if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} + continue-on-error: true run: | set -euo pipefail cd "$GITHUB_WORKSPACE" @@ -106,15 +108,44 @@ jobs: cgo_cflags="-I${GITHUB_WORKSPACE}/cgo -I${thirdparties_install_dir}/include" cgo_ldflags="-L${GITHUB_WORKSPACE}/cgo -lmo -L${thirdparties_install_dir}/lib -lusearch_c -Wl,-rpath,${thirdparties_install_dir}/lib -lm" coverage_profile="${RUNNER_TEMP}/ut-coverage.out" + coverage_report="${RUNNER_TEMP}/ut-coverage-report.json" + # Keep the same all-package coverage scope and external storage + # coverage as main. Save the verbose JSON report as an artifact + # rather than flooding the Actions console; the next step prints a + # concise parsed failure summary when this command fails. + set +e CGO_CFLAGS="${cgo_cflags}" CGO_LDFLAGS="${cgo_ldflags}" \ go test -mod=readonly -json -short -v -tags matrixone_test -p 6 \ -covermode=set -coverprofile="${coverage_profile}" -coverpkg="${cover_pkgs}" \ - ${test_scope} | tee "${RUNNER_TEMP}/ut-coverage-report.json" + ${test_scope} \ + > "${coverage_report}" 2>&1 + test_status=$? + set -e + echo "test_status=${test_status}" >> "$GITHUB_OUTPUT" + if (( test_status != 0 )); then + exit "${test_status}" + fi test -s "${coverage_profile}" grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true test -s "${coverage_profile}.filtered" mv "${coverage_profile}.filtered" "${coverage_profile}" + - name: Print failed coverage UT packages and tests + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/ut-coverage-report.json" + echo '::group::Failed coverage UT packages' + if [ -s "${report}" ]; then + jq -r 'select(.Action == "fail" and .Package != null) | "FAIL \(.Package) (\(.Elapsed // 0)s)"' "${report}" | sort -u || true + echo '::endgroup::' + echo '::group::Failure details' + jq -r 'select(.Action == "output" and (.Output | test("panic: test timed out|--- FAIL:|^FAIL\\t"))) | "\(.Package // "unknown") / \(.Test // "package"): \(.Output | gsub("\\n$"; ""))"' "${report}" \ + | tail -n 100 | cut -c 1-1000 || true + else + echo 'No JSON test report was produced; inspect the failed test step.' + fi + echo '::endgroup::' - name: Upload UT coverage if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' }} uses: actions/upload-artifact@v7 @@ -125,3 +156,6 @@ jobs: ${{ runner.temp }}/ut-coverage-report.json if-no-files-found: warn retention-days: 7 + - name: Fail on coverage UT failure + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + run: exit 1 From 57ad5823abb7caf16e9f8ef36507dbd5f189a45f Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 26 Jul 2026 22:08:35 +0800 Subject: [PATCH 03/15] ci: align coverage runner with main --- .github/workflows/coverage-ut.yaml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index 894bbb3..275ce48 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -36,9 +36,9 @@ jobs: ut-coverage-linux-x86: name: UT Coverage on Ubuntu/x86 environment: ci - # Match the existing PR BVT runner policy: use the configured runner label - # when present, otherwise use GitHub-hosted Ubuntu. - runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + # Match main's legacy coverage job. The external Aliyun/QCloud integration + # tests depend on its Guangzhou network path as well as their credentials. + runs-on: amd64-mo-guangzhou-2xlarge16 timeout-minutes: 60 outputs: coverage_ready: ${{ steps.coverage_secrets.outputs.ready }} @@ -56,21 +56,6 @@ jobs: else echo 'ready=true' >> "$GITHUB_OUTPUT" fi - - name: Free hosted-runner disk space - if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} - run: | - set -euo pipefail - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /opt/hostedtoolcache/CodeQL /usr/local/.ghcup /usr/share/swift \ - /usr/local/share/powershell - sudo docker system prune -af || true - if [ -d /mnt ]; then - sudo mkdir -p /mnt/runner-ut-workspace - sudo chmod 777 /mnt/runner-ut-workspace - sudo rm -rf /tmp/runner-ut-workspace - sudo ln -sfn /mnt/runner-ut-workspace /tmp/runner-ut-workspace - fi - df -h / /mnt || true - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} with: From 752dfbec2d7644d741793b2427c8f4bfd3838de5 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 26 Jul 2026 22:16:42 +0800 Subject: [PATCH 04/15] ci: rightsize coverage UT runner --- .github/workflows/coverage-ut.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index 275ce48..0f3e47a 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -36,9 +36,10 @@ jobs: ut-coverage-linux-x86: name: UT Coverage on Ubuntu/x86 environment: ci - # Match main's legacy coverage job. The external Aliyun/QCloud integration - # tests depend on its Guangzhou network path as well as their credentials. - runs-on: amd64-mo-guangzhou-2xlarge16 + # Keep the Guangzhou network path used by main's external Aliyun/QCloud + # tests. Unlike the legacy serial job, this job runs UT only (no BVT), so + # use the smaller runner already used by cloud-resource CI tasks. + runs-on: amd64-mo-guangzhou-medium8 timeout-minutes: 60 outputs: coverage_ready: ${{ steps.coverage_secrets.outputs.ready }} From 8a4eaa9f21daa624bd5255877ce0d24a272ad655 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 26 Jul 2026 23:12:22 +0800 Subject: [PATCH 05/15] ci: rightsize coverage and SCA runners --- .github/workflows/ci.yaml | 5 ++++- .github/workflows/coverage-ut.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f87a1f7..a5d007d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -216,7 +216,10 @@ jobs: sca: environment: ci - runs-on: ${{ vars.SCA_RUNNER_LABEL || 'arm64-mo-shanghai-8c16g' }} + # SCA is CPU-bound but does not run the database or BVT; validate it on + # the smaller standard ARM runner. A repository variable remains an + # explicit escape hatch for a larger runner. + runs-on: ${{ vars.SCA_RUNNER_LABEL || 'arm64-mo-shanghai-4c8g' }} name: SCA Test on Linux/arm64 steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index 0f3e47a..746d3b7 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -36,10 +36,10 @@ jobs: ut-coverage-linux-x86: name: UT Coverage on Ubuntu/x86 environment: ci - # Keep the Guangzhou network path used by main's external Aliyun/QCloud - # tests. Unlike the legacy serial job, this job runs UT only (no BVT), so - # use the smaller runner already used by cloud-resource CI tasks. - runs-on: amd64-mo-guangzhou-medium8 + # Preserve the Guangzhou network path and capacity used by main's + # external Aliyun/QCloud coverage UT. This is intentionally larger than + # the UT-only minimum while we validate the external-storage timeout fix. + runs-on: amd64-mo-guangzhou-2xlarge16 timeout-minutes: 60 outputs: coverage_ready: ${{ steps.coverage_secrets.outputs.ready }} From 788276b9b12c84894a060399b41da80c07e6b775 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Sun, 26 Jul 2026 23:55:44 +0800 Subject: [PATCH 06/15] fix: download BVT coverage profiles --- .github/workflows/coverage-merge.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage-merge.yaml b/.github/workflows/coverage-merge.yaml index 3336806..a4b460e 100644 --- a/.github/workflows/coverage-merge.yaml +++ b/.github/workflows/coverage-merge.yaml @@ -104,7 +104,9 @@ jobs: - name: Download test coverage artifacts uses: actions/download-artifact@v7 with: - pattern: '*-coverage' + # The producers use ut-coverage and bvt-coverage-. + # '*-coverage' only matches the former; retain all three inputs. + pattern: '*coverage*' merge-multiple: true path: ${{ github.workspace }}/coverage-artifacts - name: Merge UT and BVT coverage From 3d470a8721b738628819d20ffff53855e0d345f1 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 00:05:48 +0800 Subject: [PATCH 07/15] ci: align coverage observability --- .github/workflows/coverage-merge.yaml | 19 ++++++++ .github/workflows/coverage-ut.yaml | 65 ++++++++++++++++++++++++++- .github/workflows/e2e-compose.yaml | 31 ++++++++++++- .github/workflows/e2e-standalone.yaml | 31 ++++++++++++- 4 files changed, 142 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coverage-merge.yaml b/.github/workflows/coverage-merge.yaml index a4b460e..8740462 100644 --- a/.github/workflows/coverage-merge.yaml +++ b/.github/workflows/coverage-merge.yaml @@ -110,6 +110,7 @@ jobs: merge-multiple: true path: ${{ github.workspace }}/coverage-artifacts - name: Merge UT and BVT coverage + id: merge_coverage run: | set -euo pipefail artifact_dir="$GITHUB_WORKSPACE/coverage-artifacts" @@ -141,6 +142,24 @@ jobs: -coverage_files "${coverage_files[@]}" \ -diff_path "$GITHUB_WORKSPACE/matrixone/diff.patch" go tool cover -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" -html=merged_coverage.out + - name: Summarize coverage merge result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + echo '### Coverage merge result' >> "$GITHUB_STEP_SUMMARY" + echo "- Merge step: \`${{ steps.merge_coverage.outcome }}\`" >> "$GITHUB_STEP_SUMMARY" + echo '- Expected inputs: one UT profile and two BVT profiles.' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '#### Downloaded profiles' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + find "$GITHUB_WORKSPACE/coverage-artifacts" -type f -name '*.out' -printf '%f\n' 2>/dev/null | sort >> "$GITHUB_STEP_SUMMARY" || true + echo '```' >> "$GITHUB_STEP_SUMMARY" + if [ "${{ steps.merge_coverage.outcome }}" = 'success' ]; then + echo '- Result: `final-result-files` contains the merged profile and HTML report.' >> "$GITHUB_STEP_SUMMARY" + else + echo '- Diagnostics: download `coverage-merge-debug` for profiles, processed inputs, and the PR diff.' >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload coverage result if: ${{ always() && !cancelled() }} uses: actions/upload-artifact@v7 diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index 746d3b7..53cb4a7 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -116,6 +116,26 @@ jobs: grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true test -s "${coverage_profile}.filtered" mv "${coverage_profile}.filtered" "${coverage_profile}" + - name: Summarize 30 slowest coverage UT tests + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/ut-coverage-report.json" + echo '### Coverage UT timing' >> "$GITHUB_STEP_SUMMARY" + if [ "${{ steps.coverage_ut.outcome }}" = 'failure' ]; then + echo '- Partial result: the failed run did not complete every test.' >> "$GITHUB_STEP_SUMMARY" + fi + if [ ! -s "${report}" ]; then + echo '- No Go JSON report was produced.' >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '#### Top 30 slowest completed tests' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + jq -r 'select(.Action == "pass" and (.Test | type == "string") and .Elapsed != null) | [(.Elapsed | tonumber), .Package, .Test] | @tsv' "${report}" \ + | sort -nr -k1,1 | head -n 30 >> "$GITHUB_STEP_SUMMARY" || true + echo '```' >> "$GITHUB_STEP_SUMMARY" - name: Print failed coverage UT packages and tests if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} run: | @@ -132,14 +152,55 @@ jobs: echo 'No JSON test report was produced; inspect the failed test step.' fi echo '::endgroup::' - - name: Upload UT coverage + # Match the legacy coverage job's actionable failure report while keeping + # the large JSON stream out of successful coverage artifacts. + - name: Checkout CI repository for UT failure summary + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + uses: actions/checkout@v6 + with: + token: ${{ secrets.TOKEN_ACTION }} + repository: matrixorigin/CI + fetch-depth: "1" + path: CI + - name: Analyze and summarize failed coverage UT + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + continue-on-error: true + run: | + set -uo pipefail + report="${RUNNER_TEMP}/ut-coverage-report.json" + summary="${RUNNER_TEMP}/ut-failure-summary.md" + : > "${summary}" + if [ ! -s "${report}" ]; then + echo 'UT JSON report was not produced; inspect the failed test step.' | tee -a "${summary}" + else + go install github.com/matrixorigin/go-ut-analysis@latest || true + if command -v go-ut-analysis >/dev/null 2>&1; then + go-ut-analysis test -f "${report}" --first 10 --report-path "${RUNNER_TEMP}/ut-report" --stdout=false || true + else + echo 'go-ut-analysis is unavailable; the raw JSON report is attached.' | tee -a "${summary}" + fi + python "$GITHUB_WORKSPACE/CI/scripts/summarize_ut_report.py" "${report}" | tee -a "${summary}" || true + fi + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "${summary}" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload UT coverage profile if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' }} uses: actions/upload-artifact@v7 with: name: ut-coverage + path: ${{ runner.temp }}/ut-coverage.out + if-no-files-found: warn + retention-days: 7 + - name: Upload failed UT diagnostics + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + uses: actions/upload-artifact@v7 + with: + name: ut-coverage-diagnostics path: | - ${{ runner.temp }}/ut-coverage.out ${{ runner.temp }}/ut-coverage-report.json + ${{ runner.temp }}/ut-failure-summary.md + ${{ runner.temp }}/ut-report if-no-files-found: warn retention-days: 7 - name: Fail on coverage UT failure diff --git a/.github/workflows/e2e-compose.yaml b/.github/workflows/e2e-compose.yaml index 10a00a4..cd814f0 100644 --- a/.github/workflows/e2e-compose.yaml +++ b/.github/workflows/e2e-compose.yaml @@ -306,10 +306,39 @@ jobs: echo "- Group: ${bvt_group}" } >> "$GITHUB_STEP_SUMMARY" cd "$GITHUB_WORKSPACE" + set -o pipefail bash ./optools/run_bvt_group.sh \ "$GITHUB_WORKSPACE/mo-tester" \ "$GITHUB_WORKSPACE/test/distributed/cases" \ - "${bvt_group}" 2>&1 + "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-compose.log" + + - name: Summarize Compose BVT result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/bvt-compose.log" + echo '### Compose + Proxy BVT result' >> "$GITHUB_STEP_SUMMARY" + if [ ! -s "${report}" ]; then + echo '- No BVT execution log was produced.' >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + grep '^Run BVT group' "${report}" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" || true + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '#### Top 10 slow BVT scripts' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ + | sort -nr -k1,1 | head -n 10 >> "$GITHUB_STEP_SUMMARY" || true + echo '```' >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Compose BVT execution log + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-execution-compose + path: ${{ runner.temp }}/bvt-compose.log + if-no-files-found: warn + retention-days: 7 - name: Print Docker Info Before Container Shutdown if: ${{ always() }} diff --git a/.github/workflows/e2e-standalone.yaml b/.github/workflows/e2e-standalone.yaml index 01c41c2..aaf1fab 100644 --- a/.github/workflows/e2e-standalone.yaml +++ b/.github/workflows/e2e-standalone.yaml @@ -314,10 +314,39 @@ jobs: echo "- Group: ${bvt_group}" } >> "$GITHUB_STEP_SUMMARY" cd "$GITHUB_WORKSPACE/head" + set -o pipefail bash ./optools/run_bvt_group.sh \ "$GITHUB_WORKSPACE/mo-tester" \ "$GITHUB_WORKSPACE/head/test/distributed/cases" \ - "${bvt_group}" 2>&1 + "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log" + + - name: Summarize Launch + Pessimistic BVT result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/bvt-pessimistic.log" + echo '### Launch + Pessimistic BVT result' >> "$GITHUB_STEP_SUMMARY" + if [ ! -s "${report}" ]; then + echo '- No BVT execution log was produced.' >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + grep '^Run BVT group' "${report}" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" || true + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '#### Top 10 slow BVT scripts' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ + | sort -nr -k1,1 | head -n 10 >> "$GITHUB_STEP_SUMMARY" || true + echo '```' >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Launch + Pessimistic BVT execution log + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-execution-pessimistic + path: ${{ runner.temp }}/bvt-pessimistic.log + if-no-files-found: warn + retention-days: 7 - name: Dump mo-service goroutines if: ${{ always() && !cancelled() }} From 1e108d944c1f5d58929ca00c3ee3e59f1c98a182 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 00:32:53 +0800 Subject: [PATCH 08/15] ci: print coverage UT timing summary --- .github/workflows/coverage-ut.yaml | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index 53cb4a7..fc13474 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -121,21 +121,28 @@ jobs: run: | set -euo pipefail report="${RUNNER_TEMP}/ut-coverage-report.json" - echo '### Coverage UT timing' >> "$GITHUB_STEP_SUMMARY" + timing="${RUNNER_TEMP}/ut-coverage-top30.txt" + { + echo '### Coverage UT timing' if [ "${{ steps.coverage_ut.outcome }}" = 'failure' ]; then - echo '- Partial result: the failed run did not complete every test.' >> "$GITHUB_STEP_SUMMARY" + echo '- Partial result: the failed run did not complete every test.' fi if [ ! -s "${report}" ]; then - echo '- No Go JSON report was produced.' >> "$GITHUB_STEP_SUMMARY" - exit 0 + echo '- No Go JSON report was produced.' + exit 0 fi - echo '' >> "$GITHUB_STEP_SUMMARY" - echo '#### Top 30 slowest completed tests' >> "$GITHUB_STEP_SUMMARY" - echo '' >> "$GITHUB_STEP_SUMMARY" - echo '```text' >> "$GITHUB_STEP_SUMMARY" - jq -r 'select(.Action == "pass" and (.Test | type == "string") and .Elapsed != null) | [(.Elapsed | tonumber), .Package, .Test] | @tsv' "${report}" \ - | sort -nr -k1,1 | head -n 30 >> "$GITHUB_STEP_SUMMARY" || true - echo '```' >> "$GITHUB_STEP_SUMMARY" + if ! jq -r 'select(.Action == "pass" and (.Test | type == "string") and .Elapsed != null) | [(.Elapsed | tonumber), .Package, .Test] | @tsv' "${report}" \ + | sort -nr -k1,1 | awk 'NR <= 30 { print }' > "${timing}"; then + echo '::warning::failed to generate the UT timing summary' + exit 0 + fi + echo '' + echo '#### Top 30 slowest completed tests' + echo '' + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Print failed coverage UT packages and tests if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} run: | From c01bfd01cb6b2b7017923b46c36fdf50423d0953 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 00:41:47 +0800 Subject: [PATCH 09/15] ci: improve BVT result observability --- .github/workflows/e2e-compose.yaml | 30 ++++++++++++++++------- .github/workflows/e2e-standalone.yaml | 34 +++++++++++++++++++-------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e-compose.yaml b/.github/workflows/e2e-compose.yaml index cd814f0..2255f51 100644 --- a/.github/workflows/e2e-compose.yaml +++ b/.github/workflows/e2e-compose.yaml @@ -317,19 +317,31 @@ jobs: run: | set -euo pipefail report="${RUNNER_TEMP}/bvt-compose.log" - echo '### Compose + Proxy BVT result' >> "$GITHUB_STEP_SUMMARY" + timing="${RUNNER_TEMP}/bvt-compose-slowest.tsv" if [ ! -s "${report}" ]; then - echo '- No BVT execution log was produced.' >> "$GITHUB_STEP_SUMMARY" + echo '### Compose + Proxy BVT result' | tee -a "$GITHUB_STEP_SUMMARY" + echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" exit 0 fi - grep '^Run BVT group' "${report}" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" || true - echo '' >> "$GITHUB_STEP_SUMMARY" - echo '#### Top 10 slow BVT scripts' >> "$GITHUB_STEP_SUMMARY" - echo '' >> "$GITHUB_STEP_SUMMARY" - echo '```text' >> "$GITHUB_STEP_SUMMARY" + { + echo '### Compose + Proxy BVT result' + grep '^Run BVT group' "${report}" | sed 's/^/- /' || true + echo + echo '#### Top 10 slow BVT scripts' + echo + } | tee -a "$GITHUB_STEP_SUMMARY" + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ - | sort -nr -k1,1 | head -n 10 >> "$GITHUB_STEP_SUMMARY" || true - echo '```' >> "$GITHUB_STEP_SUMMARY" + | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" + if [ ! -s "${timing}" ]; then + echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Upload Compose BVT execution log if: ${{ always() && !cancelled() }} diff --git a/.github/workflows/e2e-standalone.yaml b/.github/workflows/e2e-standalone.yaml index aaf1fab..4c5e8a9 100644 --- a/.github/workflows/e2e-standalone.yaml +++ b/.github/workflows/e2e-standalone.yaml @@ -318,26 +318,40 @@ jobs: bash ./optools/run_bvt_group.sh \ "$GITHUB_WORKSPACE/mo-tester" \ "$GITHUB_WORKSPACE/head/test/distributed/cases" \ - "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log" + "${bvt_group}" \ + "$GITHUB_WORKSPACE/head/test/distributed/resources" \ + 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log" - name: Summarize Launch + Pessimistic BVT result if: ${{ always() && !cancelled() }} run: | set -euo pipefail report="${RUNNER_TEMP}/bvt-pessimistic.log" - echo '### Launch + Pessimistic BVT result' >> "$GITHUB_STEP_SUMMARY" + timing="${RUNNER_TEMP}/bvt-pessimistic-slowest.tsv" if [ ! -s "${report}" ]; then - echo '- No BVT execution log was produced.' >> "$GITHUB_STEP_SUMMARY" + echo '### Launch + Pessimistic BVT result' | tee -a "$GITHUB_STEP_SUMMARY" + echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" exit 0 fi - grep '^Run BVT group' "${report}" | sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" || true - echo '' >> "$GITHUB_STEP_SUMMARY" - echo '#### Top 10 slow BVT scripts' >> "$GITHUB_STEP_SUMMARY" - echo '' >> "$GITHUB_STEP_SUMMARY" - echo '```text' >> "$GITHUB_STEP_SUMMARY" + { + echo '### Launch + Pessimistic BVT result' + grep '^Run BVT group' "${report}" | sed 's/^/- /' || true + echo + echo '#### Top 10 slow BVT scripts' + echo + } | tee -a "$GITHUB_STEP_SUMMARY" + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ - | sort -nr -k1,1 | head -n 10 >> "$GITHUB_STEP_SUMMARY" || true - echo '```' >> "$GITHUB_STEP_SUMMARY" + | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" + if [ ! -s "${timing}" ]; then + echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Upload Launch + Pessimistic BVT execution log if: ${{ always() && !cancelled() }} From 358b6f205602abc5ff30eec0d9726a32b5ad5784 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 01:36:15 +0800 Subject: [PATCH 10/15] fix: compact coverage artifacts before merge --- .github/workflows/coverage-merge.yaml | 19 ++++++++++------ .github/workflows/coverage-ut.yaml | 31 +++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/.github/workflows/coverage-merge.yaml b/.github/workflows/coverage-merge.yaml index 8740462..7be6394 100644 --- a/.github/workflows/coverage-merge.yaml +++ b/.github/workflows/coverage-merge.yaml @@ -141,7 +141,12 @@ jobs: python CI/scripts/parse_coverage.py \ -coverage_files "${coverage_files[@]}" \ -diff_path "$GITHUB_WORKSPACE/matrixone/diff.patch" - go tool cover -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" -html=merged_coverage.out + # Coverage entries use MatrixOne module import paths. Run cover from + # that module root while keeping the merged profile in the workspace. + cd "$GITHUB_WORKSPACE/matrixone" + go tool cover \ + -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" \ + -html="$GITHUB_WORKSPACE/merged_coverage.out" - name: Summarize coverage merge result if: ${{ always() && !cancelled() }} run: | @@ -153,12 +158,15 @@ jobs: echo '#### Downloaded profiles' >> "$GITHUB_STEP_SUMMARY" echo '' >> "$GITHUB_STEP_SUMMARY" echo '```text' >> "$GITHUB_STEP_SUMMARY" - find "$GITHUB_WORKSPACE/coverage-artifacts" -type f -name '*.out' -printf '%f\n' 2>/dev/null | sort >> "$GITHUB_STEP_SUMMARY" || true + inventory="${RUNNER_TEMP}/coverage-inputs.txt" + find "$GITHUB_WORKSPACE/coverage-artifacts" -type f -name '*.out' \ + -printf '%f %s bytes\n' 2>/dev/null | sort > "${inventory}" || true + cat "${inventory}" >> "$GITHUB_STEP_SUMMARY" echo '```' >> "$GITHUB_STEP_SUMMARY" if [ "${{ steps.merge_coverage.outcome }}" = 'success' ]; then echo '- Result: `final-result-files` contains the merged profile and HTML report.' >> "$GITHUB_STEP_SUMMARY" else - echo '- Diagnostics: download `coverage-merge-debug` for profiles, processed inputs, and the PR diff.' >> "$GITHUB_STEP_SUMMARY" + echo '- Diagnostics: `coverage-merge-debug` contains the input inventory and PR diff; `final-result-files` retains any merged outputs.' >> "$GITHUB_STEP_SUMMARY" fi - name: Upload coverage result if: ${{ always() && !cancelled() }} @@ -171,14 +179,13 @@ jobs: ${{ github.workspace }}/merged_coverage.out if-no-files-found: warn retention-days: 7 - - name: Upload raw coverage inputs for failed merge + - name: Upload coverage merge diagnostics if: ${{ failure() && !cancelled() }} uses: actions/upload-artifact@v7 with: name: coverage-merge-debug path: | - ${{ github.workspace }}/coverage-artifacts - ${{ github.workspace }}/coverage-processed + ${{ runner.temp }}/coverage-inputs.txt ${{ github.workspace }}/matrixone/diff.patch if-no-files-found: warn retention-days: 1 diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index fc13474..af4e91f 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -113,9 +113,32 @@ jobs: exit "${test_status}" fi test -s "${coverage_profile}" - grep -Ev 'pkg/pb|pkg/sql/parsers/goyacc|yaccpar' "${coverage_profile}" > "${coverage_profile}.filtered" || true - test -s "${coverage_profile}.filtered" - mv "${coverage_profile}.filtered" "${coverage_profile}" + raw_coverage_bytes=$(wc -c < "${coverage_profile}") + # -coverpkg emits the same instrumented block once per tested + # package. Collapse those duplicate records before crossing the + # artifact boundary; the coverage gate only needs hit/not-hit + # semantics and parse_coverage.py applies the same max(hit) merge. + LC_ALL=C awk ' + NR == 1 { next } + $1 ~ /pkg\/pb|pkg\/sql\/parsers\/goyacc|yaccpar/ { next } + NF >= 3 { + key = $1 " " $2 + hit = ($3 + 0 > 0) ? 1 : 0 + if (!(key in coverage) || hit > coverage[key]) { + coverage[key] = hit + } + } + END { + print "mode: set" + for (key in coverage) { + print key, coverage[key] + } + } + ' "${coverage_profile}" > "${coverage_profile}.compacted" + test "$(wc -l < "${coverage_profile}.compacted")" -gt 1 + mv "${coverage_profile}.compacted" "${coverage_profile}" + compacted_coverage_bytes=$(wc -c < "${coverage_profile}") + echo "UT coverage profile compacted from ${raw_coverage_bytes} to ${compacted_coverage_bytes} bytes" - name: Summarize 30 slowest coverage UT tests if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' }} run: | @@ -192,7 +215,7 @@ jobs: cat "${summary}" >> "$GITHUB_STEP_SUMMARY" fi - name: Upload UT coverage profile - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' }} + if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'success' }} uses: actions/upload-artifact@v7 with: name: ut-coverage From 27e6031f944625cccfc3a65c867734732628ed4a Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 01:57:18 +0800 Subject: [PATCH 11/15] ci: gate coverage before running untrusted code --- .github/workflows/coverage-merge.yaml | 51 ---------------------- .github/workflows/coverage-ut.yaml | 63 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/.github/workflows/coverage-merge.yaml b/.github/workflows/coverage-merge.yaml index 7be6394..815c985 100644 --- a/.github/workflows/coverage-merge.yaml +++ b/.github/workflows/coverage-merge.yaml @@ -8,59 +8,8 @@ on: required: false jobs: - check_organization_user: - name: Check coverage eligibility - environment: ci - runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - outputs: - in_org: ${{ steps.check_in_org.outputs.in_org }} - safe_label: ${{ steps.check_safe_label.outputs.safe_label }} - steps: - - id: check_in_org - name: CHECK ORGANIZATION USER - env: - GH_TOKEN: ${{ secrets.TOKEN_ACTION }} - PR_AUTHOR: ${{ github.event.pull_request.user.login }} - run: | - set -euo pipefail - page=1 - user_in_org=0 - while :; do - users=$(curl -fsSL \ - -H 'Accept: application/vnd.github+json' \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "https://api.github.com/orgs/matrixorigin/members?page=${page}&per_page=100") - if jq -e --arg user "${PR_AUTHOR}" '.[] | select(.login == $user)' <<<"${users}" >/dev/null; then - user_in_org=1 - break - fi - [ "$(jq length <<<"${users}")" -eq 100 ] || break - page=$((page + 1)) - done - echo "in_org=${user_in_org}" >> "$GITHUB_OUTPUT" - - id: check_safe_label - name: CHECK PULL REQUEST LABEL - env: - GH_TOKEN: ${{ secrets.TOKEN_ACTION }} - PR_URL: ${{ github.event.pull_request.url }} - run: | - set -euo pipefail - labels=$(curl -fsSL \ - -H 'Accept: application/vnd.github+json' \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "${PR_URL}" | jq -r '.labels[].name') - if grep -Fxq 'safe-to-test' <<<"${labels}"; then - echo 'safe_label=1' >> "$GITHUB_OUTPUT" - else - echo 'safe_label=0' >> "$GITHUB_OUTPUT" - fi - coverage_merge: name: Coverage - needs: [check_organization_user] - if: ${{ needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1' }} environment: ci runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} timeout-minutes: 20 diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index af4e91f..efac4f8 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -33,8 +33,71 @@ on: required: true jobs: + check_coverage_eligibility: + name: Check coverage eligibility + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + outputs: + allowed: ${{ steps.eligibility.outputs.allowed }} + steps: + - id: check_in_org + name: Check organization membership + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + set -euo pipefail + page=1 + user_in_org=0 + while :; do + users=$(curl -fsSL \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/orgs/matrixorigin/members?page=${page}&per_page=100") + if jq -e --arg user "${PR_AUTHOR}" '.[] | select(.login == $user)' <<<"${users}" >/dev/null; then + user_in_org=1 + break + fi + [ "$(jq length <<<"${users}")" -eq 100 ] || break + page=$((page + 1)) + done + echo "in_org=${user_in_org}" >> "$GITHUB_OUTPUT" + - id: check_safe_label + name: Check pull request label + env: + GH_TOKEN: ${{ secrets.TOKEN_ACTION }} + PR_URL: ${{ github.event.pull_request.url }} + run: | + set -euo pipefail + labels=$(curl -fsSL \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "${PR_URL}" | jq -r '.labels[].name') + if grep -Fxq 'safe-to-test' <<<"${labels}"; then + echo 'safe_label=1' >> "$GITHUB_OUTPUT" + else + echo 'safe_label=0' >> "$GITHUB_OUTPUT" + fi + - id: eligibility + name: Decide coverage eligibility + env: + IN_ORG: ${{ steps.check_in_org.outputs.in_org }} + SAFE_LABEL: ${{ steps.check_safe_label.outputs.safe_label }} + run: | + set -euo pipefail + if [ "${IN_ORG}" = '1' ] || [ "${SAFE_LABEL}" = '1' ]; then + echo 'allowed=true' >> "$GITHUB_OUTPUT" + else + echo 'allowed=false' >> "$GITHUB_OUTPUT" + echo '::notice::UT coverage skipped for an untrusted pull request' + fi + ut-coverage-linux-x86: name: UT Coverage on Ubuntu/x86 + needs: [check_coverage_eligibility] + if: ${{ needs.check_coverage_eligibility.outputs.allowed == 'true' }} environment: ci # Preserve the Guangzhou network path and capacity used by main's # external Aliyun/QCloud coverage UT. This is intentionally larger than From 6beb8d8bb8a93d62f6ff02591c3eafe287d536ce Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 02:03:07 +0800 Subject: [PATCH 12/15] ci: preserve coverage during parallel rollout --- .github/workflows/utils.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/utils.yaml b/.github/workflows/utils.yaml index 37a8478..8a317d8 100644 --- a/.github/workflows/utils.yaml +++ b/.github/workflows/utils.yaml @@ -3,6 +3,12 @@ name: MatrixOne Utils CI on: workflow_call: + inputs: + enable_legacy_coverage: + description: "Keep the serial coverage job until the caller enables parallel coverage producers" + required: false + type: boolean + default: true secrets: TOKEN_ACTION: description: "A token passed from the caller workflow" @@ -111,10 +117,11 @@ jobs: fi pr_coverage: - # Replaced by coverage-ut.yaml plus coverage-merge.yaml. Keep this job - # disabled during the rollout so the legacy serial UT+BVT path cannot run. - if: ${{ false }} - name: Legacy serial coverage (disabled) + # Default-on preserves coverage for existing callers while MatrixOne rolls + # out the parallel producers. The new entrypoint disables this job in the + # same commit that starts coverage-ut.yaml and coverage-merge.yaml. + if: ${{ inputs.enable_legacy_coverage && (needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1') && needs.check_organization_user.outputs.coverage_secrets_available == 'true' }} + name: Coverage needs: [check_organization_user] environment: ci # If the runner/workspace layout changes, also update setup-env's go-version-file below. From bba0c9fca48b9b98ba9e6956cfba6db604dee369 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 07:29:09 +0800 Subject: [PATCH 13/15] ci: make parallel coverage a fail-closed gate --- .github/workflows/coverage-merge.yaml | 168 +++++++++++++++++++++++--- .github/workflows/coverage-ut.yaml | 132 ++++++++++++++------ .github/workflows/e2e-compose.yaml | 49 ++++++-- .github/workflows/e2e-standalone.yaml | 52 ++++++-- .github/workflows/utils.yaml | 2 +- scripts/parse_coverage.py | 71 +++++++++-- scripts/test_parse_coverage.py | 92 +++++++++++++- 7 files changed, 480 insertions(+), 86 deletions(-) diff --git a/.github/workflows/coverage-merge.yaml b/.github/workflows/coverage-merge.yaml index 815c985..61cc8d3 100644 --- a/.github/workflows/coverage-merge.yaml +++ b/.github/workflows/coverage-merge.yaml @@ -2,6 +2,22 @@ name: MatrixOne Coverage Merge on: workflow_call: + inputs: + prerequisites_ready: + description: "Whether every coverage producer completed successfully" + required: false + type: boolean + default: true + expected_bvt_generation: + description: "Assignment generation expected in both BVT manifests" + required: false + type: string + default: "" + ci_ref: + description: "CI ref containing the coverage parser" + required: false + type: string + default: "main" secrets: TOKEN_ACTION: description: "Token for checkout and pull-request metadata" @@ -14,18 +30,27 @@ jobs: runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} timeout-minutes: 20 steps: + - name: Verify coverage prerequisites + env: + PREREQUISITES_READY: ${{ inputs.prerequisites_ready }} + run: | + if [ "${PREREQUISITES_READY}" != 'true' ]; then + echo '::error::coverage merge skipped because at least one required producer was skipped or failed' + exit 1 + fi - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "1" path: ./matrixone repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} - name: Checkout coverage parser uses: actions/checkout@v6 with: token: ${{ secrets.TOKEN_ACTION }} repository: matrixorigin/CI + ref: ${{ inputs.ci_ref }} fetch-depth: "1" path: ./CI - name: Set up Go @@ -37,11 +62,22 @@ jobs: env: GH_TOKEN: ${{ secrets.TOKEN_ACTION }} IS_PUB_REPO: ${{ vars.IS_PUB_REPO }} + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | set -euo pipefail cd "$GITHUB_WORKSPACE/matrixone" pr_repo='${{ github.event.pull_request.base.repo.full_name }}' pr_number='${{ github.event.pull_request.number }}' + current_head_sha=$(curl -fsSL --retry 5 --retry-delay 3 \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "https://api.github.com/repos/${pr_repo}/pulls/${pr_number}" \ + | jq -r '.head.sha') + if [ "${current_head_sha}" != "${EXPECTED_HEAD_SHA}" ]; then + echo "::error::PR head moved from ${EXPECTED_HEAD_SHA} to ${current_head_sha}; refusing to merge coverage against a different diff" + exit 1 + fi if [ -n "${IS_PUB_REPO:-}" ] && [ "${IS_PUB_REPO}" != "0" ] && [ "${IS_PUB_REPO}" != "false" ]; then curl -fL --retry 5 --retry-delay 3 "https://github.com/${pr_repo}/pull/${pr_number}.diff" -o diff.patch else @@ -53,18 +89,22 @@ jobs: - name: Download test coverage artifacts uses: actions/download-artifact@v7 with: - # The producers use ut-coverage and bvt-coverage-. - # '*-coverage' only matches the former; retain all three inputs. - pattern: '*coverage*' + # A full rerun rotates the shared generation. Filter out artifacts + # left by previous attempts while retaining successful producers + # from an earlier attempt during "Re-run failed jobs". + pattern: ${{ inputs.expected_bvt_generation != '' && format('*coverage*generation-{0}-*', inputs.expected_bvt_generation) || '*coverage*' }} merge-multiple: true path: ${{ github.workspace }}/coverage-artifacts - - name: Merge UT and BVT coverage + - name: Merge profiles and check coverage rate id: merge_coverage run: | set -euo pipefail artifact_dir="$GITHUB_WORKSPACE/coverage-artifacts" processed_dir="$GITHUB_WORKSPACE/coverage-processed" mkdir -p "${processed_dir}" + find "${artifact_dir}" -type f -printf '%P %s bytes\n' | sort \ + | tee "${RUNNER_TEMP}/coverage-inputs.txt" + mapfile -t ut_profiles < <(find "${artifact_dir}" -type f -name 'ut-coverage.out' | sort) mapfile -t bvt_profiles < <(find "${artifact_dir}" -type f -name 'bvt-*.out' | sort) if [ "${#ut_profiles[@]}" -ne 1 ] || [ "${#bvt_profiles[@]}" -ne 2 ]; then @@ -72,6 +112,30 @@ jobs: find "${artifact_dir}" -type f -print || true exit 1 fi + + mapfile -t manifests < <(find "${artifact_dir}" -type f -name 'bvt-*-manifest.json' | sort) + if [ "${#manifests[@]}" -ne 2 ]; then + echo "::error::expected exactly two BVT manifests, found ${#manifests[@]}" + exit 1 + fi + jq -s '.' "${manifests[@]}" > "${RUNNER_TEMP}/bvt-group-manifests.json" + if ! jq -e \ + --arg expected_generation '${{ inputs.expected_bvt_generation }}' \ + --arg expected_sha '${{ github.event.pull_request.head.sha }}' \ + ' + length == 2 + and (map(.schema_version) == [1, 1]) + and ((map(.deployment) | sort) == ["compose-proxy", "launch-pessimistic"]) + and ((map(.group) | sort) == [0, 1]) + and ((map(.generation) | unique | length) == 1) + and ($expected_generation == "" or all(.[]; .generation == $expected_generation)) + and all(.[]; .head_sha == $expected_sha) + ' "${RUNNER_TEMP}/bvt-group-manifests.json" >/dev/null; then + echo '::error::BVT manifests do not describe complementary groups from the expected generation and PR head' + jq . "${RUNNER_TEMP}/bvt-group-manifests.json" || true + exit 1 + fi + coverage_files=() for profile in "${ut_profiles[@]}"; do output="${processed_dir}/$(basename "${profile}")" @@ -87,15 +151,35 @@ jobs: done cd "$GITHUB_WORKSPACE" cp "$GITHUB_WORKSPACE/CI/scripts/.ignore" "$GITHUB_WORKSPACE/.ignore" + set +e python CI/scripts/parse_coverage.py \ -coverage_files "${coverage_files[@]}" \ - -diff_path "$GITHUB_WORKSPACE/matrixone/diff.patch" - # Coverage entries use MatrixOne module import paths. Run cover from - # that module root while keeping the merged profile in the workspace. - cd "$GITHUB_WORKSPACE/matrixone" - go tool cover \ - -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" \ - -html="$GITHUB_WORKSPACE/merged_coverage.out" + -diff_path "$GITHUB_WORKSPACE/matrixone/diff.patch" \ + -minimal_coverage 0.75 \ + -summary_path "$GITHUB_WORKSPACE/coverage-summary.json" \ + 2>&1 | tee "${RUNNER_TEMP}/coverage-parser.log" + parser_status=${PIPESTATUS[0]} + set -e + + has_pr_coverage=false + if [ -s "$GITHUB_WORKSPACE/coverage-summary.json" ]; then + has_pr_coverage=$(jq -r '.has_pr_coverage' "$GITHUB_WORKSPACE/coverage-summary.json") + echo "overall_coverage=$(jq -r '.overall_coverage' "$GITHUB_WORKSPACE/coverage-summary.json")" >> "$GITHUB_OUTPUT" + echo "pr_coverage=$(jq -r '.pr_coverage' "$GITHUB_WORKSPACE/coverage-summary.json")" >> "$GITHUB_OUTPUT" + echo "coverage_approved=$(jq -r '.approved' "$GITHUB_WORKSPACE/coverage-summary.json")" >> "$GITHUB_OUTPUT" + fi + echo "has_pr_coverage=${has_pr_coverage}" >> "$GITHUB_OUTPUT" + + # Preserve the legacy PR-focused HTML semantics and generate it even + # when the coverage threshold rejects the PR. + if [ "${has_pr_coverage}" = 'true' ]; then + test -s "$GITHUB_WORKSPACE/pr_coverage.out" + cd "$GITHUB_WORKSPACE/matrixone" + go tool cover \ + -o "$GITHUB_WORKSPACE/matrixone/pr_coverage.html" \ + -html="$GITHUB_WORKSPACE/pr_coverage.out" + fi + exit "${parser_status}" - name: Summarize coverage merge result if: ${{ always() && !cancelled() }} run: | @@ -103,22 +187,64 @@ jobs: echo '### Coverage merge result' >> "$GITHUB_STEP_SUMMARY" echo "- Merge step: \`${{ steps.merge_coverage.outcome }}\`" >> "$GITHUB_STEP_SUMMARY" echo '- Expected inputs: one UT profile and two BVT profiles.' >> "$GITHUB_STEP_SUMMARY" + summary="$GITHUB_WORKSPACE/coverage-summary.json" + if [ -s "${summary}" ]; then + overall=$(jq -r '.overall_coverage' "${summary}") + pr_rate=$(jq -r '.pr_coverage' "${summary}") + threshold=$(jq -r '.minimal_coverage' "${summary}") + covered=$(jq -r '.covered_modified_lines' "${summary}") + total=$(jq -r '.total_modified_lines' "${summary}") + has_changes=$(jq -r '.has_go_changes' "${summary}") + has_pr_coverage=$(jq -r '.has_pr_coverage' "${summary}") + approved=$(jq -r '.approved' "${summary}") + overall_pct=$(awk -v rate="${overall}" 'BEGIN { printf "%.2f%%", rate * 100 }') + pr_pct=$(awk -v rate="${pr_rate}" 'BEGIN { printf "%.2f%%", rate * 100 }') + threshold_pct=$(awk -v rate="${threshold}" 'BEGIN { printf "%.2f%%", rate * 100 }') + echo "- Overall coverage: \`${overall_pct}\`" >> "$GITHUB_STEP_SUMMARY" + if [ "${has_pr_coverage}" = 'true' ]; then + if [ "${approved}" = 'true' ]; then + gate='PASS' + else + gate='FAIL' + fi + echo "- Changed-code coverage: \`${covered}/${total} (${pr_pct})\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Required changed-code coverage: \`>${threshold_pct}\`" >> "$GITHUB_STEP_SUMMARY" + echo "- Coverage gate: \`${gate}\`" >> "$GITHUB_STEP_SUMMARY" + elif [ "${has_changes}" = 'true' ]; then + if [ "${approved}" = 'true' ]; then + gate='PASS' + else + gate='FAIL' + fi + echo '- Changed-code coverage: Go changes were found, but no instrumented blocks matched the coverage profiles.' >> "$GITHUB_STEP_SUMMARY" + echo "- Coverage gate: \`${gate}\`" >> "$GITHUB_STEP_SUMMARY" + else + echo '- Changed-code coverage: no modified Go blocks; gate passes.' >> "$GITHUB_STEP_SUMMARY" + fi + else + echo '- Coverage rates unavailable because profile processing failed before the summary was written.' >> "$GITHUB_STEP_SUMMARY" + fi + if [ -s "${RUNNER_TEMP}/bvt-group-manifests.json" ]; then + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '#### BVT group manifests' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + echo '```json' >> "$GITHUB_STEP_SUMMARY" + jq . "${RUNNER_TEMP}/bvt-group-manifests.json" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi echo '' >> "$GITHUB_STEP_SUMMARY" echo '#### Downloaded profiles' >> "$GITHUB_STEP_SUMMARY" echo '' >> "$GITHUB_STEP_SUMMARY" echo '```text' >> "$GITHUB_STEP_SUMMARY" - inventory="${RUNNER_TEMP}/coverage-inputs.txt" - find "$GITHUB_WORKSPACE/coverage-artifacts" -type f -name '*.out' \ - -printf '%f %s bytes\n' 2>/dev/null | sort > "${inventory}" || true - cat "${inventory}" >> "$GITHUB_STEP_SUMMARY" + cat "${RUNNER_TEMP}/coverage-inputs.txt" >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true echo '```' >> "$GITHUB_STEP_SUMMARY" if [ "${{ steps.merge_coverage.outcome }}" = 'success' ]; then - echo '- Result: `final-result-files` contains the merged profile and HTML report.' >> "$GITHUB_STEP_SUMMARY" + echo '- Result: `final-result-files` contains the changed-code profile, merged profile, summary, and PR coverage HTML.' >> "$GITHUB_STEP_SUMMARY" else echo '- Diagnostics: `coverage-merge-debug` contains the input inventory and PR diff; `final-result-files` retains any merged outputs.' >> "$GITHUB_STEP_SUMMARY" fi - name: Upload coverage result - if: ${{ always() && !cancelled() }} + if: ${{ always() && !cancelled() && steps.merge_coverage.outputs.has_pr_coverage == 'true' }} uses: actions/upload-artifact@v7 with: name: final-result-files @@ -126,7 +252,8 @@ jobs: ${{ github.workspace }}/matrixone/pr_coverage.html ${{ github.workspace }}/pr_coverage.out ${{ github.workspace }}/merged_coverage.out - if-no-files-found: warn + ${{ github.workspace }}/coverage-summary.json + if-no-files-found: error retention-days: 7 - name: Upload coverage merge diagnostics if: ${{ failure() && !cancelled() }} @@ -135,6 +262,9 @@ jobs: name: coverage-merge-debug path: | ${{ runner.temp }}/coverage-inputs.txt + ${{ runner.temp }}/coverage-parser.log + ${{ runner.temp }}/bvt-group-manifests.json ${{ github.workspace }}/matrixone/diff.patch + ${{ github.workspace }}/coverage-summary.json if-no-files-found: warn retention-days: 1 diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index efac4f8..c49927d 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -2,10 +2,16 @@ name: MatrixOne UT Coverage on: workflow_call: + inputs: + bvt_generation: + description: "Shared CI generation used to select this run's coverage artifacts" + required: false + type: string + default: "" outputs: coverage_ready: description: "Whether the UT coverage profile was eligible to run" - value: ${{ jobs.ut-coverage-linux-x86.outputs.coverage_ready }} + value: ${{ jobs.check_coverage_eligibility.outputs.allowed }} secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" @@ -18,25 +24,25 @@ on: required: false S3ENDPOINT: description: "S3ENDPOINT For Test" - required: true + required: false S3REGION: description: "S3REGION For Test" - required: true + required: false S3APIKEY: description: "S3APIKEY For Test" - required: true + required: false S3APISECRET: description: "S3APISECRET For Test" - required: true + required: false S3BUCKET: description: "S3BUCKET For Test" - required: true + required: false jobs: check_coverage_eligibility: name: Check coverage eligibility environment: ci - runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + runs-on: ubuntu-latest outputs: allowed: ${{ steps.eligibility.outputs.allowed }} steps: @@ -80,18 +86,45 @@ jobs: else echo 'safe_label=0' >> "$GITHUB_OUTPUT" fi + - id: check_coverage_secrets + name: Check coverage secrets + env: + TEST_S3FS_ALIYUN: ${{ secrets.TEST_S3FS_ALIYUN }} + TEST_S3FS_QCLOUD: ${{ secrets.TEST_S3FS_QCLOUD }} + S3ENDPOINT: ${{ secrets.S3ENDPOINT }} + S3REGION: ${{ secrets.S3REGION }} + S3APIKEY: ${{ secrets.S3APIKEY }} + S3APISECRET: ${{ secrets.S3APISECRET }} + S3BUCKET: ${{ secrets.S3BUCKET }} + run: | + set -euo pipefail + missing=() + for name in TEST_S3FS_ALIYUN TEST_S3FS_QCLOUD S3ENDPOINT S3REGION S3APIKEY S3APISECRET S3BUCKET; do + if [ -z "${!name:-}" ]; then + missing+=("${name}") + fi + done + if [ "${#missing[@]}" -eq 0 ]; then + echo 'ready=true' >> "$GITHUB_OUTPUT" + else + echo 'ready=false' >> "$GITHUB_OUTPUT" + echo "::notice::UT coverage skipped because required secrets are unavailable: ${missing[*]}" + fi - id: eligibility name: Decide coverage eligibility env: IN_ORG: ${{ steps.check_in_org.outputs.in_org }} SAFE_LABEL: ${{ steps.check_safe_label.outputs.safe_label }} + SECRETS_READY: ${{ steps.check_coverage_secrets.outputs.ready }} run: | set -euo pipefail - if [ "${IN_ORG}" = '1' ] || [ "${SAFE_LABEL}" = '1' ]; then + if { [ "${IN_ORG}" = '1' ] || [ "${SAFE_LABEL}" = '1' ]; } && [ "${SECRETS_READY}" = 'true' ]; then echo 'allowed=true' >> "$GITHUB_OUTPUT" else echo 'allowed=false' >> "$GITHUB_OUTPUT" - echo '::notice::UT coverage skipped for an untrusted pull request' + if [ "${IN_ORG}" != '1' ] && [ "${SAFE_LABEL}" != '1' ]; then + echo '::notice::UT coverage skipped for an untrusted pull request' + fi fi ut-coverage-linux-x86: @@ -104,38 +137,21 @@ jobs: # the UT-only minimum while we validate the external-storage timeout fix. runs-on: amd64-mo-guangzhou-2xlarge16 timeout-minutes: 60 - outputs: - coverage_ready: ${{ steps.coverage_secrets.outputs.ready }} steps: - - id: coverage_secrets - name: Check coverage secrets - env: - TEST_S3FS_ALIYUN: ${{ secrets.TEST_S3FS_ALIYUN }} - TEST_S3FS_QCLOUD: ${{ secrets.TEST_S3FS_QCLOUD }} - run: | - set -euo pipefail - if [ -z "${TEST_S3FS_ALIYUN:-}" ] || [ -z "${TEST_S3FS_QCLOUD:-}" ]; then - echo 'ready=false' >> "$GITHUB_OUTPUT" - echo '::notice::UT coverage skipped because TEST_S3FS_ALIYUN or TEST_S3FS_QCLOUD is unavailable' - else - echo 'ready=true' >> "$GITHUB_OUTPUT" - fi - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} with: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go - if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} uses: matrixorigin/CI/actions/setup-env@main with: setup-java: true go-version-file: "${{ github.workspace }}/go.mod" - id: coverage_ut name: Run coverage unit tests - if: ${{ steps.coverage_secrets.outputs.ready == 'true' }} + timeout-minutes: 45 continue-on-error: true run: | set -euo pipefail @@ -203,7 +219,7 @@ jobs: compacted_coverage_bytes=$(wc -c < "${coverage_profile}") echo "UT coverage profile compacted from ${raw_coverage_bytes} to ${compacted_coverage_bytes} bytes" - name: Summarize 30 slowest coverage UT tests - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' }} + if: ${{ always() && !cancelled() }} run: | set -euo pipefail report="${RUNNER_TEMP}/ut-coverage-report.json" @@ -230,7 +246,7 @@ jobs: echo '```' } | tee -a "$GITHUB_STEP_SUMMARY" - name: Print failed coverage UT packages and tests - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} run: | set -euo pipefail report="${RUNNER_TEMP}/ut-coverage-report.json" @@ -248,7 +264,7 @@ jobs: # Match the legacy coverage job's actionable failure report while keeping # the large JSON stream out of successful coverage artifacts. - name: Checkout CI repository for UT failure summary - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} uses: actions/checkout@v6 with: token: ${{ secrets.TOKEN_ACTION }} @@ -256,7 +272,7 @@ jobs: fetch-depth: "1" path: CI - name: Analyze and summarize failed coverage UT - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} continue-on-error: true run: | set -uo pipefail @@ -277,25 +293,67 @@ jobs: if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat "${summary}" >> "$GITHUB_STEP_SUMMARY" fi + - name: Collect coverage UT runner diagnostics + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} + continue-on-error: true + run: | + set +e + diagnostics="${RUNNER_TEMP}/ut-runner-diagnostics.txt" + { + echo '## Memory' + free -h + echo + echo '## Disk' + df -h + echo + echo '## Limits' + ulimit -a + echo + echo '## Largest processes by RSS' + ps -eo pid,ppid,rss,vsz,stat,etime,cmd --sort=-rss | head -n 50 + echo + echo '## Cgroup memory' + for file in /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory.events; do + echo "### ${file}" + if [ -r "${file}" ]; then + cat "${file}" + else + echo unavailable + fi + done + echo + echo '## Recent kernel messages' + sudo dmesg -T 2>&1 | tail -n 1000 + } | tee "${diagnostics}" + { + echo '### Coverage UT runner diagnostics' + echo "- Full diagnostics: \`ut-coverage-diagnostics-attempt-${{ github.run_attempt }}/ut-runner-diagnostics.txt\`" + if grep -Eqi 'out of memory|oom-kill|killed process' "${diagnostics}"; then + echo '- OOM signal: detected in runner diagnostics.' + else + echo '- OOM signal: not detected in available runner diagnostics.' + fi + } >> "$GITHUB_STEP_SUMMARY" - name: Upload UT coverage profile - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'success' }} + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'success' }} uses: actions/upload-artifact@v7 with: - name: ut-coverage + name: ut-coverage-generation-${{ inputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} path: ${{ runner.temp }}/ut-coverage.out if-no-files-found: warn retention-days: 7 - name: Upload failed UT diagnostics - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} uses: actions/upload-artifact@v7 with: - name: ut-coverage-diagnostics + name: ut-coverage-diagnostics-attempt-${{ github.run_attempt }} path: | ${{ runner.temp }}/ut-coverage-report.json ${{ runner.temp }}/ut-failure-summary.md ${{ runner.temp }}/ut-report + ${{ runner.temp }}/ut-runner-diagnostics.txt if-no-files-found: warn retention-days: 7 - name: Fail on coverage UT failure - if: ${{ always() && !cancelled() && steps.coverage_secrets.outputs.ready == 'true' && steps.coverage_ut.outcome == 'failure' }} + if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} run: exit 1 diff --git a/.github/workflows/e2e-compose.yaml b/.github/workflows/e2e-compose.yaml index 2255f51..bc90e42 100644 --- a/.github/workflows/e2e-compose.yaml +++ b/.github/workflows/e2e-compose.yaml @@ -2,6 +2,17 @@ name: MatrixOne Compose Test on: workflow_call: + inputs: + bvt_group: + description: "BVT group assigned by the caller (0 or 1)" + required: false + type: string + default: "" + bvt_generation: + description: "Shared assignment generation for complementary BVT jobs" + required: false + type: string + default: "" secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" @@ -31,7 +42,7 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -181,7 +192,6 @@ jobs: retention-days: 7 multi-CN-bvt-docker-compose-proxy: - if: ${{ !github.event.pull_request.draft }} environment: ci # Match the existing PR BVT runner policy. runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} @@ -203,7 +213,7 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -296,14 +306,33 @@ jobs: cat mo.yml echo "=============================" - run_id="${{ github.run_id }}" - run_attempt="${{ github.run_attempt }}" - bvt_group=$(( (run_id + run_attempt) % 2 )) - echo "Compose + Proxy BVT: run ${run_id}, attempt ${run_attempt}, group ${bvt_group}" + bvt_group='${{ inputs.bvt_group }}' + bvt_generation='${{ inputs.bvt_generation }}' + if [ -z "${bvt_group}" ]; then + bvt_group=$(( (${{ github.run_id }} + ${{ github.run_attempt }}) % 2 )) + fi + if [ -z "${bvt_generation}" ]; then + bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' + fi + if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then + echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" + exit 1 + fi + echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" + echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" + jq -n \ + --arg deployment 'compose-proxy' \ + --argjson group "${bvt_group}" \ + --arg generation "${bvt_generation}" \ + --arg head_sha '${{ github.event.pull_request.head.sha }}' \ + '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ + > "${RUNNER_TEMP}/bvt-compose-manifest.json" + echo "Compose + Proxy BVT: generation ${bvt_generation}, group ${bvt_group}" { echo "### BVT group assignment" echo "- Deployment: Compose Multi-CN + Proxy" echo "- Group: ${bvt_group}" + echo "- Generation: ${bvt_generation}" } >> "$GITHUB_STEP_SUMMARY" cd "$GITHUB_WORKSPACE" set -o pipefail @@ -409,8 +438,10 @@ jobs: if: ${{ always() && !cancelled() }} uses: actions/upload-artifact@v7 with: - name: bvt-coverage-compose - path: ${{ runner.temp }}/bvt-compose.out + name: bvt-coverage-compose-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/bvt-compose.out + ${{ runner.temp }}/bvt-compose-manifest.json if-no-files-found: warn retention-days: 7 diff --git a/.github/workflows/e2e-standalone.yaml b/.github/workflows/e2e-standalone.yaml index 4c5e8a9..70aeb95 100644 --- a/.github/workflows/e2e-standalone.yaml +++ b/.github/workflows/e2e-standalone.yaml @@ -2,6 +2,17 @@ name: MatrixOne e2e CI(Standalone) on: workflow_call: + inputs: + bvt_group: + description: "BVT group assigned by the caller (0 or 1)" + required: false + type: string + default: "" + bvt_generation: + description: "Shared assignment generation for complementary BVT jobs" + required: false + type: string + default: "" secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" @@ -22,7 +33,7 @@ jobs: fetch-depth: "3" path: ./head repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -149,7 +160,7 @@ jobs: # native proxy launcher out of the PR matrix until it has a distinct role. if: false runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - name: Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) + name: Legacy Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) (disabled) timeout-minutes: 75 env: mo_reuse_enable_checker: true @@ -160,7 +171,7 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} path: ./head repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -251,7 +262,7 @@ jobs: fetch-depth: "3" path: ./head repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main @@ -304,14 +315,33 @@ jobs: cat mo.yml echo "=============================" - run_id="${{ github.run_id }}" - run_attempt="${{ github.run_attempt }}" - bvt_group=$(( 1 - ((run_id + run_attempt) % 2) )) - echo "Launch + Pessimistic BVT: run ${run_id}, attempt ${run_attempt}, group ${bvt_group}" + bvt_group='${{ inputs.bvt_group }}' + bvt_generation='${{ inputs.bvt_generation }}' + if [ -z "${bvt_group}" ]; then + bvt_group=$(( 1 - ((${{ github.run_id }} + ${{ github.run_attempt }}) % 2) )) + fi + if [ -z "${bvt_generation}" ]; then + bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' + fi + if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then + echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" + exit 1 + fi + echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" + echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" + jq -n \ + --arg deployment 'launch-pessimistic' \ + --argjson group "${bvt_group}" \ + --arg generation "${bvt_generation}" \ + --arg head_sha '${{ github.event.pull_request.head.sha }}' \ + '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ + > "${RUNNER_TEMP}/bvt-pessimistic-manifest.json" + echo "Launch + Pessimistic BVT: generation ${bvt_generation}, group ${bvt_group}" { echo "### BVT group assignment" echo "- Deployment: native launch (PESSIMISTIC)" echo "- Group: ${bvt_group}" + echo "- Generation: ${bvt_generation}" } >> "$GITHUB_STEP_SUMMARY" cd "$GITHUB_WORKSPACE/head" set -o pipefail @@ -405,8 +435,10 @@ jobs: if: ${{ always() && !cancelled() }} uses: actions/upload-artifact@v7 with: - name: bvt-coverage-pessimistic - path: ${{ runner.temp }}/bvt-pessimistic.out + name: bvt-coverage-pessimistic-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/bvt-pessimistic.out + ${{ runner.temp }}/bvt-pessimistic-manifest.json if-no-files-found: warn retention-days: 7 diff --git a/.github/workflows/utils.yaml b/.github/workflows/utils.yaml index 8a317d8..00efbad 100644 --- a/.github/workflows/utils.yaml +++ b/.github/workflows/utils.yaml @@ -121,7 +121,7 @@ jobs: # out the parallel producers. The new entrypoint disables this job in the # same commit that starts coverage-ut.yaml and coverage-merge.yaml. if: ${{ inputs.enable_legacy_coverage && (needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1') && needs.check_organization_user.outputs.coverage_secrets_available == 'true' }} - name: Coverage + name: ${{ inputs.enable_legacy_coverage && 'Coverage' || 'Legacy Coverage (disabled)' }} needs: [check_organization_user] environment: ci # If the runner/workspace layout changes, also update setup-env's go-version-file below. diff --git a/scripts/parse_coverage.py b/scripts/parse_coverage.py index d43dbf2..eddab13 100644 --- a/scripts/parse_coverage.py +++ b/scripts/parse_coverage.py @@ -4,6 +4,7 @@ import logging import argparse import fnmatch +import json # 设置日志配置 logging.basicConfig( @@ -56,7 +57,7 @@ def merge_coverage_files(output_path, *coverage_files): # 计算覆盖率 coverage_percentage = (covered_blocks / total_blocks) if total_blocks > 0 else 0 - logging.info(f"Total code blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {coverage_percentage}%") + logging.info(f"Total code blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {coverage_percentage:.2%}") # 将合并后的结果写入输出文件 try: @@ -278,7 +279,7 @@ def diff_coverage(diff_path, coverage_path, output_path='pr_coverage.out', ignor except Exception as e: logging.error(f"An error occurred during the process: {e}") - return 0,0,0 + raise def is_valid_code_segment(segment): """判断一个代码片段是否包含有效的代码（忽略结构符号、关键字和空白）""" @@ -452,6 +453,39 @@ def _line_matches_block(line_num, mincol, maxcol, block): return False +def write_coverage_summary( + output_path, + total_blocks, + covered_blocks, + overall_coverage, + total_modified_lines, + covered_modified_lines, + pr_coverage, + minimal_coverage, + has_go_changes=None, +): + """Write the coverage gate result before the caller enforces the threshold.""" + if has_go_changes is None: + has_go_changes = total_modified_lines > 0 + summary = { + "schema_version": 1, + "total_blocks": total_blocks, + "covered_blocks": covered_blocks, + "overall_coverage": overall_coverage, + "total_modified_lines": total_modified_lines, + "covered_modified_lines": covered_modified_lines, + "pr_coverage": pr_coverage, + "minimal_coverage": minimal_coverage, + "has_go_changes": has_go_changes, + "has_pr_coverage": total_modified_lines > 0, + "approved": pr_coverage > minimal_coverage, + } + with open(output_path, "w") as summary_file: + json.dump(summary, summary_file, indent=2, sort_keys=True) + summary_file.write("\n") + return summary + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Merge coverage files and calculate coverage based on diff.") @@ -483,20 +517,39 @@ def _line_matches_block(line_num, mincol, maxcol, block): help='Count coverage by line instead of block. Default is False (count by block).' ) + parser.add_argument( + '-summary_path', + type=str, + default='coverage_summary.json', + help='Path for the machine-readable coverage summary.' + ) + args = parser.parse_args() - total_blocks, covered_blocks, coverage_percentage = merge_coverage_files('merged_coverage.out', *args.coverage_files) - logging.info(f"Total modified blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {coverage_percentage:.2f}%") + total_blocks, covered_blocks, overall_coverage = merge_coverage_files('merged_coverage.out', *args.coverage_files) + logging.info(f"Total code blocks: {total_blocks}, Covered blocks: {covered_blocks}, Coverage: {overall_coverage:.2%}") # 调用主函数 diff_path = args.diff_path # 可以根据实际情况修改路径 coverage_path = 'merged_coverage.out' # 可以根据实际情况修改路径 - total_modified_lines, covered_modified_lines, coverage_percentage = diff_coverage(diff_path, coverage_path, count_by_line=args.count_by_line) - logging.info(f"total_modified_lines: {total_modified_lines}, covered_modified_lines: {covered_modified_lines}, coverage_percentage:{coverage_percentage}") + total_modified_lines, covered_modified_lines, pr_coverage = diff_coverage(diff_path, coverage_path, count_by_line=args.count_by_line) + logging.info(f"total_modified_lines: {total_modified_lines}, covered_modified_lines: {covered_modified_lines}, coverage_percentage:{pr_coverage:.2%}") + + summary = write_coverage_summary( + args.summary_path, + total_blocks, + covered_blocks, + overall_coverage, + total_modified_lines, + covered_modified_lines, + pr_coverage, + args.minimal_coverage, + has_go_changes=os.path.exists("pr_coverage.out"), + ) - if coverage_percentage <= args.minimal_coverage: + if not summary["approved"]: parse_file_coverage(args.minimal_coverage) - logging.warning(f"The code coverage:{coverage_percentage} is below or equal {args.minimal_coverage}, not approved.") + logging.warning(f"The code coverage:{pr_coverage:.2%} is below or equal {args.minimal_coverage:.2%}, not approved.") sys.exit(1) - logging.info(f"The code coverage:{coverage_percentage} is above {args.minimal_coverage}, pass.") \ No newline at end of file + logging.info(f"The code coverage:{pr_coverage:.2%} is above {args.minimal_coverage:.2%}, pass.") diff --git a/scripts/test_parse_coverage.py b/scripts/test_parse_coverage.py index 2708dde..761748e 100644 --- a/scripts/test_parse_coverage.py +++ b/scripts/test_parse_coverage.py @@ -1,11 +1,15 @@ import pytest import os +import json +import subprocess +import sys from parse_coverage import ( parse_diff, parse_coverage_and_generate_report, merge_coverage_files, diff_coverage, normalize_path, + write_coverage_summary, ) @@ -231,7 +235,6 @@ def test_multiple_modified_lines_one_block_by_line(self, tmp_path): assert total == 3 assert covered == 3 assert pct == 1.0 - def test_multiple_lines_partial_coverage(self, tmp_path): """按块计数(默认): 2块，1covered → 50%""" coverage_content = """\ @@ -309,6 +312,93 @@ def test_uncovered_block(self, tmp_path): assert pct == 0.0 +class TestCoverageSummary: + def test_writes_rates_counts_and_gate_result(self, tmp_path): + summary_path = tmp_path / "coverage-summary.json" + + summary = write_coverage_summary( + str(summary_path), + total_blocks=200, + covered_blocks=120, + overall_coverage=0.6, + total_modified_lines=4, + covered_modified_lines=3, + pr_coverage=0.75, + minimal_coverage=0.75, + ) + + assert summary["schema_version"] == 1 + assert summary["has_go_changes"] is True + assert summary["has_pr_coverage"] is True + assert summary["approved"] is False + assert '"overall_coverage": 0.6' in summary_path.read_text() + + def test_no_go_changes_is_explicit_and_passes(self, tmp_path): + summary = write_coverage_summary( + str(tmp_path / "coverage-summary.json"), + total_blocks=10, + covered_blocks=5, + overall_coverage=0.5, + total_modified_lines=0, + covered_modified_lines=0, + pr_coverage=1.0, + minimal_coverage=0.75, + ) + + assert summary["has_go_changes"] is False + assert summary["has_pr_coverage"] is False + assert summary["approved"] is True + + def test_diff_coverage_propagates_parser_errors(self, tmp_path): + with pytest.raises(FileNotFoundError): + diff_coverage( + str(tmp_path / "missing.diff"), + str(tmp_path / "missing-coverage.out"), + ignore_path=str(tmp_path / "missing.ignore"), + ) + + def test_cli_keeps_summary_and_pr_profile_when_gate_fails(self, tmp_path): + coverage_path = tmp_path / "coverage.out" + coverage_path.write_text( + "mode: set\n" + "github.com/matrixorigin/matrixone/pkg/example.go:1.1,2.2 1 0\n" + ) + diff_path = tmp_path / "diff.patch" + diff_path.write_text( + "diff --git a/pkg/example.go b/pkg/example.go\n" + "--- a/pkg/example.go\n" + "+++ b/pkg/example.go\n" + "@@ -1,1 +1,1 @@\n" + "-old\n" + "+new\n" + ) + (tmp_path / ".ignore").write_text("!*.go\n") + script_path = os.path.join(os.path.dirname(__file__), "parse_coverage.py") + + result = subprocess.run( + [ + sys.executable, + script_path, + "-coverage_files", + str(coverage_path), + "-diff_path", + str(diff_path), + "-summary_path", + "coverage-summary.json", + ], + cwd=tmp_path, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + summary = json.loads((tmp_path / "coverage-summary.json").read_text()) + assert summary["has_go_changes"] is True + assert summary["approved"] is False + assert (tmp_path / "pr_coverage.out").read_text().startswith("mode: set\n") + + class TestPathNormalization: """测试路径规范化""" From 08b3e6177f8b68ee87e7e31286252fb61c3d98c3 Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 07:38:15 +0800 Subject: [PATCH 14/15] ci: preserve legacy BVT for existing callers --- .github/workflows/e2e-compose.yaml | 37 ++++++++++++++------- .github/workflows/e2e-standalone.yaml | 47 ++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/.github/workflows/e2e-compose.yaml b/.github/workflows/e2e-compose.yaml index bc90e42..70f29f8 100644 --- a/.github/workflows/e2e-compose.yaml +++ b/.github/workflows/e2e-compose.yaml @@ -3,6 +3,11 @@ name: MatrixOne Compose Test on: workflow_call: inputs: + enable_parallel_bvt: + description: "Run the complementary Compose + Proxy BVT and emit coverage" + required: false + type: boolean + default: false bvt_group: description: "BVT group assigned by the caller (0 or 1)" required: false @@ -192,10 +197,11 @@ jobs: retention-days: 7 multi-CN-bvt-docker-compose-proxy: + if: ${{ inputs.enable_parallel_bvt || !github.event.pull_request.draft }} environment: ci # Match the existing PR BVT runner policy. runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - name: multi cn e2e bvt test docker compose(PROXY) + name: ${{ inputs.enable_parallel_bvt && 'multi cn e2e bvt test docker compose(PROXY)' || 'multi cn e2e bvt test docker compose(PESSIMISTIC)' }} timeout-minutes: 75 steps: - name: Pre Clean Some Unneeded Images @@ -244,11 +250,15 @@ jobs: # Handle various docker-compose volume formats: /test, /test:options, /test followed by space sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml - # Build once for this single Compose job. Splitting this image build - # into an artifact would add transfer time without reducing latency. - mkdir -p "$GITHUB_WORKSPACE/coverage" - docker build --build-arg GOBUILD_OPT=-cover \ - -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + # Existing callers keep the legacy image and full Pessimistic BVT. + # The new MatrixOne main entrypoint explicitly enables coverage. + if [ '${{ inputs.enable_parallel_bvt }}' = 'true' ]; then + mkdir -p "$GITHUB_WORKSPACE/coverage" + docker build --build-arg GOBUILD_OPT=-cover \ + -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + else + docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + fi mkdir -p ${{ github.workspace }}/docker-compose-log docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d @@ -306,6 +316,11 @@ jobs: cat mo.yml echo "=============================" + if [ '${{ inputs.enable_parallel_bvt }}' != 'true' ]; then + ./run.sh -n -g -o -p "$GITHUB_WORKSPACE/test/distributed/cases" -e optimistic 2>&1 + exit + fi + bvt_group='${{ inputs.bvt_group }}' bvt_generation='${{ inputs.bvt_generation }}' if [ -z "${bvt_group}" ]; then @@ -342,7 +357,7 @@ jobs: "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-compose.log" - name: Summarize Compose BVT result - if: ${{ always() && !cancelled() }} + if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} run: | set -euo pipefail report="${RUNNER_TEMP}/bvt-compose.log" @@ -373,7 +388,7 @@ jobs: } | tee -a "$GITHUB_STEP_SUMMARY" - name: Upload Compose BVT execution log - if: ${{ always() && !cancelled() }} + if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} uses: actions/upload-artifact@v7 with: name: bvt-execution-compose @@ -420,7 +435,7 @@ jobs: docker volume rm launch-tae-compose_minio_storage || true - name: Generate Compose BVT coverage profile - if: ${{ always() && !cancelled() }} + if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} run: | set -uo pipefail coverage_dir="$GITHUB_WORKSPACE/coverage" @@ -435,7 +450,7 @@ jobs: fi - name: Upload Compose BVT coverage - if: ${{ always() && !cancelled() }} + if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} uses: actions/upload-artifact@v7 with: name: bvt-coverage-compose-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} @@ -449,7 +464,7 @@ jobs: if: ${{ failure() || cancelled()}} continue-on-error: true with: - name: Compose-multi-cn-e2e-bvt-test-docker-log(PROXY) + name: ${{ inputs.enable_parallel_bvt && 'Compose-multi-cn-e2e-bvt-test-docker-log(PROXY)' || 'Compose-multi-cn-e2e-bvt-test-docker-log(PESSIMISTIC)' }} path: | ${{ github.workspace }}/docker-compose-log retention-days: 7 diff --git a/.github/workflows/e2e-standalone.yaml b/.github/workflows/e2e-standalone.yaml index 70aeb95..3edf443 100644 --- a/.github/workflows/e2e-standalone.yaml +++ b/.github/workflows/e2e-standalone.yaml @@ -3,6 +3,11 @@ name: MatrixOne e2e CI(Standalone) on: workflow_call: inputs: + enable_parallel_bvt: + description: "Run only complementary Launch + Pessimistic BVT and emit coverage" + required: false + type: boolean + default: false bvt_group: description: "BVT group assigned by the caller (0 or 1)" required: false @@ -158,9 +163,9 @@ jobs: multi-cn-proxy-bvt-linux-x86: # Compose + Proxy is the active proxy coverage deployment. Keep the # native proxy launcher out of the PR matrix until it has a distinct role. - if: false + if: ${{ !inputs.enable_parallel_bvt }} runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - name: Legacy Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) (disabled) + name: ${{ inputs.enable_parallel_bvt && 'Legacy Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) (disabled)' || 'Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY)' }} timeout-minutes: 75 env: mo_reuse_enable_checker: true @@ -271,7 +276,12 @@ jobs: - name: Build MatrixOne run: | - cd $GITHUB_WORKSPACE/head && make clean && make GOBUILD_OPT=-cover build + cd "$GITHUB_WORKSPACE/head" + if [ '${{ inputs.enable_parallel_bvt }}' = 'true' ]; then + make clean && make GOBUILD_OPT=-cover build + else + make clean && make build + fi git rev-parse --short HEAD - name: Add cn.txn @@ -290,9 +300,11 @@ jobs: - name: Start MO run: | cd $GITHUB_WORKSPACE/head - export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-pessimistic" - rm -rf "${GOCOVERDIR}" - mkdir -p "${GOCOVERDIR}" + if [ '${{ inputs.enable_parallel_bvt }}' = 'true' ]; then + export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-pessimistic" + rm -rf "${GOCOVERDIR}" + mkdir -p "${GOCOVERDIR}" + fi ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch - name: Clone test-tool repository @@ -315,6 +327,14 @@ jobs: cat mo.yml echo "=============================" + if [ '${{ inputs.enable_parallel_bvt }}' != 'true' ]; then + ./run.sh -n -g -o \ + -p "$GITHUB_WORKSPACE/head/test/distributed/cases" \ + -s "$GITHUB_WORKSPACE/head/test/distributed/resources" \ + -e optimistic 2>&1 + exit + fi + bvt_group='${{ inputs.bvt_group }}' bvt_generation='${{ inputs.bvt_generation }}' if [ -z "${bvt_group}" ]; then @@ -353,7 +373,7 @@ jobs: 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log" - name: Summarize Launch + Pessimistic BVT result - if: ${{ always() && !cancelled() }} + if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} run: | set -euo pipefail report="${RUNNER_TEMP}/bvt-pessimistic.log" @@ -384,7 +404,7 @@ jobs: } | tee -a "$GITHUB_STEP_SUMMARY" - name: Upload Launch + Pessimistic BVT execution log - if: ${{ always() && !cancelled() }} + if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} uses: actions/upload-artifact@v7 with: name: bvt-execution-pessimistic @@ -408,6 +428,15 @@ jobs: if: ${{ always() && !cancelled() }} run: | set -uo pipefail + if [ '${{ inputs.enable_parallel_bvt }}' != 'true' ]; then + if pgrep -f 'mo-service' >/dev/null; then + pkill -9 mo-service + else + echo 'current mo-service has already crashed' + exit 1 + fi + exit + fi coverage_dir="${RUNNER_TEMP}/bvt-coverage-pessimistic" coverage_profile="${RUNNER_TEMP}/bvt-pessimistic.out" pids=$(pgrep -f 'mo-service' || true) @@ -432,7 +461,7 @@ jobs: fi - name: Upload Launch + Pessimistic BVT coverage - if: ${{ always() && !cancelled() }} + if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} uses: actions/upload-artifact@v7 with: name: bvt-coverage-pessimistic-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} From cfd84a1c0f021c22028dcfc7ee798fb237648e4f Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Mon, 27 Jul 2026 07:47:45 +0800 Subject: [PATCH 15/15] ci: isolate parallel coverage workflows --- .github/workflows/e2e-compose-parallel.yaml | 453 +++++++++++++++++ .github/workflows/e2e-compose.yaml | 148 +----- .../workflows/e2e-standalone-parallel.yaml | 464 ++++++++++++++++++ .github/workflows/e2e-standalone.yaml | 173 +------ .github/workflows/utils-parallel.yaml | 29 ++ .github/workflows/utils.yaml | 14 +- 6 files changed, 968 insertions(+), 313 deletions(-) create mode 100644 .github/workflows/e2e-compose-parallel.yaml create mode 100644 .github/workflows/e2e-standalone-parallel.yaml create mode 100644 .github/workflows/utils-parallel.yaml diff --git a/.github/workflows/e2e-compose-parallel.yaml b/.github/workflows/e2e-compose-parallel.yaml new file mode 100644 index 0000000..5139d1a --- /dev/null +++ b/.github/workflows/e2e-compose-parallel.yaml @@ -0,0 +1,453 @@ +name: MatrixOne Compose Test + +on: + workflow_call: + inputs: + bvt_group: + description: "BVT group assigned by the caller (0 or 1)" + required: false + type: string + default: "" + bvt_generation: + description: "Shared assignment generation for complementary BVT jobs" + required: false + type: string + default: "" + secrets: + TOKEN_ACTION: + description: "Token for checkout (e.g. pull from fork/private)" + required: false + +jobs: + bvt-docker-compose-push: + if: false + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: multi cn e2e bvt test docker compose(Optimistic/PUSH) + timeout-minutes: 75 + + steps: + - name: Pre Clean Some Unneeded Images + run: | + sudo df -h /* + echo "=========================================" + sudo docker builder prune -f -a; + sudo docker images | grep -v REPOSITORY | awk '{system("sudo docker rmi "$1":"$2)}'; + sudo docker volume prune -f; + echo "=========================================" + sudo df -h /* + - name: checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/go.mod" + + - name: Print run attempt + run: echo "run attempt is ${{ github.run_attempt }}" + + - name: Print Disk Usage Before Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: docker compose launch-multi-cn + timeout-minutes: 10 + run: | + echo "" >> ./etc/launch-tae-compose/config/cn-0.toml + echo '[cn.txn]' >> ./etc/launch-tae-compose/config/cn-0.toml + echo 'mode = "Optimistic"' >> ./etc/launch-tae-compose/config/cn-0.toml + echo "" >> ./etc/launch-tae-compose/config/cn-1.toml + echo '[cn.txn]' >> ./etc/launch-tae-compose/config/cn-1.toml + echo 'mode = "Optimistic"' >> ./etc/launch-tae-compose/config/cn-1.toml + echo "" >> ./etc/launch-tae-compose/config/tn.toml + echo '[tn.txn]' >> ./etc/launch-tae-compose/config/tn.toml + echo 'mode = "Optimistic"' >> ./etc/launch-tae-compose/config/tn.toml + cat ./etc/launch-tae-compose/config/cn-0.toml + cat ./etc/launch-tae-compose/config/cn-1.toml + cat ./etc/launch-tae-compose/config/tn.toml + + # Update compose.yaml to mount test directory to $GITHUB_WORKSPACE/test instead of /test + # Replace container mount point from /test to $GITHUB_WORKSPACE/test + # Handle various docker-compose volume formats: /test, /test:options, /test followed by space + sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml + + # build matrixone docker image first + docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + + mkdir -p ${{ github.workspace }}/docker-compose-log + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d + + # wait for ready + i=1 + while [ $(mysql -h 127.0.0.1 -P 6001 -u dump -p111 --execute 'create database if not exists compose_test;use compose_test; create table if not exists compose_test_table(col1 int auto_increment primary key);show tables;' 2>&1 | tee /dev/stderr | grep 'compose_test_table' | wc -l) -lt 1 ]; do + echo "wait mo init finished...$i" + if [ $i -ge 300 ]; then + echo "wait for $i seconds, mo init not finish, so exit 1" + docker ps + exit 1; + fi + i=$(($i+1)) + sleep 1 + done + docker ps + + - name: Print Disk Usage After Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: Clean Docker Image Build Cache + if: ${{ always() }} + run: | + sudo docker builder prune -f -a; + echo "==========================================="; + sudo docker system df -v; + + - name: Clone test-tool repository + # Node.js 12 actions are deprecated. + # For more information see: + # - https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/. + # Please update the following actions to use Node.js 16: actions/checkout@v3 + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/test/distributed/cases -e pessimistic_transaction 2>&1 + + - name: Print Docker Info Before Container Shutdown + if: ${{ always() }} + run: | + set +e; + sudo docker ps; + echo "==========================================="; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + - name: Print System Dmesg + if: ${{ always() }} + continue-on-error: true + run: | + sudo dmesg -T + + - name: export log + if: ${{ failure() || cancelled()}} + run: | + mkdir -p ${{ github.workspace }}/mo-tester/report + mv ${{ github.workspace }}/mo-tester/report ${{ github.workspace }}/docker-compose-log + curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-0-dump-stacks.log + curl http://localhost:22345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-1-dump-stacks.log + + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval} + + - name: shutdown containers + if: ${{ always() }} + run: | + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans + docker volume rm launch-tae-compose_minio_storage + + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Compose-multi-cn-e2e-bvt-test-docker-log(Optimistic,PUSH) + path: | + ${{ github.workspace }}/docker-compose-log + retention-days: 7 + + multi-CN-bvt-docker-compose-proxy: + environment: ci + # Match the existing PR BVT runner policy. + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: multi cn e2e bvt test docker compose(PROXY) + timeout-minutes: 75 + steps: + - name: Pre Clean Some Unneeded Images + run: | + sudo df -h /* + echo "=========================================" + sudo docker builder prune -f -a; + sudo docker images | grep -v REPOSITORY | awk '{system("sudo docker rmi "$1":"$2)}'; + sudo docker volume prune -f; + echo "=========================================" + sudo df -h /* + - name: checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/go.mod" + + - name: Print run attempt + run: echo "run attempt is ${{ github.run_attempt }}" + + - name: Print Disk Usage Before Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: docker compose launch-multi-cn + timeout-minutes: 10 + run: | + cat ./etc/launch-tae-compose/config/cn-0.toml + cat ./etc/launch-tae-compose/config/cn-1.toml + cat ./etc/launch-tae-compose/config/tn.toml + + # Update compose.yaml to mount test directory to $GITHUB_WORKSPACE/test instead of /test + # Replace container mount point from /test to $GITHUB_WORKSPACE/test + # Handle various docker-compose volume formats: /test, /test:options, /test followed by space + sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml + + mkdir -p "$GITHUB_WORKSPACE/coverage" + docker build --build-arg GOBUILD_OPT=-cover \ + -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile + + mkdir -p ${{ github.workspace }}/docker-compose-log + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d + + # wait for ready + i=1 + while [ $(mysql -h 127.0.0.1 -P 6001 -u dump -p111 --execute 'create database if not exists compose_test;use compose_test; create table if not exists compose_test_table(col1 int auto_increment primary key);show tables;' 2>&1 | tee /dev/stderr | grep 'compose_test_table' | wc -l) -lt 1 ]; do + echo "wait mo init finished...$i" + if [ $i -ge 300 ]; then + echo "wait for $i seconds, mo init not finish, so exit 1" + docker ps + exit 1; + fi + i=$(($i+1)) + sleep 1 + done + docker ps + + - name: Print Disk Usage After Container Start + if: ${{ always() }} + run: | + set +e; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: Clean Docker Image Build Cache + if: ${{ always() }} + run: | + sudo docker builder prune -f -a; + echo "==========================================="; + sudo docker system df -v; + + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + bvt_group='${{ inputs.bvt_group }}' + bvt_generation='${{ inputs.bvt_generation }}' + if [ -z "${bvt_group}" ]; then + bvt_group=$(( (${{ github.run_id }} + ${{ github.run_attempt }}) % 2 )) + fi + if [ -z "${bvt_generation}" ]; then + bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' + fi + if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then + echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" + exit 1 + fi + echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" + echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" + jq -n \ + --arg deployment 'compose-proxy' \ + --argjson group "${bvt_group}" \ + --arg generation "${bvt_generation}" \ + --arg head_sha '${{ github.event.pull_request.head.sha }}' \ + '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ + > "${RUNNER_TEMP}/bvt-compose-manifest.json" + echo "Compose + Proxy BVT: generation ${bvt_generation}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- Deployment: Compose Multi-CN + Proxy" + echo "- Group: ${bvt_group}" + echo "- Generation: ${bvt_generation}" + } >> "$GITHUB_STEP_SUMMARY" + cd "$GITHUB_WORKSPACE" + set -o pipefail + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/test/distributed/cases" \ + "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-compose.log" + + - name: Summarize Compose BVT result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/bvt-compose.log" + timing="${RUNNER_TEMP}/bvt-compose-slowest.tsv" + if [ ! -s "${report}" ]; then + echo '### Compose + Proxy BVT result' | tee -a "$GITHUB_STEP_SUMMARY" + echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '### Compose + Proxy BVT result' + grep '^Run BVT group' "${report}" | sed 's/^/- /' || true + echo + echo '#### Top 10 slow BVT scripts' + echo + } | tee -a "$GITHUB_STEP_SUMMARY" + + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ + | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" + if [ ! -s "${timing}" ]; then + echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload Compose BVT execution log + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-execution-compose + path: ${{ runner.temp }}/bvt-compose.log + if-no-files-found: warn + retention-days: 7 + + - name: Print Docker Info Before Container Shutdown + if: ${{ always() }} + run: | + set +e; + sudo docker ps; + echo "==========================================="; + sudo df -h /*; + echo "==========================================="; + sudo du -hs /var/lib/docker; + echo "==========================================="; + sudo docker system df -v; + + - name: Print System Dmesg + if: ${{ always() }} + continue-on-error: true + run: | + sudo dmesg -T + + - name: export log + if: ${{ failure() || cancelled()}} + run: | + mkdir -p ${{ github.workspace }}/mo-tester/report + mv ${{ github.workspace }}/mo-tester/report ${{ github.workspace }}/docker-compose-log + curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-0-dump-stacks.log + curl http://localhost:22345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/docker-compose-log/cn-1-dump-stacks.log + + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval} + + - name: shutdown containers + if: ${{ always() }} + run: | + docker ps + docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans + docker volume rm launch-tae-compose_minio_storage || true + + - name: Generate Compose BVT coverage profile + if: ${{ always() && !cancelled() }} + run: | + set -uo pipefail + coverage_dir="$GITHUB_WORKSPACE/coverage" + coverage_profile="$RUNNER_TEMP/bvt-compose.out" + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo '::error::failed to generate Compose BVT coverage after a successful BVT' + exit 1 + else + echo '::warning::Compose BVT coverage unavailable because BVT did not finish successfully' + fi + + - name: Upload Compose BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-compose-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/bvt-compose.out + ${{ runner.temp }}/bvt-compose-manifest.json + if-no-files-found: warn + retention-days: 7 + + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Compose-multi-cn-e2e-bvt-test-docker-log(PROXY) + path: | + ${{ github.workspace }}/docker-compose-log + retention-days: 7 diff --git a/.github/workflows/e2e-compose.yaml b/.github/workflows/e2e-compose.yaml index 70f29f8..4ffc3ab 100644 --- a/.github/workflows/e2e-compose.yaml +++ b/.github/workflows/e2e-compose.yaml @@ -2,22 +2,6 @@ name: MatrixOne Compose Test on: workflow_call: - inputs: - enable_parallel_bvt: - description: "Run the complementary Compose + Proxy BVT and emit coverage" - required: false - type: boolean - default: false - bvt_group: - description: "BVT group assigned by the caller (0 or 1)" - required: false - type: string - default: "" - bvt_generation: - description: "Shared assignment generation for complementary BVT jobs" - required: false - type: string - default: "" secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" @@ -47,7 +31,7 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -196,12 +180,11 @@ jobs: ${{ github.workspace }}/docker-compose-log retention-days: 7 - multi-CN-bvt-docker-compose-proxy: - if: ${{ inputs.enable_parallel_bvt || !github.event.pull_request.draft }} + multi-CN-bvt-docker-compose-pessimistic: + if: ${{ !github.event.pull_request.draft }} environment: ci - # Match the existing PR BVT runner policy. runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - name: ${{ inputs.enable_parallel_bvt && 'multi cn e2e bvt test docker compose(PROXY)' || 'multi cn e2e bvt test docker compose(PESSIMISTIC)' }} + name: multi cn e2e bvt test docker compose(PESSIMISTIC) timeout-minutes: 75 steps: - name: Pre Clean Some Unneeded Images @@ -219,7 +202,7 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} fetch-depth: "3" repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -250,15 +233,8 @@ jobs: # Handle various docker-compose volume formats: /test, /test:options, /test followed by space sed -i 's|:/test\b|:'"$GITHUB_WORKSPACE"'/test|g' ./etc/launch-tae-compose/compose.yaml - # Existing callers keep the legacy image and full Pessimistic BVT. - # The new MatrixOne main entrypoint explicitly enables coverage. - if [ '${{ inputs.enable_parallel_bvt }}' = 'true' ]; then - mkdir -p "$GITHUB_WORKSPACE/coverage" - docker build --build-arg GOBUILD_OPT=-cover \ - -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile - else - docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile - fi + # build matrixone docker image first + docker build -t matrixorigin/matrixone:latest . -f ./optools/images/Dockerfile mkdir -p ${{ github.workspace }}/docker-compose-log docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn up -d @@ -316,85 +292,7 @@ jobs: cat mo.yml echo "=============================" - if [ '${{ inputs.enable_parallel_bvt }}' != 'true' ]; then - ./run.sh -n -g -o -p "$GITHUB_WORKSPACE/test/distributed/cases" -e optimistic 2>&1 - exit - fi - - bvt_group='${{ inputs.bvt_group }}' - bvt_generation='${{ inputs.bvt_generation }}' - if [ -z "${bvt_group}" ]; then - bvt_group=$(( (${{ github.run_id }} + ${{ github.run_attempt }}) % 2 )) - fi - if [ -z "${bvt_generation}" ]; then - bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' - fi - if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then - echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" - exit 1 - fi - echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" - echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" - jq -n \ - --arg deployment 'compose-proxy' \ - --argjson group "${bvt_group}" \ - --arg generation "${bvt_generation}" \ - --arg head_sha '${{ github.event.pull_request.head.sha }}' \ - '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ - > "${RUNNER_TEMP}/bvt-compose-manifest.json" - echo "Compose + Proxy BVT: generation ${bvt_generation}, group ${bvt_group}" - { - echo "### BVT group assignment" - echo "- Deployment: Compose Multi-CN + Proxy" - echo "- Group: ${bvt_group}" - echo "- Generation: ${bvt_generation}" - } >> "$GITHUB_STEP_SUMMARY" - cd "$GITHUB_WORKSPACE" - set -o pipefail - bash ./optools/run_bvt_group.sh \ - "$GITHUB_WORKSPACE/mo-tester" \ - "$GITHUB_WORKSPACE/test/distributed/cases" \ - "${bvt_group}" 2>&1 | tee "${RUNNER_TEMP}/bvt-compose.log" - - - name: Summarize Compose BVT result - if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} - run: | - set -euo pipefail - report="${RUNNER_TEMP}/bvt-compose.log" - timing="${RUNNER_TEMP}/bvt-compose-slowest.tsv" - if [ ! -s "${report}" ]; then - echo '### Compose + Proxy BVT result' | tee -a "$GITHUB_STEP_SUMMARY" - echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - { - echo '### Compose + Proxy BVT result' - grep '^Run BVT group' "${report}" | sed 's/^/- /' || true - echo - echo '#### Top 10 slow BVT scripts' - echo - } | tee -a "$GITHUB_STEP_SUMMARY" - - sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ - | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" - if [ ! -s "${timing}" ]; then - echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - { - echo '```text' - cat "${timing}" - echo '```' - } | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Upload Compose BVT execution log - if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} - uses: actions/upload-artifact@v7 - with: - name: bvt-execution-compose - path: ${{ runner.temp }}/bvt-compose.log - if-no-files-found: warn - retention-days: 7 + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/test/distributed/cases -e optimistic 2>&1 - name: Print Docker Info Before Container Shutdown if: ${{ always() }} @@ -432,39 +330,13 @@ jobs: run: | docker ps docker compose -f etc/launch-tae-compose/compose.yaml --profile launch-multi-cn down --remove-orphans - docker volume rm launch-tae-compose_minio_storage || true - - - name: Generate Compose BVT coverage profile - if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} - run: | - set -uo pipefail - coverage_dir="$GITHUB_WORKSPACE/coverage" - coverage_profile="$RUNNER_TEMP/bvt-compose.out" - if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then - test -s "${coverage_profile}" - elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then - echo '::error::failed to generate Compose BVT coverage after a successful BVT' - exit 1 - else - echo '::warning::Compose BVT coverage unavailable because BVT did not finish successfully' - fi - - - name: Upload Compose BVT coverage - if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} - uses: actions/upload-artifact@v7 - with: - name: bvt-coverage-compose-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/bvt-compose.out - ${{ runner.temp }}/bvt-compose-manifest.json - if-no-files-found: warn - retention-days: 7 + docker volume rm launch-tae-compose_minio_storage - uses: actions/upload-artifact@v7 if: ${{ failure() || cancelled()}} continue-on-error: true with: - name: ${{ inputs.enable_parallel_bvt && 'Compose-multi-cn-e2e-bvt-test-docker-log(PROXY)' || 'Compose-multi-cn-e2e-bvt-test-docker-log(PESSIMISTIC)' }} + name: Compose-multi-cn-e2e-bvt-test-docker-log(PESSIMISTIC) path: | ${{ github.workspace }}/docker-compose-log retention-days: 7 diff --git a/.github/workflows/e2e-standalone-parallel.yaml b/.github/workflows/e2e-standalone-parallel.yaml new file mode 100644 index 0000000..74b4ff9 --- /dev/null +++ b/.github/workflows/e2e-standalone-parallel.yaml @@ -0,0 +1,464 @@ +name: MatrixOne e2e CI(Standalone) + +on: + workflow_call: + inputs: + bvt_group: + description: "BVT group assigned by the caller (0 or 1)" + required: false + type: string + default: "" + bvt_generation: + description: "Shared assignment generation for complementary BVT jobs" + required: false + type: string + default: "" + secrets: + TOKEN_ACTION: + description: "Token for checkout (e.g. pull from fork/private)" + required: false + +jobs: + bvt-linux-x86: + if: false + environment: ci + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: e2e BVT Test on Linux/x64(LAUNCH,Optimistic) + timeout-minutes: 90 + steps: + - name: checkout head + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + path: ./head + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/head/go.mod" + - name: Build MatrixOne + run: | + cd $GITHUB_WORKSPACE/head && make clean && make build + git rev-parse --short HEAD + - name: echo config + run: | + cd $GITHUB_WORKSPACE/head + echo "" >> ./etc/launch/cn.toml + echo '[cn.txn]' >> ./etc/launch/cn.toml + echo 'mode = "Optimistic"' >> ./etc/launch/cn.toml + echo " enable-leak-check = 1" >> ./etc/launch/cn.toml + echo ' max-active-ages = "2m"'>> ./etc/launch/cn.toml + echo "" >> ./etc/launch/tn.toml + echo '[tn.txn]' >> ./etc/launch/tn.toml + echo 'mode = "Optimistic"' >> ./etc/launch/tn.toml + cat ./etc/launch/cn.toml + cat ./etc/launch/tn.toml + - name: Start MO + run: | + cd $GITHUB_WORKSPACE/head + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -e pessimistic_transaction 2>&1 + - name: Dump mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + max_retry=5 + retry=0 + while [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; do + curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/dump-stacks${retry}.log + pkill -9 mo-service + retry=$((retry+1)) + if [ ${retry} -ge ${max_retry} ]; then + echo 'after retry, still cannot shutdown mo-service' + exit 1 + fi + sleep 2 + done + - name: Restart the Version of Head of MO + run: | + cd $GITHUB_WORKSPACE/head + mv mo-service.log mo-service.r1.log + + # delete for clear the start finish status of mo + rm -rf mo-data/local/system_init_completed + + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch + - name: Start BVT Test for MO with Version of Head Restarted + id: bvt_on_latest_head_version_run2 + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -e pessimistic_transaction 2>&1 + - name: Dump restarted mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/restarted-dump-stacks.log; pkill -9 mo-service; else echo 'current mo-service has already crashed'; exit 1; fi + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + cd $GITHUB_WORKSPACE/head + # 4 nodes in one Process + ./optools/check_log_count.sh 4000 60 # {count threshold} {metric collected interval} + - name: generate upload files + if: ${{ always() }} + continue-on-error: true + run: | + set +e + mkdir -p ${{ github.workspace }}/upload + rm -rf ./mo-tester/.git + rm -rf ./mo-tester/lib + mv ${{ github.workspace }}/head/mo-service.r1.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/mo-service.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/mo-tester ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/dump-stacks* ${{ github.workspace }}/upload/ + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Standalone-e2e-BVT-Test-on-Linux-x64(LAUNCH,Optimistic)-reports + path: | + ${{ github.workspace }}/upload + retention-days: 7 + - uses: actions/upload-artifact@v7 + name: Upload MO DATA + if: ${{ failure() }} + continue-on-error: true + with: + name: Standalone-e2e-BVT-Test-on-Linux-x64(LAUNCH,Optimistic)-mo-data + path: | + ${{ github.workspace }}/head/mo-data + retention-days: 7 + + multi-cn-proxy-bvt-linux-x86: + # Compose + Proxy is the active proxy coverage deployment. Keep the + # native proxy launcher out of the PR matrix until it has a distinct role. + if: false + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: Legacy Launch + Proxy BVT (disabled in parallel workflow) + timeout-minutes: 75 + env: + mo_reuse_enable_checker: true + steps: + - name: checkout head + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + path: ./head + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/head/go.mod" + - name: Build MatrixOne + run: | + cd $GITHUB_WORKSPACE/head && make clean && make build + git rev-parse --short HEAD + - name: Prepare Hosts + run: | + sudo bash -c 'echo -e "127.0.0.1\tcn0" >> /etc/hosts;'; + sudo bash -c 'echo -e "127.0.0.1\tcn1" >> /etc/hosts;'; + - name: Add cn.txn + run: | + cd $GITHUB_WORKSPACE/head + echo "" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo "" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo "[cn.txn]" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo " enable-leak-check = 1" >> ./etc/launch-dynamic-with-proxy/cn.toml.base + echo ' max-active-ages = "2m"'>> ./etc/launch-dynamic-with-proxy/cn.toml.base + - name: Start MO + run: | + sudo cat /etc/hosts | grep '127.0.0.1'; + cd $GITHUB_WORKSPACE/head + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch-dynamic-with-proxy -with-proxy + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd $GITHUB_WORKSPACE/mo-tester + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -e optimistic 2>&1 + - name: Dump mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/dump-stacks.log; else echo 'current mo-service has already crashed'; exit 1; fi + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + cd $GITHUB_WORKSPACE/head + # one node in one Process + ./optools/check_log_count.sh 1000 60 # {count threshold} {metric collected interval} + - name: generate upload files + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p ${{ github.workspace }}/upload + rm -rf ./mo-tester/.git + rm -rf ./mo-tester/lib + mv ${{ github.workspace }}/head/mo-service.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/mo-tester ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/dump-stacks.log ${{ github.workspace }}/upload/ + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Standalone-Multi-CN-e2e-BVT-Test-on-Linux-x64(LAUNCH,PROXY)-reports + path: | + ${{ github.workspace }}/upload + retention-days: 7 + + pessimistic-bvt-linux-x86: + # Match the existing PR BVT runner policy. + runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} + name: e2e BVT Test on Linux/x64(LAUNCH, PESSIMISTIC) + timeout-minutes: 75 + env: + mo_reuse_enable_checker: true + steps: + - name: checkout head + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.TOKEN_ACTION }} + fetch-depth: "3" + path: ./head + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Go And Java + uses: matrixorigin/CI/actions/setup-env@main + with: + go-version-file: "${{ github.workspace }}/head/go.mod" + + - name: Build MatrixOne + run: | + cd "$GITHUB_WORKSPACE/head" + make clean && make GOBUILD_OPT=-cover build + git rev-parse --short HEAD + + - name: Add cn.txn + run: | + cd $GITHUB_WORKSPACE/head + echo "[cn.txn]" >> ./etc/launch/cn.toml + echo " enable-leak-check = 1" >> ./etc/launch/cn.toml + echo ' max-active-ages = "2m"'>> ./etc/launch/cn.toml + + - name: echo config + run: | + cd $GITHUB_WORKSPACE/head + cat ./etc/launch/cn.toml + cat ./etc/launch/tn.toml + + - name: Start MO + run: | + cd $GITHUB_WORKSPACE/head + export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-pessimistic" + rm -rf "${GOCOVERDIR}" + mkdir -p "${GOCOVERDIR}" + ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch + + - name: Clone test-tool repository + uses: actions/checkout@v6 + with: + repository: matrixorigin/mo-tester + path: ./mo-tester + ref: main + - name: Update BVT SQL Timeout + run: | + cd $GITHUB_WORKSPACE/mo-tester + sed -i "s/socketTimeout:.*/socketTimeout: 300000/g" mo.yml + - name: Start BVT Test + id: bvt_on_pr_version + run: | + export LC_ALL="C.UTF-8" + locale + cd "$GITHUB_WORKSPACE/mo-tester" + sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml + cat mo.yml + echo "=============================" + + bvt_group='${{ inputs.bvt_group }}' + bvt_generation='${{ inputs.bvt_generation }}' + if [ -z "${bvt_group}" ]; then + bvt_group=$(( 1 - ((${{ github.run_id }} + ${{ github.run_attempt }}) % 2) )) + fi + if [ -z "${bvt_generation}" ]; then + bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' + fi + if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then + echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" + exit 1 + fi + echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" + echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" + jq -n \ + --arg deployment 'launch-pessimistic' \ + --argjson group "${bvt_group}" \ + --arg generation "${bvt_generation}" \ + --arg head_sha '${{ github.event.pull_request.head.sha }}' \ + '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ + > "${RUNNER_TEMP}/bvt-pessimistic-manifest.json" + echo "Launch + Pessimistic BVT: generation ${bvt_generation}, group ${bvt_group}" + { + echo "### BVT group assignment" + echo "- Deployment: native launch (PESSIMISTIC)" + echo "- Group: ${bvt_group}" + echo "- Generation: ${bvt_generation}" + } >> "$GITHUB_STEP_SUMMARY" + cd "$GITHUB_WORKSPACE/head" + set -o pipefail + bash ./optools/run_bvt_group.sh \ + "$GITHUB_WORKSPACE/mo-tester" \ + "$GITHUB_WORKSPACE/head/test/distributed/cases" \ + "${bvt_group}" \ + "$GITHUB_WORKSPACE/head/test/distributed/resources" \ + 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log" + + - name: Summarize Launch + Pessimistic BVT result + if: ${{ always() && !cancelled() }} + run: | + set -euo pipefail + report="${RUNNER_TEMP}/bvt-pessimistic.log" + timing="${RUNNER_TEMP}/bvt-pessimistic-slowest.tsv" + if [ ! -s "${report}" ]; then + echo '### Launch + Pessimistic BVT result' | tee -a "$GITHUB_STEP_SUMMARY" + echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '### Launch + Pessimistic BVT result' + grep '^Run BVT group' "${report}" | sed 's/^/- /' || true + echo + echo '#### Top 10 slow BVT scripts' + echo + } | tee -a "$GITHUB_STEP_SUMMARY" + + sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ + | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" + if [ ! -s "${timing}" ]; then + echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + { + echo '```text' + cat "${timing}" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload Launch + Pessimistic BVT execution log + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-execution-pessimistic + path: ${{ runner.temp }}/bvt-pessimistic.log + if-no-files-found: warn + retention-days: 7 + + - name: Dump mo-service goroutines + if: ${{ always() && !cancelled() }} + run: | + if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then curl http://localhost:12345/debug/pprof/goroutine\?debug=2 -o ${{ github.workspace }}/head/dump-stacks.log; else echo 'current mo-service has already crashed'; exit 1; fi + + - name: Check Log Messages Count per second + if: ${{ always() && !cancelled() }} + run: | + cd $GITHUB_WORKSPACE/head + # 4 nodes in one Process + ./optools/check_log_count.sh 4000 60 # {count threshold} {metric collected interval} + + - name: Stop service and generate BVT coverage profile + if: ${{ always() && !cancelled() }} + run: | + set -uo pipefail + coverage_dir="${RUNNER_TEMP}/bvt-coverage-pessimistic" + coverage_profile="${RUNNER_TEMP}/bvt-pessimistic.out" + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -TERM ${pids} || true + fi + for _ in $(seq 1 30); do + pgrep -f 'mo-service' >/dev/null || break + sleep 1 + done + pids=$(pgrep -f 'mo-service' || true) + if [ -n "${pids}" ]; then + kill -KILL ${pids} || true + fi + if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then + test -s "${coverage_profile}" + elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then + echo '::error::failed to generate Launch + Pessimistic BVT coverage after a successful BVT' + exit 1 + else + echo '::warning::Launch + Pessimistic BVT coverage unavailable because BVT did not finish successfully' + fi + + - name: Upload Launch + Pessimistic BVT coverage + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: bvt-coverage-pessimistic-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/bvt-pessimistic.out + ${{ runner.temp }}/bvt-pessimistic-manifest.json + if-no-files-found: warn + retention-days: 7 + + - name: generate upload files + if: ${{ always() }} + continue-on-error: true + run: | + mkdir -p ${{ github.workspace }}/upload + rm -rf ./mo-tester/.git + rm -rf ./mo-tester/lib + mv ${{ github.workspace }}/head/mo-service.log ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/mo-tester ${{ github.workspace }}/upload/ + mv ${{ github.workspace }}/head/dump-stacks.log ${{ github.workspace }}/upload/ + + - uses: actions/upload-artifact@v7 + if: ${{ failure() || cancelled()}} + continue-on-error: true + with: + name: Standalone-e2e-BVT-Test-on-Linux-x64(LAUNCH,PESSIMISTIC)-reports + path: | + ${{ github.workspace }}/upload + retention-days: 7 diff --git a/.github/workflows/e2e-standalone.yaml b/.github/workflows/e2e-standalone.yaml index 3edf443..1ecbb25 100644 --- a/.github/workflows/e2e-standalone.yaml +++ b/.github/workflows/e2e-standalone.yaml @@ -2,22 +2,6 @@ name: MatrixOne e2e CI(Standalone) on: workflow_call: - inputs: - enable_parallel_bvt: - description: "Run only complementary Launch + Pessimistic BVT and emit coverage" - required: false - type: boolean - default: false - bvt_group: - description: "BVT group assigned by the caller (0 or 1)" - required: false - type: string - default: "" - bvt_generation: - description: "Shared assignment generation for complementary BVT jobs" - required: false - type: string - default: "" secrets: TOKEN_ACTION: description: "Token for checkout (e.g. pull from fork/private)" @@ -38,7 +22,7 @@ jobs: fetch-depth: "3" path: ./head repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -161,11 +145,8 @@ jobs: retention-days: 7 multi-cn-proxy-bvt-linux-x86: - # Compose + Proxy is the active proxy coverage deployment. Keep the - # native proxy launcher out of the PR matrix until it has a distinct role. - if: ${{ !inputs.enable_parallel_bvt }} runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} - name: ${{ inputs.enable_parallel_bvt && 'Legacy Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) (disabled)' || 'Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY)' }} + name: Multi-CN e2e BVT Test on Linux/x64(LAUNCH, PROXY) timeout-minutes: 75 env: mo_reuse_enable_checker: true @@ -176,7 +157,7 @@ jobs: token: ${{ secrets.TOKEN_ACTION }} path: ./head repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main with: @@ -253,7 +234,6 @@ jobs: retention-days: 7 pessimistic-bvt-linux-x86: - # Match the existing PR BVT runner policy. runs-on: ${{ vars.RUNNER_LABEL || 'ubuntu-22.04' }} name: e2e BVT Test on Linux/x64(LAUNCH, PESSIMISTIC) timeout-minutes: 75 @@ -267,7 +247,7 @@ jobs: fetch-depth: "3" path: ./head repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.head.ref }} - name: Set up Go And Java uses: matrixorigin/CI/actions/setup-env@main @@ -276,12 +256,7 @@ jobs: - name: Build MatrixOne run: | - cd "$GITHUB_WORKSPACE/head" - if [ '${{ inputs.enable_parallel_bvt }}' = 'true' ]; then - make clean && make GOBUILD_OPT=-cover build - else - make clean && make build - fi + cd $GITHUB_WORKSPACE/head && make clean && make build git rev-parse --short HEAD - name: Add cn.txn @@ -300,11 +275,6 @@ jobs: - name: Start MO run: | cd $GITHUB_WORKSPACE/head - if [ '${{ inputs.enable_parallel_bvt }}' = 'true' ]; then - export GOCOVERDIR="${RUNNER_TEMP}/bvt-coverage-pessimistic" - rm -rf "${GOCOVERDIR}" - mkdir -p "${GOCOVERDIR}" - fi ./optools/run_bvt.sh $GITHUB_WORKSPACE/head launch - name: Clone test-tool repository @@ -322,95 +292,12 @@ jobs: run: | export LC_ALL="C.UTF-8" locale - cd "$GITHUB_WORKSPACE/mo-tester" + cd $GITHUB_WORKSPACE/mo-tester sed -i 's/ port: [0-9]*/ port: 12345/g' mo.yml cat mo.yml echo "=============================" - if [ '${{ inputs.enable_parallel_bvt }}' != 'true' ]; then - ./run.sh -n -g -o \ - -p "$GITHUB_WORKSPACE/head/test/distributed/cases" \ - -s "$GITHUB_WORKSPACE/head/test/distributed/resources" \ - -e optimistic 2>&1 - exit - fi - - bvt_group='${{ inputs.bvt_group }}' - bvt_generation='${{ inputs.bvt_generation }}' - if [ -z "${bvt_group}" ]; then - bvt_group=$(( 1 - ((${{ github.run_id }} + ${{ github.run_attempt }}) % 2) )) - fi - if [ -z "${bvt_generation}" ]; then - bvt_generation='${{ github.run_id }}-${{ github.run_attempt }}' - fi - if [[ ! "${bvt_group}" =~ ^[01]$ ]] || [ -z "${bvt_generation}" ]; then - echo "::error::invalid BVT assignment: group=${bvt_group}, generation=${bvt_generation}" - exit 1 - fi - echo "bvt_group=${bvt_group}" >> "$GITHUB_OUTPUT" - echo "bvt_generation=${bvt_generation}" >> "$GITHUB_OUTPUT" - jq -n \ - --arg deployment 'launch-pessimistic' \ - --argjson group "${bvt_group}" \ - --arg generation "${bvt_generation}" \ - --arg head_sha '${{ github.event.pull_request.head.sha }}' \ - '{schema_version: 1, deployment: $deployment, group: $group, generation: $generation, head_sha: $head_sha}' \ - > "${RUNNER_TEMP}/bvt-pessimistic-manifest.json" - echo "Launch + Pessimistic BVT: generation ${bvt_generation}, group ${bvt_group}" - { - echo "### BVT group assignment" - echo "- Deployment: native launch (PESSIMISTIC)" - echo "- Group: ${bvt_group}" - echo "- Generation: ${bvt_generation}" - } >> "$GITHUB_STEP_SUMMARY" - cd "$GITHUB_WORKSPACE/head" - set -o pipefail - bash ./optools/run_bvt_group.sh \ - "$GITHUB_WORKSPACE/mo-tester" \ - "$GITHUB_WORKSPACE/head/test/distributed/cases" \ - "${bvt_group}" \ - "$GITHUB_WORKSPACE/head/test/distributed/resources" \ - 2>&1 | tee "${RUNNER_TEMP}/bvt-pessimistic.log" - - - name: Summarize Launch + Pessimistic BVT result - if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} - run: | - set -euo pipefail - report="${RUNNER_TEMP}/bvt-pessimistic.log" - timing="${RUNNER_TEMP}/bvt-pessimistic-slowest.tsv" - if [ ! -s "${report}" ]; then - echo '### Launch + Pessimistic BVT result' | tee -a "$GITHUB_STEP_SUMMARY" - echo '- No BVT execution log was produced.' | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - { - echo '### Launch + Pessimistic BVT result' - grep '^Run BVT group' "${report}" | sed 's/^/- /' || true - echo - echo '#### Top 10 slow BVT scripts' - echo - } | tee -a "$GITHUB_STEP_SUMMARY" - - sed -nE 's#.*script file\[(.*)\] has been executed, and cost: ([0-9.]+)s.*#\2s\t\1#p' "${report}" \ - | sort -nr -k1,1 | awk 'NR <= 10 { print }' > "${timing}" - if [ ! -s "${timing}" ]; then - echo 'No per-script timing records found in BVT output.' | tee -a "$GITHUB_STEP_SUMMARY" - exit 0 - fi - { - echo '```text' - cat "${timing}" - echo '```' - } | tee -a "$GITHUB_STEP_SUMMARY" - - - name: Upload Launch + Pessimistic BVT execution log - if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} - uses: actions/upload-artifact@v7 - with: - name: bvt-execution-pessimistic - path: ${{ runner.temp }}/bvt-pessimistic.log - if-no-files-found: warn - retention-days: 7 + ./run.sh -n -g -o -p $GITHUB_WORKSPACE/head/test/distributed/cases -s $GITHUB_WORKSPACE/head/test/distributed/resources -e optimistic 2>&1 - name: Dump mo-service goroutines if: ${{ always() && !cancelled() }} @@ -424,52 +311,10 @@ jobs: # 4 nodes in one Process ./optools/check_log_count.sh 4000 60 # {count threshold} {metric collected interval} - - name: Stop service and generate BVT coverage profile + - name: Check mo-service Status if: ${{ always() && !cancelled() }} run: | - set -uo pipefail - if [ '${{ inputs.enable_parallel_bvt }}' != 'true' ]; then - if pgrep -f 'mo-service' >/dev/null; then - pkill -9 mo-service - else - echo 'current mo-service has already crashed' - exit 1 - fi - exit - fi - coverage_dir="${RUNNER_TEMP}/bvt-coverage-pessimistic" - coverage_profile="${RUNNER_TEMP}/bvt-pessimistic.out" - pids=$(pgrep -f 'mo-service' || true) - if [ -n "${pids}" ]; then - kill -TERM ${pids} || true - fi - for _ in $(seq 1 30); do - pgrep -f 'mo-service' >/dev/null || break - sleep 1 - done - pids=$(pgrep -f 'mo-service' || true) - if [ -n "${pids}" ]; then - kill -KILL ${pids} || true - fi - if go tool covdata textfmt -i="${coverage_dir}" -o "${coverage_profile}"; then - test -s "${coverage_profile}" - elif [ "${{ steps.bvt_on_pr_version.conclusion }}" = "success" ]; then - echo '::error::failed to generate Launch + Pessimistic BVT coverage after a successful BVT' - exit 1 - else - echo '::warning::Launch + Pessimistic BVT coverage unavailable because BVT did not finish successfully' - fi - - - name: Upload Launch + Pessimistic BVT coverage - if: ${{ inputs.enable_parallel_bvt && always() && !cancelled() }} - uses: actions/upload-artifact@v7 - with: - name: bvt-coverage-pessimistic-generation-${{ steps.bvt_on_pr_version.outputs.bvt_generation || format('{0}-{1}', github.run_id, github.run_attempt) }}-attempt-${{ github.run_attempt }} - path: | - ${{ runner.temp }}/bvt-pessimistic.out - ${{ runner.temp }}/bvt-pessimistic-manifest.json - if-no-files-found: warn - retention-days: 7 + if [ "$(ps -ef | grep 'mo-service' | grep -v "grep" | wc -l)" -gt 0 ]; then pkill -9 mo-service; else echo 'current mo-service has already crashed'; exit 1; fi - name: generate upload files if: ${{ always() }} diff --git a/.github/workflows/utils-parallel.yaml b/.github/workflows/utils-parallel.yaml new file mode 100644 index 0000000..67446de --- /dev/null +++ b/.github/workflows/utils-parallel.yaml @@ -0,0 +1,29 @@ +# Utilities that remain outside the parallel coverage pipeline. +name: MatrixOne Utils CI + +on: + workflow_call: + secrets: + TOKEN_ACTION: + description: "A token passed from the caller workflow" + required: true + +jobs: + pr-size-label: + environment: ci + runs-on: arm64-mo-shanghai-4c8g + steps: + - name: size-label + uses: matrixorigin/CI/actions/label-size-action@main + with: + size_token: ${{ secrets.TOKEN_ACTION }} + ignore: ".md,.pb.go" + sizes: > + { + "XS":0, + "S":10, + "M":100, + "L":500, + "XL":1000, + "XXL":2000 + } diff --git a/.github/workflows/utils.yaml b/.github/workflows/utils.yaml index 00efbad..4656f11 100644 --- a/.github/workflows/utils.yaml +++ b/.github/workflows/utils.yaml @@ -3,12 +3,6 @@ name: MatrixOne Utils CI on: workflow_call: - inputs: - enable_legacy_coverage: - description: "Keep the serial coverage job until the caller enables parallel coverage producers" - required: false - type: boolean - default: true secrets: TOKEN_ACTION: description: "A token passed from the caller workflow" @@ -117,11 +111,9 @@ jobs: fi pr_coverage: - # Default-on preserves coverage for existing callers while MatrixOne rolls - # out the parallel producers. The new entrypoint disables this job in the - # same commit that starts coverage-ut.yaml and coverage-merge.yaml. - if: ${{ inputs.enable_legacy_coverage && (needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1') && needs.check_organization_user.outputs.coverage_secrets_available == 'true' }} - name: ${{ inputs.enable_legacy_coverage && 'Coverage' || 'Legacy Coverage (disabled)' }} + # Secrets cannot be referenced directly in job-level conditions; preflight prevents allocating the large ARC runner. + if: ${{ (needs.check_organization_user.outputs.safe_label == '1' || needs.check_organization_user.outputs.in_org == '1') && needs.check_organization_user.outputs.coverage_secrets_available == 'true' }} + name: Coverage needs: [check_organization_user] environment: ci # If the runner/workspace layout changes, also update setup-env's go-version-file below.