From 1d067bbb940d1dfa7feb3686c5f0e0c2932d36ef Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 16 Mar 2026 20:58:21 -0400 Subject: [PATCH 01/24] concurrency and cancel in progress pushes --- .github/workflows/glean-probe-scraper.yml | 1 + .github/workflows/publish-docs.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/glean-probe-scraper.yml b/.github/workflows/glean-probe-scraper.yml index 8930a4ad84..cfc222614b 100644 --- a/.github/workflows/glean-probe-scraper.yml +++ b/.github/workflows/glean-probe-scraper.yml @@ -5,6 +5,7 @@ on: branches: [master] pull_request: branches: [master] + permissions: {} # workflow-level default — deny all concurrency: diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index 7aef9a05e1..583061e45d 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -6,9 +6,11 @@ on: branches: [master, main] # paths: ['docs/**.md'] # API docs need building always +# This ensures any in-progress workflows in the same branch or PR +# are cancelled if there is a fresh push. concurrency: - group: github-pages - cancel-in-progress: false # Skip any intermediate builds but finish deploying + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true # Skip any intermediate builds but finish deploying jobs: build-mdbook: From e80066529d418df328b52df38e492de329626041 Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 20 Mar 2026 12:43:28 -0400 Subject: [PATCH 02/24] cache rust build, move clippy into a matrix strategy --- .github/actions/setup-rust/action.yml | 8 ++++++++ .github/workflows/checks.yml | 28 ++++++++++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 2d0676c4b9..163063ace9 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -18,6 +18,14 @@ runs: with: persist-credentials: false + - name: Cache Rust toolchain + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: ${{ runner.os }}-rust-toolchain-${{ inputs.rust-version }} + - name: Install Rust toolchain uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 with: diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3cd2d8f415..e2a17da0f1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -55,17 +55,27 @@ jobs: - name: Cargo Audit run: cargo audit - - name: Rust Clippy Spanner - run: make clippy_spanner - - - name: Rust Clippy MySQL - run: make clippy_mysql - - - name: Rust Clippy Postgres - run: make clippy_postgres - - name: Setup documentation checks run: make doc-install-deps - name: Documentation checks run: make doc-test + + clippy: + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + matrix: + target: [spanner, mysql, postgres] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: ./.github/actions/setup-rust + with: + workspace-path: workflow + + - name: Rust Clippy ${{ matrix.target }} + run: make clippy_${{ matrix.target }} From 7f4e90e52d945ccbebd42d251b02f9d7e5f0eecf Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 20 Mar 2026 13:50:09 -0400 Subject: [PATCH 03/24] setup steps cache --- .github/workflows/checks.yml | 102 ++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 8 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e2a17da0f1..187abebbff 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,7 +17,70 @@ concurrency: permissions: {} # workflow-level default — deny all jobs: + # Setup jobs and populate cache + + python-env: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Cache pip and Poetry virtualenv + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cache/pip + ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + + - name: Install Poetry + run: pip3 install poetry + + - name: Install Python dependencies + run: poetry install --with tokenserver-unit-tests,dev --no-interaction --no-ansi + + - name: Display Python Version Info + shell: bash + run: | + if [ "$(which python)" != "" ]; then python --version; fi + uname -a + cat /etc/os-release + + rust-env: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Cache Rust toolchain + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: ${{ runner.os }}-rust-toolchain-1.91 # RUST_VER + + - name: Install Rust toolchain + run: rustup toolchain install 1.91 --component rustfmt --component clippy --no-self-update && rustup default 1.91 + + - name: Install cargo-audit + run: cargo install --locked cargo-audit + + - name: Display Rust Version Info + shell: bash + run: | + if [ "$(which rustc)" != "" ]; then rustc --version; fi + uname -a + cat /etc/os-release + python-checks: + needs: python-env runs-on: ubuntu-latest permissions: contents: read @@ -26,17 +89,25 @@ jobs: with: persist-credentials: false - - uses: ./.github/actions/setup-python + - name: Restore pip and Poetry virtualenv + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: - workspace-path: workflow + path: | + ~/.cache/pip + ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + + - name: Install Poetry + run: pip3 install poetry - name: Python Format Check - run: make ruff-fmt-chk + run: poetry run ruff format --diff tools - name: Python Lint Check - run: make ruff-lint + run: poetry run ruff check tools rust-checks: + needs: rust-env runs-on: ubuntu-latest permissions: contents: read @@ -45,9 +116,16 @@ jobs: with: persist-credentials: false - - uses: ./.github/actions/setup-rust + - name: Restore Rust toolchain + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: - workspace-path: workflow + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: ${{ runner.os }}-rust-toolchain-1.91 # RUST_VER + + - name: Set Rust toolchain + run: rustup default 1.91 - name: Rust Format Check run: cargo fmt -- --check @@ -62,6 +140,7 @@ jobs: run: make doc-test clippy: + needs: rust-env runs-on: ubuntu-latest permissions: contents: read @@ -73,9 +152,16 @@ jobs: with: persist-credentials: false - - uses: ./.github/actions/setup-rust + - name: Restore Rust toolchain + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: - workspace-path: workflow + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: ${{ runner.os }}-rust-toolchain-1.91 # RUST_VER + + - name: Set Rust toolchain + run: rustup default 1.91 - name: Rust Clippy ${{ matrix.target }} run: make clippy_${{ matrix.target }} From 3946c395bcdeaa5a67681eed615b3e8484c27292 Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 20 Mar 2026 13:59:57 -0400 Subject: [PATCH 04/24] cargo audit fix --- .github/workflows/checks.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 187abebbff..0135a45a00 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -28,6 +28,11 @@ jobs: with: persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ inputs.python-version }} + - name: Cache pip and Poetry virtualenv uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: @@ -69,9 +74,6 @@ jobs: - name: Install Rust toolchain run: rustup toolchain install 1.91 --component rustfmt --component clippy --no-self-update && rustup default 1.91 - - name: Install cargo-audit - run: cargo install --locked cargo-audit - - name: Display Rust Version Info shell: bash run: | @@ -130,6 +132,9 @@ jobs: - name: Rust Format Check run: cargo fmt -- --check + - name: Install cargo-audit + run: cargo install --locked cargo-audit + - name: Cargo Audit run: cargo audit From fb17c110b3b88375117a07c3ca1fff6119157228 Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 20 Mar 2026 14:02:40 -0400 Subject: [PATCH 05/24] version constraints for python and rust --- .github/workflows/checks.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 0135a45a00..5df836ef67 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -9,6 +9,20 @@ on: pull_request: branches: - "**" + workflow_dispatch: + inputs: + python-version: + description: "Python version to use" + required: false + default: "3.12" # PY_VER + rust-version: + description: "Rust version to use" + required: false + default: "1.91" # RUST_VER + +env: + PYTHON_VERSION: ${{ inputs.python-version || '3.12' }} # PY_VER + RUST_VERSION: ${{ inputs.rust-version || '1.91' }} # RUST_VER concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} @@ -31,7 +45,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: - python-version: ${{ inputs.python-version }} + python-version: ${{ env.PYTHON_VERSION }} - name: Cache pip and Poetry virtualenv uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 @@ -69,10 +83,10 @@ jobs: path: | ~/.rustup/toolchains ~/.rustup/update-hashes - key: ${{ runner.os }}-rust-toolchain-1.91 # RUST_VER + key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} - name: Install Rust toolchain - run: rustup toolchain install 1.91 --component rustfmt --component clippy --no-self-update && rustup default 1.91 + run: rustup toolchain install ${{ env.RUST_VERSION }} --component rustfmt --component clippy --no-self-update && rustup default ${{ env.RUST_VERSION }} - name: Display Rust Version Info shell: bash @@ -124,10 +138,10 @@ jobs: path: | ~/.rustup/toolchains ~/.rustup/update-hashes - key: ${{ runner.os }}-rust-toolchain-1.91 # RUST_VER + key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} - name: Set Rust toolchain - run: rustup default 1.91 + run: rustup default ${{ env.RUST_VERSION }} - name: Rust Format Check run: cargo fmt -- --check @@ -163,10 +177,10 @@ jobs: path: | ~/.rustup/toolchains ~/.rustup/update-hashes - key: ${{ runner.os }}-rust-toolchain-1.91 # RUST_VER + key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} - name: Set Rust toolchain - run: rustup default 1.91 + run: rustup default ${{ env.RUST_VERSION }} - name: Rust Clippy ${{ matrix.target }} run: make clippy_${{ matrix.target }} From cf042d3b1a4e755b0e47d1a3453fe75f81d2958b Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 20 Mar 2026 14:09:19 -0400 Subject: [PATCH 06/24] update order --- .github/workflows/checks.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5df836ef67..adde982f65 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -143,18 +143,18 @@ jobs: - name: Set Rust toolchain run: rustup default ${{ env.RUST_VERSION }} - - name: Rust Format Check - run: cargo fmt -- --check - - name: Install cargo-audit run: cargo install --locked cargo-audit - - name: Cargo Audit - run: cargo audit - - name: Setup documentation checks run: make doc-install-deps + - name: Rust Format Check + run: cargo fmt -- --check + + - name: Cargo Audit + run: cargo audit + - name: Documentation checks run: make doc-test From c10f343c0e1d14d3e35b6f2b819133673ad02a66 Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 20 Mar 2026 14:43:22 -0400 Subject: [PATCH 07/24] run for any push to the main branch or when a pull request is opened or updated --- .github/workflows/checks.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index adde982f65..0472c54eca 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,12 +3,11 @@ name: Checks on: push: branches: - - "**" + - master + - main tags: - - "*" - pull_request: - branches: - "**" + pull_request: workflow_dispatch: inputs: python-version: From 56f9e5af6b308af8411057e8748c19e2e28d22e8 Mon Sep 17 00:00:00 2001 From: Taddes Date: Fri, 20 Mar 2026 15:15:20 -0400 Subject: [PATCH 08/24] try to cache mdbook and cargo audit --- .github/workflows/checks.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 0472c54eca..b77e639c37 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -142,10 +142,28 @@ jobs: - name: Set Rust toolchain run: rustup default ${{ env.RUST_VERSION }} + - name: Cache cargo-audit + id: cache-cargo-audit + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: ~/.cargo/bin/cargo-audit + key: ${{ runner.os }}-cargo-audit-${{ hashFiles('.github/workflows/checks.yml') }} + - name: Install cargo-audit + if: steps.cache-cargo-audit.outputs.cache-hit != 'true' run: cargo install --locked cargo-audit + - name: Cache mdbook + id: cache-mdbook + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cargo/bin/mdbook + ~/.cargo/bin/mdbook-mermaid + key: ${{ runner.os }}-mdbook-${{ hashFiles('Makefile') }} + - name: Setup documentation checks + if: steps.cache-mdbook.outputs.cache-hit != 'true' run: make doc-install-deps - name: Rust Format Check From d225da3ec832e71edbf54b96e4fa870444e8cd5c Mon Sep 17 00:00:00 2001 From: Taddes Date: Mon, 23 Mar 2026 15:01:23 -0400 Subject: [PATCH 09/24] reduce dependabot spam by creating dep groups --- .github/dependabot.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 967f9de21e..89688f65a7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,6 +8,12 @@ updates: open-pull-requests-limit: 1 labels: - dependencies + groups: + dev-deps: + dependency-type: "development" + prod-deps: + dependency-type: "production" + ignore: - dependency-name: actix-rt versions: @@ -69,6 +75,10 @@ updates: open-pull-requests-limit: 1 labels: - dependencies + groups: + python-deps: + patterns: + - "*" - package-ecosystem: "github-actions" directory: "/" @@ -78,3 +88,7 @@ updates: open-pull-requests-limit: 1 labels: - dependencies + groups: + actions-deps: + patterns: + - "*" From b3acae53ebc76ed2b2c7ceada0fd40eb7412ff06 Mon Sep 17 00:00:00 2001 From: Taddes Date: Wed, 25 Mar 2026 16:45:10 -0400 Subject: [PATCH 10/24] test postgres unit test --- .github/workflows/checks.yml | 134 +++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b77e639c37..e7f9158c81 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -201,3 +201,137 @@ jobs: - name: Rust Clippy ${{ matrix.target }} run: make clippy_${{ matrix.target }} + + build-and-test-postgres: + runs-on: ubuntu-latest + permissions: + contents: read + checks: write + + services: + postgres: + image: postgres:18.0 + env: + POSTGRES_USER: test + POSTGRES_PASSWORD: test + POSTGRES_DB: syncstorage + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U test" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + + env: + SYNC_SYNCSTORAGE__DATABASE_URL: postgres://test:test@127.0.0.1/syncstorage + SYNC_TOKENSERVER__DATABASE_URL: postgres://test:test@127.0.0.1/tokenserver + SYNC_TOKENSERVER__NODE_TYPE: postgres + RUST_BACKTRACE: 1 + RUST_TEST_THREADS: 1 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Restore Rust toolchain + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} + + - name: Restore pip and Poetry virtualenv + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cache/pip + ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + + - name: Install PostgreSQL client + run: sudo apt-get update && sudo apt-get install -y postgresql-client + + - name: Create Tokenserver database + run: | + PGPASSWORD=test psql -U test -h 127.0.0.1 -d syncstorage -c 'CREATE DATABASE tokenserver;' + + - name: Create version.json + run: | + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "${GITHUB_SHA}" \ + "${GITHUB_REF_NAME}" \ + "${GITHUB_REPOSITORY_OWNER}" \ + "${GITHUB_REPOSITORY_NAME}" \ + "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + > version.json + env: + GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + + - name: Install cargo-nextest + run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + + - name: Install cargo-llvm-cov + run: cargo install --locked cargo-llvm-cov + + - name: Run unit tests with coverage + run: make postgres_test_with_coverage + + - name: Run unit tests with coverage (quota enforced) + run: make postgres_test_with_coverage + env: + SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 + + - name: Run Postgres utils tests + working-directory: tools/postgres + run: | + poetry install --no-interaction --no-ansi + WORKFLOW=$(echo "${GITHUB_WORKFLOW}" | tr ' ' '-' | tr '[:upper:]' '[:lower:]') + poetry run pytest test_purge_ttl.py -v --junit-xml="../../workflow/test-results/${GITHUB_RUN_NUMBER}__$(date +%s)__$(basename ${GITHUB_REPOSITORY})__${WORKFLOW}__postgres_utils__results.xml" + env: + SYNC_SYNCSTORAGE__DATABASE_URL: postgresql://test:test@127.0.0.1/syncstorage + + - name: Publish Test Report + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + if: always() + with: + name: Postgres Unit Tests + path: workflow/test-results/*.xml + reporter: java-junit + fail-on-error: false + + - name: Upload test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: postgres-test-results + path: workflow/test-results/ + + # Upload to GCS on master + - name: Authenticate to Google Cloud + if: github.ref == 'refs/heads/master' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} + + - name: Upload JUnit results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/junit + glob: "*.xml" + parent: false + process_gcloudignore: false + + - name: Upload coverage results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage + glob: "*.json" + parent: false + process_gcloudignore: false From 95e68bbfd58c45503cc6c1389d0f3c6db89f6241 Mon Sep 17 00:00:00 2001 From: Taddes Date: Wed, 25 Mar 2026 18:15:59 -0400 Subject: [PATCH 11/24] full postgres test workflow with checks --- .github/workflows/checks.yml | 113 ++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e7f9158c81..c3d738648a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -202,7 +202,9 @@ jobs: - name: Rust Clippy ${{ matrix.target }} run: make clippy_${{ matrix.target }} - build-and-test-postgres: + # Postgres + build-and-unit-test-postgres: + needs: [rust-env, python-env] runs-on: ubuntu-latest permissions: contents: read @@ -335,3 +337,112 @@ jobs: glob: "*.json" parent: false process_gcloudignore: false + + build-postgres-image: + runs-on: ubuntu-latest + needs: [rust-env, python-env] + permissions: + contents: read + actions: write + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Create version.json + run: | + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "${GITHUB_SHA}" \ + "${GITHUB_REF_NAME}" \ + "${GITHUB_REPOSITORY_OWNER}" \ + "${GITHUB_REPOSITORY_NAME}" \ + "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + > version.json + env: + GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build Postgres Docker image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + push: false + tags: app:build + build-args: | + SYNCSTORAGE_DATABASE_BACKEND=postgres + TOKENSERVER_DATABASE_BACKEND=postgres + outputs: type=docker,dest=/tmp/postgres-image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload Docker image artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: postgres-docker-image + path: /tmp/postgres-image.tar + retention-days: 1 + + postgres-e2e-tests: + runs-on: ubuntu-latest + needs: build-postgres-image + permissions: + contents: read + checks: write + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Download Docker image + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + name: postgres-docker-image + path: /tmp + + - name: Load Docker image + run: docker load --input /tmp/postgres-image.tar + + - name: Create test results directory + run: mkdir -p workflow/test-results + + - name: Run Postgres e2e tests + run: make docker_run_postgres_e2e_tests + env: + SYNCSTORAGE_RS_IMAGE: app:build + + - name: Publish E2E Test Report + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + if: always() + with: + name: Postgres E2E Tests + path: workflow/test-results/*.xml + reporter: java-junit + fail-on-error: false + + - name: Upload e2e test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: postgres-e2e-test-results + path: workflow/test-results/ + + # Upload to GCS on master + - name: Authenticate to Google Cloud + if: github.ref == 'refs/heads/master' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} + + - name: Upload e2e test results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/junit + glob: "*.xml" + parent: false + process_gcloudignore: false From 59d168f89943c9e697c7fec4405ecdfbf39da150 Mon Sep 17 00:00:00 2001 From: Taddes Date: Wed, 25 Mar 2026 18:28:14 -0400 Subject: [PATCH 12/24] add caching --- .github/workflows/checks.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c3d738648a..5179e63966 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -362,10 +362,19 @@ jobs: env: GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + - name: Cache Docker image tar + id: cache-postgres-image + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: /tmp/postgres-image.tar + key: ${{ runner.os }}-postgres-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml') }} + - name: Set up Docker Buildx + if: steps.cache-postgres-image.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build Postgres Docker image + if: steps.cache-postgres-image.outputs.cache-hit != 'true' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . From edfb8732be9940717429521eeb2c53cd8c374466 Mon Sep 17 00:00:00 2001 From: Taddes Date: Wed, 25 Mar 2026 18:42:48 -0400 Subject: [PATCH 13/24] add mysql support and add tolls and scripts changes to hash key --- .github/workflows/checks.yml | 246 ++++++++++++++++++++++++++++++++++- 1 file changed, 245 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5179e63966..9d674ec0f1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -367,7 +367,7 @@ jobs: uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: /tmp/postgres-image.tar - key: ${{ runner.os }}-postgres-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml') }} + key: ${{ runner.os }}-postgres-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} - name: Set up Docker Buildx if: steps.cache-postgres-image.outputs.cache-hit != 'true' @@ -455,3 +455,247 @@ jobs: glob: "*.xml" parent: false process_gcloudignore: false + + # MySQL + build-and-test-mysql: + runs-on: ubuntu-latest + needs: [rust-env, python-env] + permissions: + contents: read + checks: write + + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: password + MYSQL_USER: test + MYSQL_PASSWORD: test + MYSQL_DATABASE: syncstorage + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + + env: + SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@127.0.0.1/syncstorage + SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver + SYNC_TOKENSERVER__NODE_TYPE: spanner + RUST_BACKTRACE: 1 + RUST_TEST_THREADS: 1 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Restore Rust toolchain + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} + + - name: Restore pip and Poetry virtualenv + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cache/pip + ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + + - name: Install MySQL client + run: sudo apt-get update && sudo apt-get install -y default-mysql-client + + - name: Create Tokenserver database + run: | + mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' + mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';" + + - name: Create version.json + run: | + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "${GITHUB_SHA}" \ + "${GITHUB_REF_NAME}" \ + "${GITHUB_REPOSITORY_OWNER}" \ + "${GITHUB_REPOSITORY_NAME}" \ + "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + > version.json + env: + GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + + - name: Install cargo-nextest + run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + + - name: Install cargo-llvm-cov + run: cargo install --locked cargo-llvm-cov + + - name: Run unit tests with coverage + run: make test_with_coverage + + - name: Run unit tests with coverage (quota enforced) + run: make test_with_coverage + env: + SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 + + - name: Publish Test Report + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + if: always() + with: + name: MySQL Unit Tests + path: workflow/test-results/*.xml + reporter: java-junit + fail-on-error: false + + - name: Upload test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: mysql-test-results + path: workflow/test-results/ + + # Upload to GCS on master + - name: Authenticate to Google Cloud + if: github.ref == 'refs/heads/master' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} + + - name: Upload JUnit results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/junit + glob: "*.xml" + parent: false + process_gcloudignore: false + + - name: Upload coverage results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage + glob: "*.json" + parent: false + process_gcloudignore: false + + build-mysql-image: + runs-on: ubuntu-latest + needs: [rust-env, python-env] + permissions: + contents: read + actions: write + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Create version.json + run: | + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "${GITHUB_SHA}" \ + "${GITHUB_REF_NAME}" \ + "${GITHUB_REPOSITORY_OWNER}" \ + "${GITHUB_REPOSITORY_NAME}" \ + "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + > version.json + env: + GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + + - name: Cache Docker image tar + id: cache-mysql-image + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: /tmp/mysql-image.tar + key: ${{ runner.os }}-mysql-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build MySQL Docker image + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + push: false + tags: app:build + build-args: | + SYNCSTORAGE_DATABASE_BACKEND=mysql + TOKENSERVER_DATABASE_BACKEND=mysql + outputs: type=docker,dest=/tmp/mysql-image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload Docker image artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: mysql-docker-image + path: /tmp/mysql-image.tar + retention-days: 1 + + mysql-e2e-tests: + runs-on: ubuntu-latest + needs: build-mysql-image + permissions: + contents: read + checks: write + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Download Docker image + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + name: mysql-docker-image + path: /tmp + + - name: Load Docker image + run: docker load --input /tmp/mysql-image.tar + + - name: Create test results directory + run: mkdir -p workflow/test-results + + - name: Run MySQL e2e tests + run: make docker_run_mysql_e2e_tests + env: + SYNCSTORAGE_RS_IMAGE: app:build + + - name: Publish E2E Test Report + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + if: always() + with: + name: MySQL E2E Tests + path: workflow/test-results/*.xml + reporter: java-junit + fail-on-error: false + + - name: Upload e2e test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: mysql-e2e-test-results + path: workflow/test-results/ + + # Upload to GCS on master + - name: Authenticate to Google Cloud + if: github.ref == 'refs/heads/master' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} + + - name: Upload e2e test results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/junit + glob: "*.xml" + parent: false + process_gcloudignore: false From 99f10724ae626caef5399898d963d66d3d8528f5 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 08:36:27 -0400 Subject: [PATCH 14/24] spanner --- .github/workflows/checks.yml | 289 ++++++++++++++++++++++++++++++++++- 1 file changed, 288 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9d674ec0f1..271e9e00e2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -253,6 +253,9 @@ jobs: ~/.cache/pypoetry/virtualenvs key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + - name: Create test results directory + run: mkdir -p workflow/test-results + - name: Install PostgreSQL client run: sudo apt-get update && sudo apt-get install -y postgresql-client @@ -457,7 +460,7 @@ jobs: process_gcloudignore: false # MySQL - build-and-test-mysql: + build-and-unit-test-mysql: runs-on: ubuntu-latest needs: [rust-env, python-env] permissions: @@ -508,6 +511,9 @@ jobs: ~/.cache/pypoetry/virtualenvs key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + - name: Create test results directory + run: mkdir -p workflow/test-results + - name: Install MySQL client run: sudo apt-get update && sudo apt-get install -y default-mysql-client @@ -699,3 +705,284 @@ jobs: glob: "*.xml" parent: false process_gcloudignore: false + + # Spanner + build-and-unit-test-spanner: + runs-on: ubuntu-latest + needs: [rust-env, python-env] + permissions: + contents: read + checks: write + + services: + spanner-emulator: + image: gcr.io/cloud-spanner-emulator/emulator:1.4.0 + ports: + - 9010:9010 + - 9020:9020 + + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: password + MYSQL_USER: test + MYSQL_PASSWORD: test + MYSQL_DATABASE: syncstorage + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=3 + + env: + # The code expects a spanner URL like: + SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database + RUST_BACKTRACE: 1 + RUST_TEST_THREADS: 1 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Restore Rust toolchain + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.rustup/toolchains + ~/.rustup/update-hashes + key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} + + - name: Restore pip and Poetry virtualenv + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cache/pip + ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + + - name: Create test results directory + run: mkdir -p workflow/test-results + + - name: Create version.json + run: | + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "${GITHUB_SHA}" \ + "${GITHUB_REF_NAME}" \ + "${GITHUB_REPOSITORY_OWNER}" \ + "${GITHUB_REPOSITORY_NAME}" \ + "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + > version.json + env: + GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + + - name: Install cargo-nextest + run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + + - name: Install cargo-llvm-cov + run: cargo install --locked cargo-llvm-cov + + - name: Build workspace (spanner feature) + run: | + # Build with the spanner feature so any compile-time issues surface early + cargo build --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier + + - name: Wait for Spanner Emulator to be ready + run: | + echo "Waiting for Spanner emulator to be ready..." + for i in {1..30}; do + if curl -s http://localhost:9020/ > /dev/null 2>&1; then + echo "Spanner emulator is ready (REST port 9020 responding)" + break + fi + echo "Attempt $i/30: Spanner emulator not ready yet, waiting..." + sleep 2 + done + # Verify both ports are accessible + if ! curl -s http://localhost:9020/ > /dev/null 2>&1; then + echo "ERROR: Cannot connect to Spanner emulator REST API at localhost:9020" + exit 1 + fi + echo "Spanner emulator is fully ready" + + - name: Setup Spanner schema & instance (prepare-spanner.sh) + env: + SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database + SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: http://localhost:9020 + run: | + # prepare-spanner.sh uses the REST API (port 9020) + scripts/prepare-spanner.sh + + - name: Create Tokenserver database + run: | + mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' + mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';" + + - name: Run Spanner unit tests with coverage + env: + SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database + SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: localhost:9010 + SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver + SYNC_TOKENSERVER__NODE_TYPE: spanner + RUST_TEST_THREADS: 1 + run: make spanner_test_with_coverage + + - name: Publish Test Report + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + if: always() + with: + name: Spanner Unit Tests + path: workflow/test-results/*.xml + reporter: java-junit + fail-on-error: false + + - name: Upload test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: spanner-test-results + path: workflow/test-results/ + + # Upload to GCS on master + - name: Authenticate to Google Cloud + if: github.ref == 'refs/heads/master' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} + + - name: Upload JUnit results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/junit + glob: "*.xml" + parent: false + process_gcloudignore: false + + - name: Upload coverage results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage + glob: "*.json" + parent: false + process_gcloudignore: false + + build-spanner-image: + runs-on: ubuntu-latest + needs: [rust-env, python-env] + permissions: + contents: read + actions: write + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Create version.json + run: | + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ + "${GITHUB_SHA}" \ + "${GITHUB_REF_NAME}" \ + "${GITHUB_REPOSITORY_OWNER}" \ + "${GITHUB_REPOSITORY_NAME}" \ + "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + > version.json + env: + GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + + - name: Cache Docker image tar + id: cache-spanner-image + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: /tmp/spanner-image.tar + key: ${{ runner.os }}-spanner-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Build Spanner Docker image (local artifact) + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + push: false + tags: app:build + build-args: | + SYNCSTORAGE_DATABASE_BACKEND=spanner + MYSQLCLIENT_PKG=libmysqlclient-dev + outputs: type=docker,dest=/tmp/spanner-image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload Docker image artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: spanner-docker-image + path: /tmp/spanner-image.tar + retention-days: 1 + + spanner-e2e-tests: + runs-on: ubuntu-latest + needs: build-spanner-image + permissions: + contents: read + checks: write + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Download Docker image + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + with: + name: spanner-docker-image + path: /tmp + + - name: Load Docker image + run: docker load --input /tmp/spanner-image.tar + + - name: Create test results directory + run: mkdir -p workflow/test-results + + - name: Run Spanner e2e tests + run: make docker_run_spanner_e2e_tests + env: + SYNCSTORAGE_RS_IMAGE: app:build + + - name: Publish E2E Test Report + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + if: always() + with: + name: Spanner E2E Tests + path: workflow/test-results/*.xml + reporter: java-junit + fail-on-error: false + + - name: Upload e2e test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: spanner-e2e-test-results + path: workflow/test-results/ + + # Upload to GCS on master + - name: Authenticate to Google Cloud + if: github.ref == 'refs/heads/master' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} + + - name: Upload e2e test results to GCS + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 + with: + path: workflow/test-results + destination: ecosystem-test-eng-metrics/syncstorage-rs/junit + glob: "*.xml" + parent: false + process_gcloudignore: false From d89e5e42bf08f15fb96fdee9967819b2664fdad9 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 09:35:44 -0400 Subject: [PATCH 15/24] cache cargo build artifacts and install poetry in utils test --- .github/workflows/checks.yml | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 271e9e00e2..54858d2126 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -253,6 +253,18 @@ jobs: ~/.cache/pypoetry/virtualenvs key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + - name: Cache Cargo build artifacts + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-postgres-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-postgres- + - name: Create test results directory run: mkdir -p workflow/test-results @@ -289,6 +301,9 @@ jobs: env: SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 + - name: Install Poetry + run: pip3 install poetry + - name: Run Postgres utils tests working-directory: tools/postgres run: | @@ -511,6 +526,18 @@ jobs: ~/.cache/pypoetry/virtualenvs key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + - name: Cache Cargo build artifacts + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-mysql-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-mysql- + - name: Create test results directory run: mkdir -p workflow/test-results @@ -763,6 +790,18 @@ jobs: ~/.cache/pypoetry/virtualenvs key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} + - name: Cache Cargo build artifacts + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-spanner-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-spanner- + - name: Create test results directory run: mkdir -p workflow/test-results From a253eb683a79313a048cf306fee3ccb061170689 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 10:03:29 -0400 Subject: [PATCH 16/24] cache cleanup pr --- .github/workflows/cleanup-branch-cache.yml | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/cleanup-branch-cache.yml diff --git a/.github/workflows/cleanup-branch-cache.yml b/.github/workflows/cleanup-branch-cache.yml new file mode 100644 index 0000000000..5567d299b2 --- /dev/null +++ b/.github/workflows/cleanup-branch-cache.yml @@ -0,0 +1,35 @@ +name: Cleanup branch caches on merge + +on: + pull_request: + types: [closed] + +jobs: + cleanup-cache: + # Runs on both merged PRs and PRs closed without merging + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Delete caches for merged branch + run: | + set +e + CACHE_IDS=$(gh cache list \ + --repo "${{ github.repository }}" \ + --ref "refs/heads/${{ github.head_ref }}" \ + --limit 100 \ + --json id \ + --jq '.[].id') + + if [ -z "$CACHE_IDS" ]; then + echo "No caches found for branch ${{ github.head_ref }}" + exit 0 + fi + + echo "Deleting caches for merged branch: ${{ github.head_ref }}" + echo "$CACHE_IDS" | while IFS= read -r id; do + echo " Deleting cache ID: $id" + gh cache delete "$id" --repo "${{ github.repository }}" + done + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0dccc646c6176255dfecac618a1746eb5597c870 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 13:31:52 -0400 Subject: [PATCH 17/24] remove seperate workflows --- .github/workflows/checks.yml | 37 +++-- .github/workflows/mysql.yml | 246 ----------------------------- .github/workflows/postgres.yml | 253 ----------------------------- .github/workflows/spanner.yml | 280 --------------------------------- 4 files changed, 20 insertions(+), 796 deletions(-) delete mode 100644 .github/workflows/mysql.yml delete mode 100644 .github/workflows/postgres.yml delete mode 100644 .github/workflows/spanner.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 54858d2126..4be9d6f8b0 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -30,8 +30,7 @@ concurrency: permissions: {} # workflow-level default — deny all jobs: - # Setup jobs and populate cache - + # Setup Python ====== python-env: runs-on: ubuntu-latest permissions: @@ -66,7 +65,7 @@ jobs: if [ "$(which python)" != "" ]; then python --version; fi uname -a cat /etc/os-release - + # Setup Rust ====== rust-env: runs-on: ubuntu-latest permissions: @@ -94,6 +93,7 @@ jobs: uname -a cat /etc/os-release + # Python lint and format checks ====== python-checks: needs: python-env runs-on: ubuntu-latest @@ -120,7 +120,7 @@ jobs: - name: Python Lint Check run: poetry run ruff check tools - + # Rust lint and format checks ====== rust-checks: needs: rust-env runs-on: ubuntu-latest @@ -174,7 +174,7 @@ jobs: - name: Documentation checks run: make doc-test - + # Rust clippy lint checker clippy: needs: rust-env runs-on: ubuntu-latest @@ -202,7 +202,7 @@ jobs: - name: Rust Clippy ${{ matrix.target }} run: make clippy_${{ matrix.target }} - # Postgres + # Postgres unit tests ====== build-and-unit-test-postgres: needs: [rust-env, python-env] runs-on: ubuntu-latest @@ -355,7 +355,7 @@ jobs: glob: "*.json" parent: false process_gcloudignore: false - + # Docker build Postgres Image ====== build-postgres-image: runs-on: ubuntu-latest needs: [rust-env, python-env] @@ -389,11 +389,11 @@ jobs: - name: Set up Docker Buildx if: steps.cache-postgres-image.outputs.cache-hit != 'true' - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Build Postgres Docker image if: steps.cache-postgres-image.outputs.cache-hit != 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . push: false @@ -411,7 +411,7 @@ jobs: name: postgres-docker-image path: /tmp/postgres-image.tar retention-days: 1 - + # End-to-end tests for Postgres build ====== postgres-e2e-tests: runs-on: ubuntu-latest needs: build-postgres-image @@ -474,7 +474,7 @@ jobs: parent: false process_gcloudignore: false - # MySQL + # MySQL Unit tests ====== build-and-unit-test-mysql: runs-on: ubuntu-latest needs: [rust-env, python-env] @@ -618,6 +618,7 @@ jobs: parent: false process_gcloudignore: false + # Docker build for MySQL Image build-mysql-image: runs-on: ubuntu-latest needs: [rust-env, python-env] @@ -649,10 +650,10 @@ jobs: path: /tmp/mysql-image.tar key: ${{ runner.os }}-mysql-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Build MySQL Docker image - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . push: false @@ -670,7 +671,7 @@ jobs: name: mysql-docker-image path: /tmp/mysql-image.tar retention-days: 1 - + # End-to-end tests for MySQL ====== mysql-e2e-tests: runs-on: ubuntu-latest needs: build-mysql-image @@ -733,7 +734,7 @@ jobs: parent: false process_gcloudignore: false - # Spanner + # Spanner Unit tests ====== build-and-unit-test-spanner: runs-on: ubuntu-latest needs: [rust-env, python-env] @@ -911,6 +912,7 @@ jobs: parent: false process_gcloudignore: false + # Docker image build for Spanner build-spanner-image: runs-on: ubuntu-latest needs: [rust-env, python-env] @@ -942,10 +944,10 @@ jobs: path: /tmp/spanner-image.tar key: ${{ runner.os }}-spanner-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Build Spanner Docker image (local artifact) - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . push: false @@ -964,6 +966,7 @@ jobs: path: /tmp/spanner-image.tar retention-days: 1 + # End-to-end tests for Spanner ====== spanner-e2e-tests: runs-on: ubuntu-latest needs: build-spanner-image diff --git a/.github/workflows/mysql.yml b/.github/workflows/mysql.yml deleted file mode 100644 index f09538d04f..0000000000 --- a/.github/workflows/mysql.yml +++ /dev/null @@ -1,246 +0,0 @@ -name: MySQL Build and Test - -on: - pull_request: - types: [opened, synchronize] - push: - branches: - - "**" - tags: - - "**" - workflow_dispatch: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -permissions: {} # workflow-level default — deny all - -jobs: - build-and-test-mysql: - runs-on: ubuntu-latest - permissions: - contents: read - checks: write - - services: - mysql: - image: mysql:8.0 - env: - MYSQL_ROOT_PASSWORD: password - MYSQL_USER: test - MYSQL_PASSWORD: test - MYSQL_DATABASE: syncstorage - ports: - - 3306:3306 - options: >- - --health-cmd="mysqladmin ping" - --health-interval=10s - --health-timeout=5s - --health-retries=5 - - env: - SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@127.0.0.1/syncstorage - SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver - SYNC_TOKENSERVER__NODE_TYPE: spanner - RUST_BACKTRACE: 1 - RUST_TEST_THREADS: 1 - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/setup-rust - with: - workspace-path: workflow/test-results - - - uses: ./.github/actions/setup-python - with: - workspace-path: workflow/test-results - - - name: Install MySQL client - run: sudo apt-get update && sudo apt-get install -y default-mysql-client - - - name: Create Tokenserver database - run: | - mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' - mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';" - - - name: Create version.json - run: | - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "${GITHUB_SHA}" \ - "${GITHUB_REF_NAME}" \ - "${GITHUB_REPOSITORY_OWNER}" \ - "${GITHUB_REPOSITORY_NAME}" \ - "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - > version.json - env: - GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} - - - name: Install cargo-nextest - run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - - - name: Install cargo-llvm-cov - run: cargo install --locked cargo-llvm-cov - - - name: Run unit tests with coverage - run: make test_with_coverage - - - name: Run unit tests with coverage (quota enforced) - run: make test_with_coverage - env: - SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 - - - name: Publish Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 - if: always() - with: - name: MySQL Unit Tests - path: workflow/test-results/*.xml - reporter: java-junit - fail-on-error: false - - - name: Upload test results - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: mysql-test-results - path: workflow/test-results/ - - # Upload to GCS on master - - name: Authenticate to Google Cloud - if: github.ref == 'refs/heads/master' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} - - - name: Upload JUnit results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/junit - glob: "*.xml" - parent: false - process_gcloudignore: false - - - name: Upload coverage results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage - glob: "*.json" - parent: false - process_gcloudignore: false - - build-mysql-image: - runs-on: ubuntu-latest - needs: build-and-test-mysql - permissions: - contents: read - actions: write - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Create version.json - run: | - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "${GITHUB_SHA}" \ - "${GITHUB_REF_NAME}" \ - "${GITHUB_REPOSITORY_OWNER}" \ - "${GITHUB_REPOSITORY_NAME}" \ - "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - > version.json - env: - GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - - name: Build MySQL Docker image - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 - with: - context: . - push: false - tags: app:build - build-args: | - SYNCSTORAGE_DATABASE_BACKEND=mysql - TOKENSERVER_DATABASE_BACKEND=mysql - outputs: type=docker,dest=/tmp/mysql-image.tar - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Upload Docker image artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: mysql-docker-image - path: /tmp/mysql-image.tar - retention-days: 1 - - mysql-e2e-tests: - runs-on: ubuntu-latest - needs: build-mysql-image - permissions: - contents: read - checks: write - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Download Docker image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 - with: - name: mysql-docker-image - path: /tmp - - - name: Load Docker image - run: docker load --input /tmp/mysql-image.tar - - - name: Create test results directory - run: mkdir -p workflow/test-results - - - name: Run MySQL e2e tests - run: make docker_run_mysql_e2e_tests - env: - SYNCSTORAGE_RS_IMAGE: app:build - - - name: Publish E2E Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 - if: always() - with: - name: MySQL E2E Tests - path: workflow/test-results/*.xml - reporter: java-junit - fail-on-error: false - - - name: Upload e2e test results - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: mysql-e2e-test-results - path: workflow/test-results/ - - # Upload to GCS on master - - name: Authenticate to Google Cloud - if: github.ref == 'refs/heads/master' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} - - - name: Upload e2e test results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/junit - glob: "*.xml" - parent: false - process_gcloudignore: false diff --git a/.github/workflows/postgres.yml b/.github/workflows/postgres.yml deleted file mode 100644 index 7c74aa9c36..0000000000 --- a/.github/workflows/postgres.yml +++ /dev/null @@ -1,253 +0,0 @@ -name: Postgres Build and Test - -on: - pull_request: - types: [opened, synchronize] - push: - branches: - - "**" - tags: - - "**" - workflow_dispatch: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -permissions: {} # workflow-level default — deny all - -jobs: - build-and-test-postgres: - runs-on: ubuntu-latest - permissions: - contents: read - checks: write - - services: - postgres: - image: postgres:18.0 - env: - POSTGRES_USER: test - POSTGRES_PASSWORD: test - POSTGRES_DB: syncstorage - ports: - - 5432:5432 - options: >- - --health-cmd="pg_isready -U test" - --health-interval=10s - --health-timeout=5s - --health-retries=5 - - env: - SYNC_SYNCSTORAGE__DATABASE_URL: postgres://test:test@127.0.0.1/syncstorage - SYNC_TOKENSERVER__DATABASE_URL: postgres://test:test@127.0.0.1/tokenserver - SYNC_TOKENSERVER__NODE_TYPE: postgres - RUST_BACKTRACE: 1 - RUST_TEST_THREADS: 1 - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/setup-rust - with: - workspace-path: workflow/test-results - - - uses: ./.github/actions/setup-python - with: - workspace-path: workflow/test-results - - - name: Install PostgreSQL client - run: sudo apt-get update && sudo apt-get install -y postgresql-client - - - name: Create Tokenserver database - run: | - PGPASSWORD=test psql -U test -h 127.0.0.1 -d syncstorage -c 'CREATE DATABASE tokenserver;' - - - name: Create version.json - run: | - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "${GITHUB_SHA}" \ - "${GITHUB_REF_NAME}" \ - "${GITHUB_REPOSITORY_OWNER}" \ - "${GITHUB_REPOSITORY_NAME}" \ - "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - > version.json - env: - GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} - - - name: Install cargo-nextest - run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - - - name: Install cargo-llvm-cov - run: cargo install --locked cargo-llvm-cov - - - name: Run unit tests with coverage - run: make postgres_test_with_coverage - - - name: Run unit tests with coverage (quota enforced) - run: make postgres_test_with_coverage - env: - SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 - - - name: Run Postgres utils tests - working-directory: tools/postgres - run: | - poetry install --no-interaction --no-ansi - WORKFLOW=$(echo "${GITHUB_WORKFLOW}" | tr ' ' '-' | tr '[:upper:]' '[:lower:]') - poetry run pytest test_purge_ttl.py -v --junit-xml="../../workflow/test-results/${GITHUB_RUN_NUMBER}__$(date +%s)__$(basename ${GITHUB_REPOSITORY})__${WORKFLOW}__postgres_utils__results.xml" - env: - SYNC_SYNCSTORAGE__DATABASE_URL: postgresql://test:test@127.0.0.1/syncstorage - - - name: Publish Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 - if: always() - with: - name: Postgres Unit Tests - path: workflow/test-results/*.xml - reporter: java-junit - fail-on-error: false - - - name: Upload test results - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: postgres-test-results - path: workflow/test-results/ - - # Upload to GCS on master - - name: Authenticate to Google Cloud - if: github.ref == 'refs/heads/master' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} - - - name: Upload JUnit results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/junit - glob: "*.xml" - parent: false - process_gcloudignore: false - - - name: Upload coverage results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage - glob: "*.json" - parent: false - process_gcloudignore: false - - build-postgres-image: - runs-on: ubuntu-latest - needs: build-and-test-postgres - permissions: - contents: read - actions: write - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Create version.json - run: | - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "${GITHUB_SHA}" \ - "${GITHUB_REF_NAME}" \ - "${GITHUB_REPOSITORY_OWNER}" \ - "${GITHUB_REPOSITORY_NAME}" \ - "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - > version.json - env: - GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - - name: Build Postgres Docker image - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 - with: - context: . - push: false - tags: app:build - build-args: | - SYNCSTORAGE_DATABASE_BACKEND=postgres - TOKENSERVER_DATABASE_BACKEND=postgres - outputs: type=docker,dest=/tmp/postgres-image.tar - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Upload Docker image artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: postgres-docker-image - path: /tmp/postgres-image.tar - retention-days: 1 - - postgres-e2e-tests: - runs-on: ubuntu-latest - needs: build-postgres-image - permissions: - contents: read - checks: write - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Download Docker image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 - with: - name: postgres-docker-image - path: /tmp - - - name: Load Docker image - run: docker load --input /tmp/postgres-image.tar - - - name: Create test results directory - run: mkdir -p workflow/test-results - - - name: Run Postgres e2e tests - run: make docker_run_postgres_e2e_tests - env: - SYNCSTORAGE_RS_IMAGE: app:build - - - name: Publish E2E Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 - if: always() - with: - name: Postgres E2E Tests - path: workflow/test-results/*.xml - reporter: java-junit - fail-on-error: false - - - name: Upload e2e test results - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: postgres-e2e-test-results - path: workflow/test-results/ - - # Upload to GCS on master - - name: Authenticate to Google Cloud - if: github.ref == 'refs/heads/master' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} - - - name: Upload e2e test results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/junit - glob: "*.xml" - parent: false - process_gcloudignore: false diff --git a/.github/workflows/spanner.yml b/.github/workflows/spanner.yml deleted file mode 100644 index 62efbe2cd9..0000000000 --- a/.github/workflows/spanner.yml +++ /dev/null @@ -1,280 +0,0 @@ -name: Spanner Build, Test, and Push - -on: - pull_request: - types: [opened, synchronize] - push: - branches: - - "**" - tags: - - "**" - workflow_dispatch: {} - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -permissions: {} # workflow-level default — deny all - -jobs: - build-and-test-spanner: - runs-on: ubuntu-latest - permissions: - contents: read - checks: write - - services: - spanner-emulator: - image: gcr.io/cloud-spanner-emulator/emulator:1.4.0 - ports: - - 9010:9010 - - 9020:9020 - - mysql: - image: mysql:8.0 - env: - MYSQL_ROOT_PASSWORD: password - MYSQL_USER: test - MYSQL_PASSWORD: test - MYSQL_DATABASE: syncstorage - ports: - - 3306:3306 - options: >- - --health-cmd="mysqladmin ping" - --health-interval=10s - --health-timeout=5s - --health-retries=3 - - env: - # The code expects a spanner URL like: - SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database - RUST_BACKTRACE: 1 - RUST_TEST_THREADS: 1 - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/setup-rust - with: - workspace-path: workflow/test-results - - - uses: ./.github/actions/setup-python - with: - workspace-path: workflow/test-results - - - name: Create version.json - run: | - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "${GITHUB_SHA}" \ - "${GITHUB_REF_NAME}" \ - "${GITHUB_REPOSITORY_OWNER}" \ - "${GITHUB_REPOSITORY_NAME}" \ - "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - > version.json - env: - GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} - - - name: Install cargo-nextest - run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - - - name: Install cargo-llvm-cov - run: cargo install --locked cargo-llvm-cov - - - name: Build workspace (spanner feature) - run: | - # Build with the spanner feature so any compile-time issues surface early - cargo build --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier - - - name: Wait for Spanner Emulator to be ready - run: | - echo "Waiting for Spanner emulator to be ready..." - for i in {1..30}; do - if curl -s http://localhost:9020/ > /dev/null 2>&1; then - echo "Spanner emulator is ready (REST port 9020 responding)" - break - fi - echo "Attempt $i/30: Spanner emulator not ready yet, waiting..." - sleep 2 - done - # Verify both ports are accessible - if ! curl -s http://localhost:9020/ > /dev/null 2>&1; then - echo "ERROR: Cannot connect to Spanner emulator REST API at localhost:9020" - exit 1 - fi - echo "Spanner emulator is fully ready" - - - name: Setup Spanner schema & instance (prepare-spanner.sh) - env: - SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database - SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: http://localhost:9020 - run: | - # prepare-spanner.sh uses the REST API (port 9020) - scripts/prepare-spanner.sh - - - name: Create Tokenserver database - run: | - mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' - mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';" - - - name: Run Spanner unit tests with coverage - env: - SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database - SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: localhost:9010 - SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver - SYNC_TOKENSERVER__NODE_TYPE: spanner - RUST_TEST_THREADS: 1 - run: make spanner_test_with_coverage - - - name: Publish Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 - if: always() - with: - name: Spanner Unit Tests - path: workflow/test-results/*.xml - reporter: java-junit - fail-on-error: false - - - name: Upload test results - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: spanner-test-results - path: workflow/test-results/ - - # Upload to GCS on master - - name: Authenticate to Google Cloud - if: github.ref == 'refs/heads/master' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} - - - name: Upload JUnit results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/junit - glob: "*.xml" - parent: false - process_gcloudignore: false - - - name: Upload coverage results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage - glob: "*.json" - parent: false - process_gcloudignore: false - - build-spanner-image: - runs-on: ubuntu-latest - needs: build-and-test-spanner - permissions: - contents: read - actions: write - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Create version.json - run: | - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ - "${GITHUB_SHA}" \ - "${GITHUB_REF_NAME}" \ - "${GITHUB_REPOSITORY_OWNER}" \ - "${GITHUB_REPOSITORY_NAME}" \ - "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - > version.json - env: - GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - - - name: Build Spanner Docker image (local artifact) - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 - with: - context: . - push: false - tags: app:build - build-args: | - SYNCSTORAGE_DATABASE_BACKEND=spanner - MYSQLCLIENT_PKG=libmysqlclient-dev - outputs: type=docker,dest=/tmp/spanner-image.tar - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Upload Docker image artifact - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: spanner-docker-image - path: /tmp/spanner-image.tar - retention-days: 1 - - spanner-e2e-tests: - runs-on: ubuntu-latest - needs: build-spanner-image - permissions: - contents: read - checks: write - - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Download Docker image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 - with: - name: spanner-docker-image - path: /tmp - - - name: Load Docker image - run: docker load --input /tmp/spanner-image.tar - - - name: Create test results directory - run: mkdir -p workflow/test-results - - - name: Run Spanner e2e tests - run: make docker_run_spanner_e2e_tests - env: - SYNCSTORAGE_RS_IMAGE: app:build - - - name: Publish E2E Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 - if: always() - with: - name: Spanner E2E Tests - path: workflow/test-results/*.xml - reporter: java-junit - fail-on-error: false - - - name: Upload e2e test results - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: spanner-e2e-test-results - path: workflow/test-results/ - - # Upload to GCS on master - - name: Authenticate to Google Cloud - if: github.ref == 'refs/heads/master' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} - - - name: Upload e2e test results to GCS - if: github.ref == 'refs/heads/master' - uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 - with: - path: workflow/test-results - destination: ecosystem-test-eng-metrics/syncstorage-rs/junit - glob: "*.xml" - parent: false - process_gcloudignore: false From cabaf68c0c93e210b5a8d552c774f2dedb1f8077 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 13:41:22 -0400 Subject: [PATCH 18/24] rename main workflow --- .github/workflows/{checks.yml => main-workflow.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{checks.yml => main-workflow.yml} (100%) diff --git a/.github/workflows/checks.yml b/.github/workflows/main-workflow.yml similarity index 100% rename from .github/workflows/checks.yml rename to .github/workflows/main-workflow.yml From d79fa5a47cf1f724fa4a24d0b50b03130189f766 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 13:47:30 -0400 Subject: [PATCH 19/24] rename workflow --- .github/workflows/main-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml index 4be9d6f8b0..dfd9b42eeb 100644 --- a/.github/workflows/main-workflow.yml +++ b/.github/workflows/main-workflow.yml @@ -1,4 +1,4 @@ -name: Checks +name: Main Workflow - Lint, Build, Test on: push: From deac5069e21698b6e1df0dd02d77d319d4da4008 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 14:13:46 -0400 Subject: [PATCH 20/24] add missed cache hits, rmv needless checkout, add nexttest and llvm cache --- .github/workflows/cleanup-branch-cache.yml | 9 ++- .github/workflows/main-workflow.yml | 78 +++++++++++++++++++--- 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cleanup-branch-cache.yml b/.github/workflows/cleanup-branch-cache.yml index 5567d299b2..191860d9ed 100644 --- a/.github/workflows/cleanup-branch-cache.yml +++ b/.github/workflows/cleanup-branch-cache.yml @@ -11,25 +11,24 @@ jobs: permissions: actions: write steps: - - name: Delete caches for merged branch + - name: Delete caches for closed branch run: | - set +e CACHE_IDS=$(gh cache list \ --repo "${{ github.repository }}" \ --ref "refs/heads/${{ github.head_ref }}" \ --limit 100 \ --json id \ - --jq '.[].id') + --jq '.[].id') || true if [ -z "$CACHE_IDS" ]; then echo "No caches found for branch ${{ github.head_ref }}" exit 0 fi - echo "Deleting caches for merged branch: ${{ github.head_ref }}" + echo "Deleting caches for closed branch: ${{ github.head_ref }}" echo "$CACHE_IDS" | while IFS= read -r id; do echo " Deleting cache ID: $id" - gh cache delete "$id" --repo "${{ github.repository }}" + gh cache delete "$id" --repo "${{ github.repository }}" || true done env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml index dfd9b42eeb..5815919a56 100644 --- a/.github/workflows/main-workflow.yml +++ b/.github/workflows/main-workflow.yml @@ -60,7 +60,6 @@ jobs: run: poetry install --with tokenserver-unit-tests,dev --no-interaction --no-ansi - name: Display Python Version Info - shell: bash run: | if [ "$(which python)" != "" ]; then python --version; fi uname -a @@ -68,13 +67,8 @@ jobs: # Setup Rust ====== rust-env: runs-on: ubuntu-latest - permissions: - contents: read + permissions: {} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - name: Cache Rust toolchain uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: @@ -104,6 +98,11 @@ jobs: with: persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Restore pip and Poetry virtualenv uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: @@ -147,7 +146,7 @@ jobs: uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: ~/.cargo/bin/cargo-audit - key: ${{ runner.os }}-cargo-audit-${{ hashFiles('.github/workflows/checks.yml') }} + key: ${{ runner.os }}-cargo-audit-${{ hashFiles('.github/workflows/main-workflow.yml') }} - name: Install cargo-audit if: steps.cache-cargo-audit.outputs.cache-hit != 'true' @@ -245,6 +244,9 @@ jobs: ~/.rustup/update-hashes key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} + - name: Set Rust toolchain + run: rustup default ${{ env.RUST_VERSION }} + - name: Restore pip and Poetry virtualenv uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: @@ -287,10 +289,26 @@ jobs: env: GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + - name: Cache cargo-nextest + id: cache-cargo-nextest-postgres + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: ~/.cargo/bin/cargo-nextest + key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }} + - name: Install cargo-nextest + if: steps.cache-cargo-nextest-postgres.outputs.cache-hit != 'true' run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + - name: Cache cargo-llvm-cov + id: cache-cargo-llvm-cov-postgres + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: ~/.cargo/bin/cargo-llvm-cov + key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }} + - name: Install cargo-llvm-cov + if: steps.cache-cargo-llvm-cov-postgres.outputs.cache-hit != 'true' run: cargo install --locked cargo-llvm-cov - name: Run unit tests with coverage @@ -518,6 +536,9 @@ jobs: ~/.rustup/update-hashes key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} + - name: Set Rust toolchain + run: rustup default ${{ env.RUST_VERSION }} + - name: Restore pip and Poetry virtualenv uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: @@ -561,10 +582,26 @@ jobs: env: GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + - name: Cache cargo-nextest + id: cache-cargo-nextest-mysql + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: ~/.cargo/bin/cargo-nextest + key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }} + - name: Install cargo-nextest + if: steps.cache-cargo-nextest-mysql.outputs.cache-hit != 'true' run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + - name: Cache cargo-llvm-cov + id: cache-cargo-llvm-cov-mysql + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: ~/.cargo/bin/cargo-llvm-cov + key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }} + - name: Install cargo-llvm-cov + if: steps.cache-cargo-llvm-cov-mysql.outputs.cache-hit != 'true' run: cargo install --locked cargo-llvm-cov - name: Run unit tests with coverage @@ -649,10 +686,13 @@ jobs: with: path: /tmp/mysql-image.tar key: ${{ runner.os }}-mysql-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} + - name: Set up Docker Buildx + if: steps.cache-mysql-image.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Build MySQL Docker image + if: steps.cache-mysql-image.outputs.cache-hit != 'true' uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . @@ -783,6 +823,9 @@ jobs: ~/.rustup/update-hashes key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} + - name: Set Rust toolchain + run: rustup default ${{ env.RUST_VERSION }} + - name: Restore pip and Poetry virtualenv uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: @@ -818,10 +861,26 @@ jobs: env: GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} + - name: Cache cargo-nextest + id: cache-cargo-nextest-spanner + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: ~/.cargo/bin/cargo-nextest + key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }} + - name: Install cargo-nextest + if: steps.cache-cargo-nextest-spanner.outputs.cache-hit != 'true' run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + - name: Cache cargo-llvm-cov + id: cache-cargo-llvm-cov-spanner + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 + with: + path: ~/.cargo/bin/cargo-llvm-cov + key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }} + - name: Install cargo-llvm-cov + if: steps.cache-cargo-llvm-cov-spanner.outputs.cache-hit != 'true' run: cargo install --locked cargo-llvm-cov - name: Build workspace (spanner feature) @@ -943,10 +1002,13 @@ jobs: with: path: /tmp/spanner-image.tar key: ${{ runner.os }}-spanner-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} + - name: Set up Docker Buildx + if: steps.cache-spanner-image.outputs.cache-hit != 'true' uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Build Spanner Docker image (local artifact) + if: steps.cache-spanner-image.outputs.cache-hit != 'true' uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 with: context: . From 76a7b1eb4de76ec219a945a88507bff4138c3eae Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 14:49:32 -0400 Subject: [PATCH 21/24] update test-reporter and download-artifact to remove node deprecation warning, add llvm to rust toolchain --- .github/workflows/main-workflow.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml index 5815919a56..4a4e3ea661 100644 --- a/.github/workflows/main-workflow.yml +++ b/.github/workflows/main-workflow.yml @@ -78,7 +78,7 @@ jobs: key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} - name: Install Rust toolchain - run: rustup toolchain install ${{ env.RUST_VERSION }} --component rustfmt --component clippy --no-self-update && rustup default ${{ env.RUST_VERSION }} + run: rustup toolchain install ${{ env.RUST_VERSION }} --component rustfmt --component clippy --component llvm-tools-preview --no-self-update && rustup default ${{ env.RUST_VERSION }} - name: Display Rust Version Info shell: bash @@ -230,6 +230,7 @@ jobs: SYNC_TOKENSERVER__NODE_TYPE: postgres RUST_BACKTRACE: 1 RUST_TEST_THREADS: 1 + CARGO_INCREMENTAL: "0" steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -332,7 +333,7 @@ jobs: SYNC_SYNCSTORAGE__DATABASE_URL: postgresql://test:test@127.0.0.1/syncstorage - name: Publish Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 if: always() with: name: Postgres Unit Tests @@ -443,7 +444,7 @@ jobs: persist-credentials: false - name: Download Docker image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 with: name: postgres-docker-image path: /tmp @@ -460,7 +461,7 @@ jobs: SYNCSTORAGE_RS_IMAGE: app:build - name: Publish E2E Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 if: always() with: name: Postgres E2E Tests @@ -522,6 +523,7 @@ jobs: SYNC_TOKENSERVER__NODE_TYPE: spanner RUST_BACKTRACE: 1 RUST_TEST_THREADS: 1 + CARGO_INCREMENTAL: "0" steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -613,7 +615,7 @@ jobs: SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 - name: Publish Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 if: always() with: name: MySQL Unit Tests @@ -725,7 +727,7 @@ jobs: persist-credentials: false - name: Download Docker image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 with: name: mysql-docker-image path: /tmp @@ -742,7 +744,7 @@ jobs: SYNCSTORAGE_RS_IMAGE: app:build - name: Publish E2E Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 if: always() with: name: MySQL E2E Tests @@ -809,6 +811,7 @@ jobs: SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database RUST_BACKTRACE: 1 RUST_TEST_THREADS: 1 + CARGO_INCREMENTAL: "0" steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -929,7 +932,7 @@ jobs: run: make spanner_test_with_coverage - name: Publish Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 if: always() with: name: Spanner Unit Tests @@ -1042,7 +1045,7 @@ jobs: persist-credentials: false - name: Download Docker image - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 with: name: spanner-docker-image path: /tmp @@ -1059,7 +1062,7 @@ jobs: SYNCSTORAGE_RS_IMAGE: app:build - name: Publish E2E Test Report - uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 if: always() with: name: Spanner E2E Tests From 5451a5c152abb875892c132db8ce6ea5232cf535 Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 14:51:03 -0400 Subject: [PATCH 22/24] nvm, zizmor doesn't like new test-reporter --- .github/workflows/main-workflow.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml index 4a4e3ea661..6406432908 100644 --- a/.github/workflows/main-workflow.yml +++ b/.github/workflows/main-workflow.yml @@ -333,7 +333,7 @@ jobs: SYNC_SYNCSTORAGE__DATABASE_URL: postgresql://test:test@127.0.0.1/syncstorage - name: Publish Test Report - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 if: always() with: name: Postgres Unit Tests @@ -461,7 +461,7 @@ jobs: SYNCSTORAGE_RS_IMAGE: app:build - name: Publish E2E Test Report - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 if: always() with: name: Postgres E2E Tests @@ -615,7 +615,7 @@ jobs: SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 - name: Publish Test Report - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 if: always() with: name: MySQL Unit Tests @@ -744,7 +744,7 @@ jobs: SYNCSTORAGE_RS_IMAGE: app:build - name: Publish E2E Test Report - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 if: always() with: name: MySQL E2E Tests @@ -932,7 +932,7 @@ jobs: run: make spanner_test_with_coverage - name: Publish Test Report - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 if: always() with: name: Spanner Unit Tests @@ -1062,7 +1062,7 @@ jobs: SYNCSTORAGE_RS_IMAGE: app:build - name: Publish E2E Test Report - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 + uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 if: always() with: name: Spanner E2E Tests From f4e98a5a27052a63ce724870169273fbb145567c Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 20:50:30 -0400 Subject: [PATCH 23/24] cache for rust toolchain install --- .github/workflows/main-workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml index 6406432908..60585bdbc2 100644 --- a/.github/workflows/main-workflow.yml +++ b/.github/workflows/main-workflow.yml @@ -70,6 +70,7 @@ jobs: permissions: {} steps: - name: Cache Rust toolchain + id: cache-rust-toolchain uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: | @@ -78,6 +79,7 @@ jobs: key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} - name: Install Rust toolchain + if: steps.cache-rust-toolchain.outputs.cache-hit != 'true' run: rustup toolchain install ${{ env.RUST_VERSION }} --component rustfmt --component clippy --component llvm-tools-preview --no-self-update && rustup default ${{ env.RUST_VERSION }} - name: Display Rust Version Info From ef9ec1062c94d3ac85b86b22f30ff5e5ec14a46c Mon Sep 17 00:00:00 2001 From: Taddes Date: Thu, 26 Mar 2026 20:51:04 -0400 Subject: [PATCH 24/24] rmv actions for rust and py setup --- .github/actions/setup-python/action.yml | 46 ----------------------- .github/actions/setup-rust/action.yml | 50 ------------------------- 2 files changed, 96 deletions(-) delete mode 100644 .github/actions/setup-python/action.yml delete mode 100644 .github/actions/setup-rust/action.yml diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml deleted file mode 100644 index 9d33767b18..0000000000 --- a/.github/actions/setup-python/action.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Setup Python Environment -description: Sets up Python with Poetry dependencies - -inputs: - python-version: - description: "Python version to use" - required: false - default: "3.14" # PY_VER - workspace-path: - description: "The directory path to store test results" - required: false - default: "workflow" - -runs: - using: "composite" - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 - with: - python-version: ${{ inputs.python-version }} - - - name: Install Poetry - shell: bash - run: pip install poetry - - - name: Install Python dependencies with Poetry - shell: bash - run: | - poetry install --with tokenserver-unit-tests,dev --no-interaction --no-ansi - - - name: Create workspace directory - shell: bash - run: mkdir -p ${WORKSPACE_PATH} - env: - WORKSPACE_PATH: ${{ inputs.workspace-path }} - - - name: Display Version Info - shell: bash - run: | - if [ "$(which python)" != "" ]; then python --version; fi - uname -a - cat /etc/os-release diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml deleted file mode 100644 index 163063ace9..0000000000 --- a/.github/actions/setup-rust/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Setup Rust Environment -description: Sets up Rust - -inputs: - rust-version: - description: "Rust version to install" - required: false - default: "1.91" # RUST_VER - workspace-path: - description: "The directory path to store test results" - required: false - default: "workflow" - -runs: - using: "composite" - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Cache Rust toolchain - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 - with: - path: | - ~/.rustup/toolchains - ~/.rustup/update-hashes - key: ${{ runner.os }}-rust-toolchain-${{ inputs.rust-version }} - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 - with: - toolchain: ${{ inputs.rust-version }} - components: rustfmt, clippy - - - name: Install cargo-audit - shell: bash - run: cargo install --locked cargo-audit - - - name: Create workspace directory - shell: bash - run: mkdir -p ${INPUTS_WORKSPACE_PATH}/test-results - env: - INPUTS_WORKSPACE_PATH: ${{ inputs.workspace-path }} - - - name: Display Version Info - shell: bash - run: | - if [ "$(which rustc)" != "" ]; then rustc --version; fi - uname -a - cat /etc/os-release