diff --git a/.github/scripts/run-e2e.sh b/.github/scripts/run-e2e.sh new file mode 100644 index 0000000000..87cd2ca1f8 --- /dev/null +++ b/.github/scripts/run-e2e.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -euo pipefail + +variant="${1:?E2E variant is required}" +image="${2:?OpenMeter image is required}" +pull_policy="${3:?image pull policy is required}" +dependency_source="${4:?dependency image source is required}" +repository="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" + +case "${dependency_source}" in + depot|upstream) ;; + *) + echo "Unsupported dependency image source: ${dependency_source}" + exit 1 + ;; +esac + +cd "${repository}/e2e" + +export OPENMETER_ADDRESS=http://localhost:38888 +export TZ=UTC + +compose=( + docker compose + -f docker-compose.infra.yaml + -f docker-compose.openmeter.yaml + -f docker-compose.override.yaml +) + +case "${variant}" in + base) + test_target=test-base + ;; + credits-disabled) + compose+=(-f docker-compose.credits-disabled.yaml) + test_target=test-credits-disabled + ;; + *) + echo "Unsupported E2E variant: ${variant}" + exit 1 + ;; +esac + +if [ "${dependency_source}" = depot ]; then + compose+=(-f ../.github/docker-compose.depot-registry.yaml) +fi + +log_dir="artifacts/logs/docker-compose/${variant}" +log_pid="" + +cleanup() { + status=$? + trap - EXIT + set +e + if [ -n "${log_pid}" ]; then + kill "${log_pid}" 2>/dev/null + wait "${log_pid}" 2>/dev/null + fi + mkdir -p "${log_dir}" + "${compose[@]}" ps --all > "${log_dir}/compose-ps.txt" 2>&1 + mapfile -t services < <("${compose[@]}" config --services) + for service in "${services[@]}"; do + "${compose[@]}" logs --no-color --timestamps "${service}" > "${log_dir}/${service}.log" 2>&1 + done + "${compose[@]}" down -v + exit "${status}" +} +trap cleanup EXIT + +cat > docker-compose.override.yaml < "${log_dir}/compose-follow.log" 2>&1 & +log_pid=$! +curl --fail --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:30000/healthz +curl --fail --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:30001/healthz +make "${test_target}" diff --git a/.github/scripts/run-quickstart.sh b/.github/scripts/run-quickstart.sh new file mode 100644 index 0000000000..fb058da2ee --- /dev/null +++ b/.github/scripts/run-quickstart.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +image="${1:?OpenMeter image is required}" +pull_policy="${2:?image pull policy is required}" +repository="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" + +cd "${repository}/quickstart" + +export COMPOSE_PROFILES=webhook +export OPENMETER_ADDRESS=http://localhost:48888 + +compose=(docker compose -f docker-compose.yaml -f docker-compose.override.yaml) +log_dir="artifacts/logs/docker-compose/quickstart" + +cleanup() { + status=$? + trap - EXIT + set +e + mkdir -p "${log_dir}" + "${compose[@]}" ps --all > "${log_dir}/compose-ps.txt" 2>&1 + "${compose[@]}" logs --no-color --timestamps > "${log_dir}/compose.log" 2>&1 + "${compose[@]}" down -v + exit "${status}" +} +trap cleanup EXIT + +cat > docker-compose.override.yaml <> "$GITHUB_OUTPUT" - - - name: Gather build metadata - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - with: - images: ${{ steps.image-name.outputs.value }} - flavor: | - latest = false - tags: | - type=ref,event=branch - type=ref,event=pr,prefix=pr- - type=semver,pattern={{raw}} - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch,suffix=-{{sha}}-{{date 'X'}},enable={{is_default_branch}} - - - name: Login to GitHub Container Registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - if: inputs.publish - - - name: Build and push image - id: build - uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 - with: - context: . - build-args: | - VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} - platforms: linux/amd64,linux/arm64 # The confluent library doesn't support ARMv7 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - push: ${{ inputs.publish }} - save: true - project: ${{ vars.DEPOT_PROJECT }} - - - name: Set image ref - id: image-ref - run: echo "value=${STEPS_IMAGE_NAME_OUTPUTS_VALUE}@${STEPS_BUILD_OUTPUTS_DIGEST}" >> "$GITHUB_OUTPUT" - env: - STEPS_IMAGE_NAME_OUTPUTS_VALUE: ${{ steps.image-name.outputs.value }} - STEPS_BUILD_OUTPUTS_DIGEST: ${{ steps.build.outputs.digest }} - - - name: Retrieve pull token - id: pull-token - run: | - PULL_TOKEN="$(depot pull-token --project "${VARS_DEPOT_PROJECT}")" - echo "token=$PULL_TOKEN" >> "$GITHUB_OUTPUT" - echo "::add-mask::$PULL_TOKEN" - env: - VARS_DEPOT_PROJECT: ${{ vars.DEPOT_PROJECT }} - - ### Trivy is comporomised - # - # - name: Run Trivy vulnerability scanner - # uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # 0.33.1 - # with: - # image-ref: registry.depot.dev/${{ vars.DEPOT_PROJECT }}:${{ steps.build.outputs.build-id }} - # format: sarif - # output: trivy-results.sarif - # env: - # TRIVY_USERNAME: x-token - # TRIVY_PASSWORD: ${{ steps.pull-token.outputs.token }} - # TRIVY_DB_REPOSITORY: ghcr.io/openmeterio/trivy-db:2 - - # - name: Upload Trivy scan results as artifact - # uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - # with: - # name: "[${{ github.job }}] Trivy scan results" - # path: trivy-results.sarif - # retention-days: 5 - - # - name: Upload Trivy scan results to GitHub Security tab - # uses: github/codeql-action/upload-sarif@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v3.29.5 - # with: - # sarif_file: trivy-results.sarif - - benthos-collector-container-image: - name: Benthos Collector Container image - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - id-token: write - security-events: write - - outputs: - name: ${{ steps.image-name.outputs.value }} - digest: ${{ steps.build.outputs.digest }} - tag: ${{ steps.meta.outputs.version }} - ref: ${{ steps.image-ref.outputs.value }} - - steps: - - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} - api-key: ${{ secrets.STEP_SECURITY_API_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up Depot CLI - uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 - - - name: Set image name - id: image-name - run: echo "value=ghcr.io/openmeterio/benthos-collector" >> "$GITHUB_OUTPUT" - - - name: Gather build metadata - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - with: - images: ${{ steps.image-name.outputs.value }} - flavor: | - latest = false - tags: | - type=ref,event=branch - type=ref,event=pr,prefix=pr- - type=semver,pattern={{raw}} - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch,suffix=-{{sha}}-{{date 'X'}},enable={{is_default_branch}} - - - name: Login to GitHub Container Registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - if: inputs.publish - - - name: Build and push image - id: build - uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 - with: - context: . - file: benthos-collector.Dockerfile - build-args: | - VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - push: ${{ inputs.publish }} - save: true - project: ${{ vars.DEPOT_PROJECT }} - - - name: Set image ref - id: image-ref - run: echo "value=${STEPS_IMAGE_NAME_OUTPUTS_VALUE}@${STEPS_BUILD_OUTPUTS_DIGEST}" >> "$GITHUB_OUTPUT" - env: - STEPS_IMAGE_NAME_OUTPUTS_VALUE: ${{ steps.image-name.outputs.value }} - STEPS_BUILD_OUTPUTS_DIGEST: ${{ steps.build.outputs.digest }} - - - name: Retrieve pull token - id: pull-token - run: | - PULL_TOKEN="$(depot pull-token --project "${VARS_DEPOT_PROJECT}")" - echo "token=$PULL_TOKEN" >> "$GITHUB_OUTPUT" - echo "::add-mask::$PULL_TOKEN" - env: - VARS_DEPOT_PROJECT: ${{ vars.DEPOT_PROJECT }} - - # - name: Run Trivy vulnerability scanner - # uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # 0.33.1 - # with: - # image-ref: registry.depot.dev/${{ vars.DEPOT_PROJECT }}:${{ steps.build.outputs.build-id }} - # format: sarif - # output: trivy-results.sarif - # env: - # TRIVY_USERNAME: x-token - # TRIVY_PASSWORD: ${{ steps.pull-token.outputs.token }} - # TRIVY_DB_REPOSITORY: ghcr.io/openmeterio/trivy-db:2 - - # - name: Upload Trivy scan results as artifact - # uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - # with: - # name: "[${{ github.job }}] Trivy scan results" - # path: trivy-results.sarif - # retention-days: 5 - - # - name: Upload Trivy scan results to GitHub Security tab - # uses: github/codeql-action/upload-sarif@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v3.29.5 - # with: - # sarif_file: trivy-results.sarif diff --git a/.github/workflows/benthos-collector.yaml b/.github/workflows/benthos-collector.yaml new file mode 100644 index 0000000000..ea7cec933d --- /dev/null +++ b/.github/workflows/benthos-collector.yaml @@ -0,0 +1,94 @@ +name: Benthos Collector + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + depot: + name: Build + if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]' }} + runs-on: depot-ubuntu-24.04-4 + permissions: + contents: read + id-token: write + + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Depot CLI + uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 + + - name: Build Benthos Collector image + uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 + with: + context: . + file: benthos-collector.Dockerfile + build-args: VERSION=${{ github.sha }} + platforms: linux/amd64 + tags: benthos-collector:ci + push: false + project: ${{ vars.DEPOT_PROJECT }} + + fork: + name: Build (Fork) + if: ${{ github.event.pull_request.head.repo.full_name != github.repository || github.event.pull_request.user.login == 'dependabot[bot]' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build Benthos Collector image + run: docker build --file benthos-collector.Dockerfile --build-arg "VERSION=${GITHUB_SHA}" --tag benthos-collector:ci . + + # Preserve one check name while selecting the cached Depot build for + # same-repository pull requests and the credentialless build for forks. + result: + name: Result + needs: + - depot + - fork + if: always() + runs-on: ubuntu-24.04 + + steps: + - name: Verify Benthos Collector build + env: + DEPOT_RESULT: ${{ needs.depot.result }} + FORK_RESULT: ${{ needs.fork.result }} + run: | + case "${DEPOT_RESULT}:${FORK_RESULT}" in + success:skipped|skipped:success) + exit 0 + ;; + *) + echo "Benthos Collector build did not complete successfully" + exit 1 + ;; + esac diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e22aa43cf..d8dbf7bb46 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,268 +9,14 @@ permissions: contents: read concurrency: - # Supersede stale pull request runs, but never let a queued main run block a - # later push from rebuilding the shared caches. + # Supersede stale pull request runs without grouping distinct main pushes. group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - nix-changes: - name: Detect Nix environment changes - runs-on: depot-ubuntu-24.04 - outputs: - cache-key: ${{ steps.cache-key.outputs.cache-key }} - rebuild: ${{ steps.cache-key.outputs.rebuild }} - - steps: - - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} - api-key: ${{ secrets.STEP_SECURITY_API_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 2 - persist-credentials: false - - - name: Detect Nix input changes - id: changed-files - uses: kong/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf - with: - files_yaml: | - nix: - # flake.lock captures remote inputs; every local Nix expression - # can change the realized development environment. - - 'flake.lock' - - '**/*.nix' - - - name: Fingerprint Nix inputs - id: nix-inputs - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - EVENT_NAME: ${{ github.event_name }} - run: | - set -euo pipefail - - nix_input_hash() { - # Hash the relevant Git tree entries so a fork can select the base - # branch's cache without checking out or executing fork-owned code. - GIT_OPTIONAL_LOCKS=0 git ls-tree -r "$1" \ - | awk -F '\t' '$2 == "flake.lock" || $2 ~ /\.nix$/' \ - | sha256sum \ - | cut -d ' ' -f 1 - } - - current_hash="$(nix_input_hash HEAD)" - main_hash="${current_hash}" - - if [ "${EVENT_NAME}" = "pull_request" ]; then - main_hash="$(nix_input_hash "${BASE_SHA}")" - fi - - echo "current-hash=${current_hash}" >> "${GITHUB_OUTPUT}" - echo "main-hash=${main_hash}" >> "${GITHUB_OUTPUT}" - - - name: Select Nix cache key - id: cache-key - env: - BASE_REPOSITORY: ${{ github.repository }} - EVENT_NAME: ${{ github.event_name }} - HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} - MAIN_NIX_INPUT_HASH: ${{ steps.nix-inputs.outputs.main-hash }} - # any_modified covers additions, changes, renames, and deletions. - NIX_INPUTS_CHANGED: ${{ steps.changed-files.outputs.nix_any_modified }} - NIX_INPUT_HASH: ${{ steps.nix-inputs.outputs.current-hash }} - PR_NUMBER: ${{ github.event.pull_request.number }} - RUN_ATTEMPT: ${{ github.run_attempt }} - RUN_ID: ${{ github.run_id }} - run: | - main_key="${RUNNER_OS}-openmeter-nix-build-v4-main-${MAIN_NIX_INPUT_HASH}" - cache_key="${main_key}" - rebuild="false" - trusted="true" - - if [ "${EVENT_NAME}" = "pull_request" ] && [ "${HEAD_REPOSITORY}" != "${BASE_REPOSITORY}" ]; then - trusted="false" - fi - - if [ "${NIX_INPUTS_CHANGED}" = "true" ] && [ "${trusted}" = "true" ]; then - rebuild="true" - - # A PR publishes to an immutable, per-attempt key so a validated - # patch cannot overwrite main's stable input-addressed cache. - if [ "${EVENT_NAME}" = "pull_request" ]; then - cache_key="${RUNNER_OS}-openmeter-nix-build-v4-pr-${PR_NUMBER}-${RUN_ID}-${RUN_ATTEMPT}-${NIX_INPUT_HASH}" - fi - fi - - echo "cache-key=${cache_key}" >> "${GITHUB_OUTPUT}" - echo "rebuild=${rebuild}" >> "${GITHUB_OUTPUT}" - echo "Selected Nix cache: ${cache_key} (rebuild: ${rebuild}, trusted: ${trusted})" - - cache-rebuild: - name: Select or rebuild Nix cache - needs: nix-changes - if: needs.nix-changes.outputs.rebuild == 'true' - runs-on: depot-ubuntu-24.04-16 - env: - # Nix namespaces GOCACHE by its Go toolchain and dynamic loader. Depot's - # shared Go cache does not include those Nix store identities. - GOCACHEPROG: "" - - steps: - - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} - api-key: ${{ secrets.STEP_SECURITY_API_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up Nix - uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - nix_conf: | - access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - keep-env-derivations = true - keep-outputs = true - # Nix defaults to one local build at a time. Balance independent - # builds with per-build parallelism across this 16-vCPU runner. - max-jobs = 8 - cores = 2 - - - name: Build nix environment - run: | - nix flake check --impure - nix develop --impure .#ci - git diff --exit-code -- .nvmrc .go-version .golangci.version .pre-commit-config.yaml - - - name: Restore Go module cache - id: go-module-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-openmeter-go-modules-${{ hashFiles('go.*', 'collector/go.*', 'api/v3/client/go.*', 'e2e/go.*') }} - restore-keys: | - ${{ runner.os }}-openmeter-go-modules- - - - name: Populate Go module cache - if: steps.go-module-cache.outputs.cache-hit != 'true' - run: | - nix develop --impure .#ci -c sh -eu -c ' - go mod download - go -C collector mod download - go -C api/v3/client mod download - go -C e2e mod download - ' - - - name: Save Go module cache - if: steps.go-module-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: ${{ steps.go-module-cache.outputs.cache-primary-key }} - - - name: Verify Go runtime loader - run: | - runtime_dir="${RUNNER_TEMP}/openmeter-go-runtime" - mkdir -p "${runtime_dir}/go-tmp" - - GOTMPDIR="${runtime_dir}/go-tmp" nix develop --impure .#ci -c bash -euo pipefail <<'EOF' - runtime_probe="${RUNNER_TEMP}/openmeter-go-runtime/client.test" - expected_interpreter="$(cat "${NIX_CC}/nix-support/dynamic-linker")" - go env -json GOCACHE GOCACHEPROG GOROOT GOTOOLDIR - printf 'GO_LDSO=%s\n' "${GO_LDSO:-}" - - if [[ -n "$(go env GOCACHEPROG)" ]]; then - echo "Nix builds must not use Depot's loader-agnostic remote Go cache" - exit 1 - fi - - if [[ "${GO_LDSO:-}" != "${expected_interpreter}" ]]; then - echo "GO_LDSO uses ${GO_LDSO:-}; expected ${expected_interpreter}" - exit 1 - fi - - # GO_LDSO is not part of Go's link-action key. Start publication from - # an empty namespace so the assertion exercises the wrapped linker. - go clean -cache - go -C api/v3/client test -c -o "${runtime_probe}" . - - # A cached Go link action can retain the loader from an older Nixpkgs - # generation. Never publish a cache containing such an executable. - interpreter="$(readelf -l "${runtime_probe}" | sed -n 's/.*interpreter: \(.*\)]/\1/p')" - if [[ "${interpreter}" != "${expected_interpreter}" ]]; then - echo "Go runtime probe uses ${interpreter:-}; expected ${expected_interpreter}" - exit 1 - fi - - "${runtime_probe}" -test.run '^$' - EOF - - - name: Save Nix store cache - uses: nix-community/cache-nix-action/save@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 - with: - primary-key: ${{ needs.nix-changes.outputs.cache-key }} - save: "true" - - build: - name: Build - runs-on: depot-ubuntu-24.04-8 - - steps: - - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} - api-key: ${{ secrets.STEP_SECURITY_API_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: .go-version - cache: false - - - name: Verify Depot Go cache - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - run: | - cache_program="$(go env GOCACHEPROG)" - - if [[ "${cache_program}" != *"depot gocache"* ]]; then - echo "Depot Go cache is unavailable with upstream Go" - exit 1 - fi - - echo "Using ${cache_program}" - - - name: Build components - run: | - # On Depot runners, cgo external linking can spill large temporary linker - # files into /run via the default temp dir. Keep Go and system temp files - # on the workspace disk for this step to avoid "no space left on device", - # while still allowing each parallel go build to get its own temp dir. - mkdir -p \ - "$GITHUB_WORKSPACE/.tmp/go-work" \ - "$GITHUB_WORKSPACE/.tmp/system" - env \ - GOTMPDIR="$GITHUB_WORKSPACE/.tmp/go-work" \ - TMPDIR="$GITHUB_WORKSPACE/.tmp/system" \ - make -j 4 build GO_BUILD_FLAGS= - generators-openapi: name: Code Generators / OpenAPI - runs-on: depot-ubuntu-24.04-8 + runs-on: ubuntu-24.04 steps: - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -288,7 +34,12 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: .go-version - cache: false + cache: true + cache-dependency-path: | + go.sum + collector/go.sum + api/v3/client/go.sum + e2e/go.sum - name: Set up pnpm uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 @@ -306,17 +57,8 @@ jobs: cache: pnpm cache-dependency-path: api/spec/pnpm-lock.yaml - - name: Verify Depot Go cache - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - run: | - cache_program="$(go env GOCACHEPROG)" - - if [[ "${cache_program}" != *"depot gocache"* ]]; then - echo "Depot Go cache is unavailable with upstream Go" - exit 1 - fi - - echo "Using ${cache_program}" + - name: Record Go cache configuration + run: go env GOCACHE GOCACHEPROG - name: Verify runner tools run: | @@ -345,7 +87,7 @@ jobs: generators-javascript-sdk: name: Code Generators / JavaScript SDK - runs-on: depot-ubuntu-24.04 + runs-on: ubuntu-24.04 steps: - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -464,7 +206,7 @@ jobs: go-sdk: name: Go SDK - runs-on: depot-ubuntu-24.04 + runs-on: ubuntu-24.04 steps: - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -482,19 +224,15 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: .go-version - cache: false - - - name: Verify Depot Go cache - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - run: | - cache_program="$(go env GOCACHEPROG)" - - if [[ "${cache_program}" != *"depot gocache"* ]]; then - echo "Depot Go cache is unavailable with upstream Go" - exit 1 - fi + cache: true + cache-dependency-path: | + go.sum + collector/go.sum + api/v3/client/go.sum + e2e/go.sum - echo "Using ${cache_program}" + - name: Record Go cache configuration + run: go env GOCACHE GOCACHEPROG - name: Run Go SDK checks run: make test-go-sdk @@ -572,7 +310,7 @@ jobs: migrations: name: Migration Checks - runs-on: depot-ubuntu-24.04-4 + runs-on: ubuntu-24.04 env: ATLAS_SHA256_AMD64: d88aae186a55e5893c318f3b11c838a3372adf4dac1e2fd3bc7d2b55944c5797 ATLAS_SHA256_ARM64: bb8a22a08ccd9a6cb93f0a16205b7bbeb067dd7fec1f60227c687d8de93d6088 @@ -595,19 +333,15 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: .go-version - cache: false - - - name: Verify Depot Go cache - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - run: | - cache_program="$(go env GOCACHEPROG)" - - if [[ "${cache_program}" != *"depot gocache"* ]]; then - echo "Depot Go cache is unavailable with upstream Go" - exit 1 - fi + cache: true + cache-dependency-path: | + go.sum + collector/go.sum + api/v3/client/go.sum + e2e/go.sum - echo "Using ${cache_program}" + - name: Record Go cache configuration + run: go env GOCACHE GOCACHEPROG - name: Install Atlas run: | @@ -698,7 +432,7 @@ jobs: lint-other: name: Lint / Other - runs-on: depot-ubuntu-24.04-4 + runs-on: ubuntu-24.04 steps: - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -744,35 +478,6 @@ jobs: HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: pipx run --spec commitizen==4.16.5 cz check --allow-abort --rev-range "${BASE_SHA}..${HEAD_SHA}" - trusted-artifacts: - name: Artifacts - uses: $/.github/workflows/artifacts.yaml - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - with: - publish: ${{ github.event_name == 'push' }} - permissions: - contents: read - packages: write - id-token: write - security-events: write - - untrusted-artifacts: - name: Untrusted Artifacts - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} - uses: $/.github/workflows/untrusted-artifacts.yaml - permissions: - contents: read - - artifacts-pass: - name: Artifacts - needs: - - trusted-artifacts - - untrusted-artifacts - if: ${{ always() }} - uses: $/.github/workflows/workflow-result.yaml - with: - result: ${{ (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || !contains(needs.*.result, 'success')) && 'fail' || 'pass' }} - dependency-review: name: Dependency review runs-on: ubuntu-latest @@ -792,404 +497,3 @@ jobs: - name: Dependency Review uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 - - quickstart: - name: Quickstart - runs-on: depot-ubuntu-24.04-4 - env: - COMPOSE_PROFILES: depot-registry-svix - needs: - - trusted-artifacts - - untrusted-artifacts - if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && contains(needs.*.result, 'success') }} - - steps: - - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} - api-key: ${{ secrets.STEP_SECURITY_API_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Create override files for quickstart - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - env: - DEPOT_IMAGE_URL: ${{ needs.trusted-artifacts.outputs.container-image-url-depot }} - run: | - cat > quickstart/docker-compose.override.yaml < quickstart/docker-compose.override.yaml < e2e/docker-compose.override.yaml < e2e/docker-compose.override.yaml < artifacts/logs/docker-compose/base/compose-follow.log 2>&1 & - echo "$!" > artifacts/logs/docker-compose/base/compose-follow.pid - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: .go-version - cache: false - - - name: Verify Depot Go cache - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - run: | - cache_program="$(go env GOCACHEPROG)" - - if [[ "${cache_program}" != *"depot gocache"* ]]; then - echo "Depot Go cache is unavailable with upstream Go" - exit 1 - fi - - echo "Using ${cache_program}" - - - name: Check container health - run: docker inspect --format "{{json .State.Health }}" $(docker container list --all --filter 'name=^*-openmeter-*' --format '{{.Names}}') - if: always() - continue-on-error: true - - - name: Wait for worker to become ready - run: | - curl --fail --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:30000/healthz - curl --fail --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:30001/healthz - docker ps - - - name: Run base tests - id: run_base_tests - env: - OPENMETER_ADDRESS: http://localhost:38888 - TZ: UTC - run: make -C e2e test-base - - - name: Capture Docker Compose logs after base tests - if: always() && steps.launch_e2e.outcome != 'skipped' - working-directory: e2e - run: | - mkdir -p artifacts/logs/docker-compose/base - if [ -f artifacts/logs/docker-compose/base/compose-follow.pid ]; then - kill "$(cat artifacts/logs/docker-compose/base/compose-follow.pid)" 2>/dev/null || true - sleep 1 - fi - docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f ../.github/docker-compose.depot-registry.yaml ps --all > artifacts/logs/docker-compose/base/compose-ps.txt 2>&1 || true - for service in $(docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f ../.github/docker-compose.depot-registry.yaml config --services); do - docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f ../.github/docker-compose.depot-registry.yaml logs --no-color --timestamps "$service" > "artifacts/logs/docker-compose/base/${service}.log" 2>&1 || true - done - - - name: Cleanup Docker Compose - run: docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f ../.github/docker-compose.depot-registry.yaml down -v - working-directory: e2e - if: always() - - - name: Upload Openmeter logs as artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() - with: - name: "[${{ github.job }}] Openmeter logs" - path: | - e2e/logs/** - e2e/artifacts/logs/** - retention-days: 14 - - e2e-credits-disabled: - name: E2E / Credits disabled - runs-on: depot-ubuntu-24.04-8 - # Note: This check is running against the image that is going to be pushed. - needs: - - trusted-artifacts - - untrusted-artifacts - if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && contains(needs.*.result, 'success') }} - - steps: - - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} - api-key: ${{ secrets.STEP_SECURITY_API_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Create override files for e2e - env: - DEPOT_IMAGE_URL: ${{ needs.trusted-artifacts.outputs.container-image-url-depot }} - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - run: | - cat > e2e/docker-compose.override.yaml < e2e/docker-compose.override.yaml < artifacts/logs/docker-compose/credits-disabled/compose-follow.log 2>&1 & - echo "$!" > artifacts/logs/docker-compose/credits-disabled/compose-follow.pid - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: .go-version - cache: false - - - name: Verify Depot Go cache - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - run: | - cache_program="$(go env GOCACHEPROG)" - - if [[ "${cache_program}" != *"depot gocache"* ]]; then - echo "Depot Go cache is unavailable with upstream Go" - exit 1 - fi - - echo "Using ${cache_program}" - - - name: Check container health - run: docker inspect --format "{{json .State.Health }}" $(docker container list --all --filter 'name=^*-openmeter-*' --format '{{.Names}}') - if: always() - continue-on-error: true - - - name: Wait for worker to become ready with credits disabled - run: | - curl --fail --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:30000/healthz - curl --fail --retry 10 --retry-max-time 120 --retry-all-errors http://localhost:30001/healthz - docker ps - - - name: Run credits-disabled tests - id: run_credits_disabled_tests - env: - OPENMETER_ADDRESS: http://localhost:38888 - TZ: UTC - run: make -C e2e test-credits-disabled - - - name: Capture Docker Compose logs after credits-disabled tests - if: always() && steps.launch_credits_disabled_e2e.outcome != 'skipped' - working-directory: e2e - run: | - mkdir -p artifacts/logs/docker-compose/credits-disabled - if [ -f artifacts/logs/docker-compose/credits-disabled/compose-follow.pid ]; then - kill "$(cat artifacts/logs/docker-compose/credits-disabled/compose-follow.pid)" 2>/dev/null || true - sleep 1 - fi - docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f docker-compose.credits-disabled.yaml -f ../.github/docker-compose.depot-registry.yaml ps --all > artifacts/logs/docker-compose/credits-disabled/compose-ps.txt 2>&1 || true - for service in $(docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f docker-compose.credits-disabled.yaml -f ../.github/docker-compose.depot-registry.yaml config --services); do - docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f docker-compose.credits-disabled.yaml -f ../.github/docker-compose.depot-registry.yaml logs --no-color --timestamps "$service" > "artifacts/logs/docker-compose/credits-disabled/${service}.log" 2>&1 || true - done - - - name: Cleanup Docker Compose - run: docker compose -f docker-compose.infra.yaml -f docker-compose.openmeter.yaml -f docker-compose.override.yaml -f docker-compose.credits-disabled.yaml -f ../.github/docker-compose.depot-registry.yaml down -v - working-directory: e2e - if: always() - - - name: Upload Openmeter logs as artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() - with: - name: "[${{ github.job }}] Openmeter logs" - path: | - e2e/logs/** - e2e/artifacts/logs/** - retention-days: 14 diff --git a/.github/workflows/container-tests.yaml b/.github/workflows/container-tests.yaml new file mode 100644 index 0000000000..c90ebc9959 --- /dev/null +++ b/.github/workflows/container-tests.yaml @@ -0,0 +1,265 @@ +name: Container Tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + # Supersede stale pull request runs without grouping distinct main pushes. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + fork: + name: Fork + if: ${{ github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || github.event.pull_request.user.login == 'dependabot[bot]') }} + runs-on: ubuntu-24.04 + env: + OPENMETER_IMAGE: openmeter-ci:${{ github.sha }} + permissions: + contents: read + + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build OpenMeter image + run: docker build --build-arg "VERSION=${GITHUB_SHA}" --tag "${OPENMETER_IMAGE}" . + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: .go-version + cache: true + cache-dependency-path: | + go.sum + collector/go.sum + api/v3/client/go.sum + e2e/go.sum + + - name: Quickstart + id: quickstart + continue-on-error: true + run: bash .github/scripts/run-quickstart.sh "${OPENMETER_IMAGE}" never + + - name: Base E2E + id: e2e + continue-on-error: true + run: bash .github/scripts/run-e2e.sh base "${OPENMETER_IMAGE}" never upstream + + - name: Credits-disabled E2E + id: e2e-credits-disabled + continue-on-error: true + run: bash .github/scripts/run-e2e.sh credits-disabled "${OPENMETER_IMAGE}" never upstream + + - name: Upload OpenMeter logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: "[${{ github.job }}] OpenMeter logs" + path: | + quickstart/artifacts/logs/** + e2e/logs/** + e2e/artifacts/logs/** + retention-days: 14 + + - name: Verify container test results + if: always() + env: + QUICKSTART_RESULT: ${{ steps.quickstart.outcome }} + E2E_RESULT: ${{ steps.e2e.outcome }} + CREDITS_DISABLED_RESULT: ${{ steps.e2e-credits-disabled.outcome }} + run: | + failed="false" + + for result in \ + "Quickstart:${QUICKSTART_RESULT}" \ + "Base E2E:${E2E_RESULT}" \ + "Credits-disabled E2E:${CREDITS_DISABLED_RESULT}" + do + name="${result%%:*}" + outcome="${result#*:}" + echo "${name}: ${outcome}" + + if [ "${outcome}" != "success" ]; then + failed="true" + fi + done + + if [ "${failed}" = "true" ]; then + exit 1 + fi + + registry-image: + name: Build + if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]') }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + outputs: + image: registry.depot.dev/${{ steps.build.outputs.project-id }}@${{ steps.build.outputs.imageid }} + + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Depot CLI + uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 + + - name: Build OpenMeter image + id: build + uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 + with: + context: . + build-args: VERSION=${{ github.sha }} + platforms: linux/amd64 + tags: openmeter-ci:${{ github.sha }} + push: false + save: true + project: ${{ vars.DEPOT_PROJECT }} + + registry-container-tests: + name: ${{ matrix.name }} + needs: registry-image + strategy: + fail-fast: false + matrix: + include: + - name: Quickstart + suite: quickstart + runner: ubuntu-24.04 + go_cache: true + verify_depot_cache: false + dependency_source: upstream + - name: Base E2E + suite: base + runner: depot-ubuntu-24.04-8 + go_cache: false + verify_depot_cache: true + dependency_source: depot + - name: Credits-disabled E2E + suite: credits-disabled + runner: ubuntu-24.04 + go_cache: true + verify_depot_cache: false + dependency_source: upstream + runs-on: ${{ matrix.runner }} + permissions: + contents: read + id-token: write + + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: .go-version + cache: ${{ matrix.go_cache }} + cache-dependency-path: | + go.sum + collector/go.sum + api/v3/client/go.sum + e2e/go.sum + + - name: Verify Depot Go cache + if: matrix.verify_depot_cache + run: | + cache_program="$(go env GOCACHEPROG)" + + if [[ "${cache_program}" != *"depot gocache"* ]]; then + echo "Depot Go cache is unavailable with upstream Go" + exit 1 + fi + + echo "Using ${cache_program}" + + - name: Set up Depot CLI + uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 + + - name: Log in to Depot build registry + env: + DEPOT_PROJECT: ${{ vars.DEPOT_PROJECT }} + run: depot pull-token --project "${DEPOT_PROJECT}" | docker login registry.depot.dev --username x-token --password-stdin + + - name: Run ${{ matrix.name }} + env: + OPENMETER_IMAGE: ${{ needs.registry-image.outputs.image }} + run: | + case "${{ matrix.suite }}" in + quickstart) + bash .github/scripts/run-quickstart.sh "${OPENMETER_IMAGE}" always + ;; + base|credits-disabled) + bash .github/scripts/run-e2e.sh "${{ matrix.suite }}" "${OPENMETER_IMAGE}" always "${{ matrix.dependency_source }}" + ;; + esac + + - name: Upload OpenMeter logs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: "[${{ github.job }}-${{ matrix.suite }}] OpenMeter logs" + path: | + quickstart/artifacts/logs/** + e2e/logs/** + e2e/artifacts/logs/** + retention-days: 14 + + # Branch protection needs one stable result even though fork and registry + # builds intentionally use different execution graphs. + result: + name: Result + needs: + - fork + - registry-image + - registry-container-tests + if: always() + runs-on: ubuntu-24.04 + + steps: + - name: Verify container test path + env: + FORK_RESULT: ${{ needs.fork.result }} + REGISTRY_IMAGE_RESULT: ${{ needs.registry-image.result }} + REGISTRY_TESTS_RESULT: ${{ needs.registry-container-tests.result }} + run: | + case "${FORK_RESULT}:${REGISTRY_IMAGE_RESULT}:${REGISTRY_TESTS_RESULT}" in + success:skipped:skipped|skipped:success:success) + exit 0 + ;; + *) + echo "Container tests did not complete successfully" + exit 1 + ;; + esac diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml new file mode 100644 index 0000000000..35d4db6e73 --- /dev/null +++ b/.github/workflows/nix.yaml @@ -0,0 +1,139 @@ +name: Nix + +on: + push: + branches: [main] + pull_request: + paths: + - .github/workflows/nix.yaml + - flake.lock + - "**/*.nix" + +permissions: + contents: read + +concurrency: + # Supersede stale pull request runs without grouping distinct main pushes. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + nix: + name: Build Nix environment + runs-on: depot-ubuntu-24.04-16 + env: + # Nix namespaces GOCACHE by its Go toolchain and dynamic loader. Depot's + # shared Go cache does not include those Nix store identities. + GOCACHEPROG: "" + + steps: + - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + use-policy-store: ${{ secrets.STEP_SECURITY_API_KEY != '' }} + api-key: ${{ secrets.STEP_SECURITY_API_KEY }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Nix + uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 + with: + # Authenticate public flake fetches to avoid GitHub API rate limits, + # but never expose the token to fork or Dependabot code. + github_access_token: ${{ (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]')) && github.token || null }} + nix_conf: | + keep-env-derivations = true + keep-outputs = true + # Nix defaults to one local build at a time. Balance independent + # builds with per-build parallelism across this 16-vCPU runner. + max-jobs = 8 + cores = 2 + + - name: Build nix environment + run: | + nix flake check --impure + nix develop --impure .#ci + git diff --exit-code -- .nvmrc .go-version .golangci.version .pre-commit-config.yaml + + - name: Restore Go module cache + id: go-module-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-openmeter-go-modules-${{ hashFiles('go.*', 'collector/go.*', 'api/v3/client/go.*', 'e2e/go.*') }} + restore-keys: | + ${{ runner.os }}-openmeter-go-modules- + + - name: Populate Go module cache + if: steps.go-module-cache.outputs.cache-hit != 'true' + run: | + nix develop --impure .#ci -c sh -eu -c ' + go mod download + go -C collector mod download + go -C api/v3/client mod download + go -C e2e mod download + ' + + - name: Save Go module cache + if: steps.go-module-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: ${{ steps.go-module-cache.outputs.cache-primary-key }} + + - name: Verify Go runtime loader + run: | + runtime_dir="${RUNNER_TEMP}/openmeter-go-runtime" + mkdir -p "${runtime_dir}/go-tmp" + + GOTMPDIR="${runtime_dir}/go-tmp" nix develop --impure .#ci -c env GOCACHEPROG= bash -euo pipefail <<'EOF' + runtime_probe="${RUNNER_TEMP}/openmeter-go-runtime/client.test" + expected_interpreter="$(cat "${NIX_CC}/nix-support/dynamic-linker")" + go env -json GOCACHE GOCACHEPROG GOROOT GOTOOLDIR + printf 'GO_LDSO=%s\n' "${GO_LDSO:-}" + + if [[ -n "$(go env GOCACHEPROG)" ]]; then + echo "Nix builds must not use Depot's loader-agnostic remote Go cache" + exit 1 + fi + + if [[ "${GO_LDSO:-}" != "${expected_interpreter}" ]]; then + echo "GO_LDSO uses ${GO_LDSO:-}; expected ${expected_interpreter}" + exit 1 + fi + + # GO_LDSO is not part of Go's link-action key. Start publication from + # an empty namespace so the assertion exercises the wrapped linker. + go clean -cache + go -C api/v3/client test -c -o "${runtime_probe}" . + + # A cached Go link action can retain the loader from an older Nixpkgs + # generation. Never publish a cache containing such an executable. + interpreter="$(readelf -l "${runtime_probe}" | sed -n 's/.*interpreter: \(.*\)]/\1/p')" + if [[ "${interpreter}" != "${expected_interpreter}" ]]; then + echo "Go runtime probe uses ${interpreter:-}; expected ${expected_interpreter}" + exit 1 + fi + + "${runtime_probe}" -test.run '^$' + EOF + + - name: Build components in Nix environment + run: | + mkdir -p \ + "$GITHUB_WORKSPACE/.tmp/go-work" \ + "$GITHUB_WORKSPACE/.tmp/system" + env \ + GOTMPDIR="$GITHUB_WORKSPACE/.tmp/go-work" \ + TMPDIR="$GITHUB_WORKSPACE/.tmp/system" \ + nix develop --impure .#ci -c env GOCACHEPROG= make -j 4 build + + - name: Save Nix store cache + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: nix-community/cache-nix-action/save@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 + with: + primary-key: ${{ runner.os }}-openmeter-nix-build-v5-main-${{ hashFiles('.github/workflows/nix.yaml', 'flake.lock', '**/*.nix') }} + save: "true" diff --git a/.github/workflows/aip-npm-release.yaml b/.github/workflows/release-aip-npm.yaml similarity index 99% rename from .github/workflows/aip-npm-release.yaml rename to .github/workflows/release-aip-npm.yaml index daf046fdec..06e29f31fb 100644 --- a/.github/workflows/aip-npm-release.yaml +++ b/.github/workflows/release-aip-npm.yaml @@ -6,7 +6,7 @@ # configured against the caller workflow (e.g. release.yaml) + environment prod. # See: https://github.com/npm/documentation/issues/1755 -name: AIP NPM Release +name: AIP npm Release on: workflow_call: diff --git a/.github/workflows/untrusted-artifacts.yaml b/.github/workflows/release-docker.yaml similarity index 60% rename from .github/workflows/untrusted-artifacts.yaml rename to .github/workflows/release-docker.yaml index 49cc6bcba6..75b9f7d933 100644 --- a/.github/workflows/untrusted-artifacts.yaml +++ b/.github/workflows/release-docker.yaml @@ -1,18 +1,26 @@ -name: Untrusted Artifacts +name: Docker Release on: - workflow_call: + push: + branches: + - main + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-dev.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+" permissions: contents: read jobs: - container-image: - name: Container image - runs-on: ubuntu-latest + openmeter: + name: OpenMeter image + runs-on: depot-ubuntu-24.04 permissions: contents: read + packages: write + id-token: write steps: - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -26,6 +34,9 @@ jobs: with: persist-credentials: false + - name: Set up Depot CLI + uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 + - name: Set image name id: image-name run: echo "value=ghcr.io/${{ github.repository }}" >> "$GITHUB_OUTPUT" @@ -39,31 +50,37 @@ jobs: latest = false tags: | type=ref,event=branch - type=ref,event=pr,prefix=pr- type=semver,pattern={{raw}} type=raw,value=latest,enable={{is_default_branch}} type=ref,event=branch,suffix=-{{sha}}-{{date 'X'}},enable={{is_default_branch}} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Log in to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} - - name: Build image - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + - name: Build and push image + uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 with: context: . build-args: | VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 # The confluent library doesn't support ARMv7 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - push: false + push: true + project: ${{ vars.DEPOT_PROJECT }} - benthos-collector-container-image: - name: Benthos Collector Container image - runs-on: ubuntu-latest + benthos-collector: + name: Benthos Collector image + runs-on: depot-ubuntu-24.04 permissions: contents: read + packages: write + id-token: write steps: - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -77,35 +94,38 @@ jobs: with: persist-credentials: false - - name: Set image name - id: image-name - run: echo "value=ghcr.io/openmeterio/benthos-collector" >> "$GITHUB_OUTPUT" + - name: Set up Depot CLI + uses: depot/setup-action@91bc8495a33ebfc504ffc89e5674379ccf23c29c # v1.7.2 - name: Gather build metadata id: meta uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: - images: ${{ steps.image-name.outputs.value }} + images: ghcr.io/openmeterio/benthos-collector flavor: | latest = false tags: | type=ref,event=branch - type=ref,event=pr,prefix=pr- type=semver,pattern={{raw}} type=raw,value=latest,enable={{is_default_branch}} type=ref,event=branch,suffix=-{{sha}}-{{date 'X'}},enable={{is_default_branch}} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Log in to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} - - name: Build image - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + - name: Build and push image + uses: depot/build-push-action@98e78adca7817480b8185f474a400b451d74e287 # v1.18.0 with: context: . file: benthos-collector.Dockerfile build-args: | VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - push: false + push: true + project: ${{ vars.DEPOT_PROJECT }} diff --git a/.github/workflows/npm-release.yaml b/.github/workflows/release-npm.yaml similarity index 99% rename from .github/workflows/npm-release.yaml rename to .github/workflows/release-npm.yaml index 08f1a64f71..5b7b35bb6f 100644 --- a/.github/workflows/npm-release.yaml +++ b/.github/workflows/release-npm.yaml @@ -5,7 +5,7 @@ # configured against the caller workflow (e.g. release.yaml) + environment prod. # See: https://github.com/npm/documentation/issues/1755 -name: NPM Release +name: npm Release on: workflow_call: diff --git a/.github/workflows/sdk-python-dev-release.yaml b/.github/workflows/release-sdk-python-dev.yaml similarity index 82% rename from .github/workflows/sdk-python-dev-release.yaml rename to .github/workflows/release-sdk-python-dev.yaml index 306d042a8b..a17912e16b 100644 --- a/.github/workflows/sdk-python-dev-release.yaml +++ b/.github/workflows/release-sdk-python-dev.yaml @@ -36,6 +36,7 @@ jobs: - name: Set up Nix uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 with: + # Authenticate public flake fetches to avoid GitHub API rate limits. github_access_token: ${{ secrets.GITHUB_TOKEN }} nix_conf: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} @@ -45,12 +46,9 @@ jobs: - name: Restore Nix store uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 with: - primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }} + primary-key: ${{ runner.os }}-openmeter-nix-build-v5-main-${{ hashFiles('.github/workflows/nix.yaml', 'flake.lock', '**/*.nix') }} restore-prefixes-first-match: | - ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- - ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} - ${{ runner.os }}-openmeter-nix-build-main- - ${{ runner.os }}-openmeter-nix-build- + ${{ runner.os }}-openmeter-nix-build-v5-main- - name: Publish Python package via Nix make target run: | diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e992ed001a..a35da75bcf 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,19 +15,6 @@ permissions: contents: read jobs: - artifacts: - name: Artifacts - # Artifact publishing (container images, etc.) is tag-only. - if: github.ref_type == 'tag' - uses: $/.github/workflows/artifacts.yaml - with: - publish: true - permissions: - contents: read - packages: write - id-token: write - security-events: write - helm-release: name: Helm Charts # Helm chart releases are tag-only. @@ -59,6 +46,7 @@ jobs: - name: Set up Nix uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 with: + # Authenticate public flake fetches to avoid GitHub API rate limits. github_access_token: ${{ secrets.GITHUB_TOKEN }} nix_conf: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} @@ -68,13 +56,9 @@ jobs: - name: Restore Nix store uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 with: - primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ - hashFiles('flake.*') }} + primary-key: ${{ runner.os }}-openmeter-nix-build-v5-main-${{ hashFiles('.github/workflows/nix.yaml', 'flake.lock', '**/*.nix') }} restore-prefixes-first-match: | - ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- - ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} - ${{ runner.os }}-openmeter-nix-build-main- - ${{ runner.os }}-openmeter-nix-build- + ${{ runner.os }}-openmeter-nix-build-v5-main- - name: Package chart # Untrusted values (github.ref_name, matrix.chart) passed via env to @@ -135,6 +119,7 @@ jobs: - name: Set up Nix uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 with: + # Authenticate public flake fetches to avoid GitHub API rate limits. github_access_token: ${{ secrets.GITHUB_TOKEN }} nix_conf: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} @@ -144,13 +129,9 @@ jobs: - name: Restore Nix store uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 with: - primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ - hashFiles('flake.*') }} + primary-key: ${{ runner.os }}-openmeter-nix-build-v5-main-${{ hashFiles('.github/workflows/nix.yaml', 'flake.lock', '**/*.nix') }} restore-prefixes-first-match: | - ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- - ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} - ${{ runner.os }}-openmeter-nix-build-main- - ${{ runner.os }}-openmeter-nix-build- + ${{ runner.os }}-openmeter-nix-build-v5-main- - name: Restore Go module cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -268,7 +249,7 @@ jobs: # npm's trusted publisher entry is keyed on caller workflow file + environment, # so this single caller must serve both channels. needs: [ sdk-javascript-meta ] - uses: $/.github/workflows/npm-release.yaml + uses: $/.github/workflows/release-npm.yaml with: version: ${{ needs.sdk-javascript-meta.outputs.version }} dist-tag: ${{ needs.sdk-javascript-meta.outputs.dist-tag }} @@ -284,7 +265,7 @@ jobs: # publisher entry is keyed on caller workflow file + environment, so this # caller must serve both channels. needs: [ sdk-javascript-meta ] - uses: $/.github/workflows/aip-npm-release.yaml + uses: $/.github/workflows/release-aip-npm.yaml with: version: ${{ needs.sdk-javascript-meta.outputs.version }} dist-tag: ${{ needs.sdk-javascript-meta.outputs.dist-tag }} @@ -294,7 +275,7 @@ jobs: sdk-python-release: name: Python SDK Release - # Python SDK releases are tag-only (dev Python releases live in sdk-python-dev-release.yaml). + # Python SDK releases are tag-only (dev Python releases live in release-sdk-python-dev.yaml). if: github.ref_type == 'tag' runs-on: depot-ubuntu-latest-8 environment: prod @@ -319,6 +300,7 @@ jobs: - name: Set up Nix uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 with: + # Authenticate public flake fetches to avoid GitHub API rate limits. github_access_token: ${{ secrets.GITHUB_TOKEN }} nix_conf: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} @@ -328,13 +310,9 @@ jobs: - name: Restore Nix store uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 with: - primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ - hashFiles('flake.*') }} + primary-key: ${{ runner.os }}-openmeter-nix-build-v5-main-${{ hashFiles('.github/workflows/nix.yaml', 'flake.lock', '**/*.nix') }} restore-prefixes-first-match: | - ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- - ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} - ${{ runner.os }}-openmeter-nix-build-main- - ${{ runner.os }}-openmeter-nix-build- + ${{ runner.os }}-openmeter-nix-build-v5-main- - name: Publish Python package via Nix make target run: | diff --git a/.github/workflows/workflow-result.yaml b/.github/workflows/workflow-result.yaml deleted file mode 100644 index bb2f8e58a4..0000000000 --- a/.github/workflows/workflow-result.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: Workflow Result for Required Check - -on: - workflow_call: - inputs: - result: - description: Workflow result - required: true - type: string - -jobs: - workflow_result: - name: Workflow Result - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - name: Pass or Fail - run: | - if [[ "$INPUTS_RESULT" == true ]]; then - exit 1 - else - exit 0 - fi - env: - INPUTS_RESULT: ${{ inputs.result == 'fail' }}