diff --git a/.config/rail.toml b/.config/rail.toml index 14114221da..8089448ba3 100644 --- a/.config/rail.toml +++ b/.config/rail.toml @@ -15,14 +15,39 @@ # specific language governing permissions and limitations # under the License. -# cargo-rail change detection configuration -# See: https://github.com/loadingalias/cargo-rail - -[change-detection] -# Changes to these paths trigger a full workspace rebuild/retest -infrastructure = [ - "Cargo.lock", - "rust-toolchain.toml", - ".cargo/**", - ".github/**", +[plan.work.runtime-artifacts] +scope = "cargo" +cargo_prerequisites = [ + { source_work = "cargo.test", when = [ + { package = "integration" }, + ], require = [ + { package = "server", target = { name = "iggy-server", kind = "bin" } }, + { package = "iggy-cli", target = { name = "iggy", kind = "bin" } }, + { package = "iggy-bench", target = { name = "iggy-bench", kind = "bin" } }, + { package = "iggy-connectors", target = { name = "iggy-connectors", kind = "bin" } }, + { package = "iggy-mcp", target = { name = "iggy-mcp", kind = "bin" } }, + { package = "iggy_connector_clickhouse_sink", target = { name = "iggy_connector_clickhouse_sink", kind = "cdylib" } }, + { package = "iggy_connector_delta_sink", target = { name = "iggy_connector_delta_sink", kind = "cdylib" } }, + { package = "iggy_connector_doris_sink", target = { name = "iggy_connector_doris_sink", kind = "cdylib" } }, + { package = "iggy_connector_elasticsearch_sink", target = { name = "iggy_connector_elasticsearch_sink", kind = "cdylib" } }, + { package = "iggy_connector_elasticsearch_source", target = { name = "iggy_connector_elasticsearch_source", kind = "cdylib" } }, + { package = "iggy_connector_http_sink", target = { name = "iggy_connector_http_sink", kind = "cdylib" } }, + { package = "iggy_connector_iceberg_sink", target = { name = "iggy_connector_iceberg_sink", kind = "cdylib" } }, + { package = "iggy_connector_influxdb_sink", target = { name = "iggy_connector_influxdb_sink", kind = "cdylib" } }, + { package = "iggy_connector_influxdb_source", target = { name = "iggy_connector_influxdb_source", kind = "cdylib" } }, + { package = "iggy_connector_meilisearch_sink", target = { name = "iggy_connector_meilisearch_sink", kind = "cdylib" } }, + { package = "iggy_connector_mongodb_sink", target = { name = "iggy_connector_mongodb_sink", kind = "cdylib" } }, + { package = "iggy_connector_postgres_sink", target = { name = "iggy_connector_postgres_sink", kind = "cdylib" } }, + { package = "iggy_connector_postgres_source", target = { name = "iggy_connector_postgres_source", kind = "cdylib" } }, + { package = "iggy_connector_quickwit_sink", target = { name = "iggy_connector_quickwit_sink", kind = "cdylib" } }, + { package = "iggy_connector_random_source", target = { name = "iggy_connector_random_source", kind = "cdylib" } }, + { package = "iggy_connector_redshift_sink", target = { name = "iggy_connector_redshift_sink", kind = "cdylib" } }, + { package = "iggy_connector_s3_sink", target = { name = "iggy_connector_s3_sink", kind = "cdylib" } }, + { package = "iggy_connector_stdout_sink", target = { name = "iggy_connector_stdout_sink", kind = "cdylib" } }, + { package = "iggy_connector_surrealdb_sink", target = { name = "iggy_connector_surrealdb_sink", kind = "cdylib" } }, + ] }, ] + +[plan.work.edge-images] +scope = "variants" +variant_catalog = ".github/config/edge-image-variants.json" diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 97efa4d3af..1f45e4f7eb 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -79,63 +79,13 @@ runs: fi shell: bash - # DAG-based test scoping: use cargo-rail to compute affected crates from the - # workspace dependency graph + git diff, avoiding the full test suite when only - # a subset of crates changed. - # Safety: cargo check/clippy run on the full workspace separately, catching all - # compilation errors. This only scopes test BUILD and EXECUTION. - - name: Fetch base branch for DAG analysis + # A planner failure must not reduce coverage. The test step below restores + # Iggy's full-workspace behavior unless the plan and checkout both validate. + - name: Plan affected Rust work + id: rail if: startsWith(inputs.task, 'test-') - run: git fetch origin master --depth=1 2>/dev/null || true - shell: bash - - - name: Install cargo-rail - if: startsWith(inputs.task, 'test-') - uses: taiki-e/install-action@v2 - with: - tool: cargo-rail - - - name: Compute affected crates (cargo-rail) - if: startsWith(inputs.task, 'test-') - run: | - METADATA_JSON=$(cargo metadata --format-version 1 --no-deps 2>/dev/null || echo "{}") - TOTAL_CRATES=$(echo "$METADATA_JSON" | jq '.workspace_members | length' 2>/dev/null || echo "?") - echo "$TOTAL_CRATES" > /tmp/total-crates.txt - # Extract packages with binary or cdylib targets — integration tests may - # invoke binaries via assert_cmd (CARGO_BIN_EXE_), and connector - # plugins (cdylib) are loaded at runtime via dlopen by iggy-connectors. - # Both must always be compiled alongside affected crates. - echo "$METADATA_JSON" | jq -r '.packages[] | select(.targets[] | .kind[] | (. == "bin" or . == "cdylib")) | .name' 2>/dev/null > /tmp/bin-packages.txt || true - - PLAN_JSON=$(cargo rail plan --since origin/master -f json 2>/tmp/affected-stderr.txt || echo "") - - if [[ -n "$PLAN_JSON" ]]; then - MODE=$(echo "$PLAN_JSON" | jq -r '.scope.mode') - if [[ "$MODE" == "crates" ]]; then - CRATES=$(echo "$PLAN_JSON" | jq -r '.scope.crates[]') - CRATE_COUNT=$(echo "$CRATES" | wc -l) - # Build nextest filter expression for cargo nextest run (affected crates only) - echo "$CRATES" | sed 's/^/package(/; s/$/)/' | paste -sd '|' | sed 's/|/ | /g' > /tmp/nextest-filter.txt - # Save affected-only -p flags for cargo test fallback (no nextest filter) - echo "$CRATES" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/test-packages.txt - # Build -p flags: affected crates + packages with bin/cdylib targets. - # Binary packages: nextest sets CARGO_BIN_EXE_ from compiled - # artifacts; cdylib packages: connector plugins loaded via dlopen. - # Both must be in the build even if not directly in the DAG scope. - BIN_PKGS=$(cat /tmp/bin-packages.txt 2>/dev/null || echo "") - ALL_BUILD_PKGS=$(printf '%s\n%s\n' "$CRATES" "$BIN_PKGS" | sort -u | grep -v '^$') - echo "$ALL_BUILD_PKGS" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/packages.txt - BUILD_COUNT=$(echo "$ALL_BUILD_PKGS" | wc -l) - echo "::notice::DAG analysis: testing ${CRATE_COUNT} crates, building ${BUILD_COUNT} (of ${TOTAL_CRATES} total, +$(( BUILD_COUNT - CRATE_COUNT )) binary pkgs)" - else - echo "::notice::Full workspace affected (${TOTAL_CRATES} crates)" - fi - else - STDERR=$(cat /tmp/affected-stderr.txt 2>/dev/null || echo "") - echo "::warning::Could not compute affected crates, running full test suite. ${STDERR}" - rm -f /tmp/nextest-filter.txt /tmp/packages.txt - fi - shell: bash + continue-on-error: true + uses: loadingalias/cargo-rail-action@78ad385a85627484a5b634cf1bc3caa1de872c6a # v8.2.0 # Individual lint tasks for parallel execution - name: Cargo check @@ -209,44 +159,77 @@ runs: - name: Build and test with coverage if: startsWith(inputs.task, 'test-') + env: + PLAN_FILE: ${{ steps.rail.outputs.plan-file }} + PLAN_READER: ${{ steps.rail.outputs.plan-reader }} + PLAN_STATUS: ${{ steps.rail.outcome }} + TASK: ${{ inputs.task }} run: | + set -euo pipefail + # Parse partition index from task name (test-1 -> hash:1/3, test-2 -> hash:2/3, ...). # TEST_PARTITIONS must match the number of test-N tasks in # .github/config/components.yml. Cluster bootstrap makes each test # CPU-heavy, so partitions stay small. TEST_PARTITIONS=3 - TASK="${{ inputs.task }}" - PARTITION_FLAG="" + PARTITION_ARGS=() if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then PARTITION_INDEX="${BASH_REMATCH[1]}" - PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/${TEST_PARTITIONS}" + PARTITION_ARGS=(--partition "hash:${PARTITION_INDEX}/${TEST_PARTITIONS}") echo "::notice::Running test partition ${PARTITION_INDEX}/${TEST_PARTITIONS}" fi - # Read DAG-based affected crate filter (computed in earlier step) - NEXTEST_FILTER="" - PACKAGE_FLAGS="" - TEST_PACKAGE_FLAGS="" - TOTAL_CRATES="?" - if [[ -f /tmp/nextest-filter.txt ]]; then - NEXTEST_FILTER=$(cat /tmp/nextest-filter.txt) - fi - if [[ -f /tmp/packages.txt ]]; then - PACKAGE_FLAGS=$(cat /tmp/packages.txt) - fi - if [[ -f /tmp/test-packages.txt ]]; then - TEST_PACKAGE_FLAGS=$(cat /tmp/test-packages.txt) + # Consume only a verified plan through the bundled typed reader. Any + # planner or reader failure restores the previous full-workspace path. + PLAN_OUTPUT_DIR="${RUNNER_TEMP:?RUNNER_TEMP is required}/cargo-rail-reader-${TASK}" + mkdir -p "$PLAN_OUTPUT_DIR" + PLAN_USABLE=false + TEST_SCOPE=workspace + RUNTIME_SCOPE=workspace + TEST_CARGO_ARGS=() + TEST_TARGET_ARGS=() + TEST_PACKAGES=() + RUNTIME_PACKAGES=() + + if [[ "$PLAN_STATUS" == "success" ]] && + python3 "$PLAN_READER" verify-checkout "$PLAN_FILE" && + TEST_SCOPE=$(python3 "$PLAN_READER" cargo-scope "$PLAN_FILE" cargo.test) && + RUNTIME_SCOPE=$(python3 "$PLAN_READER" cargo-scope "$PLAN_FILE" runtime-artifacts) && + python3 "$PLAN_READER" cargo-args "$PLAN_FILE" cargo.test > "$PLAN_OUTPUT_DIR/test-args" && + python3 "$PLAN_READER" target-args "$PLAN_FILE" cargo.test > "$PLAN_OUTPUT_DIR/test-target-args" && + python3 "$PLAN_READER" package-names "$PLAN_FILE" cargo.test > "$PLAN_OUTPUT_DIR/test-packages" && + python3 "$PLAN_READER" package-names "$PLAN_FILE" runtime-artifacts > "$PLAN_OUTPUT_DIR/runtime-packages"; then + mapfile -d '' -t TEST_CARGO_ARGS < "$PLAN_OUTPUT_DIR/test-args" + mapfile -d '' -t TEST_TARGET_ARGS < "$PLAN_OUTPUT_DIR/test-target-args" + mapfile -d '' -t TEST_PACKAGES < "$PLAN_OUTPUT_DIR/test-packages" + mapfile -d '' -t RUNTIME_PACKAGES < "$PLAN_OUTPUT_DIR/runtime-packages" + PLAN_USABLE=true + else + echo "::warning::Cargo-Rail plan unavailable or invalid; running the full workspace" fi - if [[ -f /tmp/total-crates.txt ]]; then - TOTAL_CRATES=$(cat /tmp/total-crates.txt) + + if [[ "$PLAN_USABLE" == true && "$TEST_SCOPE" == "skipped" ]]; then + echo "::notice::Cargo-Rail reports no affected Rust tests" + exit 0 fi - if [[ -n "$PACKAGE_FLAGS" ]]; then - TEST_CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l) - BUILD_CRATE_COUNT=$(echo "$PACKAGE_FLAGS" | grep -o '\-p ' | wc -l) - echo "::notice::DAG-scoped: testing ${TEST_CRATE_COUNT} crates, building ${BUILD_CRATE_COUNT} (cargo check/clippy cover full workspace separately)" + BUILD_ARGS=() + BUILD_SCOPE=workspace + if [[ "$PLAN_USABLE" == true && "$TEST_SCOPE" == "packages" && + ( "$RUNTIME_SCOPE" == "packages" || "$RUNTIME_SCOPE" == "skipped" ) ]]; then + declare -A SEEN_BUILD_PACKAGES=() + BUILD_PACKAGES=() + for package in "${TEST_PACKAGES[@]}" "${RUNTIME_PACKAGES[@]}"; do + if [[ -n "$package" && -z "${SEEN_BUILD_PACKAGES[$package]:-}" ]]; then + SEEN_BUILD_PACKAGES["$package"]=1 + BUILD_PACKAGES+=("$package") + BUILD_ARGS+=(-p "$package") + fi + done + BUILD_SCOPE=packages + echo "::notice::Cargo-Rail: testing ${#TEST_PACKAGES[@]} packages, building ${#BUILD_PACKAGES[@]} including runtime prerequisites" else - echo "::notice::Full workspace build (no DAG filter available)" + echo "::notice::Full workspace build and test" fi source <(cargo llvm-cov show-env --export-prefix) @@ -262,33 +245,30 @@ runs: fi bins_start=$(date +%s) - if [[ -n "$PACKAGE_FLAGS" ]]; then - cargo build --locked $PACKAGE_FLAGS - else - cargo build --locked - fi + cargo build --locked "${BUILD_ARGS[@]}" bins_end=$(date +%s) bins_duration=$((bins_end - bins_start)) echo "::notice::Binaries and libraries built in ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" compile_start=$(date +%s) - if [[ -n "$PACKAGE_FLAGS" ]]; then - cargo test --locked --no-run $PACKAGE_FLAGS - else - cargo test --locked --no-run - fi + cargo test --locked --no-run "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" compile_end=$(date +%s) compile_duration=$((compile_end - compile_start)) echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" # api_handler_tests, version_firewall_tests, and server_e2e_tests need gitignored - # wire fixtures. Generate when iggy-gateway-kafka is in the DAG test scope, or (on - # a full-workspace run) when gateways/** changed vs origin/master — avoid building - # kafka-message-gen / kafka-protocol when the gateway was not touched. + # wire fixtures. Generate when iggy-gateway-kafka is in the Cargo-Rail test scope, + # or on a full-workspace run when gateways/** changed vs origin/master. This avoids + # building kafka-message-gen / kafka-protocol when the gateway was not touched. NEEDS_KAFKA_FIXTURES=false - if grep -q 'package(iggy-gateway-kafka)' <<< "$NEXTEST_FILTER"; then - NEEDS_KAFKA_FIXTURES=true - elif [[ -z "$NEXTEST_FILTER" ]]; then + if [[ "$TEST_SCOPE" == "packages" ]]; then + for package in "${TEST_PACKAGES[@]}"; do + if [[ "$package" == "iggy-gateway-kafka" ]]; then + NEEDS_KAFKA_FIXTURES=true + break + fi + done + else # `2>&1` into a variable, not `2>/dev/null` piped straight to grep: the old form # silenced *any* git failure (e.g. a shallow checkout with no merge-base history for # origin/master) into empty output, which `grep -q` then reads identically to "diff @@ -332,27 +312,22 @@ runs: test_start=$(date +%s) if command -v cargo-nextest &> /dev/null; then - if [[ -n "$NEXTEST_FILTER" ]]; then + if [[ "$TEST_SCOPE" == "packages" ]]; then # Every test target in the `bdd` crate is `required-features = # ["bdd"]`, so the `cargo test --no-run` above builds no binaries # for it. A PR touching only that crate then produces a valid - # filter that matches zero tests, and nextest's default + # package scope that matches zero tests, and nextest's default # `--no-tests=fail` turns that into exit 4 on a green PR. - cargo nextest run --locked --no-fail-fast --no-tests=warn --profile ci $PARTITION_FLAG $PACKAGE_FLAGS -E "$NEXTEST_FILTER" + cargo nextest run --locked --no-fail-fast --no-tests=warn --profile ci \ + "${PARTITION_ARGS[@]}" "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" else - cargo nextest run --locked --no-fail-fast --profile ci $PARTITION_FLAG + cargo nextest run --locked --no-fail-fast --profile ci "${PARTITION_ARGS[@]}" fi else - if [[ -n "$PARTITION_FLAG" ]]; then + if [[ ${#PARTITION_ARGS[@]} -gt 0 ]]; then echo "::error::cargo-nextest not found, falling back to cargo test without partitioning (all tests will run on every partition)" fi - # Use TEST_PACKAGE_FLAGS (affected crates only), not PACKAGE_FLAGS - # (which includes binary packages whose tests should NOT run). - if [[ -n "$TEST_PACKAGE_FLAGS" ]]; then - cargo test --locked --no-fail-fast $TEST_PACKAGE_FLAGS - else - cargo test --locked --no-fail-fast - fi + cargo test --locked --no-fail-fast "${TEST_CARGO_ARGS[@]}" "${TEST_TARGET_ARGS[@]}" fi test_end=$(date +%s) test_duration=$((test_end - test_start)) @@ -362,13 +337,11 @@ runs: total_duration=$((build_duration + test_duration)) echo "" echo "=========================================" - if [[ -n "$PACKAGE_FLAGS" ]]; then - TEST_CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l) - BUILD_CRATE_COUNT=$(echo "$PACKAGE_FLAGS" | grep -o '\-p ' | wc -l) - echo "DAG scope (test): ${TEST_CRATE_COUNT}/${TOTAL_CRATES} crates" - echo "DAG scope (build): ${BUILD_CRATE_COUNT}/${TOTAL_CRATES} crates" + if [[ "$BUILD_SCOPE" == "packages" ]]; then + echo "Cargo-Rail scope (test): ${#TEST_PACKAGES[@]} packages" + echo "Cargo-Rail scope (build): ${#BUILD_PACKAGES[@]} packages" else - echo "DAG scope: full workspace (${TOTAL_CRATES} crates)" + echo "Cargo-Rail scope: full workspace" fi echo "All targets build: ${bins_duration}s ($(date -ud @${bins_duration} +'%M:%S'))" echo "Tests compile: ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" diff --git a/.github/config/edge-image-variants.json b/.github/config/edge-image-variants.json new file mode 100644 index 0000000000..bda838b320 --- /dev/null +++ b/.github/config/edge-image-variants.json @@ -0,0 +1,117 @@ +{ + "variant_catalog_version": 2, + "work": "edge-images", + "variants": [ + { + "id": "rust-server", + "dimensions": { "component": "rust-server" }, + "cargo_roots": [ + { "package": "server", "target": { "name": "iggy-server", "kind": "bin" } }, + { "package": "iggy-cli", "target": { "name": "iggy", "kind": "bin" } } + ], + "external_paths": [ + ".dockerignore", + ".github/actions/utils/docker-buildx/**", + ".github/actions/utils/docker-login/**", + ".github/config/publish.yml", + ".github/workflows/post-merge.yml", + ".github/workflows/publish.yml", + "LICENSE", + "NOTICE", + "about.toml", + "about.hbs", + "core/server/Dockerfile", + "web/**", + "scripts/extract-version.sh", + "scripts/ci/third-party-licenses.sh", + "scripts/ci/render-node-licenses.mjs" + ] + }, + { + "id": "rust-mcp", + "dimensions": { "component": "rust-mcp" }, + "cargo_roots": [ + { "package": "iggy-mcp", "target": { "name": "iggy-mcp", "kind": "bin" } } + ], + "external_paths": [ + ".dockerignore", + ".github/actions/utils/docker-buildx/**", + ".github/actions/utils/docker-login/**", + ".github/config/publish.yml", + ".github/workflows/post-merge.yml", + ".github/workflows/publish.yml", + "LICENSE", + "NOTICE", + "about.toml", + "about.hbs", + "core/ai/mcp/Dockerfile", + "scripts/extract-version.sh", + "scripts/ci/third-party-licenses.sh" + ] + }, + { + "id": "rust-bench-dashboard", + "dimensions": { "component": "rust-bench-dashboard" }, + "cargo_roots": [ + { "package": "iggy-bench-dashboard-server", "target": { "name": "iggy-bench-dashboard-server", "kind": "bin" } }, + { "package": "bench-dashboard-frontend", "target": { "name": "bench-dashboard-frontend", "kind": "bin" } } + ], + "external_paths": [ + ".dockerignore", + ".github/actions/utils/docker-buildx/**", + ".github/actions/utils/docker-login/**", + ".github/config/publish.yml", + ".github/workflows/post-merge.yml", + ".github/workflows/publish.yml", + "LICENSE", + "NOTICE", + "about.toml", + "about.hbs", + "core/bench/dashboard/server/Dockerfile", + "core/bench/dashboard/server/docker-entrypoint.sh", + "scripts/extract-version.sh", + "scripts/ci/third-party-licenses.sh" + ] + }, + { + "id": "rust-connectors", + "dimensions": { "component": "rust-connectors" }, + "cargo_roots": [ + { "package": "iggy-connectors", "target": { "name": "iggy-connectors", "kind": "bin" } } + ], + "external_paths": [ + ".dockerignore", + ".github/actions/utils/docker-buildx/**", + ".github/actions/utils/docker-login/**", + ".github/config/publish.yml", + ".github/workflows/post-merge.yml", + ".github/workflows/publish.yml", + "LICENSE", + "NOTICE", + "about.toml", + "about.hbs", + "core/connectors/runtime/Dockerfile", + "scripts/extract-version.sh", + "scripts/ci/third-party-licenses.sh" + ] + }, + { + "id": "web-ui", + "dimensions": { "component": "web-ui" }, + "external_paths": [ + ".dockerignore", + ".github/actions/utils/docker-buildx/**", + ".github/actions/utils/docker-login/**", + ".github/config/publish.yml", + ".github/workflows/post-merge.yml", + ".github/workflows/publish.yml", + "LICENSE", + "NOTICE", + "web/**", + "scripts/extract-version.sh", + "scripts/ci/third-party-licenses.sh", + "scripts/ci/render-node-licenses.mjs" + ] + } + ] +} diff --git a/.github/config/publish.yml b/.github/config/publish.yml index 36376d34c6..1e6c9b0ee3 100644 --- a/.github/config/publish.yml +++ b/.github/config/publish.yml @@ -55,14 +55,6 @@ components: platforms: ["linux/amd64", "linux/arm64"] version_file: "core/server/Cargo.toml" version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' - # :edge refresh gate (scripts/ci/edge-affected-images.sh). Ships server + CLI - # with the web UI embedded at compile time, so web-only changes must rebuild - # it. crates: matched against `cargo rail plan`; paths: extra git pathspecs - # (the dockerfile: above is gated automatically). - gate: - crates: [server, iggy-cli] - paths: [web] - rust-mcp: tag_pattern: "^mcp-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: dockerhub @@ -71,9 +63,6 @@ components: platforms: ["linux/amd64", "linux/arm64"] version_file: "core/ai/mcp/Cargo.toml" version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' - gate: - crates: [iggy-mcp] - rust-bench-dashboard: tag_pattern: "^bench-dashboard-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: dockerhub @@ -82,12 +71,6 @@ components: platforms: ["linux/amd64", "linux/arm64"] version_file: "core/bench/dashboard/server/Cargo.toml" version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' - # Image bundles the server binary plus the built WASM frontend. - gate: - crates: [iggy-bench-dashboard-server, bench-dashboard-frontend] - paths: - - core/bench/dashboard/server/docker-entrypoint.sh - rust-connectors: tag_pattern: "^connectors-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: dockerhub @@ -96,10 +79,6 @@ components: platforms: ["linux/amd64", "linux/arm64"] version_file: "core/connectors/runtime/Cargo.toml" version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"' - # Image ships only the runtime binary; plugin .so files are not bundled. - gate: - crates: [iggy-connectors] - web-ui: tag_pattern: "^web-ui-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: dockerhub @@ -108,14 +87,6 @@ components: platforms: ["linux/amd64", "linux/arm64"] version_file: "web/package.json" version_regex: '"version"\s*:\s*"([^"]+)"' - # Not a cargo crate: path-gated only. The Dockerfile also COPYs two - # license scripts from scripts/ci/ into the build. - gate: - paths: - - web - - scripts/ci/third-party-licenses.sh - - scripts/ci/render-node-licenses.mjs - # ── Other SDKs ───────────────────────────────────────────────────────────── sdk-python: tag_pattern: "^python-sdk-([0-9]+\\.[0-9]+\\.[0-9]+(?:[.-][0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index f45a4b64d5..9d8554a99a 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -60,23 +60,51 @@ jobs: chmod +x /usr/local/bin/yq fi - # cargo-rail computes the affected crate set for the Docker :edge gate. - # Metadata-only `cargo rail plan` (no compile), so it runs on the runner's - # preinstalled cargo (version pinned by rust-toolchain.toml) and skips the - # heavyweight build-cache restore. - - name: Install cargo-rail - uses: taiki-e/install-action@v2.86.3 - with: - tool: cargo-rail + # Planning is an optimization. A failure falls back to refreshing every + # Docker image in the component registry. + - name: Plan affected edge images + id: rail + continue-on-error: true + uses: loadingalias/cargo-rail-action@78ad385a85627484a5b634cf1bc3caa1de872c6a # v8.2.0 - name: Check all components id: check + env: + PLAN_FILE: ${{ steps.rail.outputs.plan-file }} + PLAN_READER: ${{ steps.rail.outputs.plan-reader }} + PLAN_STATUS: ${{ steps.rail.outcome }} run: | - chmod +x scripts/extract-version.sh + set -euo pipefail + + COMPONENTS_JSON=$(yq -o=json -I=0 '.components' .github/config/publish.yml) + ALL_DOCKER_COMPONENTS=$(jq -r \ + '[to_entries[] | select(.value.registry == "dockerhub") | .key] | join(",")' \ + <<< "$COMPONENTS_JSON") + + IMAGE_MATRIX=all + if [[ "$PLAN_STATUS" == "success" ]] && + python3 "$PLAN_READER" verify-checkout "$PLAN_FILE" && + IMAGE_MATRIX=$(python3 "$PLAN_READER" matrix "$PLAN_FILE" edge-images); then + echo "::notice::Using validated Cargo-Rail edge image plan" + else + echo "::warning::Cargo-Rail plan unavailable or invalid; refreshing all edge images" + fi + + if [[ "$IMAGE_MATRIX" == "all" ]]; then + DOCKER_COMPONENTS="$ALL_DOCKER_COMPONENTS" + elif DOCKER_COMPONENTS=$(jq -er --argjson configured "$COMPONENTS_JSON" ' + [.include[].component] as $selected + | if ($selected | all(. as $component | $configured[$component].registry == "dockerhub")) + then $selected | join(",") + else error("Cargo-Rail selected an unknown Docker component") + end + ' <<< "$IMAGE_MATRIX"); then + : + else + echo "::warning::Cargo-Rail edge image matrix is invalid; refreshing all edge images" + DOCKER_COMPONENTS="$ALL_DOCKER_COMPONENTS" + fi - # Refresh :edge only for Docker images whose crate closure or build - # context changed in this push (DAG gate). Fail-open to all images. - DOCKER_COMPONENTS=$(scripts/ci/edge-affected-images.sh "${{ github.event.before }}" "${{ github.sha }}") echo "docker_components=$DOCKER_COMPONENTS" >> "$GITHUB_OUTPUT" echo "Docker components to refresh: ${DOCKER_COMPONENTS:-}" diff --git a/scripts/ci/edge-affected-images.sh b/scripts/ci/edge-affected-images.sh deleted file mode 100755 index b69ba99073..0000000000 --- a/scripts/ci/edge-affected-images.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -# Decide which DockerHub :edge images a master push actually changed, so -# post-merge only refreshes the affected ones instead of all of them. -# -# Each Rust image builds via cargo-chef over the whole workspace (COPY . .), -# so the shipped binary changes only when its crate dependency closure does. -# That closure is exactly what `cargo rail plan` reports, the same DAG that -# scopes test runs in .github/actions/rust/pre-merge. web-ui is not a crate, -# and a Dockerfile-only edit touches no crate source, so each image also -# declares fallback `gate.paths`. -# -# Fail-open: on any uncertainty (force push, workspace-global change, -# cargo-rail failure, unconfigured image) emit the full image list so a needed -# edge refresh is never skipped. -# -# Usage: edge-affected-images.sh -# Output: comma-separated publish.yml component keys (e.g. "rust-server,web-ui") - -BASE="${1:-}" -HEAD="${2:-}" -CONFIG=".github/config/publish.yml" -ZERO="0000000000000000000000000000000000000000" - -if [[ -z "$BASE" || -z "$HEAD" ]]; then - echo "usage: $0 " >&2 - exit 2 -fi - -CFG_JSON="$(yq -o=json -I=0 '.components' "$CONFIG")" -mapfile -t ALL_IMAGES < <(jq -r 'to_entries[] | select(.value.registry == "dockerhub") | .key' <<<"$CFG_JSON") - -emit() { (IFS=,; echo "$*"); } -emit_all() { emit "${ALL_IMAGES[@]}"; } - -# Unusable base: initial push, force-push, or a commit not in history. -if [[ "$BASE" == "$ZERO" ]] || ! git cat-file -e "${BASE}^{commit}" 2>/dev/null; then - echo "::notice::edge-gate: unusable base '$BASE', refreshing all images" >&2 - emit_all - exit 0 -fi - -# Workspace-global edits rebuild every crate. cargo-rail does not flag -# toolchain/lockfile changes (they are not crate sources), so escalate here. -if ! git diff --quiet "$BASE" "$HEAD" -- Cargo.toml Cargo.lock rust-toolchain.toml .cargo; then - echo "::notice::edge-gate: workspace-global change, refreshing all images" >&2 - emit_all - exit 0 -fi - -RAIL_ERR="$(mktemp)" -trap 'rm -f "$RAIL_ERR"' EXIT - -# Capture cargo-rail's exit status explicitly: a nonzero exit means the plan is -# untrustworthy, so fall open. `|| true` would hide the failure and let -# malformed stdout reach the parsing below. -if ! PLAN="$(cargo rail plan --since "$BASE" -f json 2>"$RAIL_ERR")"; then - echo "::warning::edge-gate: cargo-rail exited nonzero, refreshing all images. $(cat "$RAIL_ERR" 2>/dev/null || true)" >&2 - emit_all - exit 0 -fi - -# Parse defensively: malformed output must fall OPEN, never abort under set -e -# (that fails closed and skips the publish). cargo-rail's ExecutionScopeMode is -# one of three snake_case values: -# crates -> a specific affected subset; gate each image on it below. -# empty -> no build/test/bench surface was touched (docs / infra / non-Rust -# files only), so zero crates ship new code. Non-crate build inputs -# (the embedded web UI, Dockerfiles) are still gated per image via -# gate.paths, so fall through with an EMPTY crate set rather than -# refreshing everything. A node/docs/SDK-only push lands here. -# workspace -> a package-scoped surface pins to all crates (or none resolvable); -# rebuild every image. -# An absent or unparsable mode falls open to emit_all. -MODE="$(jq -r '.scope.mode // ""' <<<"$PLAN" 2>/dev/null || true)" -AFFECTED=() -case "$MODE" in - crates) - mapfile -t AFFECTED < <(jq -r '.scope.crates // [] | .[]' <<<"$PLAN") - ;; - empty) - echo "::notice::edge-gate: no crate impact, gating images on paths only" >&2 - ;; - workspace) - echo "::notice::edge-gate: cargo-rail reports full workspace, refreshing all images" >&2 - emit_all - exit 0 - ;; - *) - echo "::warning::edge-gate: unparsable or unexpected cargo-rail mode '${MODE:-}', refreshing all images" >&2 - emit_all - exit 0 - ;; -esac - -declare -A AFFECTED_SET=() -for crate in ${AFFECTED[@]+"${AFFECTED[@]}"}; do - AFFECTED_SET["$crate"]=1 -done - -SELECTED=() -for img in "${ALL_IMAGES[@]}"; do - # No gate block fails open (always refreshed). An incomplete gate fails closed - # (under-publishes): enumerate every embedded/COPYed source. - if [[ "$(jq -r --arg k "$img" '.[$k] | has("gate")' <<<"$CFG_JSON")" != "true" ]]; then - SELECTED+=("$img") - continue - fi - - keep=false - - while IFS= read -r crate; do - if [[ -n "${AFFECTED_SET[$crate]:-}" ]]; then - keep=true - break - fi - done < <(jq -r --arg k "$img" '.[$k].gate.crates // [] | .[]' <<<"$CFG_JSON") - - if [[ "$keep" == false ]]; then - # dockerfile: is the build context's primary input; gate.paths lists any - # extra sources COPYed or embedded beyond the crate closure. - mapfile -t gate_paths < <(jq -r --arg k "$img" \ - '[.[$k].dockerfile] + (.[$k].gate.paths // []) | .[] | select(. != null)' <<<"$CFG_JSON") - if [[ ${#gate_paths[@]} -gt 0 ]] && ! git diff --quiet "$BASE" "$HEAD" -- "${gate_paths[@]}"; then - keep=true - fi - fi - - [[ "$keep" == true ]] && SELECTED+=("$img") -done - -emit ${SELECTED[@]+"${SELECTED[@]}"}