From 66623bc1c59e50a7d7055b07a4f1a6349bb2f104 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Sun, 17 May 2026 12:00:49 +0300 Subject: [PATCH 1/4] Add aggressive IB cache experiment workflow Co-authored-by: Cursor --- .../ib-aggressive-cache-experiment.yml | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 .github/workflows/ib-aggressive-cache-experiment.yml diff --git a/.github/workflows/ib-aggressive-cache-experiment.yml b/.github/workflows/ib-aggressive-cache-experiment.yml new file mode 100644 index 000000000..707d2dcff --- /dev/null +++ b/.github/workflows/ib-aggressive-cache-experiment.yml @@ -0,0 +1,202 @@ +name: ib aggressive cache experiment + +on: + workflow_dispatch: {} + push: + branches: + - ci/ib-python-orchestrator-wrap + +permissions: {} + +env: + COLUMNS: 150 + UV_PYTHON: '3.14' + UV_FROZEN: '1' + DEBUG: 'napi:*' + LLVM_COV_IGNORE_FILENAME_REGEX: '(tests/|test_cases/|/tests\.rs$$)' + +jobs: + aggressive-cache: + runs-on: incredibuild-runner + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + with: + toolchain: stable + components: llvm-tools + + - uses: taiki-e/install-action@1ed3272338f573e042a2e6bca3893aa19f43b47a # v2.71.3 + with: + tool: cargo-llvm-cov + + - name: set up python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.14' + + - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 + with: + enable-cache: false + + - name: Run safe vs aggressive IB cache experiment + shell: bash + run: | + set -euo pipefail + + rm -f .cargo/config.toml + + as_root() { + if command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + "$@" + fi + } + + as_root mkdir -p /etc/incredibuild/log + as_root chmod 1777 /etc/incredibuild/log + + cat > "$RUNNER_TEMP/ib_profile_safe_orchestrators.xml" <<'XML' + + + + + + + + + + + + + + + + + XML + + cat > "$RUNNER_TEMP/ib_profile_aggressive_cache.xml" <<'XML' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + XML + + common_ib_args() { + local profile="$1" + local log="$2" + echo \ + --standalone \ + --build-cache-local-shared \ + --build-cache-force \ + --build-cache-basedir="$PWD" \ + --build-cache-local-logfile="$log" \ + --build-cache-report-all-miss \ + --no-monitor \ + --profile="$profile" + } + + run_ib() { + local phase="$1" + local label="$2" + shift 2 + local start end + echo "::group::$phase $label" + start=$(date +%s) + /usr/bin/ib_console $(common_ib_args "$IB_PROFILE_UNDER_TEST" "$IB_CACHE_LOG") "$@" + end=$(date +%s) + echo "::endgroup::" + echo "$phase,$label,$((end - start))" | tee -a "$RUNNER_TEMP/ib_aggressive_timings.csv" + } + + summarize_log() { + local phase="$1" + local hits=0 misses=0 + if [ -f "$IB_CACHE_LOG" ]; then + hits=$(grep -c -E '^HIT[[:space:]]' "$IB_CACHE_LOG" 2>/dev/null || true) + misses=$(grep -c -E '^MISS[[:space:]]' "$IB_CACHE_LOG" 2>/dev/null || true) + fi + { + echo "" + echo "### $phase IB cache log" + echo "" + echo "| metric | value |" + echo "|---|---:|" + echo "| HIT | $hits |" + echo "| MISS | $misses |" + echo "| log | \`$IB_CACHE_LOG\` |" + } >> "$GITHUB_STEP_SUMMARY" + } + + run_phase() { + local phase="$1" + local profile="$2" + export IB_PROFILE_UNDER_TEST="$profile" + export IB_CACHE_LOG="/etc/incredibuild/log/ib_aggressive_${phase}_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}.log" + rm -f "$IB_CACHE_LOG" + rm -rf target .venv + + run_ib "$phase" "uv sync" uv sync --all-packages --only-dev + + eval "$(cargo llvm-cov show-env --export-prefix)" + cargo llvm-cov clean --workspace + + run_ib "$phase" "maturin develop" uv run maturin develop --uv -m crates/monty-python/Cargo.toml + run_ib "$phase" "pytest" uv run --package pydantic-monty --only-dev pytest crates/monty-python/tests + run_ib "$phase" "make dev-bench" make dev-bench + + summarize_log "$phase" + } + + echo "phase,step,seconds" > "$RUNNER_TEMP/ib_aggressive_timings.csv" + + echo "::group::Clear local IB build cache" + as_root rm -rf /etc/incredibuild/cache/build_cache/shared/* /etc/incredibuild/cache/build_cache/builds/* 2>/dev/null || true + echo "::endgroup::" + + run_phase "safe-orchestrators" "$RUNNER_TEMP/ib_profile_safe_orchestrators.xml" + + echo "::group::Clear local IB build cache before aggressive cold" + as_root rm -rf /etc/incredibuild/cache/build_cache/shared/* /etc/incredibuild/cache/build_cache/builds/* 2>/dev/null || true + echo "::endgroup::" + + run_phase "aggressive-cold" "$RUNNER_TEMP/ib_profile_aggressive_cache.xml" + run_phase "aggressive-hot" "$RUNNER_TEMP/ib_profile_aggressive_cache.xml" + + { + echo "" + echo "### Safe vs aggressive process caching" + echo "" + echo "| phase | step | seconds |" + echo "|---|---|---:|" + awk -F, 'NR > 1 { printf "| %s | %s | %s |\n", $1, $2, $3 }' "$RUNNER_TEMP/ib_aggressive_timings.csv" + awk -F, 'NR > 1 { sum[$1] += $3 } END { for (phase in sum) printf "| %s | total | %s |\n", phase, sum[phase] }' "$RUNNER_TEMP/ib_aggressive_timings.csv" + } >> "$GITHUB_STEP_SUMMARY" From fe7810ee7cc449ebca37e31aa301fd20ed94c225 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Sun, 17 May 2026 12:03:27 +0300 Subject: [PATCH 2/4] Compare rust-cache against IB rust cache Co-authored-by: Cursor --- .../ib-aggressive-cache-experiment.yml | 279 ++++++++++++++++++ 1 file changed, 279 insertions(+) diff --git a/.github/workflows/ib-aggressive-cache-experiment.yml b/.github/workflows/ib-aggressive-cache-experiment.yml index 707d2dcff..2d3d8d39b 100644 --- a/.github/workflows/ib-aggressive-cache-experiment.yml +++ b/.github/workflows/ib-aggressive-cache-experiment.yml @@ -200,3 +200,282 @@ jobs: awk -F, 'NR > 1 { printf "| %s | %s | %s |\n", $1, $2, $3 }' "$RUNNER_TEMP/ib_aggressive_timings.csv" awk -F, 'NR > 1 { sum[$1] += $3 } END { for (phase in sum) printf "| %s | total | %s |\n", phase, sum[phase] }' "$RUNNER_TEMP/ib_aggressive_timings.csv" } >> "$GITHUB_STEP_SUMMARY" + + rust-cache-prime: + runs-on: incredibuild-runner + + env: + __IB_CARGO_WRAPPED: '1' + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + with: + toolchain: stable + components: llvm-tools + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: ib-rust-cache-comparison + lookup-only: false + cache-on-failure: true + + - uses: taiki-e/install-action@1ed3272338f573e042a2e6bca3893aa19f43b47a # v2.71.3 + with: + tool: cargo-llvm-cov + + - name: set up python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.14' + + - name: Prime GitHub rust-cache without IB wrapping + shell: bash + run: | + set -euo pipefail + rm -f .cargo/config.toml + csv="$RUNNER_TEMP/rust_cache_prime_timings.csv" + echo "phase,step,seconds" > "$csv" + + run_timed() { + local label="$1" + shift + local start end + echo "::group::rust-cache-prime $label" + start=$(date +%s) + "$@" + end=$(date +%s) + echo "::endgroup::" + echo "rust-cache-prime,$label,$((end - start))" | tee -a "$csv" + } + + run_timed "clean" cargo llvm-cov clean --workspace + run_timed "monty no features" cargo llvm-cov --no-report -p monty + run_timed "datatest no features" cargo llvm-cov run --no-report -p monty-datatest + run_timed "monty memory-model-checks" cargo llvm-cov --no-report -p monty --features memory-model-checks + run_timed "datatest memory-model-checks" cargo llvm-cov run --no-report -p monty-datatest --features memory-model-checks + run_timed "monty ref-count-return" cargo llvm-cov --no-report -p monty --features ref-count-return + run_timed "datatest ref-count-return" cargo llvm-cov run --no-report -p monty-datatest --features ref-count-return + run_timed "type-checking" cargo llvm-cov --no-report -p monty_type_checking -p monty_typeshed + run_timed "report text" cargo llvm-cov report --ignore-filename-regex "$LLVM_COV_IGNORE_FILENAME_REGEX" + run_timed "report codecov" cargo llvm-cov report --codecov --output-path=rust-coverage.json --ignore-filename-regex "$LLVM_COV_IGNORE_FILENAME_REGEX" + + { + echo "### GitHub rust-cache prime, IB cargo shim bypassed" + echo "" + echo "| phase | step | seconds |" + echo "|---|---|---:|" + awk -F, 'NR > 1 { printf "| %s | %s | %s |\n", $1, $2, $3 }' "$csv" + awk -F, 'NR > 1 { sum[$1] += $3 } END { for (phase in sum) printf "| %s | total | %s |\n", phase, sum[phase] }' "$csv" + } >> "$GITHUB_STEP_SUMMARY" + + rust-cache-warm: + runs-on: incredibuild-runner + needs: rust-cache-prime + + env: + __IB_CARGO_WRAPPED: '1' + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + with: + toolchain: stable + components: llvm-tools + + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + shared-key: ib-rust-cache-comparison + lookup-only: false + cache-on-failure: true + + - uses: taiki-e/install-action@1ed3272338f573e042a2e6bca3893aa19f43b47a # v2.71.3 + with: + tool: cargo-llvm-cov + + - name: set up python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.14' + + - name: Measure warm GitHub rust-cache without IB wrapping + shell: bash + run: | + set -euo pipefail + rm -f .cargo/config.toml + csv="$RUNNER_TEMP/rust_cache_warm_timings.csv" + echo "phase,step,seconds" > "$csv" + + run_timed() { + local label="$1" + shift + local start end + echo "::group::rust-cache-warm $label" + start=$(date +%s) + "$@" + end=$(date +%s) + echo "::endgroup::" + echo "rust-cache-warm,$label,$((end - start))" | tee -a "$csv" + } + + run_timed "clean" cargo llvm-cov clean --workspace + run_timed "monty no features" cargo llvm-cov --no-report -p monty + run_timed "datatest no features" cargo llvm-cov run --no-report -p monty-datatest + run_timed "monty memory-model-checks" cargo llvm-cov --no-report -p monty --features memory-model-checks + run_timed "datatest memory-model-checks" cargo llvm-cov run --no-report -p monty-datatest --features memory-model-checks + run_timed "monty ref-count-return" cargo llvm-cov --no-report -p monty --features ref-count-return + run_timed "datatest ref-count-return" cargo llvm-cov run --no-report -p monty-datatest --features ref-count-return + run_timed "type-checking" cargo llvm-cov --no-report -p monty_type_checking -p monty_typeshed + run_timed "report text" cargo llvm-cov report --ignore-filename-regex "$LLVM_COV_IGNORE_FILENAME_REGEX" + run_timed "report codecov" cargo llvm-cov report --codecov --output-path=rust-coverage.json --ignore-filename-regex "$LLVM_COV_IGNORE_FILENAME_REGEX" + + { + echo "### GitHub rust-cache warm, IB cargo shim bypassed" + echo "" + echo "| phase | step | seconds |" + echo "|---|---|---:|" + awk -F, 'NR > 1 { printf "| %s | %s | %s |\n", $1, $2, $3 }' "$csv" + awk -F, 'NR > 1 { sum[$1] += $3 } END { for (phase in sum) printf "| %s | total | %s |\n", phase, sum[phase] }' "$csv" + } >> "$GITHUB_STEP_SUMMARY" + + ib-rust-cache-only: + runs-on: incredibuild-runner + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + with: + toolchain: stable + components: llvm-tools + + - uses: taiki-e/install-action@1ed3272338f573e042a2e6bca3893aa19f43b47a # v2.71.3 + with: + tool: cargo-llvm-cov + + - name: set up python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.14' + + - name: Measure IB rustc cache without GitHub rust-cache + shell: bash + run: | + set -euo pipefail + rm -f .cargo/config.toml + + as_root() { + if command -v sudo >/dev/null 2>&1; then + sudo "$@" + else + "$@" + fi + } + + as_root mkdir -p /etc/incredibuild/log + as_root chmod 1777 /etc/incredibuild/log + + cat > "$RUNNER_TEMP/ib_profile_rust_only.xml" <<'XML' + + + + + + + + + + + XML + + common_ib_args() { + local log="$1" + echo \ + --standalone \ + --build-cache-local-shared \ + --build-cache-force \ + --build-cache-basedir="$PWD" \ + --build-cache-local-logfile="$log" \ + --build-cache-report-all-miss \ + --no-monitor \ + --profile="$RUNNER_TEMP/ib_profile_rust_only.xml" + } + + run_ib_timed() { + local phase="$1" + local label="$2" + shift 2 + local start end + echo "::group::$phase $label" + start=$(date +%s) + __IB_CARGO_WRAPPED=1 /usr/bin/ib_console $(common_ib_args "$IB_CACHE_LOG") "$@" + end=$(date +%s) + echo "::endgroup::" + echo "$phase,$label,$((end - start))" | tee -a "$RUNNER_TEMP/ib_rust_only_timings.csv" + } + + summarize_log() { + local phase="$1" + local hits=0 misses=0 + if [ -f "$IB_CACHE_LOG" ]; then + hits=$(grep -c -E '^HIT[[:space:]]' "$IB_CACHE_LOG" 2>/dev/null || true) + misses=$(grep -c -E '^MISS[[:space:]]' "$IB_CACHE_LOG" 2>/dev/null || true) + fi + { + echo "" + echo "### $phase IB rustc cache log" + echo "" + echo "| metric | value |" + echo "|---|---:|" + echo "| HIT | $hits |" + echo "| MISS | $misses |" + echo "| log | \`$IB_CACHE_LOG\` |" + } >> "$GITHUB_STEP_SUMMARY" + } + + run_phase() { + local phase="$1" + export IB_CACHE_LOG="/etc/incredibuild/log/ib_rust_only_${phase}_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}.log" + rm -f "$IB_CACHE_LOG" + rm -rf target + + cargo llvm-cov clean --workspace + run_ib_timed "$phase" "monty no features" cargo llvm-cov --no-report -p monty + run_ib_timed "$phase" "datatest no features" cargo llvm-cov run --no-report -p monty-datatest + run_ib_timed "$phase" "monty memory-model-checks" cargo llvm-cov --no-report -p monty --features memory-model-checks + run_ib_timed "$phase" "datatest memory-model-checks" cargo llvm-cov run --no-report -p monty-datatest --features memory-model-checks + run_ib_timed "$phase" "monty ref-count-return" cargo llvm-cov --no-report -p monty --features ref-count-return + run_ib_timed "$phase" "datatest ref-count-return" cargo llvm-cov run --no-report -p monty-datatest --features ref-count-return + run_ib_timed "$phase" "type-checking" cargo llvm-cov --no-report -p monty_type_checking -p monty_typeshed + run_ib_timed "$phase" "report text" cargo llvm-cov report --ignore-filename-regex "$LLVM_COV_IGNORE_FILENAME_REGEX" + run_ib_timed "$phase" "report codecov" cargo llvm-cov report --codecov --output-path=rust-coverage.json --ignore-filename-regex "$LLVM_COV_IGNORE_FILENAME_REGEX" + + summarize_log "$phase" + } + + echo "phase,step,seconds" > "$RUNNER_TEMP/ib_rust_only_timings.csv" + + echo "::group::Clear local IB build cache for cold phase" + as_root rm -rf /etc/incredibuild/cache/build_cache/shared/* /etc/incredibuild/cache/build_cache/builds/* 2>/dev/null || true + echo "::endgroup::" + + run_phase "ib-rust-cold" + run_phase "ib-rust-hot" + + { + echo "" + echo "### IB rustc cache only, no GitHub rust-cache" + echo "" + echo "| phase | step | seconds |" + echo "|---|---|---:|" + awk -F, 'NR > 1 { printf "| %s | %s | %s |\n", $1, $2, $3 }' "$RUNNER_TEMP/ib_rust_only_timings.csv" + awk -F, 'NR > 1 { sum[$1] += $3 } END { for (phase in sum) printf "| %s | total | %s |\n", phase, sum[phase] }' "$RUNNER_TEMP/ib_rust_only_timings.csv" + } >> "$GITHUB_STEP_SUMMARY" From c333981341663026cbccb7150a8e77b9fdf61f08 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Sun, 17 May 2026 12:09:31 +0300 Subject: [PATCH 3/4] Stabilize IB cache experiment workflow Co-authored-by: Cursor --- .../workflows/ib-aggressive-cache-experiment.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ib-aggressive-cache-experiment.yml b/.github/workflows/ib-aggressive-cache-experiment.yml index 2d3d8d39b..e819befb0 100644 --- a/.github/workflows/ib-aggressive-cache-experiment.yml +++ b/.github/workflows/ib-aggressive-cache-experiment.yml @@ -8,6 +8,10 @@ on: permissions: {} +concurrency: + group: ib-aggressive-cache-experiment + cancel-in-progress: true + env: COLUMNS: 150 UV_PYTHON: '3.14' @@ -47,6 +51,8 @@ jobs: run: | set -euo pipefail + ulimit -S -n 10000 || true + rm -f .cargo/config.toml as_root() { @@ -131,7 +137,7 @@ jobs: local start end echo "::group::$phase $label" start=$(date +%s) - /usr/bin/ib_console $(common_ib_args "$IB_PROFILE_UNDER_TEST" "$IB_CACHE_LOG") "$@" + __IB_CARGO_WRAPPED=1 /usr/bin/ib_console $(common_ib_args "$IB_PROFILE_UNDER_TEST" "$IB_CACHE_LOG") "$@" end=$(date +%s) echo "::endgroup::" echo "$phase,$label,$((end - start))" | tee -a "$RUNNER_TEMP/ib_aggressive_timings.csv" @@ -203,6 +209,7 @@ jobs: rust-cache-prime: runs-on: incredibuild-runner + needs: aggressive-cache env: __IB_CARGO_WRAPPED: '1' @@ -236,6 +243,7 @@ jobs: shell: bash run: | set -euo pipefail + ulimit -S -n 10000 || true rm -f .cargo/config.toml csv="$RUNNER_TEMP/rust_cache_prime_timings.csv" echo "phase,step,seconds" > "$csv" @@ -308,6 +316,7 @@ jobs: shell: bash run: | set -euo pipefail + ulimit -S -n 10000 || true rm -f .cargo/config.toml csv="$RUNNER_TEMP/rust_cache_warm_timings.csv" echo "phase,step,seconds" > "$csv" @@ -346,6 +355,7 @@ jobs: ib-rust-cache-only: runs-on: incredibuild-runner + needs: rust-cache-warm steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -370,6 +380,7 @@ jobs: shell: bash run: | set -euo pipefail + ulimit -S -n 10000 || true rm -f .cargo/config.toml as_root() { From 769fc264c25d8af59f66df6b9acce269497a7b31 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Sun, 17 May 2026 12:14:43 +0300 Subject: [PATCH 4/4] Exclude rustc metadata probes from IB cache experiment Co-authored-by: Cursor --- .github/workflows/ib-aggressive-cache-experiment.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ib-aggressive-cache-experiment.yml b/.github/workflows/ib-aggressive-cache-experiment.yml index e819befb0..5d1fa7f97 100644 --- a/.github/workflows/ib-aggressive-cache-experiment.yml +++ b/.github/workflows/ib-aggressive-cache-experiment.yml @@ -78,7 +78,7 @@ jobs: - + @@ -109,7 +109,7 @@ jobs: - + @@ -210,6 +210,7 @@ jobs: rust-cache-prime: runs-on: incredibuild-runner needs: aggressive-cache + if: always() env: __IB_CARGO_WRAPPED: '1' @@ -283,6 +284,7 @@ jobs: rust-cache-warm: runs-on: incredibuild-runner needs: rust-cache-prime + if: always() env: __IB_CARGO_WRAPPED: '1' @@ -356,6 +358,7 @@ jobs: ib-rust-cache-only: runs-on: incredibuild-runner needs: rust-cache-warm + if: always() steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -400,7 +403,7 @@ jobs: - +