From 9a5309eb8f6057145ed5c11d53f60e9172dddcaf Mon Sep 17 00:00:00 2001 From: Brenton Strickler Date: Tue, 6 Oct 2026 09:52:33 -0400 Subject: [PATCH] ci: add measurement-only workflow for standard hosted runners We have no numbers for how this repository's CI steps behave on GitHub's standard hosted runners: whether the cold cargo release build, the Java suite and the Rust unit tests fit in the disk and memory of ubuntu-24.04 (x86_64, 4 vCPU / 16 GB) and macos-15 (arm64, 3 vCPU / 7 GB), and how long each step takes there. This adds a separate workflow that runs the same steps as ci.yml (setup.sh, make compile, make test, cargo test) on those two labels and reports, per leg, the wall time of each phase, free disk before / lowest during / after, peak memory and swap, the parallelism used against the cores and RAM reported, test counts as surefire and cargo print them, toolchain versions and the size of the build directories. It is measurement only: it runs on workflow_dispatch and on pull requests that touch the workflow file itself, uses no secrets or environments, and is named so it cannot be read as the CI result. ci.yml, the scripts, the Makefile, the pom and the native crate are unchanged. Runs are cold (no caching) so the numbers are the worst case. The timing helper is the one already used for the same measurement in indextables_spark, so results are comparable. Two things differ because this build needs them: the phase log is timestamped by perl rather than a bash read loop, because surefire prints all test output to the console and the loop would throttle the tests being timed; and the sampler also records Rust toolchain memory, since the native build is the heavy part. Parallelism: the release build keeps cargo's default of one job per core, which has been measured to fit on macos-15 for this crate. The dev-profile build behind cargo test has not been measured on 7 GB, so it defaults to 2 jobs on macOS; both can be overridden per run. The Rust tests run even if the Java tests fail so that one run measures both. --- .github/workflows/ci-hosted-measure.yml | 738 ++++++++++++++++++++++++ 1 file changed, 738 insertions(+) create mode 100644 .github/workflows/ci-hosted-measure.yml diff --git a/.github/workflows/ci-hosted-measure.yml b/.github/workflows/ci-hosted-measure.yml new file mode 100644 index 0000000..0556f38 --- /dev/null +++ b/.github/workflows/ci-hosted-measure.yml @@ -0,0 +1,738 @@ +# MEASUREMENT ONLY - this is not the project's CI. +# +# Question it answers: do the steps of this repository's CI (ci.yml: +# ./scripts/setup.sh, make compile, make test, cargo test) fit on GitHub's +# standard hosted runners, on Linux x86_64 and on macOS arm64, and how long +# does each step take there? The real CI result is the `CI` workflow +# (ci.yml), which this file does not replace or modify. +# +# Shape: +# - Same steps as ci.yml, with Java 11 from setup-java in place of the fixed +# JAVA_HOME paths ci.yml uses, on standard hosted runners only. Never use +# a larger-runner label here: those are billed. +# - Cold run on purpose: no dependency caching, so the numbers are the +# worst case. A cached variant can be added later. +# - Each phase is timed and sampled (free disk, memory, swap, Java RSS, Rust +# toolchain RSS) by a small helper that this workflow writes to +# $RUNNER_TEMP; project scripts, the Makefile, the pom and the native +# crate are used unchanged. Results go to the job summary and to a +# `measure-data-*` artifact. +# - Runs on workflow_dispatch, and on pull requests that touch this file +# only, so it never appears as a check on ordinary PRs. +# +# The helper is the one from the same workflow in indextables/indextables_spark +# (samplers, phase runner, table format), so the two repositories' numbers are +# comparable. Differences: the phase log is timestamped by perl instead of a +# bash loop with a `date` fork per line (surefire prints every test's output +# to the console here), a Rust toolchain RSS column is sampled, and the facts +# and summary sections describe this repository's build. +name: "Measurement: hosted-runner fit (not CI)" + +on: + workflow_dispatch: + inputs: + cargo_jobs_linux: + description: "cargo jobs for both cargo builds on ubuntu-24.04 (empty = cargo default, one per core)" + required: false + type: string + default: '' + cargo_jobs_macos: + description: "cargo jobs for both cargo builds on macos-15 (empty = release build: one per core, Rust test build: 2)" + required: false + type: string + default: '' + pull_request: + paths: + - .github/workflows/ci-hosted-measure.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + # Explicit so every run: block gets `bash -eo pipefail`. + shell: bash + +env: + # cargo jobs for the Rust unit-test build (`cargo test`, dev profile with + # full debug info) on macos-15 (3 vCPU / 7 GB) when no input is given. + # + # The release build keeps cargo's default of one job per core on both + # runners: the same `cargo build --release` of this crate (v0.34.4) has been + # measured on macos-15 at 3 jobs, peaking at about 5.5 GB of 7 GB used with + # no swap. The dev-profile test build has no such measurement, and it + # compiles the whole dependency tree a second time with debuginfo, so it + # starts at 2 jobs: fewer rustc processes side by side while dependencies + # compile, at the cost of a longer build. A slower run that completes is + # worth more here than a faster one that exhausts memory. Linux (4 vCPU / + # 16 GB) keeps the cargo default for both builds. + # + # Java tests have no knob to set: the pom runs surefire with forkCount=1 and + # reuseForks=false (one forked test JVM at a time, a new one per test class) + # and no -Xmx, so the JVM default heap (a quarter of RAM) applies. + DEFAULT_CARGO_TEST_JOBS_MACOS: '2' + +jobs: + measure: + name: measure-only (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + # Backstop only. The per-phase step timeouts below are sized so that a + # stuck phase fails its own step first and the summary still gets written. + # Hosted jobs are capped at 360 minutes. + timeout-minutes: ${{ matrix.job-timeout }} + strategy: + fail-fast: false + matrix: + # Standard (free for public repos) labels only. + # ubuntu-24.04: x86_64, 4 vCPU / 16 GB + # macos-15: arm64 (M1), 3 vCPU / 7 GB + os: [ubuntu-24.04, macos-15] + include: + # Linux: the cold release build (opt-level z, thin LTO, one codegen + # unit) has not been timed on a hosted x86_64 runner; 120 minutes is + # several times the expected duration. + - os: ubuntu-24.04 + job-timeout: 345 + setup-timeout: 20 + compile-timeout: 120 + test-timeout: 90 + rust-test-timeout: 90 + # macOS: the release build has been measured at about 17 minutes on + # this runner type; setup includes `brew install protobuf`. + - os: macos-15 + job-timeout: 315 + setup-timeout: 20 + compile-timeout: 90 + test-timeout: 90 + rust-test-timeout: 90 + env: + MATRIX_OS: ${{ matrix.os }} + + steps: + - name: Install measurement helper and start the clock + run: | + mkdir -p "$RUNNER_TEMP/measure" + echo "MEASURE_DIR=$RUNNER_TEMP/measure" >> "$GITHUB_ENV" + echo "MEASURE=$RUNNER_TEMP/measure.sh" >> "$GITHUB_ENV" + cat > "$RUNNER_TEMP/measure.sh" <<'MEASURE_EOF' + #!/usr/bin/env bash + # Measurement helper for .github/workflows/ci-hosted-measure.yml. + # Kept bash 3.2 compatible (macOS system bash). Every value is written to a + # small file under $MEASURE_DIR as soon as it is known, so the summary can + # still be rendered when a phase fails, times out or is cancelled. + set -uo pipefail + + MEASURE_DIR="${MEASURE_DIR:?MEASURE_DIR is not set}" + WORKSPACE="${GITHUB_WORKSPACE:-$PWD}" + OS_NAME="$(uname -s)" + SAMPLE_INTERVAL="${SAMPLE_INTERVAL:-5}" + mkdir -p "$MEASURE_DIR" + + put() { printf '%s\n' "$2" > "$MEASURE_DIR/$1"; } + get() { if [[ -f "$MEASURE_DIR/$1" ]]; then cat "$MEASURE_DIR/$1"; fi; } + + # --- samplers --------------------------------------------------------------- + # Free space (MB) on the volume holding the workspace (same volume as $HOME, + # ~/.cargo and the temp dir on both hosted images). + free_disk_mb() { df -Pk "$WORKSPACE" | awk 'NR==2 {printf "%d", $4/1024}'; } + + # Prints " " for the whole machine. + # Linux: MemTotal - MemAvailable (reclaimable page cache is not counted). + # macOS: active + wired + compressor pages (approximates "Memory Used"). + mem_sample() { + local used swap + case "$OS_NAME" in + Linux) + awk '/^MemTotal:/ {t=$2} /^MemAvailable:/ {a=$2} + /^SwapTotal:/ {st=$2} /^SwapFree:/ {sf=$2} + END {printf "%d %d", (t-a)/1024, (st-sf)/1024}' /proc/meminfo + ;; + Darwin) + used="$(vm_stat | awk ' + /page size of/ {for (i = 1; i <= NF; i++) if ($i == "of") ps = $(i+1)} + /^Pages active:/ {v = $3; sub(/\.$/, "", v); a = v} + /^Pages wired down:/ {v = $4; sub(/\.$/, "", v); w = v} + /^Pages occupied by compressor:/ {v = $5; sub(/\.$/, "", v); c = v} + END {printf "%d", (a + w + c) * ps / 1048576}')" + swap="$(sysctl -n vm.swapusage 2>/dev/null | awk ' + {for (i = 1; i <= NF; i++) if ($i == "used") {v = $(i+2); sub(/M$/, "", v); printf "%d", v}}')" + printf '%s %s' "${used:-0}" "${swap:-0}" + ;; + *) printf '0 0' ;; + esac + } + + # Prints " ". + java_sample() { + ps -eo rss=,comm= 2>/dev/null \ + | awk '$NF ~ /(^|\/)java$/ {s += $1; n++} END {printf "%d %d", s/1024, n}' + } + + # Prints " " for the native build: cargo, rustc, + # the linker and C compiler they start, build scripts, and the Rust test + # binary (tantivy4java-; Linux truncates process names to 15 chars). + rust_sample() { + ps -eo rss=,comm= 2>/dev/null \ + | awk '{name = $NF; sub(/.*\//, "", name)} + name ~ /^(cargo|rustc|rust-lld|lld|ld|ld\.[a-z]+|collect2|cc1|cc1plus|clang|build-script-[a-z-]*|tantivy4java-[0-9a-f]*)$/ {s += $1; n++} + END {printf "%d %d", s/1024, n}' + } + + # Background sampler. Columns: + # epoch free_disk_mb used_mb swap_mb java_rss_mb java_count rust_rss_mb rust_count + # Exits by itself if the helper process that started it goes away. + sampler() { + local out="$MEASURE_DIR/$1.samples" + while kill -0 "$$" 2>/dev/null; do + printf '%s %s %s %s %s\n' "$(date +%s)" "$(free_disk_mb)" "$(mem_sample)" "$(java_sample)" "$(rust_sample)" >> "$out" + sleep "$SAMPLE_INTERVAL" + done + } + + # --- phase runner ----------------------------------------------------------- + # cmd_phase : runs the command unchanged, streams its + # output (stdout and stderr merged) to the step log, keeps an epoch-prefixed + # copy in .log and returns the command's exit status. + cmd_phase() { + local phase="$1" + shift + local log="$MEASURE_DIR/$phase.log" rc=0 spid lpid line i base + # The FIFO lives outside $MEASURE_DIR so it can never end up in the + # uploaded artifact directory if the step is killed. + local fifo="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/measure-$phase.fifo" + : > "$log" + rm -f "$fifo" + mkfifo "$fifo" + put "$phase.cmd" "$*" + put "$phase.disk_before" "$(free_disk_mb)" + put "$phase.start" "$(date +%s)" + sampler "$phase" > /dev/null 2>&1 & + spid=$! + # The logger must not throttle the command being measured: surefire prints + # every test's output to the console. A bash `read` loop manages only a + # few thousand lines per second on a pipe (and a few hundred with a `date` + # fork per line), so perl does the copying where it exists (both hosted + # images). The bash loop is the fallback; its epoch comes from SECONDS. + if [[ -z "${MEASURE_BASH_LOGGER:-}" ]] && command -v perl > /dev/null 2>&1; then + perl -e ' + $| = 1; + open(my $log, ">>", $ARGV[0]) or die "cannot open $ARGV[0]: $!"; + select((select($log), $| = 1)[0]); + while (my $line = ) { + print $line; + chomp $line; + print $log time(), " ", $line, "\n"; + }' "$log" < "$fifo" & + else + base=$(($(date +%s) - SECONDS)) + while IFS= read -r line || [[ -n "$line" ]]; do + printf '%s\n' "$line" + printf '%s %s\n' "$((base + SECONDS))" "$line" >> "$log" + done < "$fifo" & + fi + lpid=$! + "$@" > "$fifo" 2>&1 + rc=$? + # Timing stops when the command exits, as it would for a plain step. + put "$phase.end" "$(date +%s)" + put "$phase.rc" "$rc" + put "$phase.disk_after" "$(free_disk_mb)" + kill "$spid" 2> /dev/null + wait "$spid" 2> /dev/null + # Let the logger drain, but do not wait for EOF: a leftover child of the + # command can hold the pipe open, which would hang the step. + i=0 + while kill -0 "$lpid" 2> /dev/null && [[ "$i" -lt 25 ]]; do + sleep 0.2 + i=$((i + 1)) + done + kill "$lpid" 2> /dev/null + wait "$lpid" 2> /dev/null + rm -f "$fifo" + return "$rc" + } + + # --- runner facts and parallelism decision ---------------------------------- + # Selected settings of one cargo profile, as written in native/Cargo.toml. + profile_line() { + awk -v want="[profile.$1]" ' + $0 == want {f = 1; next} + /^\[/ {f = 0} + f && /^(opt-level|lto|codegen-units|debug|incremental) *=/ { + sub(/ *#.*$/, ""); gsub(/ +/, " "); printf "%s%s", sep, $0; sep = "; " + }' native/Cargo.toml 2>/dev/null + } + + cmd_facts() { + local cores ram_mb swap_mb cpu_model jobs_input cargo_default + local jobs_build jobs_test src_build src_test want + case "$OS_NAME" in + Linux) + cores="$(nproc)" + ram_mb="$(awk '/^MemTotal:/ {printf "%d", $2/1024}' /proc/meminfo)" + swap_mb="$(awk '/^SwapTotal:/ {printf "%d", $2/1024}' /proc/meminfo)" + cpu_model="$(awk -F': *' '/^model name/ {print $2; exit}' /proc/cpuinfo)" + jobs_input="${CARGO_JOBS_INPUT_LINUX:-}" + ;; + Darwin) + cores="$(sysctl -n hw.ncpu)" + ram_mb="$(($(sysctl -n hw.memsize) / 1048576))" + swap_mb="$(sysctl -n vm.swapusage 2>/dev/null | awk ' + {for (i = 1; i <= NF; i++) if ($i == "total") {v = $(i+2); sub(/M$/, "", v); printf "%d", v}}')" + cpu_model="$(sysctl -n machdep.cpu.brand_string 2>/dev/null || echo unknown)" + jobs_input="${CARGO_JOBS_INPUT_MACOS:-}" + ;; + *) + echo "::error::Unsupported OS for this measurement: $OS_NAME" + return 1 + ;; + esac + + # cargo's default for build.jobs is the number of logical CPUs. The value + # is always passed explicitly (CARGO_BUILD_JOBS) so the log shows it. + cargo_default="$cores" + jobs_build="$cargo_default" + jobs_test="$cargo_default" + src_build="same as the cargo default (one job per core)" + src_test="same as the cargo default (one job per core)" + if [[ -n "$jobs_input" ]]; then + if ! [[ "$jobs_input" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::cargo jobs override must be a positive integer, got '$jobs_input'" + return 1 + fi + jobs_build="$jobs_input" + jobs_test="$jobs_input" + src_build="workflow_dispatch input" + src_test="workflow_dispatch input" + elif [[ "$OS_NAME" == "Darwin" ]]; then + want="${DEFAULT_CARGO_TEST_JOBS_MACOS:-2}" + if [[ "$want" =~ ^[1-9][0-9]*$ && "$want" -lt "$cores" ]]; then + jobs_test="$want" + src_test="workflow default for macOS, deliberately below the cargo default (rationale below)" + fi + fi + + # Surefire settings and heap cap, read from the pom rather than assumed. + local fork_count reuse_forks xmx jvm_heap_mb + fork_count="$(sed -n 's/.*\(.*\)<\/forkCount>.*/\1/p' pom.xml | head -1)" + reuse_forks="$(sed -n 's/.*\(.*\)<\/reuseForks>.*/\1/p' pom.xml | head -1)" + xmx="$(grep -o -- '-Xmx[0-9]*[gGmM]' pom.xml | sort -u | tr '\n' ' ' | sed 's/ *$//')" + # What the JVM itself picks as max heap on this machine when no -Xmx is given. + jvm_heap_mb="$(java -XX:+PrintFlagsFinal -version 2>/dev/null \ + | awk '$2 == "MaxHeapSize" {printf "%d", $4 / 1048576}')" + + put facts.uname "$(uname -sm)" + put facts.image "${ImageOS:-unknown} ${ImageVersion:-}" + put facts.cores "$cores" + put facts.cpu_model "${cpu_model:-unknown}" + put facts.ram_mb "$ram_mb" + put facts.swap_mb "${swap_mb:-0}" + put facts.disk_free_mb "$(free_disk_mb)" + put facts.commit "$(git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" + put facts.cargo_default_jobs "$cargo_default" + put facts.cargo_jobs_build "$jobs_build" + put facts.cargo_jobs_build_source "$src_build" + put facts.cargo_jobs_test "$jobs_test" + put facts.cargo_jobs_test_source "$src_test" + put facts.fork_count "${fork_count:-not set in pom.xml}" + put facts.reuse_forks "${reuse_forks:-not set in pom.xml}" + put facts.xmx "${xmx:-none}" + put facts.jvm_heap_mb "$jvm_heap_mb" + put facts.profile_release "$(profile_line release)" + put facts.profile_dev "$(profile_line dev)" + put facts.java "$(java -version 2>&1 | head -1)" + put facts.maven "$(mvn --version 2>/dev/null | head -1)" + put facts.rustc "$(rustc --version 2>/dev/null || echo 'not preinstalled')" + put facts.cargo "$(cargo --version 2>/dev/null || echo 'not preinstalled')" + put facts.protoc "$(protoc --version 2>/dev/null || echo 'not preinstalled')" + + if [[ -n "${GITHUB_ENV:-}" ]]; then + echo "CARGO_JOBS_BUILD=$jobs_build" >> "$GITHUB_ENV" + echo "CARGO_JOBS_TEST=$jobs_test" >> "$GITHUB_ENV" + fi + + echo "cores=$cores ram_mb=$ram_mb swap_mb=${swap_mb:-0} free_disk_mb=$(free_disk_mb)" + echo "cargo default jobs=$cargo_default" + echo "release build: CARGO_BUILD_JOBS=$jobs_build ($src_build)" + echo "Rust test build: CARGO_BUILD_JOBS=$jobs_test ($src_test)" + echo "surefire forkCount=${fork_count:-?} reuseForks=${reuse_forks:-?} -Xmx in pom: ${xmx:-none}; JVM default max heap: ${jvm_heap_mb:-?} MB" + echo "--- df -h" + df -h + } + + # --- summary ---------------------------------------------------------------- + fmt_dur() { + if [[ -z "${1:-}" ]]; then printf 'n/a'; else printf '%dm %02ds' "$(($1 / 60))" "$(($1 % 60))"; fi + } + fmt_gb() { + if [[ -z "${1:-}" ]]; then printf 'n/a'; else awk -v m="$1" 'BEGIN {printf "%.1f GB", m / 1024}'; fi + } + # col_stat + col_stat() { + if [[ ! -s "$1" ]]; then return 0; fi + awk -v c="$2" -v op="$3" ' + NR == 1 {v = $c} + op == "min" && $c < v {v = $c} + op == "max" && $c > v {v = $c} + END {if (NR > 0) printf "%d", v}' "$1" + } + # First epoch at which a line matching was logged in . + log_epoch() { + if [[ ! -f "$1" ]]; then return 0; fi + grep -E -m1 -- "$2" "$1" | awk '{print $1}' + } + # span : duration between two epochs, n/a if either is missing. + span() { + if [[ -n "${1:-}" && -n "${2:-}" && "$2" -ge "$1" ]]; then fmt_dur "$(($2 - $1))"; else printf 'n/a'; fi + } + # Size (MB) of a path, empty if it does not exist. + dir_mb() { + if [[ -e "$1" ]]; then du -sk "$1" 2>/dev/null | awk '{printf "%d", $1/1024}'; fi + } + # cargo's own "Finished ... in