diff --git a/.github/workflows/ci-hosted-measure.yml b/.github/workflows/ci-hosted-measure.yml new file mode 100644 index 0000000..0556f38 --- /dev/null +++ b/.github/workflows/ci-hosted-measure.yml @@ -0,0 +1,738 @@ +# MEASUREMENT ONLY - this is not the project's CI. +# +# Question it answers: do the steps of this repository's CI (ci.yml: +# ./scripts/setup.sh, make compile, make test, cargo test) fit on GitHub's +# standard hosted runners, on Linux x86_64 and on macOS arm64, and how long +# does each step take there? The real CI result is the `CI` workflow +# (ci.yml), which this file does not replace or modify. +# +# Shape: +# - Same steps as ci.yml, with Java 11 from setup-java in place of the fixed +# JAVA_HOME paths ci.yml uses, on standard hosted runners only. Never use +# a larger-runner label here: those are billed. +# - Cold run on purpose: no dependency caching, so the numbers are the +# worst case. A cached variant can be added later. +# - Each phase is timed and sampled (free disk, memory, swap, Java RSS, Rust +# toolchain RSS) by a small helper that this workflow writes to +# $RUNNER_TEMP; project scripts, the Makefile, the pom and the native +# crate are used unchanged. Results go to the job summary and to a +# `measure-data-*` artifact. +# - Runs on workflow_dispatch, and on pull requests that touch this file +# only, so it never appears as a check on ordinary PRs. +# +# The helper is the one from the same workflow in indextables/indextables_spark +# (samplers, phase runner, table format), so the two repositories' numbers are +# comparable. Differences: the phase log is timestamped by perl instead of a +# bash loop with a `date` fork per line (surefire prints every test's output +# to the console here), a Rust toolchain RSS column is sampled, and the facts +# and summary sections describe this repository's build. +name: "Measurement: hosted-runner fit (not CI)" + +on: + workflow_dispatch: + inputs: + cargo_jobs_linux: + description: "cargo jobs for both cargo builds on ubuntu-24.04 (empty = cargo default, one per core)" + required: false + type: string + default: '' + cargo_jobs_macos: + description: "cargo jobs for both cargo builds on macos-15 (empty = release build: one per core, Rust test build: 2)" + required: false + type: string + default: '' + pull_request: + paths: + - .github/workflows/ci-hosted-measure.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + # Explicit so every run: block gets `bash -eo pipefail`. + shell: bash + +env: + # cargo jobs for the Rust unit-test build (`cargo test`, dev profile with + # full debug info) on macos-15 (3 vCPU / 7 GB) when no input is given. + # + # The release build keeps cargo's default of one job per core on both + # runners: the same `cargo build --release` of this crate (v0.34.4) has been + # measured on macos-15 at 3 jobs, peaking at about 5.5 GB of 7 GB used with + # no swap. The dev-profile test build has no such measurement, and it + # compiles the whole dependency tree a second time with debuginfo, so it + # starts at 2 jobs: fewer rustc processes side by side while dependencies + # compile, at the cost of a longer build. A slower run that completes is + # worth more here than a faster one that exhausts memory. Linux (4 vCPU / + # 16 GB) keeps the cargo default for both builds. + # + # Java tests have no knob to set: the pom runs surefire with forkCount=1 and + # reuseForks=false (one forked test JVM at a time, a new one per test class) + # and no -Xmx, so the JVM default heap (a quarter of RAM) applies. + DEFAULT_CARGO_TEST_JOBS_MACOS: '2' + +jobs: + measure: + name: measure-only (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + # Backstop only. The per-phase step timeouts below are sized so that a + # stuck phase fails its own step first and the summary still gets written. + # Hosted jobs are capped at 360 minutes. + timeout-minutes: ${{ matrix.job-timeout }} + strategy: + fail-fast: false + matrix: + # Standard (free for public repos) labels only. + # ubuntu-24.04: x86_64, 4 vCPU / 16 GB + # macos-15: arm64 (M1), 3 vCPU / 7 GB + os: [ubuntu-24.04, macos-15] + include: + # Linux: the cold release build (opt-level z, thin LTO, one codegen + # unit) has not been timed on a hosted x86_64 runner; 120 minutes is + # several times the expected duration. + - os: ubuntu-24.04 + job-timeout: 345 + setup-timeout: 20 + compile-timeout: 120 + test-timeout: 90 + rust-test-timeout: 90 + # macOS: the release build has been measured at about 17 minutes on + # this runner type; setup includes `brew install protobuf`. + - os: macos-15 + job-timeout: 315 + setup-timeout: 20 + compile-timeout: 90 + test-timeout: 90 + rust-test-timeout: 90 + env: + MATRIX_OS: ${{ matrix.os }} + + steps: + - name: Install measurement helper and start the clock + run: | + mkdir -p "$RUNNER_TEMP/measure" + echo "MEASURE_DIR=$RUNNER_TEMP/measure" >> "$GITHUB_ENV" + echo "MEASURE=$RUNNER_TEMP/measure.sh" >> "$GITHUB_ENV" + cat > "$RUNNER_TEMP/measure.sh" <<'MEASURE_EOF' + #!/usr/bin/env bash + # Measurement helper for .github/workflows/ci-hosted-measure.yml. + # Kept bash 3.2 compatible (macOS system bash). Every value is written to a + # small file under $MEASURE_DIR as soon as it is known, so the summary can + # still be rendered when a phase fails, times out or is cancelled. + set -uo pipefail + + MEASURE_DIR="${MEASURE_DIR:?MEASURE_DIR is not set}" + WORKSPACE="${GITHUB_WORKSPACE:-$PWD}" + OS_NAME="$(uname -s)" + SAMPLE_INTERVAL="${SAMPLE_INTERVAL:-5}" + mkdir -p "$MEASURE_DIR" + + put() { printf '%s\n' "$2" > "$MEASURE_DIR/$1"; } + get() { if [[ -f "$MEASURE_DIR/$1" ]]; then cat "$MEASURE_DIR/$1"; fi; } + + # --- samplers --------------------------------------------------------------- + # Free space (MB) on the volume holding the workspace (same volume as $HOME, + # ~/.cargo and the temp dir on both hosted images). + free_disk_mb() { df -Pk "$WORKSPACE" | awk 'NR==2 {printf "%d", $4/1024}'; } + + # Prints " " for the whole machine. + # Linux: MemTotal - MemAvailable (reclaimable page cache is not counted). + # macOS: active + wired + compressor pages (approximates "Memory Used"). + mem_sample() { + local used swap + case "$OS_NAME" in + Linux) + awk '/^MemTotal:/ {t=$2} /^MemAvailable:/ {a=$2} + /^SwapTotal:/ {st=$2} /^SwapFree:/ {sf=$2} + END {printf "%d %d", (t-a)/1024, (st-sf)/1024}' /proc/meminfo + ;; + Darwin) + used="$(vm_stat | awk ' + /page size of/ {for (i = 1; i <= NF; i++) if ($i == "of") ps = $(i+1)} + /^Pages active:/ {v = $3; sub(/\.$/, "", v); a = v} + /^Pages wired down:/ {v = $4; sub(/\.$/, "", v); w = v} + /^Pages occupied by compressor:/ {v = $5; sub(/\.$/, "", v); c = v} + END {printf "%d", (a + w + c) * ps / 1048576}')" + swap="$(sysctl -n vm.swapusage 2>/dev/null | awk ' + {for (i = 1; i <= NF; i++) if ($i == "used") {v = $(i+2); sub(/M$/, "", v); printf "%d", v}}')" + printf '%s %s' "${used:-0}" "${swap:-0}" + ;; + *) printf '0 0' ;; + esac + } + + # Prints " ". + java_sample() { + ps -eo rss=,comm= 2>/dev/null \ + | awk '$NF ~ /(^|\/)java$/ {s += $1; n++} END {printf "%d %d", s/1024, n}' + } + + # Prints " " for the native build: cargo, rustc, + # the linker and C compiler they start, build scripts, and the Rust test + # binary (tantivy4java-; Linux truncates process names to 15 chars). + rust_sample() { + ps -eo rss=,comm= 2>/dev/null \ + | awk '{name = $NF; sub(/.*\//, "", name)} + name ~ /^(cargo|rustc|rust-lld|lld|ld|ld\.[a-z]+|collect2|cc1|cc1plus|clang|build-script-[a-z-]*|tantivy4java-[0-9a-f]*)$/ {s += $1; n++} + END {printf "%d %d", s/1024, n}' + } + + # Background sampler. Columns: + # epoch free_disk_mb used_mb swap_mb java_rss_mb java_count rust_rss_mb rust_count + # Exits by itself if the helper process that started it goes away. + sampler() { + local out="$MEASURE_DIR/$1.samples" + while kill -0 "$$" 2>/dev/null; do + printf '%s %s %s %s %s\n' "$(date +%s)" "$(free_disk_mb)" "$(mem_sample)" "$(java_sample)" "$(rust_sample)" >> "$out" + sleep "$SAMPLE_INTERVAL" + done + } + + # --- phase runner ----------------------------------------------------------- + # cmd_phase : runs the command unchanged, streams its + # output (stdout and stderr merged) to the step log, keeps an epoch-prefixed + # copy in .log and returns the command's exit status. + cmd_phase() { + local phase="$1" + shift + local log="$MEASURE_DIR/$phase.log" rc=0 spid lpid line i base + # The FIFO lives outside $MEASURE_DIR so it can never end up in the + # uploaded artifact directory if the step is killed. + local fifo="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/measure-$phase.fifo" + : > "$log" + rm -f "$fifo" + mkfifo "$fifo" + put "$phase.cmd" "$*" + put "$phase.disk_before" "$(free_disk_mb)" + put "$phase.start" "$(date +%s)" + sampler "$phase" > /dev/null 2>&1 & + spid=$! + # The logger must not throttle the command being measured: surefire prints + # every test's output to the console. A bash `read` loop manages only a + # few thousand lines per second on a pipe (and a few hundred with a `date` + # fork per line), so perl does the copying where it exists (both hosted + # images). The bash loop is the fallback; its epoch comes from SECONDS. + if [[ -z "${MEASURE_BASH_LOGGER:-}" ]] && command -v perl > /dev/null 2>&1; then + perl -e ' + $| = 1; + open(my $log, ">>", $ARGV[0]) or die "cannot open $ARGV[0]: $!"; + select((select($log), $| = 1)[0]); + while (my $line = ) { + print $line; + chomp $line; + print $log time(), " ", $line, "\n"; + }' "$log" < "$fifo" & + else + base=$(($(date +%s) - SECONDS)) + while IFS= read -r line || [[ -n "$line" ]]; do + printf '%s\n' "$line" + printf '%s %s\n' "$((base + SECONDS))" "$line" >> "$log" + done < "$fifo" & + fi + lpid=$! + "$@" > "$fifo" 2>&1 + rc=$? + # Timing stops when the command exits, as it would for a plain step. + put "$phase.end" "$(date +%s)" + put "$phase.rc" "$rc" + put "$phase.disk_after" "$(free_disk_mb)" + kill "$spid" 2> /dev/null + wait "$spid" 2> /dev/null + # Let the logger drain, but do not wait for EOF: a leftover child of the + # command can hold the pipe open, which would hang the step. + i=0 + while kill -0 "$lpid" 2> /dev/null && [[ "$i" -lt 25 ]]; do + sleep 0.2 + i=$((i + 1)) + done + kill "$lpid" 2> /dev/null + wait "$lpid" 2> /dev/null + rm -f "$fifo" + return "$rc" + } + + # --- runner facts and parallelism decision ---------------------------------- + # Selected settings of one cargo profile, as written in native/Cargo.toml. + profile_line() { + awk -v want="[profile.$1]" ' + $0 == want {f = 1; next} + /^\[/ {f = 0} + f && /^(opt-level|lto|codegen-units|debug|incremental) *=/ { + sub(/ *#.*$/, ""); gsub(/ +/, " "); printf "%s%s", sep, $0; sep = "; " + }' native/Cargo.toml 2>/dev/null + } + + cmd_facts() { + local cores ram_mb swap_mb cpu_model jobs_input cargo_default + local jobs_build jobs_test src_build src_test want + case "$OS_NAME" in + Linux) + cores="$(nproc)" + ram_mb="$(awk '/^MemTotal:/ {printf "%d", $2/1024}' /proc/meminfo)" + swap_mb="$(awk '/^SwapTotal:/ {printf "%d", $2/1024}' /proc/meminfo)" + cpu_model="$(awk -F': *' '/^model name/ {print $2; exit}' /proc/cpuinfo)" + jobs_input="${CARGO_JOBS_INPUT_LINUX:-}" + ;; + Darwin) + cores="$(sysctl -n hw.ncpu)" + ram_mb="$(($(sysctl -n hw.memsize) / 1048576))" + swap_mb="$(sysctl -n vm.swapusage 2>/dev/null | awk ' + {for (i = 1; i <= NF; i++) if ($i == "total") {v = $(i+2); sub(/M$/, "", v); printf "%d", v}}')" + cpu_model="$(sysctl -n machdep.cpu.brand_string 2>/dev/null || echo unknown)" + jobs_input="${CARGO_JOBS_INPUT_MACOS:-}" + ;; + *) + echo "::error::Unsupported OS for this measurement: $OS_NAME" + return 1 + ;; + esac + + # cargo's default for build.jobs is the number of logical CPUs. The value + # is always passed explicitly (CARGO_BUILD_JOBS) so the log shows it. + cargo_default="$cores" + jobs_build="$cargo_default" + jobs_test="$cargo_default" + src_build="same as the cargo default (one job per core)" + src_test="same as the cargo default (one job per core)" + if [[ -n "$jobs_input" ]]; then + if ! [[ "$jobs_input" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::cargo jobs override must be a positive integer, got '$jobs_input'" + return 1 + fi + jobs_build="$jobs_input" + jobs_test="$jobs_input" + src_build="workflow_dispatch input" + src_test="workflow_dispatch input" + elif [[ "$OS_NAME" == "Darwin" ]]; then + want="${DEFAULT_CARGO_TEST_JOBS_MACOS:-2}" + if [[ "$want" =~ ^[1-9][0-9]*$ && "$want" -lt "$cores" ]]; then + jobs_test="$want" + src_test="workflow default for macOS, deliberately below the cargo default (rationale below)" + fi + fi + + # Surefire settings and heap cap, read from the pom rather than assumed. + local fork_count reuse_forks xmx jvm_heap_mb + fork_count="$(sed -n 's/.*\(.*\)<\/forkCount>.*/\1/p' pom.xml | head -1)" + reuse_forks="$(sed -n 's/.*\(.*\)<\/reuseForks>.*/\1/p' pom.xml | head -1)" + xmx="$(grep -o -- '-Xmx[0-9]*[gGmM]' pom.xml | sort -u | tr '\n' ' ' | sed 's/ *$//')" + # What the JVM itself picks as max heap on this machine when no -Xmx is given. + jvm_heap_mb="$(java -XX:+PrintFlagsFinal -version 2>/dev/null \ + | awk '$2 == "MaxHeapSize" {printf "%d", $4 / 1048576}')" + + put facts.uname "$(uname -sm)" + put facts.image "${ImageOS:-unknown} ${ImageVersion:-}" + put facts.cores "$cores" + put facts.cpu_model "${cpu_model:-unknown}" + put facts.ram_mb "$ram_mb" + put facts.swap_mb "${swap_mb:-0}" + put facts.disk_free_mb "$(free_disk_mb)" + put facts.commit "$(git rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" + put facts.cargo_default_jobs "$cargo_default" + put facts.cargo_jobs_build "$jobs_build" + put facts.cargo_jobs_build_source "$src_build" + put facts.cargo_jobs_test "$jobs_test" + put facts.cargo_jobs_test_source "$src_test" + put facts.fork_count "${fork_count:-not set in pom.xml}" + put facts.reuse_forks "${reuse_forks:-not set in pom.xml}" + put facts.xmx "${xmx:-none}" + put facts.jvm_heap_mb "$jvm_heap_mb" + put facts.profile_release "$(profile_line release)" + put facts.profile_dev "$(profile_line dev)" + put facts.java "$(java -version 2>&1 | head -1)" + put facts.maven "$(mvn --version 2>/dev/null | head -1)" + put facts.rustc "$(rustc --version 2>/dev/null || echo 'not preinstalled')" + put facts.cargo "$(cargo --version 2>/dev/null || echo 'not preinstalled')" + put facts.protoc "$(protoc --version 2>/dev/null || echo 'not preinstalled')" + + if [[ -n "${GITHUB_ENV:-}" ]]; then + echo "CARGO_JOBS_BUILD=$jobs_build" >> "$GITHUB_ENV" + echo "CARGO_JOBS_TEST=$jobs_test" >> "$GITHUB_ENV" + fi + + echo "cores=$cores ram_mb=$ram_mb swap_mb=${swap_mb:-0} free_disk_mb=$(free_disk_mb)" + echo "cargo default jobs=$cargo_default" + echo "release build: CARGO_BUILD_JOBS=$jobs_build ($src_build)" + echo "Rust test build: CARGO_BUILD_JOBS=$jobs_test ($src_test)" + echo "surefire forkCount=${fork_count:-?} reuseForks=${reuse_forks:-?} -Xmx in pom: ${xmx:-none}; JVM default max heap: ${jvm_heap_mb:-?} MB" + echo "--- df -h" + df -h + } + + # --- summary ---------------------------------------------------------------- + fmt_dur() { + if [[ -z "${1:-}" ]]; then printf 'n/a'; else printf '%dm %02ds' "$(($1 / 60))" "$(($1 % 60))"; fi + } + fmt_gb() { + if [[ -z "${1:-}" ]]; then printf 'n/a'; else awk -v m="$1" 'BEGIN {printf "%.1f GB", m / 1024}'; fi + } + # col_stat + col_stat() { + if [[ ! -s "$1" ]]; then return 0; fi + awk -v c="$2" -v op="$3" ' + NR == 1 {v = $c} + op == "min" && $c < v {v = $c} + op == "max" && $c > v {v = $c} + END {if (NR > 0) printf "%d", v}' "$1" + } + # First epoch at which a line matching was logged in . + log_epoch() { + if [[ ! -f "$1" ]]; then return 0; fi + grep -E -m1 -- "$2" "$1" | awk '{print $1}' + } + # span : duration between two epochs, n/a if either is missing. + span() { + if [[ -n "${1:-}" && -n "${2:-}" && "$2" -ge "$1" ]]; then fmt_dur "$(($2 - $1))"; else printf 'n/a'; fi + } + # Size (MB) of a path, empty if it does not exist. + dir_mb() { + if [[ -e "$1" ]]; then du -sk "$1" 2>/dev/null | awk '{printf "%d", $1/1024}'; fi + } + # cargo's own "Finished ... in