diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..d8d1d10 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,63 @@ +name: prototype tests + +on: + push: + branches: [main] + paths: + - "prototype/**" + - "schemas/**" + - ".github/workflows/**" + pull_request: + branches: [main] + paths: + - "prototype/**" + - "schemas/**" + - ".github/workflows/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.os }} / py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.11", "3.13"] + defaults: + run: + working-directory: prototype + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: prototype/uv.lock + + - name: Install Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} + + - name: Install package and test dependencies + run: uv sync --extra test --python ${{ matrix.python-version }} + + - name: Run tests + run: uv run --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra + + - name: Run the pyperf-bench example + shell: bash + run: uv run --extra test --python ${{ matrix.python-version }} bash examples/pyperf-bench/run.sh + + # Needs cmake and a C++ compiler (preinstalled on the Linux and macOS + # runners) and fetches Google Benchmark; the example is bash and CMake + # with Unix paths, so it is not run on Windows. + - name: Run the archery-local example + if: runner.os != 'Windows' + shell: bash + run: uv run --extra test --python ${{ matrix.python-version }} bash examples/archery-local/run.sh diff --git a/prototype/benchx/snapshot.py b/prototype/benchx/snapshot.py index 5d8d17a..519333c 100644 --- a/prototype/benchx/snapshot.py +++ b/prototype/benchx/snapshot.py @@ -78,7 +78,7 @@ def observed_context(child_env: dict, env_names=None) -> dict: facts["governor"] = governor try: facts["load_avg_1m"] = round(os.getloadavg()[0], 2) - except OSError: + except (OSError, AttributeError): # AttributeError: no os.getloadavg on Windows pass env = {k: child_env[k] for k in (env_names or env_allowlist(child_env)) if k in child_env} if env: diff --git a/prototype/examples/archery-local/README.md b/prototype/examples/archery-local/README.md index 2e233d9..ac57368 100644 --- a/prototype/examples/archery-local/README.md +++ b/prototype/examples/archery-local/README.md @@ -31,14 +31,14 @@ installed). From the repository root: ```console $ cd prototype -$ uv venv && uv pip install -e '.[test]' # or any virtualenv with `pip install -e .` -$ cd examples/archery-local -$ ./run.sh +$ python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like +$ pip install -e . +$ examples/archery-local/run.sh ``` It takes about two minutes the first time (two CMake builds and a measured -run) and prints every command before running it, as you would type it. With -another interpreter: `PYTHON=/path/to/python ./run.sh`. More rounds: +run) and prints every command before running it, as you would type it. The +script runs `bx` and `python` from the active environment. More rounds: `ROUNDS=10 ./run.sh`. Everything it creates is under `_work/` in this directory, including the local @@ -96,8 +96,7 @@ over. ## Try it by hand After `./run.sh` (the build directories and the store stay in `_work/`). From -this directory, with the virtualenv's `bx` on your `PATH` or as -`../../.venv/bin/python -m benchx.cli`: +this directory, with the environment still active: ```console $ source activate.sh diff --git a/prototype/examples/archery-local/provider.sh b/prototype/examples/archery-local/provider.sh index ce49fbe..a5226e0 100755 --- a/prototype/examples/archery-local/provider.sh +++ b/prototype/examples/archery-local/provider.sh @@ -58,7 +58,7 @@ if [[ ! -d $SRC/.git ]]; then fi mkdir -p "$WORK/deps/lib" # stand-in for the dependencies activate.sh points at -step cmake -S "$SRC" -B "$BUILD" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" "${extra[@]}" +step cmake -S "$SRC" -B "$BUILD" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" ${extra[@]+"${extra[@]}"} step cmake --build "$BUILD" -j BUILD=$BUILD SRC=$SRC HOW=$HOW CONFIGURATION=$configuration HERE=$HERE python3 - >"$BUILD/.benchx-target.json" <<'PY' diff --git a/prototype/examples/archery-local/run.sh b/prototype/examples/archery-local/run.sh index 15b5a5b..ff6c919 100755 --- a/prototype/examples/archery-local/run.sh +++ b/prototype/examples/archery-local/run.sh @@ -9,32 +9,32 @@ # nothing, and runs none of that text: you activate the environment yourself # (activate.sh) and benchx records the text beside the environment it inherited. # -# Every command is printed before it runs, as you would type it; `bx` stands -# for `$PYTHON -m benchx.cli`. Needs cmake and a C++ compiler on PATH, and -# network access the first time, to fetch Google Benchmark unless installed. +# Every command is printed before it runs, as you would type it. Run it in an +# activated environment that has benchx installed (from prototype/: +# `pip install -e .`, or `uv sync --extra test`). Needs cmake and a C++ +# compiler on PATH, and network access the first time, to fetch Google +# Benchmark unless installed. # # ROUNDS=5 alternation rounds (default 5) -# PYTHON the interpreter with benchx installed (default ../../.venv/bin/python) # # The local store is kept inside this directory (_work/benchx-home), not in # ~/.benchx. Delete _work/ to start over. set -euo pipefail cd "$(dirname "$0")" HERE=$PWD -PYTHON=${PYTHON:-$HERE/../../.venv/bin/python} ROUNDS=${ROUNDS:-5} export BENCHX_HOME=${BENCHX_HOME:-$HERE/_work/benchx-home} STAMP=$(date -u +%Y%m%dT%H%M%SZ) RUN="hardening-$STAMP" -for tool in cmake c++; do +for tool in cmake c++ bx python; do command -v "$tool" >/dev/null || { echo "run.sh: $tool not found on PATH (see README.md)" >&2; exit 1; } done -"$PYTHON" -c 'import benchx' 2>/dev/null || { echo "run.sh: benchx is not installed for $PYTHON (see README.md)" >&2; exit 1; } +bx --help >/dev/null 2>&1 || { echo "run.sh: benchx is not installed in the active environment (see README.md)" >&2; exit 1; } step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } say() { printf '\033[36m$ %s\033[0m\n' "$*" >&2; } -bx() { say bx "$@"; "$PYTHON" -m benchx.cli "$@"; } +bx() { say bx "$@"; command bx "$@"; } step "user: activate the environment (benchx will not do this for you)" say source activate.sh @@ -60,13 +60,13 @@ bx compare @plain @hardened --profile environments --label build=plain,hardened step "the comparison names what was built and how; before, it was two opaque directory paths" echo "# the first work order's target, as the runner received it:" -"$PYTHON" -c ' +python -c ' import json, sys print(json.dumps(json.load(open(sys.argv[1]))["target"], indent=1))' _work/results/orders/order-0001.json step "and every result carries it, next to what the runner saw (declared, observed: nothing judges)" say bx head --json -n 1 '| …' -"$PYTHON" -m benchx.cli head --json -n 1 | "$PYTHON" -c ' +command bx head --json -n 1 | python -c ' import json, sys doc = json.loads(sys.stdin.readline()) info = doc["provenance"]["info"] @@ -79,7 +79,7 @@ print("coordinates.subject.configuration:", doc["coordinates"]["subject"].get("c step "forget to activate: the declaration still says what you should have run, the observation says what you did" say env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --rounds 3 ... -env -u LD_LIBRARY_PATH "$PYTHON" -m benchx.cli compare @plain @hardened --no-build --profile environments \ +env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --profile environments \ --label build=plain,hardened --suite demo-bench --rounds 3 --project archery-local \ --run-key "$RUN-unactivated" --out _work/results-unactivated diff --git a/prototype/examples/pyperf-bench/README.md b/prototype/examples/pyperf-bench/README.md index 9ec886f..2b8342e 100644 --- a/prototype/examples/pyperf-bench/README.md +++ b/prototype/examples/pyperf-bench/README.md @@ -7,15 +7,16 @@ writes benchx result files, delivers them to a local store, and prints a table. ## Run it ```sh -cd prototype/examples/pyperf-bench -python3 -m pip install pyperf pyarrow jsonschema referencing rfc8785 # Python >= 3.11 -./run.sh +cd prototype +python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like +pip install -e . pyperf +examples/pyperf-bench/run.sh ``` -`PYTHON=/path/to/python ./run.sh` picks another interpreter; pyperf must be importable -by the `python` on your `PATH`, which is the one the workers use. Extra arguments go to +The script runs `bx` and `python` from the active environment, so benchx and pyperf must +be installed there; the workers use that same `python`. Extra arguments go to `bx bench` in step 1 and win over the config file, e.g. `./run.sh --processes 20`. -It takes about 15 seconds. Nothing outside this directory changes: the local store is +It takes about 20 seconds. Nothing outside this directory changes: the local store is `out/home/store.parquet` (the script sets `BENCHX_HOME`), and `out/` is deleted at the start of each run. @@ -74,8 +75,13 @@ It is the work-order `precision` block under the adapter's name: 1. `bx bench bench_suite.py` runs all five cases with the config file's precision. 2. `bx bench bench_suite.py bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2` runs one case, sets a variable for the workers, and overrides the file's process count. -3. `bx head` lists the latest rows in the local store. -4. It lists the files written. +3. `bx bench bench_suite.py bench_join --run-key example-again` runs one case a second time + under a new run key, so its series gets a second point. +4. `ls -l` and `bx head` show the local store: one Parquet file, and its latest rows. +5. `bx series` lists the series in the store. +6. `bx history` lists the points of the `bench_join` series, one per run. The script fails + if there are not two, which is what CI relies on. +7. It lists the files written. ## What comes out @@ -103,9 +109,27 @@ Files, under `out/all/` and `out/env/`: script that ran the function; `.worker-env.json`: the environment a worker really had; `.stdout` and `.stderr`. -The local store, `out/home/store.parquet`, holds the results. Read it back with -`bx head`, `bx series`, `bx history`, or `bx ingest` more result files; comparing two -runs is not part of this step. +The local store, `out/home/store.parquet`, holds the results. `run.sh` reads it back +(your numbers and fingerprints will differ): + +``` +$ bx series +d780c9564c72 local/be8372c04606 bench_dict_build wall-time [s] est=benchx/median/v1 +e6eaa02c8309 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 +4be59ef6debc local/be8372c04606 bench_join wall-time [s] est=benchx/median/v1 +... +85a6aa9c6996 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 + +$ bx history 4be59ef6debc +revision order unavailable without a revision graph; listed by measurement start +d83d37b69010 2026-10-13T11:04:28.656515Z 3.32107e-06 (server) example-all:bench_join:wall-time +d83d37b69010 2026-10-13T11:04:39.045277Z 3.76987e-06 (server) example-again:bench_join:wall-time +``` + +`bench_env_var` has two series because step 2 sets `EXAMPLE_N`, which changes the +environment the case ran in. `bench_join` has one series with two points, one for each +run key. You can also `bx ingest` more result files; comparing two runs is not part of +this step. ## Notes diff --git a/prototype/examples/pyperf-bench/run.sh b/prototype/examples/pyperf-bench/run.sh index f742124..994016e 100755 --- a/prototype/examples/pyperf-bench/run.sh +++ b/prototype/examples/pyperf-bench/run.sh @@ -4,18 +4,17 @@ # ./run.sh uses .benchx/config.json here (5 processes x 3 values) # ./run.sh --processes 20 extra arguments go to `bx bench` and win over the file # -# Needs Python >= 3.11 with pyperf, pyarrow, jsonschema, referencing, rfc8785. -# PYTHON=/path/to/python picks the interpreter. Results go to ./out, and the -# local store to ./out/home (BENCHX_HOME), so nothing outside this directory changes. +# Run it in an activated environment that has benchx installed with pyperf +# (from prototype/: `pip install -e . pyperf`, or `uv sync --extra test`). +# Results go to ./out, and the local store to ./out/home (BENCHX_HOME), so +# nothing outside this directory changes. set -euo pipefail cd "$(dirname "$0")" -PYTHON=${PYTHON:-python3} -export PYTHONPATH="$PWD/../..${PYTHONPATH:+:$PYTHONPATH}" export BENCHX_HOME="$PWD/out/home" -bx() { printf '\033[36m$ bx %s\033[0m\n' "$*" >&2; "$PYTHON" -m benchx.cli "$@"; } +bx() { printf '\033[36m$ bx %s\033[0m\n' "$*" >&2; command bx "$@"; } -"$PYTHON" -c "import pyperf, pyarrow, jsonschema, referencing, rfc8785" 2>/dev/null || { - echo "missing dependencies: $PYTHON -m pip install pyperf pyarrow jsonschema referencing rfc8785" >&2 +command bx --help >/dev/null 2>&1 && python -c "import pyperf" 2>/dev/null || { + echo "missing dependencies: install benchx and pyperf into the active environment (see README.md)" >&2 exit 1; } rm -rf out; mkdir -p out @@ -26,8 +25,24 @@ echo; echo "== 2. one function, an environment variable set for the workers, fla bx bench bench_suite.py bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2 \ --run-key example-env --out out/env -echo; echo "== 3. what is in the local store" +echo; echo "== 3. the same case again under a new run key, so its series gets a second point" +bx bench bench_suite.py bench_join --run-key example-again --out out/again + +echo; echo "== 4. what is in the local store: one Parquet file" +ls -l "$BENCHX_HOME/store.parquet" bx head -n 10 -echo; echo "== 4. what was written" -find out/all out/env -type f | sort | sed 's/^/ /' +echo; echo "== 5. its series (bench_env_var has two: the -e setting changes the environment)" +bx series > out/series.txt +cat out/series.txt + +echo; echo "== 6. the history of bench_join: one point per run" +bx series --workload bench_join > out/join-series.txt +SERIES=$(awk 'NR == 1 {print $1}' out/join-series.txt) +[ -n "$SERIES" ] || { echo "no bench_join series in the store" >&2; exit 1; } +bx history "$SERIES" > out/history.txt +cat out/history.txt +[ "$(grep -c example- out/history.txt)" -eq 2 ] || { echo "expected two history points for $SERIES" >&2; exit 1; } + +echo; echo "== 7. what was written" +find out/all out/env out/again -type f | sort | sed 's/^/ /'