diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..2727c71 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,63 @@ +name: prototype tests + +on: + push: + branches: [main] + paths: + - "prototype/**" + - "schemas/**" + - ".github/workflows/**" + pull_request: + branches: [main] + paths: + - "prototype/**" + - "schemas/**" + - ".github/workflows/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.os }} / py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ["3.11", "3.13"] + defaults: + run: + working-directory: prototype + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: prototype/uv.lock + + - name: Install Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} + + - name: Install package and test dependencies + run: uv sync --extra test --python ${{ matrix.python-version }} + + - name: Run tests + run: uv run --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra + + - name: Run the pyperf-bench example + shell: bash + run: uv run --extra test --python ${{ matrix.python-version }} bash examples/pyperf-bench/run.sh + + # Needs cmake and a C++ compiler (preinstalled on the Linux and macOS + # runners) and fetches Google Benchmark; the example is bash and CMake + # with Unix paths, so it is not run on Windows. + - name: Run the archery-local example + if: runner.os != 'Windows' + shell: bash + run: uv run --extra test --python ${{ matrix.python-version }} bash examples/archery-local/run.sh diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 0f9dde0..567a263 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -21,25 +21,53 @@ bx series [--workload … --quantity …] # list series bx history # one series' points (a query) bx head [-n 10] [--json] # the latest results in the store, newest first bx compare --run KEY --profile revisions|environments --baseline VALUE - [--label NAME] [--results DIR] [--k 3] [--json] + [--label NAME] [--results DIR] [--k 3] [--min-rounds 3] [--json] # read mode: compare results already measured +bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) bx compare BASELINE CONTENDER --profile revisions|environments --suite NAME [--filter REGEX] [--quantity wall-time] --rounds R [--repetitions N] [--min-time S] [--project P] [--label NAME=BASE,CONTENDER] - [--run-key KEY] [--out DIR] [--k 3] [--json] + [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] + [--k 3] [--min-rounds 3] [--json] # measure mode: run both sides, then compare +bx bench MODULE [FUNCTION …] [--processes N] [--values N] [--min-time S | --loops N] [--warmups N] + [-e NAME=VALUE] [--tree DIR] [--project P] [--run-key KEY] [--source-uri URI] + [--case-timeout S] [--out DIR] [--no-ingest] + # Python functions under pyperf → result files → store → table +bx config init [--dir DIR] # write .benchx/config.json stating pyperf's defaults ``` +`bx bench` is a second front end to the same runner. It builds one work order +whose target is the `working_tree` of the git checkout containing MODULE (or +`--tree`), as found, with the dirty state and tree id recorded; the suite is the +module, a case is a function in it (`bench_*` and `benchmark_*` by default, or the +functions named), and `-e` fills `environment_variables`. Precision is resolved +before the order is written, flags over `.benchx/config.json` (found by walking up +from the current directory) over pyperf's defaults, so the order always states +what applied. The pyperf adapter takes `python` from the `PATH` the workers start +with and refuses the order if pyperf is not importable there. Delivery is as for +`bx run`. There is no comparison step: `bx bench` produces results, and comparing +them is not part of it. + `bx compare` has two modes, chosen by its arguments. Two positional targets -(`BUILD_DIR[:SOURCE_DIR]`) mean measure mode; `--run` means read mode; giving +(`BUILD_DIR[:SOURCE_DIR]` or `@CONFIG[=SOURCE_REF]`, a call to the project's +target provider) mean measure mode; `--run` means read mode; giving both or neither is an error, as is a flag that belongs to the other mode (`--baseline` and `--results` are read mode only, since measure mode derives the baseline from its first target; `--suite`, `--rounds`, `--project`, -`--run-key`, `--out`, `--repetitions`, and `--min-time` are measure mode -only). `--label` names the label in read mode, and carries the two values in +`--run-key`, `--out`, `--repetitions`, `--min-time`, `--no-build`, and +`--source-uri` are measure mode only). `--label` names the label in read mode, and carries the two values in measure mode. §5 describes measure mode. +A source URI given any way (`--source-uri`, a provider's description, the +checkout's `origin`) goes through one normalization before sides are +compared or recorded, and identity does the same: git's SCP-style +`user@host:/abs/path` becomes `ssh://user@host/abs/path`, a relative +`host:path` becomes `ssh://host/~/path` (relative to the login directory, so it +is not mistaken for the absolute `/path`), and a local path becomes a +`file://` URI. Windows drive paths and bracketed IPv6 hosts are recognized. + The CLI is a thin layer: each command is one call into the `benchx` package, so the workbench, a CI script, or a notebook can do the same without a subprocess. Measure mode is a call into the session function, which calls the @@ -62,9 +90,10 @@ not post to the original store. **Work order.** The prototype reads the work order of `runner-schema.md` (`schemas/work-order/0.1.0`, `benchx/work-order/0.1.0`) and defines no format of its own. One order is one side of one round: the runner executes one order -per invocation (`runner.md` §7), and the prototype accepts one target kind, -`build`, an existing build directory with an optional `source_dir`. The runner -never builds it (`runner.md` §1). +per invocation (`runner.md` §7), and the prototype accepts two target kinds: +`build`, an existing build directory with an optional `source_dir`, and +`working_tree`, a checkout measured as found, which only `bx bench` writes. The +runner never builds either (`runner.md` §1). Interleaving needs two order fields. `round` is native to the schema and is copied to `procedure.round`. `slot`, the realized position across all sides, @@ -203,6 +232,11 @@ landing where its §3.4 puts it: §5.5); OS, kernel, libc, governor, and load as observed context; an allowlist of thread-control environment variables as observed context, which is the allowlist of `benchmark-environments.md` §3.1. + The `env` key reads differently by adapter. Google Benchmark takes the + runner's own environment and omits `env` when no allowlisted variable is set. + pyperf records what the workers actually had, so `env` is `{}` when the + capture worked and none was set, and absent only when nothing was captured + (`env-capture-missing`). Both are kept as they are. - **Source identity**, from `target.source_dir`, or from the `CMAKE_HOME_DIRECTORY` the build directory records when the order names none: revision, `subject_dirty`, and `subject_tree` (schema §4.1). When @@ -219,6 +253,11 @@ the same way report the same configuration. Nobody declared these, so the detect values are recorded as the intended ones in `coordinates.subject.configuration` (§3.4). This is what makes the hardened and plain directories differ in coordinates, which the `environments` profile requires. + The pyperf adapter's `python_cflags` is the exception to "never paths": it is + recorded verbatim in `comparison_context.runtime`, as `harness-adapter.md` §6.8 + has it, so the same Python build installed under two prefixes reports different + flags (conda adds its `-isystem` and `-fdebug-prefix-map` paths) and lands in + different series. - **Declared facts**, from `.benchx/setup.json` or `BENCHX_SETUP_FILE` (`harness-adapter.md` §5.1), for what the cache cannot tell, such as how a dependency was installed. The prototype has no placement rules, so they @@ -380,9 +419,15 @@ compares them; it is the workbench's session loop (`system-decomposition.md` first value of `--label NAME=BASE,CONTENDER` for `environments`. It is exactly `bx run` 2R times followed by read-mode `bx compare`; it adds no -document, schema, or contract. It builds nothing: both targets must exist, -and whatever produced them plays the target-provider role -(`system-decomposition.md` §3.4). If it stops partway, the results already +document, schema, or contract. It builds nothing itself: both targets must exist +when it looks. A target given as `@CONFIG` is first requested from the +project's target provider (`system-decomposition.md` §3.4), which `prepare`s +it (and may build it) or, with `--no-build`, only `describe`s it; a target +given as a build directory is read from its `.benchx-target.json` sidecar if +it has one. The description's declared text (`how_built`, `activation`, +`shell`, `provider`) is copied into each order, recorded in each result's +`provenance.info`, and shown in the comparison; it is never executed or +checked. If it stops partway, the results already delivered stay in the store, and a read-mode comparison of the run key reports missing results or a failed alternation invariant, as UC-03 §10 requires. Allowing a local-only `revisions` comparison of a dirty tree is open @@ -394,7 +439,10 @@ Both profiles use one shape: two prepared build directories, one run key, R rounds, and `bx compare` in measure mode, which alternates the sides and assigns `round` and `slot`. The demo script plays the user, and in that role the target provider: it prepares the sources and build directories with git -and CMake, since benchx builds nothing. +and CMake, since benchx builds nothing. `prototype/examples/archery-local` +shows the other route, where a provider script named in +`.benchx/provider.json` builds and describes the targets and the sides are +given as `@plain` and `@hardened`. - **Revisions:** two git worktrees of one source at the base and head commits, each built into its own directory with the same configuration. @@ -425,12 +473,17 @@ history grow from run to run, and the demo ends with `bx head`. ## 7. Deliberate simplifications -- One harness and one target kind. The runner builds nothing, ever: the user +- Two harnesses (Google Benchmark, and pyperf through `bx bench`) and two target + kinds (`build`, `working_tree`). The runner builds nothing, ever: the user prepares both sides (`benchmark-environments.md` §2). The prototype applies `environment_variables` and defers the rest of the environment policy, and it refuses an order that names one rather than ignoring it (§3). -- No target-provider hook: the demo prepares its builds by hand, and the - provider contract is open (`system-decomposition.md` §6 question 5). +- The target-provider hook is minimal: one provider command per project + (`.benchx/provider.json`), `describe` and `prepare` only, `build` targets + only, and a source ref other than `WORKSPACE` is the provider's business. + How cached builds are named and expired stays open + (`system-decomposition.md` §6 question 5). The demo prepares its builds by + hand. - Environment identity is the snapshot's host name, CPU model, and core count; continuity events, quarantine, and artifact storage are absent. - One identity policy and one derived estimator, fixed in code. diff --git a/docs/design/prototype-scope.md b/docs/design/prototype-scope.md index fd04ccd..c25bff8 100644 --- a/docs/design/prototype-scope.md +++ b/docs/design/prototype-scope.md @@ -40,11 +40,11 @@ one run: | Component | In prototype | Deferred | Out of scope | |---|---|---|---| -| Runner | execute a `build` target per a work order; apply the order's `environment_variables` to the launched process; capture the zero-configuration snapshot, the build configuration, and source identity; account for planned cases; refuse an order it cannot apply exactly, including one that names a deferred field | `python_env` targets, context collectors beyond the built-in ones, environment policy (enforcing launch settings, verifying requested hardware, refusing an unsatisfiable run) | building revisions, tuning the machine (`benchmark-environments.md` §2.3) | -| Adapter | Google Benchmark, both halves: drive filter and repetitions, translate output | all other harnesses | — | +| Runner | execute a `build` or `working_tree` target per a work order; apply the order's `environment_variables` to the launched process; capture the zero-configuration snapshot, the build configuration, and source identity; account for planned cases; refuse an order it cannot apply exactly, including one that names a deferred field | `python_env` targets, context collectors beyond the built-in ones, environment policy (enforcing launch settings, verifying requested hardware, refusing an unsatisfiable run) | building revisions, tuning the machine (`benchmark-environments.md` §2.3) | +| Adapter | Google Benchmark, both halves: drive filter and repetitions, translate output; pyperf, both halves, for `bx bench` (Python functions in the working tree; no comparison step) | all other harnesses; pyperf's `tags`, `date`, per-run load and frequency, and `pyperf check` | — | | Store | one Parquet file; ingest per schema §5.4 (validated and durable, auto-create, unit-conflict rejection, idempotency, attempt integrity); one default identity policy; queries: list series, series history, get result | continuity events, quarantine, artifact storage, identity-policy authoring | — | | Comparator | run mode with the `revisions` and `environments` profiles; eligibility guard; comparison document | history mode; `single-run`, `variants`, and `cross-machine` profiles; batch requests; provisional noise rules | — | -| Workbench session loop | `bx compare` in measure mode: writes one work order per side and round, alternates them, delivers the results, then compares; takes each side as a build directory or as `@CONFIG`, a call to the project's target provider (`describe` or `prepare`); `bx target` asks for a description directly; copies the description's declared text (`how_built`, `activation`, `shell`, `provider`) into the order | provider-built revisions (a source ref other than `WORKSPACE` is the provider's business), `artifact` and `working_tree` targets, a declared `build` | workbench packaging, front ends such as `spin bench` | +| Workbench session loop | `bx compare` in measure mode: writes one work order per side and round, alternates them, delivers the results, then compares; takes each side as a build directory or as `@CONFIG`, a call to the project's target provider (`describe` or `prepare`); `bx target` asks for a description directly; copies the description's declared text (`how_built`, `activation`, `shell`, `provider`) into the order | provider-built revisions (a source ref other than `WORKSPACE` is the provider's business), an `artifact` target, `working_tree` targets in `bx compare`, a declared `build` | workbench packaging, front ends such as `spin bench` | | Documents | work order (`schemas/work-order/0.1.0`), result (schema §5.2), comparison document | — | — | **Out entirely:** scheduler, change detector, alerting, views beyond query diff --git a/prototype/README.md b/prototype/README.md index 1fd4361..db59853 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -22,6 +22,11 @@ bx compare BASELINE CONTENDER --profile revisions|environments [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json] # measure mode: run both sides, then compare bx validate … [--kind KIND] # every problem in each document, by schema and rules +bx bench MODULE [FUNCTION …] [--processes N] [--values N] [--min-time S | --loops N] [--warmups N] + [-e NAME=VALUE] [--tree DIR] [--project P] [--run-key KEY] [--source-uri URI] + [--case-timeout S] [--out DIR] [--no-ingest] + # Python functions under pyperf: order → results → store → table +bx config init [--dir DIR] # write .benchx/config.json stating pyperf's defaults ``` Each target is `BUILD_DIR[:SOURCE_DIR]` or `@CONFIG[=SOURCE_REF]`. Two targets @@ -41,6 +46,18 @@ executed or checked: you activate the environment yourself, and the runner records the allowlisted variables it inherited beside your declaration. `examples/archery-local` is a complete walk-through. +`bx bench` is the other entry point: it benchmarks the functions of a Python file +with [pyperf](https://pyperf.readthedocs.io), as one `working_tree` order for the +checkout containing the module (uncommitted changes included, recorded as dirty). +Each `bench_*` or `benchmark_*` function, or the ones named, is a case; `-e` sets +environment variables for the workers and lands in the workload's parameters. +Precision resolves as flags, then `.benchx/config.json` (found by walking up from the +current directory; `bx config init` writes a template), then pyperf's defaults, and the +resolved values are written into the order. The workers run the `python` first on +`PATH`, which must have pyperf. It builds the order, runs it through the same runner and +delivers to the same store as `bx run`; it has no comparison step. +`examples/pyperf-bench` is a runnable walk-through. + ## What it does - **Work orders** are the `docs/design/runner-schema.md` schema @@ -86,7 +103,7 @@ records the allowlisted variables it inherited beside your declaration. ```console $ uv venv && uv pip install -e '.[test]' # editable: schemas are read from ../schemas -$ .venv/bin/python -m pytest tests # 222 tests, fake Google Benchmark binary, no compiler +$ .venv/bin/python -m pytest tests # 228 tests, fake Google Benchmark binary, no compiler $ ./demo.sh # real builds; needs cmake, a C++ compiler, network once ``` @@ -134,6 +151,9 @@ benchx/ hashing.py canonical form, hashes, order reference snapshot.py what the runner records (benchmark-environments.md §3) adapters/gbench.py Google Benchmark, driving and translating halves + adapters/pyperf.py pyperf, both halves, for Python functions (bx bench) + bench.py bx bench: module + flags + config.json → one working_tree work order + config.py .benchx/config.json: the pyperf precision defaults and their template runner.py one work order in, results out session.py the session loop: alternate two targets over rounds, then compare target.py target descriptions: the provider call, the sidecar, the declared text @@ -144,6 +164,7 @@ benchx/ cli.py bx (compare has a read mode and a measure mode) ../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) examples/demo-suite a two-benchmark Google Benchmark suite for the demo +examples/pyperf-bench Python functions under `bx bench` (own README) examples/archery-local an end-to-end, Arrow-local-shaped example with a target provider (own README) tests/ test_prototype.py: the success criteria and the runner and store rules, on a fake Google Benchmark binary; test_comparator.py: the diff --git a/prototype/benchx/adapters/pyperf.py b/prototype/benchx/adapters/pyperf.py index 42ca5f0..f22d7fb 100644 --- a/prototype/benchx/adapters/pyperf.py +++ b/prototype/benchx/adapters/pyperf.py @@ -177,7 +177,7 @@ def list_cases(runnable, case_filter, env) -> list[str]: if runner.args.worker: # Only the allowlisted names: the digest above covers the whole environment, # the file is shared, and CI environments carry secrets. - Path({worker_env!r}).write_text(json.dumps({{k: _ENV[k] for k in {env_names!r} if k in _ENV}}, sort_keys=True)) + Path({worker_env!r}).write_text(json.dumps({{k: _ENV[k] for k in {env_names!r} if k in _ENV}}, sort_keys=True, indent=1)) sys.path[:0] = [{module_dir!r}, {root!r}] spec = importlib.util.spec_from_file_location("benchx_suite", {module!r}) @@ -217,6 +217,7 @@ def run_case(runnable, case, flags, env, timeout, native_path, env_names=None) - if native_path.exists(): try: native = json.loads(native_path.read_text()) + native_path.write_text(json.dumps(native, indent=1) + "\n") except ValueError: native = None captured = None diff --git a/prototype/benchx/bench.py b/prototype/benchx/bench.py index 9f4d73c..3271078 100644 --- a/prototype/benchx/bench.py +++ b/prototype/benchx/bench.py @@ -14,6 +14,7 @@ from pathlib import Path from . import config, core, runner, snapshot +from .identity import normalize_remote from .session import CASE_TIMEOUT_S, ORDER_TIMEOUT_S, SessionError, _source_uri ADAPTER = "pyperf" @@ -50,7 +51,7 @@ def flags_to_precision(processes=None, values=None, min_time=None, loops=None, w def build_order(module, functions, *, tree=None, precision_flags=None, environment_variables=None, - project=None, run_key=None, labels=None, case_s=CASE_TIMEOUT_S, order_s=ORDER_TIMEOUT_S, cwd="."): + project=None, run_key=None, labels=None, source_uri=None, case_s=CASE_TIMEOUT_S, order_s=ORDER_TIMEOUT_S, cwd="."): """The resolved work order, and the config file it read (or None).""" module = Path(module).resolve() if not module.is_file(): @@ -71,7 +72,7 @@ def build_order(module, functions, *, tree=None, precision_flags=None, environme "state": "resolved", "suites": [entry], "target": {"kind": "working_tree", "path": str(root), "source_dir": str(root), - "source": {"uri": _source_uri(str(root)), "type": "git"}}, + "source": {"uri": normalize_remote(source_uri) if source_uri else _source_uri(str(root)), "type": "git"}}, "quantities": ["wall-time"], "precision": precision, "timeouts": {"case_s": case_s, "order_s": order_s}, diff --git a/prototype/benchx/cli.py b/prototype/benchx/cli.py index d987b52..577db8a 100644 --- a/prototype/benchx/cli.py +++ b/prototype/benchx/cli.py @@ -62,7 +62,8 @@ def cmd_bench(args): flags = bench_mod.flags_to_precision(args.processes, args.values, args.min_time, args.loops, args.warmups) order, config_path = bench_mod.build_order( args.module, args.functions, tree=args.tree, precision_flags=flags, environment_variables=env, - project=args.project, run_key=args.run_key, case_s=args.case_timeout) + project=args.project, run_key=args.run_key, source_uri=args.source_uri, + case_s=args.case_timeout) out = Path(args.out) if args.out else Path("results") / order["run_key"] reps = {lv["unit"]: lv["n"] for lv in order["precision"]["repetitions"]["levels"]} print(f"precision: {reps['process']} processes x {reps['value']} values " @@ -131,12 +132,16 @@ def cmd_head(args): print(row["document"]) return EXIT_OK print(f"{store.path}: {total} result(s); latest {len(rows)}, newest first") + cells = [] for r in rows: median = f"{r['median']:.6g} {r['unit']}" if r["median"] is not None else "-" started = r["started_at"].strftime("%Y-%m-%d %H:%M:%S") labels = f" {r['labels']}" if r["labels"] else "" - print(f"{r['row']:>5} {started} {r['run_key']:<24} {r['workload']:<16} {r['quantity']:<9} " - f"{r['status']:<8} {median:<14} {r['revision'][:10]} {r['project']}{labels}") + cells.append((f"{r['row']:>5}", started, r["run_key"], r["workload"], r["quantity"], r["status"], median, + r["revision"][:10], f"{r['project']}{labels}")) + widths = [max([len(cell[i]) for cell in cells], default=0) for i in range(len(cells[0]))] if cells else [] + for cell in cells: + print(" ".join(text.ljust(width) for text, width in zip(cell, widths)).rstrip()) return EXIT_OK @@ -282,6 +287,7 @@ def main(argv=None): help="set an environment variable for the benchmark (repeatable)") p.add_argument("--project", help="name a project, so results join its series; omit for an ad hoc run") p.add_argument("--run-key", help="default: bench-") + p.add_argument("--source-uri", help="the source's URI (default: the checkout's origin remote)") p.add_argument("--out", help="directory for the order and result files (default ./results/RUN_KEY)") p.add_argument("--case-timeout", type=float, default=bench_mod.CASE_TIMEOUT_S, help="seconds allowed per function (default %(default)s)") diff --git a/prototype/benchx/config.py b/prototype/benchx/config.py index 19a9d10..4b0d9a4 100644 --- a/prototype/benchx/config.py +++ b/prototype/benchx/config.py @@ -94,5 +94,5 @@ def write_template(directory=".", adapter_name="pyperf") -> Path: if path.exists(): raise ConfigError(f"{path} already exists") path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(template(adapter_name), indent=2) + "\n") + path.write_text(json.dumps(template(adapter_name), indent=1) + "\n") return path diff --git a/prototype/benchx/core/__init__.py b/prototype/benchx/core/__init__.py index b0f4622..4f27478 100644 --- a/prototype/benchx/core/__init__.py +++ b/prototype/benchx/core/__init__.py @@ -12,7 +12,7 @@ from . import catalog, loader, rules, validation from .errors import DocumentError, Refusal, Rejected, SchemaNotFound -from .hashing import canonical, order_ref, sha256_hex +from .hashing import canonical, order_ref, pretty, sha256_hex from .validation import Kind __all__ = [ @@ -20,6 +20,7 @@ "Refusal", "Rejected", "canonical", + "pretty", "check", "check_precision", "inspect", diff --git a/prototype/benchx/core/hashing.py b/prototype/benchx/core/hashing.py index a62e22e..037d5f1 100644 --- a/prototype/benchx/core/hashing.py +++ b/prototype/benchx/core/hashing.py @@ -1,6 +1,7 @@ """Canonical JSON (RFC 8785) and the hashes built on it.""" import hashlib +import json import rfc8785 @@ -9,6 +10,12 @@ def canonical(value) -> bytes: return rfc8785.dumps(value) +def pretty(value) -> bytes: + """The readable form for files on disk: two-space indent, trailing newline. + Not for hashing; that is canonical().""" + return (json.dumps(value, indent=1, ensure_ascii=False, allow_nan=False) + "\n").encode() + + def sha256_hex(data: bytes) -> str: return hashlib.sha256(data).hexdigest() diff --git a/prototype/benchx/identity.py b/prototype/benchx/identity.py index 03e459c..d4cc667 100644 --- a/prototype/benchx/identity.py +++ b/prototype/benchx/identity.py @@ -1,7 +1,9 @@ """Fingerprints, the prototype identity policy, and series points (schema §4.3, §4.5).""" +import re import statistics import urllib.parse +from pathlib import Path from .core import canonical, sha256_hex @@ -26,8 +28,33 @@ def reported_coordinates_fingerprint(doc: dict) -> str: return _hash("benchmark-reported-coordinates-v1", canonical(reported_coordinates(doc))) +_WINDOWS_PATH = re.compile(r"^[A-Za-z]:[\\/]") +_SCP_REMOTE = re.compile(r"^(?:(?P[^@/:\s\[\]]+)@)?(?P\[[0-9A-Fa-f:.]+\]|[^@/:\s\[\]]+):(?P.+)$") + + +def normalize_remote(remote: str, base: str | None = None) -> str: + """A git remote as a URI, whatever form it was given in; idempotent. + + A `scheme://` remote is unchanged. A Windows drive path or a local path + (resolved against `base`, default the current directory) becomes a + `file://` URI. git's SCP-style `[user@]host:path` becomes `ssh://`: an + absolute path stays absolute, and a relative one (relative to the login + directory) is kept relative as `ssh://host/~/path`, since `ssh://host/path` + would name a different repository.""" + if not remote or "://" in remote: + return remote + if _WINDOWS_PATH.match(remote): + return "file:///" + urllib.parse.quote(remote.replace("\\", "/"), safe="/:") + match = _SCP_REMOTE.match(remote) + if match: + user = f"{match['user']}@" if match["user"] else "" + path = match["path"] if match["path"].startswith("/") else "/~/" + match["path"] + return f"ssh://{user}{match['host']}{path}" + return Path(base or ".", remote).resolve().as_uri() + + def _normalize_uri(uri: str) -> str: - parts = urllib.parse.urlsplit(uri) + parts = urllib.parse.urlsplit(normalize_remote(uri)) path = parts.path.rstrip("/").removesuffix(".git") return urllib.parse.urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, "", "")) diff --git a/prototype/benchx/runner.py b/prototype/benchx/runner.py index 1316252..8f8bab5 100644 --- a/prototype/benchx/runner.py +++ b/prototype/benchx/runner.py @@ -145,7 +145,7 @@ def run(order_path, out_dir) -> dict: artifacts_dir = out_dir / "artifacts" / (f"slot-{slot}" if slot is not None else ref[7:19]) artifacts_dir.mkdir(parents=True, exist_ok=True) order_file = out_dir / f"workorder-{ref[7:]}.json" - order_file.write_bytes(core.canonical(order)) + order_file.write_bytes(core.pretty(order)) attempted = adapter.attempted(applied_protocol) try: @@ -249,7 +249,7 @@ def run(order_path, out_dir) -> dict: if errors: raise RuntimeError(f"the error result is itself invalid: {errors[0]}") path = out_dir / file_name(key) - path.write_bytes(core.canonical(document)) + path.write_bytes(core.pretty(document)) written.append(path) return {"order": ref, "cases": len(cases), "results": len(written), "out": str(out_dir), "files": [order_file, *written]} diff --git a/prototype/benchx/session.py b/prototype/benchx/session.py index 3a4f895..09cc6db 100644 --- a/prototype/benchx/session.py +++ b/prototype/benchx/session.py @@ -19,6 +19,7 @@ from . import compare as compare_mod from . import runner, snapshot, target as target_mod +from .identity import normalize_remote PROFILES = ("revisions", "environments") CASE_TIMEOUT_S, ORDER_TIMEOUT_S = 60, 600 @@ -50,7 +51,7 @@ def _source_uri(source_dir: str) -> str: comparison report the same source.""" origin = snapshot._run(["git", "remote", "get-url", "origin"], cwd=source_dir) if origin: - return origin + return normalize_remote(origin, base=source_dir) common = snapshot._run(["git", "rev-parse", "--git-common-dir"], cwd=source_dir) return Path(source_dir, common).resolve().parent.as_uri() if common else Path(source_dir).resolve().as_uri() @@ -157,7 +158,11 @@ def compare_targets(baseline: str, contender: str, *, profile: str, suite: str, if side["identity"]["dirty"] != "clean": raise SessionError(f"{name} tree {side['source']} is {side['identity']['dirty']}; " "a revisions comparison needs clean trees (UC-03 §10)") - uris = {source_uri or side["uri"] or _source_uri(side["source"]) for side in sides} + if source_uri: + uris = {normalize_remote(source_uri)} + else: + uris = {normalize_remote(side["uri"], base=side["source"]) if side["uri"] else _source_uri(side["source"]) + for side in sides} if len(uris) != 1: raise SessionError(f"the sides report different source URIs {sorted(uris)}; pass --source-uri") source_uri = uris.pop() @@ -174,7 +179,7 @@ def compare_targets(baseline: str, contender: str, *, profile: str, suite: str, min_time=min_time, project=project, run_key=run_key, source_uri=source_uri, round_=round_, slot=slot, labels=side_labels) path = out / "orders" / f"order-{slot:04d}.json" - path.write_text(json.dumps(order, indent=1)) + path.write_text(json.dumps(order, indent=1) + "\n") summary = runner.run(path, out) outcome = store.ingest_paths(summary["files"]) if outcome["rejected"]: diff --git a/prototype/benchx/snapshot.py b/prototype/benchx/snapshot.py index 5d8d17a..519333c 100644 --- a/prototype/benchx/snapshot.py +++ b/prototype/benchx/snapshot.py @@ -78,7 +78,7 @@ def observed_context(child_env: dict, env_names=None) -> dict: facts["governor"] = governor try: facts["load_avg_1m"] = round(os.getloadavg()[0], 2) - except OSError: + except (OSError, AttributeError): # AttributeError: no os.getloadavg on Windows pass env = {k: child_env[k] for k in (env_names or env_allowlist(child_env)) if k in child_env} if env: diff --git a/prototype/examples/archery-local/README.md b/prototype/examples/archery-local/README.md index 2e233d9..ac57368 100644 --- a/prototype/examples/archery-local/README.md +++ b/prototype/examples/archery-local/README.md @@ -31,14 +31,14 @@ installed). From the repository root: ```console $ cd prototype -$ uv venv && uv pip install -e '.[test]' # or any virtualenv with `pip install -e .` -$ cd examples/archery-local -$ ./run.sh +$ python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like +$ pip install -e . +$ examples/archery-local/run.sh ``` It takes about two minutes the first time (two CMake builds and a measured -run) and prints every command before running it, as you would type it. With -another interpreter: `PYTHON=/path/to/python ./run.sh`. More rounds: +run) and prints every command before running it, as you would type it. The +script runs `bx` and `python` from the active environment. More rounds: `ROUNDS=10 ./run.sh`. Everything it creates is under `_work/` in this directory, including the local @@ -96,8 +96,7 @@ over. ## Try it by hand After `./run.sh` (the build directories and the store stay in `_work/`). From -this directory, with the virtualenv's `bx` on your `PATH` or as -`../../.venv/bin/python -m benchx.cli`: +this directory, with the environment still active: ```console $ source activate.sh diff --git a/prototype/examples/archery-local/provider.sh b/prototype/examples/archery-local/provider.sh index ce49fbe..a5226e0 100755 --- a/prototype/examples/archery-local/provider.sh +++ b/prototype/examples/archery-local/provider.sh @@ -58,7 +58,7 @@ if [[ ! -d $SRC/.git ]]; then fi mkdir -p "$WORK/deps/lib" # stand-in for the dependencies activate.sh points at -step cmake -S "$SRC" -B "$BUILD" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" "${extra[@]}" +step cmake -S "$SRC" -B "$BUILD" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" ${extra[@]+"${extra[@]}"} step cmake --build "$BUILD" -j BUILD=$BUILD SRC=$SRC HOW=$HOW CONFIGURATION=$configuration HERE=$HERE python3 - >"$BUILD/.benchx-target.json" <<'PY' diff --git a/prototype/examples/archery-local/run.sh b/prototype/examples/archery-local/run.sh index 15b5a5b..d1e5edd 100755 --- a/prototype/examples/archery-local/run.sh +++ b/prototype/examples/archery-local/run.sh @@ -9,32 +9,32 @@ # nothing, and runs none of that text: you activate the environment yourself # (activate.sh) and benchx records the text beside the environment it inherited. # -# Every command is printed before it runs, as you would type it; `bx` stands -# for `$PYTHON -m benchx.cli`. Needs cmake and a C++ compiler on PATH, and -# network access the first time, to fetch Google Benchmark unless installed. +# Every command is printed before it runs, as you would type it. Run it in an +# activated environment that has benchx installed (from prototype/: +# `pip install -e .`, or `uv sync --extra test`). Needs cmake and a C++ +# compiler on PATH, and network access the first time, to fetch Google +# Benchmark unless installed. # # ROUNDS=5 alternation rounds (default 5) -# PYTHON the interpreter with benchx installed (default ../../.venv/bin/python) # # The local store is kept inside this directory (_work/benchx-home), not in # ~/.benchx. Delete _work/ to start over. set -euo pipefail cd "$(dirname "$0")" HERE=$PWD -PYTHON=${PYTHON:-$HERE/../../.venv/bin/python} ROUNDS=${ROUNDS:-5} export BENCHX_HOME=${BENCHX_HOME:-$HERE/_work/benchx-home} STAMP=$(date -u +%Y%m%dT%H%M%SZ) RUN="hardening-$STAMP" -for tool in cmake c++; do +for tool in cmake c++ bx python; do command -v "$tool" >/dev/null || { echo "run.sh: $tool not found on PATH (see README.md)" >&2; exit 1; } done -"$PYTHON" -c 'import benchx' 2>/dev/null || { echo "run.sh: benchx is not installed for $PYTHON (see README.md)" >&2; exit 1; } +bx --help >/dev/null 2>&1 || { echo "run.sh: benchx is not installed in the active environment (see README.md)" >&2; exit 1; } step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } say() { printf '\033[36m$ %s\033[0m\n' "$*" >&2; } -bx() { say bx "$@"; "$PYTHON" -m benchx.cli "$@"; } +bx() { say bx "$@"; command bx "$@"; } step "user: activate the environment (benchx will not do this for you)" say source activate.sh @@ -59,14 +59,14 @@ bx compare @plain @hardened --profile environments --label build=plain,hardened --suite demo-bench --rounds "$ROUNDS" --project archery-local --run-key "$RUN" --out _work/results step "the comparison names what was built and how; before, it was two opaque directory paths" -echo "# the first work order's target, as the runner received it:" -"$PYTHON" -c ' +echo "# the target of work order 0001 (round 1, the hardened side), as the runner received it:" +python -c ' import json, sys print(json.dumps(json.load(open(sys.argv[1]))["target"], indent=1))' _work/results/orders/order-0001.json step "and every result carries it, next to what the runner saw (declared, observed: nothing judges)" say bx head --json -n 1 '| …' -"$PYTHON" -m benchx.cli head --json -n 1 | "$PYTHON" -c ' +command bx head --json -n 1 | python -c ' import json, sys doc = json.loads(sys.stdin.readline()) info = doc["provenance"]["info"] @@ -79,7 +79,7 @@ print("coordinates.subject.configuration:", doc["coordinates"]["subject"].get("c step "forget to activate: the declaration still says what you should have run, the observation says what you did" say env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --rounds 3 ... -env -u LD_LIBRARY_PATH "$PYTHON" -m benchx.cli compare @plain @hardened --no-build --profile environments \ +env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --profile environments \ --label build=plain,hardened --suite demo-bench --rounds 3 --project archery-local \ --run-key "$RUN-unactivated" --out _work/results-unactivated diff --git a/prototype/examples/pyperf-bench/README.md b/prototype/examples/pyperf-bench/README.md index 9ec886f..0f9daf8 100644 --- a/prototype/examples/pyperf-bench/README.md +++ b/prototype/examples/pyperf-bench/README.md @@ -7,15 +7,16 @@ writes benchx result files, delivers them to a local store, and prints a table. ## Run it ```sh -cd prototype/examples/pyperf-bench -python3 -m pip install pyperf pyarrow jsonschema referencing rfc8785 # Python >= 3.11 -./run.sh +cd prototype +python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like +pip install -e . pyperf +examples/pyperf-bench/run.sh ``` -`PYTHON=/path/to/python ./run.sh` picks another interpreter; pyperf must be importable -by the `python` on your `PATH`, which is the one the workers use. Extra arguments go to +The script runs `bx` and `python` from the active environment, so benchx and pyperf must +be installed there; the workers use that same `python`. Extra arguments go to `bx bench` in step 1 and win over the config file, e.g. `./run.sh --processes 20`. -It takes about 15 seconds. Nothing outside this directory changes: the local store is +It takes about 20 seconds. Nothing outside this directory changes: the local store is `out/home/store.parquet` (the script sets `BENCHX_HOME`), and `out/` is deleted at the start of each run. @@ -74,8 +75,13 @@ It is the work-order `precision` block under the adapter's name: 1. `bx bench bench_suite.py` runs all five cases with the config file's precision. 2. `bx bench bench_suite.py bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2` runs one case, sets a variable for the workers, and overrides the file's process count. -3. `bx head` lists the latest rows in the local store. -4. It lists the files written. +3. `bx bench bench_suite.py bench_join --run-key example-again` runs one case a second time + under a new run key, so its series gets a second point. +4. `ls -l` and `bx head` show the local store: one Parquet file, and its latest rows. +5. `bx series` lists the series in the store. +6. `bx history` lists the points of the `bench_join` series, one per run. The script fails + if there are not two, which is what CI relies on. +7. It lists the files written. ## What comes out @@ -94,7 +100,9 @@ such as `env-mismatch`. Files, under `out/all/` and `out/env/`: -- `__wall-time.json`: one benchx result per case (schema `measurement-result`): +- `__wall-time.json`: one benchx result per case (schema `measurement-result`; + with `-e` the name also carries the percent-encoded parameters, as in + `example-env_bench_env_var_%7B%22EXAMPLE_N%22%3A%22100000%22%7D_wall-time.json`): the observations grouped by worker process, the pyperf protocol that applied, the environment, the revision of the tree, and warnings. - `workorder-.json`: the work order that was run, and `orders/order.json`, the @@ -103,15 +111,35 @@ Files, under `out/all/` and `out/env/`: script that ran the function; `.worker-env.json`: the environment a worker really had; `.stdout` and `.stderr`. -The local store, `out/home/store.parquet`, holds the results. Read it back with -`bx head`, `bx series`, `bx history`, or `bx ingest` more result files; comparing two -runs is not part of this step. +The local store, `out/home/store.parquet`, holds the results. `run.sh` reads it back +(your numbers and fingerprints will differ): + +``` +$ bx series +d780c9564c72 local/be8372c04606 bench_dict_build wall-time [s] est=benchx/median/v1 +e6eaa02c8309 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 +4be59ef6debc local/be8372c04606 bench_join wall-time [s] est=benchx/median/v1 +... +85a6aa9c6996 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 + +$ bx history 4be59ef6debc +revision order unavailable without a revision graph; listed by measurement start +d83d37b69010 2026-10-13T11:04:28.656515Z 3.32107e-06 (server) example-all:bench_join:wall-time +d83d37b69010 2026-10-13T11:04:39.045277Z 3.76987e-06 (server) example-again:bench_join:wall-time +``` + +`bench_env_var` has two series because step 2 sets `EXAMPLE_N`, which changes the +environment the case ran in. `bench_join` has one series with two points, one for each +run key. You can also `bx ingest` more result files; comparing two runs is not part of +this step. ## Notes - The tree measured is the git checkout containing the module (here, this repository), as found, uncommitted changes included. The result records the revision and whether the tree - was dirty. Outside git, it records a hash of the directory's contents. `--tree DIR` picks + was dirty. Because that tree is the whole repository, an uncommitted change anywhere in it, + in a doc or another example as much as in `bench_suite.py`, makes the result `dirty` and + changes its tree id; commit or stash first for a clean result. Outside git, it records a hash of the directory's contents. `--tree DIR` picks the tree. - pyperf prints advice when the machine is noisy; it is in `.stderr`. On a laptop, close other programs. `python -m pyperf system tune` (needs root) quiets the CPU further. diff --git a/prototype/examples/pyperf-bench/run.sh b/prototype/examples/pyperf-bench/run.sh index f742124..994016e 100755 --- a/prototype/examples/pyperf-bench/run.sh +++ b/prototype/examples/pyperf-bench/run.sh @@ -4,18 +4,17 @@ # ./run.sh uses .benchx/config.json here (5 processes x 3 values) # ./run.sh --processes 20 extra arguments go to `bx bench` and win over the file # -# Needs Python >= 3.11 with pyperf, pyarrow, jsonschema, referencing, rfc8785. -# PYTHON=/path/to/python picks the interpreter. Results go to ./out, and the -# local store to ./out/home (BENCHX_HOME), so nothing outside this directory changes. +# Run it in an activated environment that has benchx installed with pyperf +# (from prototype/: `pip install -e . pyperf`, or `uv sync --extra test`). +# Results go to ./out, and the local store to ./out/home (BENCHX_HOME), so +# nothing outside this directory changes. set -euo pipefail cd "$(dirname "$0")" -PYTHON=${PYTHON:-python3} -export PYTHONPATH="$PWD/../..${PYTHONPATH:+:$PYTHONPATH}" export BENCHX_HOME="$PWD/out/home" -bx() { printf '\033[36m$ bx %s\033[0m\n' "$*" >&2; "$PYTHON" -m benchx.cli "$@"; } +bx() { printf '\033[36m$ bx %s\033[0m\n' "$*" >&2; command bx "$@"; } -"$PYTHON" -c "import pyperf, pyarrow, jsonschema, referencing, rfc8785" 2>/dev/null || { - echo "missing dependencies: $PYTHON -m pip install pyperf pyarrow jsonschema referencing rfc8785" >&2 +command bx --help >/dev/null 2>&1 && python -c "import pyperf" 2>/dev/null || { + echo "missing dependencies: install benchx and pyperf into the active environment (see README.md)" >&2 exit 1; } rm -rf out; mkdir -p out @@ -26,8 +25,24 @@ echo; echo "== 2. one function, an environment variable set for the workers, fla bx bench bench_suite.py bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2 \ --run-key example-env --out out/env -echo; echo "== 3. what is in the local store" +echo; echo "== 3. the same case again under a new run key, so its series gets a second point" +bx bench bench_suite.py bench_join --run-key example-again --out out/again + +echo; echo "== 4. what is in the local store: one Parquet file" +ls -l "$BENCHX_HOME/store.parquet" bx head -n 10 -echo; echo "== 4. what was written" -find out/all out/env -type f | sort | sed 's/^/ /' +echo; echo "== 5. its series (bench_env_var has two: the -e setting changes the environment)" +bx series > out/series.txt +cat out/series.txt + +echo; echo "== 6. the history of bench_join: one point per run" +bx series --workload bench_join > out/join-series.txt +SERIES=$(awk 'NR == 1 {print $1}' out/join-series.txt) +[ -n "$SERIES" ] || { echo "no bench_join series in the store" >&2; exit 1; } +bx history "$SERIES" > out/history.txt +cat out/history.txt +[ "$(grep -c example- out/history.txt)" -eq 2 ] || { echo "expected two history points for $SERIES" >&2; exit 1; } + +echo; echo "== 7. what was written" +find out/all out/env out/again -type f | sort | sed 's/^/ /' diff --git a/prototype/tests/test_bench.py b/prototype/tests/test_bench.py index f43e619..76dfc60 100644 --- a/prototype/tests/test_bench.py +++ b/prototype/tests/test_bench.py @@ -5,6 +5,8 @@ import pytest +from benchx.identity import normalize_remote + pytest.importorskip("pyperf") from benchx import bench, cli, core @@ -99,3 +101,29 @@ def test_bench_rejects_a_missing_module_and_bad_env(tree, capsys): assert cli.main(["bench", "nope.py"]) == cli.EXIT_REFUSED assert cli.main(["bench", "mod.py", "-e", "NOEQUALS"]) == cli.EXIT_REFUSED assert not (tree / "results").exists() + + +@pytest.mark.parametrize("remote, uri", [ + ("git@github.com:ev-br/benchx.git", "ssh://git@github.com/~/ev-br/benchx.git"), + ("git@host:/srv/git/benchx.git", "ssh://git@host/srv/git/benchx.git"), + ("host:repos/x.git", "ssh://host/~/repos/x.git"), + ("git@[::1]:repo.git", "ssh://git@[::1]/~/repo.git"), + ("git@[::1]:/srv/repo.git", "ssh://git@[::1]/srv/repo.git"), + ("https://github.com/ev-br/benchx.git", "https://github.com/ev-br/benchx.git"), + ("ssh://git@github.com/ev-br/benchx.git", "ssh://git@github.com/ev-br/benchx.git"), + ("/srv/git/benchx.git", "file:///srv/git/benchx.git"), + ("C:\\src\\benchx", "file:///C:/src/benchx"), + ("file:///srv/git/benchx.git", "file:///srv/git/benchx.git"), +]) +def test_remotes_become_uris(remote, uri): + assert normalize_remote(remote) == uri + assert normalize_remote(uri) == uri # idempotent + + +def test_relative_local_remotes_resolve_against_base(tmp_path): + assert normalize_remote("../upstream", base=str(tmp_path / "work")) == (tmp_path / "upstream").as_uri() + + +def test_source_uri_flag_overrides_the_remote(tree): + order, _ = bench.build_order(tree / "mod.py", ["bench_a"], source_uri="https://example.org/r.git") + assert order["target"]["source"]["uri"] == "https://example.org/r.git" diff --git a/prototype/tests/test_target.py b/prototype/tests/test_target.py index e06fdd4..876de4e 100644 --- a/prototype/tests/test_target.py +++ b/prototype/tests/test_target.py @@ -178,3 +178,32 @@ def test_allowlist_covers_loader_variables_and_a_project_can_extend_it(monkeypat env["BENCHX_ENV_ALLOWLIST"] = "MY_TOOL_PREFIX, ,LD_LIBRARY_PATH" seen = snapshot.observed_context(env)["env"] assert seen == {"LD_LIBRARY_PATH": "/opt/lib", "MY_TOOL_PREFIX": "/x"} # never the whole environment + + +def _stub_sides(monkeypatch, uris): + sides = iter({"uri": u, "source": "/nonexistent", "identity": {"dirty": "clean"}, "text": {}} for u in uris) + monkeypatch.setattr(session, "_side", lambda *a, **k: next(sides)) + + +class _Stop(Exception): + pass + + +def _compare(tmp): + return session.compare_targets("a", "b", profile="revisions", suite="s", rounds=1, store=Store(tmp / "s.parquet"), + run_key="u", out=tmp / "out") + + +def test_equivalent_remote_spellings_are_one_source(monkeypatch, tmp): + _stub_sides(monkeypatch, ["git@host:/srv/x.git", "ssh://git@host/srv/x.git"]) + monkeypatch.setattr(session.runner, "run", lambda *a, **k: (_ for _ in ()).throw(_Stop())) + monkeypatch.setattr(session, "_order", lambda *a, **k: {}) + monkeypatch.setattr(session.json, "dumps", lambda *a, **k: "{}") + with pytest.raises(_Stop): # got past the source-URI check + _compare(tmp) + + +def test_different_sources_are_still_refused(monkeypatch, tmp): + _stub_sides(monkeypatch, ["git@host:/srv/x.git", "git@host:/srv/y.git"]) + with pytest.raises(session.SessionError, match="different source URIs"): + _compare(tmp)