diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 0f9dde0..567a263 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -21,25 +21,53 @@ bx series [--workload … --quantity …] # list series bx history # one series' points (a query) bx head [-n 10] [--json] # the latest results in the store, newest first bx compare --run KEY --profile revisions|environments --baseline VALUE - [--label NAME] [--results DIR] [--k 3] [--json] + [--label NAME] [--results DIR] [--k 3] [--min-rounds 3] [--json] # read mode: compare results already measured +bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) bx compare BASELINE CONTENDER --profile revisions|environments --suite NAME [--filter REGEX] [--quantity wall-time] --rounds R [--repetitions N] [--min-time S] [--project P] [--label NAME=BASE,CONTENDER] - [--run-key KEY] [--out DIR] [--k 3] [--json] + [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] + [--k 3] [--min-rounds 3] [--json] # measure mode: run both sides, then compare +bx bench MODULE [FUNCTION …] [--processes N] [--values N] [--min-time S | --loops N] [--warmups N] + [-e NAME=VALUE] [--tree DIR] [--project P] [--run-key KEY] [--source-uri URI] + [--case-timeout S] [--out DIR] [--no-ingest] + # Python functions under pyperf → result files → store → table +bx config init [--dir DIR] # write .benchx/config.json stating pyperf's defaults ``` +`bx bench` is a second front end to the same runner. It builds one work order +whose target is the `working_tree` of the git checkout containing MODULE (or +`--tree`), as found, with the dirty state and tree id recorded; the suite is the +module, a case is a function in it (`bench_*` and `benchmark_*` by default, or the +functions named), and `-e` fills `environment_variables`. Precision is resolved +before the order is written, flags over `.benchx/config.json` (found by walking up +from the current directory) over pyperf's defaults, so the order always states +what applied. The pyperf adapter takes `python` from the `PATH` the workers start +with and refuses the order if pyperf is not importable there. Delivery is as for +`bx run`. There is no comparison step: `bx bench` produces results, and comparing +them is not part of it. + `bx compare` has two modes, chosen by its arguments. Two positional targets -(`BUILD_DIR[:SOURCE_DIR]`) mean measure mode; `--run` means read mode; giving +(`BUILD_DIR[:SOURCE_DIR]` or `@CONFIG[=SOURCE_REF]`, a call to the project's +target provider) mean measure mode; `--run` means read mode; giving both or neither is an error, as is a flag that belongs to the other mode (`--baseline` and `--results` are read mode only, since measure mode derives the baseline from its first target; `--suite`, `--rounds`, `--project`, -`--run-key`, `--out`, `--repetitions`, and `--min-time` are measure mode -only). `--label` names the label in read mode, and carries the two values in +`--run-key`, `--out`, `--repetitions`, `--min-time`, `--no-build`, and +`--source-uri` are measure mode only). `--label` names the label in read mode, and carries the two values in measure mode. §5 describes measure mode. +A source URI given any way (`--source-uri`, a provider's description, the +checkout's `origin`) goes through one normalization before sides are +compared or recorded, and identity does the same: git's SCP-style +`user@host:/abs/path` becomes `ssh://user@host/abs/path`, a relative +`host:path` becomes `ssh://host/~/path` (relative to the login directory, so it +is not mistaken for the absolute `/path`), and a local path becomes a +`file://` URI. Windows drive paths and bracketed IPv6 hosts are recognized. + The CLI is a thin layer: each command is one call into the `benchx` package, so the workbench, a CI script, or a notebook can do the same without a subprocess. Measure mode is a call into the session function, which calls the @@ -62,9 +90,10 @@ not post to the original store. **Work order.** The prototype reads the work order of `runner-schema.md` (`schemas/work-order/0.1.0`, `benchx/work-order/0.1.0`) and defines no format of its own. One order is one side of one round: the runner executes one order -per invocation (`runner.md` §7), and the prototype accepts one target kind, -`build`, an existing build directory with an optional `source_dir`. The runner -never builds it (`runner.md` §1). +per invocation (`runner.md` §7), and the prototype accepts two target kinds: +`build`, an existing build directory with an optional `source_dir`, and +`working_tree`, a checkout measured as found, which only `bx bench` writes. The +runner never builds either (`runner.md` §1). Interleaving needs two order fields. `round` is native to the schema and is copied to `procedure.round`. `slot`, the realized position across all sides, @@ -203,6 +232,11 @@ landing where its §3.4 puts it: §5.5); OS, kernel, libc, governor, and load as observed context; an allowlist of thread-control environment variables as observed context, which is the allowlist of `benchmark-environments.md` §3.1. + The `env` key reads differently by adapter. Google Benchmark takes the + runner's own environment and omits `env` when no allowlisted variable is set. + pyperf records what the workers actually had, so `env` is `{}` when the + capture worked and none was set, and absent only when nothing was captured + (`env-capture-missing`). Both are kept as they are. - **Source identity**, from `target.source_dir`, or from the `CMAKE_HOME_DIRECTORY` the build directory records when the order names none: revision, `subject_dirty`, and `subject_tree` (schema §4.1). When @@ -219,6 +253,11 @@ the same way report the same configuration. Nobody declared these, so the detect values are recorded as the intended ones in `coordinates.subject.configuration` (§3.4). This is what makes the hardened and plain directories differ in coordinates, which the `environments` profile requires. + The pyperf adapter's `python_cflags` is the exception to "never paths": it is + recorded verbatim in `comparison_context.runtime`, as `harness-adapter.md` §6.8 + has it, so the same Python build installed under two prefixes reports different + flags (conda adds its `-isystem` and `-fdebug-prefix-map` paths) and lands in + different series. - **Declared facts**, from `.benchx/setup.json` or `BENCHX_SETUP_FILE` (`harness-adapter.md` §5.1), for what the cache cannot tell, such as how a dependency was installed. The prototype has no placement rules, so they @@ -380,9 +419,15 @@ compares them; it is the workbench's session loop (`system-decomposition.md` first value of `--label NAME=BASE,CONTENDER` for `environments`. It is exactly `bx run` 2R times followed by read-mode `bx compare`; it adds no -document, schema, or contract. It builds nothing: both targets must exist, -and whatever produced them plays the target-provider role -(`system-decomposition.md` §3.4). If it stops partway, the results already +document, schema, or contract. It builds nothing itself: both targets must exist +when it looks. A target given as `@CONFIG` is first requested from the +project's target provider (`system-decomposition.md` §3.4), which `prepare`s +it (and may build it) or, with `--no-build`, only `describe`s it; a target +given as a build directory is read from its `.benchx-target.json` sidecar if +it has one. The description's declared text (`how_built`, `activation`, +`shell`, `provider`) is copied into each order, recorded in each result's +`provenance.info`, and shown in the comparison; it is never executed or +checked. If it stops partway, the results already delivered stay in the store, and a read-mode comparison of the run key reports missing results or a failed alternation invariant, as UC-03 §10 requires. Allowing a local-only `revisions` comparison of a dirty tree is open @@ -394,7 +439,10 @@ Both profiles use one shape: two prepared build directories, one run key, R rounds, and `bx compare` in measure mode, which alternates the sides and assigns `round` and `slot`. The demo script plays the user, and in that role the target provider: it prepares the sources and build directories with git -and CMake, since benchx builds nothing. +and CMake, since benchx builds nothing. `prototype/examples/archery-local` +shows the other route, where a provider script named in +`.benchx/provider.json` builds and describes the targets and the sides are +given as `@plain` and `@hardened`. - **Revisions:** two git worktrees of one source at the base and head commits, each built into its own directory with the same configuration. @@ -425,12 +473,17 @@ history grow from run to run, and the demo ends with `bx head`. ## 7. Deliberate simplifications -- One harness and one target kind. The runner builds nothing, ever: the user +- Two harnesses (Google Benchmark, and pyperf through `bx bench`) and two target + kinds (`build`, `working_tree`). The runner builds nothing, ever: the user prepares both sides (`benchmark-environments.md` §2). The prototype applies `environment_variables` and defers the rest of the environment policy, and it refuses an order that names one rather than ignoring it (§3). -- No target-provider hook: the demo prepares its builds by hand, and the - provider contract is open (`system-decomposition.md` §6 question 5). +- The target-provider hook is minimal: one provider command per project + (`.benchx/provider.json`), `describe` and `prepare` only, `build` targets + only, and a source ref other than `WORKSPACE` is the provider's business. + How cached builds are named and expired stays open + (`system-decomposition.md` §6 question 5). The demo prepares its builds by + hand. - Environment identity is the snapshot's host name, CPU model, and core count; continuity events, quarantine, and artifact storage are absent. - One identity policy and one derived estimator, fixed in code. diff --git a/docs/design/prototype-scope.md b/docs/design/prototype-scope.md index fd04ccd..c25bff8 100644 --- a/docs/design/prototype-scope.md +++ b/docs/design/prototype-scope.md @@ -40,11 +40,11 @@ one run: | Component | In prototype | Deferred | Out of scope | |---|---|---|---| -| Runner | execute a `build` target per a work order; apply the order's `environment_variables` to the launched process; capture the zero-configuration snapshot, the build configuration, and source identity; account for planned cases; refuse an order it cannot apply exactly, including one that names a deferred field | `python_env` targets, context collectors beyond the built-in ones, environment policy (enforcing launch settings, verifying requested hardware, refusing an unsatisfiable run) | building revisions, tuning the machine (`benchmark-environments.md` §2.3) | -| Adapter | Google Benchmark, both halves: drive filter and repetitions, translate output | all other harnesses | — | +| Runner | execute a `build` or `working_tree` target per a work order; apply the order's `environment_variables` to the launched process; capture the zero-configuration snapshot, the build configuration, and source identity; account for planned cases; refuse an order it cannot apply exactly, including one that names a deferred field | `python_env` targets, context collectors beyond the built-in ones, environment policy (enforcing launch settings, verifying requested hardware, refusing an unsatisfiable run) | building revisions, tuning the machine (`benchmark-environments.md` §2.3) | +| Adapter | Google Benchmark, both halves: drive filter and repetitions, translate output; pyperf, both halves, for `bx bench` (Python functions in the working tree; no comparison step) | all other harnesses; pyperf's `tags`, `date`, per-run load and frequency, and `pyperf check` | — | | Store | one Parquet file; ingest per schema §5.4 (validated and durable, auto-create, unit-conflict rejection, idempotency, attempt integrity); one default identity policy; queries: list series, series history, get result | continuity events, quarantine, artifact storage, identity-policy authoring | — | | Comparator | run mode with the `revisions` and `environments` profiles; eligibility guard; comparison document | history mode; `single-run`, `variants`, and `cross-machine` profiles; batch requests; provisional noise rules | — | -| Workbench session loop | `bx compare` in measure mode: writes one work order per side and round, alternates them, delivers the results, then compares; takes each side as a build directory or as `@CONFIG`, a call to the project's target provider (`describe` or `prepare`); `bx target` asks for a description directly; copies the description's declared text (`how_built`, `activation`, `shell`, `provider`) into the order | provider-built revisions (a source ref other than `WORKSPACE` is the provider's business), `artifact` and `working_tree` targets, a declared `build` | workbench packaging, front ends such as `spin bench` | +| Workbench session loop | `bx compare` in measure mode: writes one work order per side and round, alternates them, delivers the results, then compares; takes each side as a build directory or as `@CONFIG`, a call to the project's target provider (`describe` or `prepare`); `bx target` asks for a description directly; copies the description's declared text (`how_built`, `activation`, `shell`, `provider`) into the order | provider-built revisions (a source ref other than `WORKSPACE` is the provider's business), an `artifact` target, `working_tree` targets in `bx compare`, a declared `build` | workbench packaging, front ends such as `spin bench` | | Documents | work order (`schemas/work-order/0.1.0`), result (schema §5.2), comparison document | — | — | **Out entirely:** scheduler, change detector, alerting, views beyond query diff --git a/prototype/README.md b/prototype/README.md index 1fd4361..db59853 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -22,6 +22,11 @@ bx compare BASELINE CONTENDER --profile revisions|environments [--run-key KEY] [--out DIR] [--no-build] [--k 3] [--json] # measure mode: run both sides, then compare bx validate … [--kind KIND] # every problem in each document, by schema and rules +bx bench MODULE [FUNCTION …] [--processes N] [--values N] [--min-time S | --loops N] [--warmups N] + [-e NAME=VALUE] [--tree DIR] [--project P] [--run-key KEY] [--source-uri URI] + [--case-timeout S] [--out DIR] [--no-ingest] + # Python functions under pyperf: order → results → store → table +bx config init [--dir DIR] # write .benchx/config.json stating pyperf's defaults ``` Each target is `BUILD_DIR[:SOURCE_DIR]` or `@CONFIG[=SOURCE_REF]`. Two targets @@ -41,6 +46,18 @@ executed or checked: you activate the environment yourself, and the runner records the allowlisted variables it inherited beside your declaration. `examples/archery-local` is a complete walk-through. +`bx bench` is the other entry point: it benchmarks the functions of a Python file +with [pyperf](https://pyperf.readthedocs.io), as one `working_tree` order for the +checkout containing the module (uncommitted changes included, recorded as dirty). +Each `bench_*` or `benchmark_*` function, or the ones named, is a case; `-e` sets +environment variables for the workers and lands in the workload's parameters. +Precision resolves as flags, then `.benchx/config.json` (found by walking up from the +current directory; `bx config init` writes a template), then pyperf's defaults, and the +resolved values are written into the order. The workers run the `python` first on +`PATH`, which must have pyperf. It builds the order, runs it through the same runner and +delivers to the same store as `bx run`; it has no comparison step. +`examples/pyperf-bench` is a runnable walk-through. + ## What it does - **Work orders** are the `docs/design/runner-schema.md` schema @@ -86,7 +103,7 @@ records the allowlisted variables it inherited beside your declaration. ```console $ uv venv && uv pip install -e '.[test]' # editable: schemas are read from ../schemas -$ .venv/bin/python -m pytest tests # 222 tests, fake Google Benchmark binary, no compiler +$ .venv/bin/python -m pytest tests # 228 tests, fake Google Benchmark binary, no compiler $ ./demo.sh # real builds; needs cmake, a C++ compiler, network once ``` @@ -134,6 +151,9 @@ benchx/ hashing.py canonical form, hashes, order reference snapshot.py what the runner records (benchmark-environments.md §3) adapters/gbench.py Google Benchmark, driving and translating halves + adapters/pyperf.py pyperf, both halves, for Python functions (bx bench) + bench.py bx bench: module + flags + config.json → one working_tree work order + config.py .benchx/config.json: the pyperf precision defaults and their template runner.py one work order in, results out session.py the session loop: alternate two targets over rounds, then compare target.py target descriptions: the provider call, the sidecar, the declared text @@ -144,6 +164,7 @@ benchx/ cli.py bx (compare has a read mode and a measure mode) ../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) examples/demo-suite a two-benchmark Google Benchmark suite for the demo +examples/pyperf-bench Python functions under `bx bench` (own README) examples/archery-local an end-to-end, Arrow-local-shaped example with a target provider (own README) tests/ test_prototype.py: the success criteria and the runner and store rules, on a fake Google Benchmark binary; test_comparator.py: the diff --git a/prototype/benchx/bench.py b/prototype/benchx/bench.py index 9f4d73c..3271078 100644 --- a/prototype/benchx/bench.py +++ b/prototype/benchx/bench.py @@ -14,6 +14,7 @@ from pathlib import Path from . import config, core, runner, snapshot +from .identity import normalize_remote from .session import CASE_TIMEOUT_S, ORDER_TIMEOUT_S, SessionError, _source_uri ADAPTER = "pyperf" @@ -50,7 +51,7 @@ def flags_to_precision(processes=None, values=None, min_time=None, loops=None, w def build_order(module, functions, *, tree=None, precision_flags=None, environment_variables=None, - project=None, run_key=None, labels=None, case_s=CASE_TIMEOUT_S, order_s=ORDER_TIMEOUT_S, cwd="."): + project=None, run_key=None, labels=None, source_uri=None, case_s=CASE_TIMEOUT_S, order_s=ORDER_TIMEOUT_S, cwd="."): """The resolved work order, and the config file it read (or None).""" module = Path(module).resolve() if not module.is_file(): @@ -71,7 +72,7 @@ def build_order(module, functions, *, tree=None, precision_flags=None, environme "state": "resolved", "suites": [entry], "target": {"kind": "working_tree", "path": str(root), "source_dir": str(root), - "source": {"uri": _source_uri(str(root)), "type": "git"}}, + "source": {"uri": normalize_remote(source_uri) if source_uri else _source_uri(str(root)), "type": "git"}}, "quantities": ["wall-time"], "precision": precision, "timeouts": {"case_s": case_s, "order_s": order_s}, diff --git a/prototype/benchx/cli.py b/prototype/benchx/cli.py index d987b52..577db8a 100644 --- a/prototype/benchx/cli.py +++ b/prototype/benchx/cli.py @@ -62,7 +62,8 @@ def cmd_bench(args): flags = bench_mod.flags_to_precision(args.processes, args.values, args.min_time, args.loops, args.warmups) order, config_path = bench_mod.build_order( args.module, args.functions, tree=args.tree, precision_flags=flags, environment_variables=env, - project=args.project, run_key=args.run_key, case_s=args.case_timeout) + project=args.project, run_key=args.run_key, source_uri=args.source_uri, + case_s=args.case_timeout) out = Path(args.out) if args.out else Path("results") / order["run_key"] reps = {lv["unit"]: lv["n"] for lv in order["precision"]["repetitions"]["levels"]} print(f"precision: {reps['process']} processes x {reps['value']} values " @@ -131,12 +132,16 @@ def cmd_head(args): print(row["document"]) return EXIT_OK print(f"{store.path}: {total} result(s); latest {len(rows)}, newest first") + cells = [] for r in rows: median = f"{r['median']:.6g} {r['unit']}" if r["median"] is not None else "-" started = r["started_at"].strftime("%Y-%m-%d %H:%M:%S") labels = f" {r['labels']}" if r["labels"] else "" - print(f"{r['row']:>5} {started} {r['run_key']:<24} {r['workload']:<16} {r['quantity']:<9} " - f"{r['status']:<8} {median:<14} {r['revision'][:10]} {r['project']}{labels}") + cells.append((f"{r['row']:>5}", started, r["run_key"], r["workload"], r["quantity"], r["status"], median, + r["revision"][:10], f"{r['project']}{labels}")) + widths = [max([len(cell[i]) for cell in cells], default=0) for i in range(len(cells[0]))] if cells else [] + for cell in cells: + print(" ".join(text.ljust(width) for text, width in zip(cell, widths)).rstrip()) return EXIT_OK @@ -282,6 +287,7 @@ def main(argv=None): help="set an environment variable for the benchmark (repeatable)") p.add_argument("--project", help="name a project, so results join its series; omit for an ad hoc run") p.add_argument("--run-key", help="default: bench-") + p.add_argument("--source-uri", help="the source's URI (default: the checkout's origin remote)") p.add_argument("--out", help="directory for the order and result files (default ./results/RUN_KEY)") p.add_argument("--case-timeout", type=float, default=bench_mod.CASE_TIMEOUT_S, help="seconds allowed per function (default %(default)s)") diff --git a/prototype/benchx/identity.py b/prototype/benchx/identity.py index 03e459c..d4cc667 100644 --- a/prototype/benchx/identity.py +++ b/prototype/benchx/identity.py @@ -1,7 +1,9 @@ """Fingerprints, the prototype identity policy, and series points (schema §4.3, §4.5).""" +import re import statistics import urllib.parse +from pathlib import Path from .core import canonical, sha256_hex @@ -26,8 +28,33 @@ def reported_coordinates_fingerprint(doc: dict) -> str: return _hash("benchmark-reported-coordinates-v1", canonical(reported_coordinates(doc))) +_WINDOWS_PATH = re.compile(r"^[A-Za-z]:[\\/]") +_SCP_REMOTE = re.compile(r"^(?:(?P[^@/:\s\[\]]+)@)?(?P\[[0-9A-Fa-f:.]+\]|[^@/:\s\[\]]+):(?P.+)$") + + +def normalize_remote(remote: str, base: str | None = None) -> str: + """A git remote as a URI, whatever form it was given in; idempotent. + + A `scheme://` remote is unchanged. A Windows drive path or a local path + (resolved against `base`, default the current directory) becomes a + `file://` URI. git's SCP-style `[user@]host:path` becomes `ssh://`: an + absolute path stays absolute, and a relative one (relative to the login + directory) is kept relative as `ssh://host/~/path`, since `ssh://host/path` + would name a different repository.""" + if not remote or "://" in remote: + return remote + if _WINDOWS_PATH.match(remote): + return "file:///" + urllib.parse.quote(remote.replace("\\", "/"), safe="/:") + match = _SCP_REMOTE.match(remote) + if match: + user = f"{match['user']}@" if match["user"] else "" + path = match["path"] if match["path"].startswith("/") else "/~/" + match["path"] + return f"ssh://{user}{match['host']}{path}" + return Path(base or ".", remote).resolve().as_uri() + + def _normalize_uri(uri: str) -> str: - parts = urllib.parse.urlsplit(uri) + parts = urllib.parse.urlsplit(normalize_remote(uri)) path = parts.path.rstrip("/").removesuffix(".git") return urllib.parse.urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, "", "")) diff --git a/prototype/benchx/session.py b/prototype/benchx/session.py index a926202..09cc6db 100644 --- a/prototype/benchx/session.py +++ b/prototype/benchx/session.py @@ -19,6 +19,7 @@ from . import compare as compare_mod from . import runner, snapshot, target as target_mod +from .identity import normalize_remote PROFILES = ("revisions", "environments") CASE_TIMEOUT_S, ORDER_TIMEOUT_S = 60, 600 @@ -50,7 +51,7 @@ def _source_uri(source_dir: str) -> str: comparison report the same source.""" origin = snapshot._run(["git", "remote", "get-url", "origin"], cwd=source_dir) if origin: - return origin + return normalize_remote(origin, base=source_dir) common = snapshot._run(["git", "rev-parse", "--git-common-dir"], cwd=source_dir) return Path(source_dir, common).resolve().parent.as_uri() if common else Path(source_dir).resolve().as_uri() @@ -157,7 +158,11 @@ def compare_targets(baseline: str, contender: str, *, profile: str, suite: str, if side["identity"]["dirty"] != "clean": raise SessionError(f"{name} tree {side['source']} is {side['identity']['dirty']}; " "a revisions comparison needs clean trees (UC-03 §10)") - uris = {source_uri or side["uri"] or _source_uri(side["source"]) for side in sides} + if source_uri: + uris = {normalize_remote(source_uri)} + else: + uris = {normalize_remote(side["uri"], base=side["source"]) if side["uri"] else _source_uri(side["source"]) + for side in sides} if len(uris) != 1: raise SessionError(f"the sides report different source URIs {sorted(uris)}; pass --source-uri") source_uri = uris.pop() diff --git a/prototype/examples/archery-local/run.sh b/prototype/examples/archery-local/run.sh index ff6c919..d1e5edd 100755 --- a/prototype/examples/archery-local/run.sh +++ b/prototype/examples/archery-local/run.sh @@ -59,7 +59,7 @@ bx compare @plain @hardened --profile environments --label build=plain,hardened --suite demo-bench --rounds "$ROUNDS" --project archery-local --run-key "$RUN" --out _work/results step "the comparison names what was built and how; before, it was two opaque directory paths" -echo "# the first work order's target, as the runner received it:" +echo "# the target of work order 0001 (round 1, the hardened side), as the runner received it:" python -c ' import json, sys print(json.dumps(json.load(open(sys.argv[1]))["target"], indent=1))' _work/results/orders/order-0001.json diff --git a/prototype/examples/pyperf-bench/README.md b/prototype/examples/pyperf-bench/README.md index 2b8342e..0f9daf8 100644 --- a/prototype/examples/pyperf-bench/README.md +++ b/prototype/examples/pyperf-bench/README.md @@ -100,7 +100,9 @@ such as `env-mismatch`. Files, under `out/all/` and `out/env/`: -- `__wall-time.json`: one benchx result per case (schema `measurement-result`): +- `__wall-time.json`: one benchx result per case (schema `measurement-result`; + with `-e` the name also carries the percent-encoded parameters, as in + `example-env_bench_env_var_%7B%22EXAMPLE_N%22%3A%22100000%22%7D_wall-time.json`): the observations grouped by worker process, the pyperf protocol that applied, the environment, the revision of the tree, and warnings. - `workorder-.json`: the work order that was run, and `orders/order.json`, the @@ -135,7 +137,9 @@ this step. - The tree measured is the git checkout containing the module (here, this repository), as found, uncommitted changes included. The result records the revision and whether the tree - was dirty. Outside git, it records a hash of the directory's contents. `--tree DIR` picks + was dirty. Because that tree is the whole repository, an uncommitted change anywhere in it, + in a doc or another example as much as in `bench_suite.py`, makes the result `dirty` and + changes its tree id; commit or stash first for a clean result. Outside git, it records a hash of the directory's contents. `--tree DIR` picks the tree. - pyperf prints advice when the machine is noisy; it is in `.stderr`. On a laptop, close other programs. `python -m pyperf system tune` (needs root) quiets the CPU further. diff --git a/prototype/tests/test_bench.py b/prototype/tests/test_bench.py index f43e619..76dfc60 100644 --- a/prototype/tests/test_bench.py +++ b/prototype/tests/test_bench.py @@ -5,6 +5,8 @@ import pytest +from benchx.identity import normalize_remote + pytest.importorskip("pyperf") from benchx import bench, cli, core @@ -99,3 +101,29 @@ def test_bench_rejects_a_missing_module_and_bad_env(tree, capsys): assert cli.main(["bench", "nope.py"]) == cli.EXIT_REFUSED assert cli.main(["bench", "mod.py", "-e", "NOEQUALS"]) == cli.EXIT_REFUSED assert not (tree / "results").exists() + + +@pytest.mark.parametrize("remote, uri", [ + ("git@github.com:ev-br/benchx.git", "ssh://git@github.com/~/ev-br/benchx.git"), + ("git@host:/srv/git/benchx.git", "ssh://git@host/srv/git/benchx.git"), + ("host:repos/x.git", "ssh://host/~/repos/x.git"), + ("git@[::1]:repo.git", "ssh://git@[::1]/~/repo.git"), + ("git@[::1]:/srv/repo.git", "ssh://git@[::1]/srv/repo.git"), + ("https://github.com/ev-br/benchx.git", "https://github.com/ev-br/benchx.git"), + ("ssh://git@github.com/ev-br/benchx.git", "ssh://git@github.com/ev-br/benchx.git"), + ("/srv/git/benchx.git", "file:///srv/git/benchx.git"), + ("C:\\src\\benchx", "file:///C:/src/benchx"), + ("file:///srv/git/benchx.git", "file:///srv/git/benchx.git"), +]) +def test_remotes_become_uris(remote, uri): + assert normalize_remote(remote) == uri + assert normalize_remote(uri) == uri # idempotent + + +def test_relative_local_remotes_resolve_against_base(tmp_path): + assert normalize_remote("../upstream", base=str(tmp_path / "work")) == (tmp_path / "upstream").as_uri() + + +def test_source_uri_flag_overrides_the_remote(tree): + order, _ = bench.build_order(tree / "mod.py", ["bench_a"], source_uri="https://example.org/r.git") + assert order["target"]["source"]["uri"] == "https://example.org/r.git" diff --git a/prototype/tests/test_target.py b/prototype/tests/test_target.py index e06fdd4..876de4e 100644 --- a/prototype/tests/test_target.py +++ b/prototype/tests/test_target.py @@ -178,3 +178,32 @@ def test_allowlist_covers_loader_variables_and_a_project_can_extend_it(monkeypat env["BENCHX_ENV_ALLOWLIST"] = "MY_TOOL_PREFIX, ,LD_LIBRARY_PATH" seen = snapshot.observed_context(env)["env"] assert seen == {"LD_LIBRARY_PATH": "/opt/lib", "MY_TOOL_PREFIX": "/x"} # never the whole environment + + +def _stub_sides(monkeypatch, uris): + sides = iter({"uri": u, "source": "/nonexistent", "identity": {"dirty": "clean"}, "text": {}} for u in uris) + monkeypatch.setattr(session, "_side", lambda *a, **k: next(sides)) + + +class _Stop(Exception): + pass + + +def _compare(tmp): + return session.compare_targets("a", "b", profile="revisions", suite="s", rounds=1, store=Store(tmp / "s.parquet"), + run_key="u", out=tmp / "out") + + +def test_equivalent_remote_spellings_are_one_source(monkeypatch, tmp): + _stub_sides(monkeypatch, ["git@host:/srv/x.git", "ssh://git@host/srv/x.git"]) + monkeypatch.setattr(session.runner, "run", lambda *a, **k: (_ for _ in ()).throw(_Stop())) + monkeypatch.setattr(session, "_order", lambda *a, **k: {}) + monkeypatch.setattr(session.json, "dumps", lambda *a, **k: "{}") + with pytest.raises(_Stop): # got past the source-URI check + _compare(tmp) + + +def test_different_sources_are_still_refused(monkeypatch, tmp): + _stub_sides(monkeypatch, ["git@host:/srv/x.git", "git@host:/srv/y.git"]) + with pytest.raises(session.SessionError, match="different source URIs"): + _compare(tmp)