Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions devtools/command_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -759,6 +759,24 @@ def to_dict(self) -> dict[str, object]:
"devtools bench slo --skip-benchmarks --json",
),
),
CommandSpec(
"bench help-latency",
"benchmarking",
"Check `--help` wall-clock latency against the interactive-tier cold-CLI budget (polylogue-20d.2).",
"devtools.help_latency_probe",
use_when=(
"Catch CLI import-tax regressions continuously. Runs `polylogue <cmd> --help` for a curated "
"set of root and nested subcommands as fresh subprocesses and compares the minimum wall time "
"against the 700ms cold-CLI budget from the 20d.14 interactive SLO tier. Fails when any "
"'required' target exceeds budget; 'informational' targets (currently `ops maintenance`, "
"known slow pending a lazy-import refactor) are reported but never block."
),
examples=(
"devtools bench help-latency",
"devtools bench help-latency --json",
"devtools bench help-latency --repeats 5 --out .local/help-latency.json",
),
),
CommandSpec(
"verify manifests",
"verification",
Expand Down
158 changes: 158 additions & 0 deletions devtools/help_latency_probe.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
"""Measure ``--help`` wall-clock latency against the interactive-tier cold-CLI budget.

Use ``devtools bench help-latency`` to catch import-tax regressions on the
CLI ``--help`` path continuously (polylogue-20d.2). The interactive SLO tier
(polylogue-20d.14, ``docs/plans/slo-catalog.yaml``) states a <700ms budget for
a cold (no warm daemon) CLI invocation; ``--help`` is the cheapest possible
invocation of any command (no archive I/O, no query execution) so it is the
tightest floor on Python/import overhead. A regression here means every
daemonless invocation of that command pays the same tax.

Each target is run as a fresh subprocess (``python -m polylogue.cli <args>``)
several times; the *minimum* wall time is compared against budget rather than
the mean, because process-launch jitter (scheduler contention, page-cache
misses) only ever adds latency on a shared dev host, never subtracts it. This
mirrors the host-variable framing of the other ``devtools bench`` probes:
wall-clock is diagnostic and campaign-comparable, but the budget comparison
here IS a CI gate for "required" targets (unlike the wall-clock-only probes),
because import cost is deterministic given the source tree, not host load.

Targets marked ``gate="informational"`` are measured and reported but never
fail the check — they document a known-slow path with an open follow-up
(currently: the ``ops maintenance`` command group; see the module docstring
in ``polylogue/cli/commands/maintenance.py``).
"""

from __future__ import annotations

import argparse
import json
import subprocess
import sys
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from time import perf_counter
from typing import Literal, cast

Gate = Literal["required", "informational"]


@dataclass(frozen=True, slots=True)
class HelpLatencyTarget:
label: str
args: tuple[str, ...]
budget_ms: int
gate: Gate


# Budget follows the 20d.14 interactive-tier "cold CLI (no daemon) <700ms"
# line in docs/plans/slo-catalog.yaml.
_DEFAULT_BUDGET_MS = 700

TARGETS: tuple[HelpLatencyTarget, ...] = (
HelpLatencyTarget("root", ("--help",), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("find", ("find", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("read", ("read", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("mark", ("mark", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("select", ("select", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("analyze", ("analyze", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("import", ("import", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("config", ("config", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("dashboard", ("dashboard", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("ops", ("ops", "--help"), _DEFAULT_BUDGET_MS, "required"),
HelpLatencyTarget("reset", ("reset", "--help"), _DEFAULT_BUDGET_MS, "required"),
# `ops maintenance` (any leaf) fully imports the ~2800-line maintenance
# command module at module scope (ArchiveStore, blob_gc, blob_integrity,
# embeddings.reconcile, migration_runner, ...) because `_LazyGroup`
# resolution needs the module to enumerate subcommands. Known slow;
# informational until the module's imports are pushed into per-command
# bodies. Do not silently promote to "required" without doing that work
# first — it will flap the gate red on every host.
HelpLatencyTarget("ops-maintenance", ("ops", "maintenance", "--help"), _DEFAULT_BUDGET_MS, "informational"),
HelpLatencyTarget(
"ops-maintenance-archive-read",
("ops", "maintenance", "archive-read", "--help"),
_DEFAULT_BUDGET_MS,
"informational",
),
)


def _time_invocation(args: tuple[str, ...], *, repeats: int) -> float:
"""Return the minimum wall-clock ms across ``repeats`` fresh subprocess runs."""

best: float | None = None
for _ in range(repeats):
started = perf_counter()
subprocess.run(
[sys.executable, "-m", "polylogue.cli", *args],
check=False,
capture_output=True,
text=True,
)
elapsed_ms = (perf_counter() - started) * 1_000
if best is None or elapsed_ms < best:
best = elapsed_ms
assert best is not None
return best


def measure(*, repeats: int = 3, targets: tuple[HelpLatencyTarget, ...] = TARGETS) -> dict[str, object]:
results = []
for target in targets:
elapsed_ms = _time_invocation(target.args, repeats=repeats)
within_budget = elapsed_ms <= target.budget_ms
results.append(
{
"label": target.label,
"args": list(target.args),
"gate": target.gate,
"budget_ms": target.budget_ms,
"elapsed_ms": round(elapsed_ms, 1),
"within_budget": within_budget,
}
)
violations = [r["label"] for r in results if r["gate"] == "required" and not r["within_budget"]]
return {
"version": 1,
"generated_at": datetime.now(UTC).isoformat(),
"repeats": repeats,
"results": results,
"violations": violations,
"ok": not violations,
}


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repeats", type=int, default=3, help="Subprocess runs per target; minimum wins.")
parser.add_argument("--json", action="store_true", help="Emit the full JSON report instead of a table.")
parser.add_argument("--out", type=Path, default=None, help="Also write the JSON report to this path.")
args = parser.parse_args(argv)

report = measure(repeats=max(1, args.repeats))

if args.out is not None:
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")

if args.json:
print(json.dumps(report, indent=2, sort_keys=True))
else:
results = cast("list[dict[str, object]]", report["results"])
for result in results:
marker = "OK" if result["within_budget"] else "OVER"
flag = "" if result["gate"] == "required" else " (informational)"
print(f"{marker:>4} {result['label']:<32} {result['elapsed_ms']:>7.1f}ms / {result['budget_ms']}ms{flag}")
violations = cast("list[str]", report["violations"])
if violations:
print(f"\nBudget violations (required): {', '.join(violations)}")
else:
print("\nAll required targets within budget.")

return 0 if report["ok"] else 1


if __name__ == "__main__":
raise SystemExit(main())
2 changes: 1 addition & 1 deletion docs/cli-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -683,7 +683,7 @@ Commands:
action-affordances Print shared query-action affordance metadata as JSON.
completions Generate shell completion scripts.
paths Print canonical archive paths and filesystem topology.
query-completions Print shared query-builder completion metadata as...
query-completions Print shared query-builder completion metadata as JSON.
```

## Completions
Expand Down
1 change: 1 addition & 0 deletions docs/devtools.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ These are the commands worth remembering during normal repo work:
| --- | --- |
| `devtools bench campaign` | Run or compare benchmark campaigns. |
| `devtools bench coordination-latency` | Measure compact coordination status p50/p95 with raw stage samples. |
| `devtools bench help-latency` | Check `--help` wall-clock latency against the interactive-tier cold-CLI budget (polylogue-20d.2). |
| `devtools bench ingest-amplification` | Measure deterministic per-tier ingest write amplification on a synthetic fixture (#1851). |
| `devtools bench ingest-throughput` | Measure ingest wall-clock throughput on a synthetic fixture. |
| `devtools bench memory` | Measure query-memory envelopes on generated fixtures. |
Expand Down
43 changes: 43 additions & 0 deletions docs/plans/slo-catalog.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,3 +79,46 @@ surfaces:
p95_ms: 60000
gate: "informational"
tier: "lab"

# --- Interactive tier (polylogue-20d.14) -----------------------------
#
# Named latency budgets for the CLI-to-daemon hot path (polylogue-20d.1).
# These are the epic's own measurement contract: sibling beads (daemon
# result cache 20d.12, SSE push 20d.13, ingest-to-searchable lag 20d.6)
# cite these budgets rather than inventing their own. Only the surfaces
# that are actually implemented and benchmarkable today are "required";
# the rest are informational placeholders naming the bead that must land
# before the row can gate (mirrors the existing context/cost pattern
# above, which defers to #838/#803/#870 the same way).

daemon_cli_query:
description: "CLI-to-daemon UDS round trip for an ordinary find-mode session page (20d.1 fast path)"
benchmark_test: "tests/benchmarks/test_daemon_uds.py::test_bench_daemon_uds_cli_query"
p50_ms: 100
p95_ms: 400
gate: "required"
tier: "cheap-local"

daemon_health_probe:
description: "Config-matched UDS health probe every fast-path call pays before the query itself"
benchmark_test: "tests/benchmarks/test_daemon_uds.py::test_bench_daemon_uds_health_probe"
p50_ms: 30
p95_ms: 100
gate: "required"
tier: "cheap-local"

daemon_cached_facets:
description: "Cached facets/status read from the daemon result-cache memo layer (deferred until 20d.12 lands)"
benchmark_test: "tests/benchmarks/test_reader_api.py::test_bench_reader_facets"
p50_ms: 30
p95_ms: 100
gate: "informational"
tier: "cheap-local"

ingest_to_searchable:
description: "JSONL write to find/webui visibility, end to end (deferred until 20d.6/20d.12/20d.13 land)"
benchmark_test: "tests/benchmarks/test_daemon_convergence.py::test_convergence_single_file_perf"
p50_ms: 5000
p95_ms: 10000
gate: "informational"
tier: "lab"
4 changes: 4 additions & 0 deletions docs/plans/test-clock-allowlist.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,7 @@ files:
reason: "Daemon resilience integration tests (#1735) measure real elapsed wall-clock for process lifecycle events (SIGKILL delivery, subprocess startup, concurrency timing). frozen_clock cannot substitute for real time when waiting on OS process state."
- path: tests/integration/test_ingest_pipeline_correctness.py
reason: "Ingest pipeline correctness tests (#1736) use time.monotonic for real process wait/retry timing that cannot be satisfied by a frozen clock."
- path: tests/benchmarks/test_daemon_uds.py
reason: "polylogue-20d.1 daemon UDS benchmark fixture polls a real background-thread HTTP server's readiness with a bounded wall-clock deadline; frozen_clock cannot substitute for waiting on real socket/thread startup."
- path: tests/unit/cli/test_daemon_golden_parity.py
reason: "polylogue-20d.1 golden-parity test starts a real UDS daemon server in a background thread and polls its readiness with a bounded wall-clock deadline before comparing direct vs daemon-proxied CLI output; frozen_clock cannot substitute for waiting on real socket/thread startup."
47 changes: 46 additions & 1 deletion polylogue/cli/archive_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -1313,6 +1313,47 @@ def _daemon_query_pairs(query_params: Mapping[str, object]) -> Iterable[tuple[st
yield key, str(value)


_DAEMON_LIST_ITEM_KEEP_KEYS = (
"id",
"origin",
"title",
"target_ref",
"anchor",
"actions",
"created_at",
"updated_at",
"message_count",
"tags",
"summary",
"words",
"repo",
"cwd_display",
"flags",
)


def _normalize_daemon_list_item(item: Mapping[str, object]) -> dict[str, object]:
"""Reshape a daemon web-reader session row into the CLI's native list-row shape.

The daemon's ``/api/sessions`` wire contract (``_archive_summary_payload`` /
``_do_list`` in ``daemon/http.py``) is the stable webui row shape —
``word_count`` naming, a ``date`` convenience field, and a ``session_id``
duplicate of ``id``. The CLI's direct-path renderer
(``archive_query._summary_payload`` -> ``SessionListRowPayload``) predates
that contract and uses ``words`` with no ``date``/``session_id`` fields.
Golden parity (polylogue-20d.1) requires the two to render identically, so
the CLI-side proxy adapts the wire shape here rather than either surface
changing its stable contract.
"""

normalized = dict(item)
if "words" not in normalized and "word_count" in normalized:
normalized["words"] = normalized.get("word_count")
return {
key: normalized[key] for key in _DAEMON_LIST_ITEM_KEEP_KEYS if key in normalized and normalized[key] is not None
}


def _emit_daemon_list_payload(
payload: Mapping[str, object],
*,
Expand All @@ -1322,7 +1363,11 @@ def _emit_daemon_list_payload(
origin: str | None,
fields: str | None,
) -> None:
items = [dict(item) for item in cast(list[object], payload.get("items") or []) if isinstance(item, Mapping)]
items = [
_normalize_daemon_list_item(item)
for item in cast(list[object], payload.get("items") or [])
if isinstance(item, Mapping)
]
total = _object_int(payload.get("total") or len(items))
next_offset = offset + limit if total > offset + limit else None
envelope: dict[str, object] = {
Expand Down
49 changes: 39 additions & 10 deletions polylogue/cli/commands/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,7 @@

import click

from polylogue.cli.commands.completions import (
action_affordances_command,
completions_command,
query_completions_command,
)
from polylogue.cli.commands.paths import paths_command
from polylogue.cli.click_command_registration import _LazyCommand
from polylogue.cli.shared.types import AppEnv


Expand Down Expand Up @@ -38,10 +33,44 @@ def config_command(ctx: click.Context, output_format: str, show_layers: bool) ->
_show_config(env, output_format, show_layers)


config_command.add_command(completions_command)
config_command.add_command(query_completions_command)
config_command.add_command(action_affordances_command)
config_command.add_command(paths_command)
# Deferred: `completions.py` transitively imports the whole insights/storage
# stack (polylogue.operations.action_contracts -> operations.archive ->
# insights.archive -> storage.repair), ~650ms alone. `config --help` only
# needs each subcommand's name + short help, not its implementation, so these
# register as lazy proxies (matches the root-level pattern in
# click_command_registration.py) instead of importing eagerly at module scope.
config_command.add_command(
_LazyCommand(
"completions",
"polylogue.cli.commands.completions",
"completions_command",
short_help="Generate shell completion scripts.",
)
)
config_command.add_command(
_LazyCommand(
"query-completions",
"polylogue.cli.commands.completions",
"query_completions_command",
short_help="Print shared query-builder completion metadata as JSON.",
)
)
config_command.add_command(
_LazyCommand(
"action-affordances",
"polylogue.cli.commands.completions",
"action_affordances_command",
short_help="Print shared query-action affordance metadata as JSON.",
)
)
config_command.add_command(
_LazyCommand(
"paths",
"polylogue.cli.commands.paths",
"paths_command",
short_help="Print canonical archive paths and filesystem topology.",
)
)


def _show_config(env: AppEnv, output_format: str, show_layers: bool) -> None:
Expand Down
4 changes: 2 additions & 2 deletions polylogue/daemon/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -2748,8 +2748,8 @@ def _archive_summary_payload(self, summary: ArchiveSessionSummary) -> dict[str,
"updated_at": summary.updated_at,
"message_count": summary.message_count,
"word_count": summary.word_count,
"repo": None,
"cwd_display": None,
"repo": summary.git_repository_url,
"cwd_display": next(iter(summary.working_directories), None),
"tags": list(summary.tags),
"flags": None,
"summary": None,
Expand Down
Loading