fix: e2e harness restores the stack, runs via uv

.github/workflows/ci.yml

@@ -118,6 +118,15 @@ jobs:
       - name: Tests
         run: make check-tests PYTEST=pytest
 
+      # `make test-e2e` runs its scripts with `uv run --locked`, and Renovate's
+      # pep723 manager can bump a script's pin but has no step that rewrites its
+      # lock, so a pin and lock that disagree are caught here, not mid-suite.
+      - name: Install uv
+        run: pipx install --python "${{ steps.python.outputs.python-path }}" "uv==$(make -s print-uv-version)"
+
+      - name: Script locks
+        run: make check-locks LOCK_UV=uv
+
   # The slim and single profiles are projections of the Kubernetes tiers of the
   # same name, so they go stale when those tiers move and nothing here notices.
   # dfe-infra is PRIVATE, so the checkout reads it with a short-lived GitHub App

Makefile

@@ -52,7 +52,7 @@ endif
 # any more (both sweep every profile), and requiring a resolvable profile to STOP
 # a stack is the same lockout the compose secret comments argue against -- set
 # DFE_PROFILE to something that does not exist and you could not tear down.
-BOOTSTRAP_GOALS := init env-files creds dev-posture help login stack dial modes down clean limits check check-compose check-hardfail check-dockerfile check-docs check-proxy check-python check-tests
+BOOTSTRAP_GOALS := init env-files creds dev-posture help login stack dial modes down clean limits check check-compose check-hardfail check-dockerfile check-docs check-locks check-proxy check-python check-tests
 
 # Resolve the active profile only when a goal actually needs the compose stack
 ifneq (,$(filter-out $(BOOTSTRAP_GOALS),$(or $(MAKECMDGOALS),help)))
@@ -536,7 +536,7 @@ infra: .env storage-dirs ## Start infrastructure services
 # ---------------------------------------------------------------------------
 
 .PHONY: check
-check: check-compose check-hardfail check-dockerfile check-docs check-proxy check-python check-tests ## Run every static check CI runs
+check: check-compose check-hardfail check-dockerfile check-docs check-proxy check-python check-tests check-locks ## Run every static check CI runs
 
 .PHONY: limits
 limits: ## Show the resource limits and totals, computed from the resolved config
@@ -571,6 +571,20 @@ print-pytest-version:
 check-tests: ## Run the helper-script unit tests (scripts/tests)
 	$(PYTEST) -q scripts/tests
 
+# Each inline-metadata script runs with `uv run --locked`, and Renovate's pep723 manager rewrites a pin but never its lock, so this fails a pin and lock that disagree.
+# Pinned and installed in CI the way RUFF is (LOCK_UV=uv).
+UV_VERSION := 0.13.0
+LOCK_UV ?= uvx uv@$(UV_VERSION)
+
+.PHONY: print-uv-version
+print-uv-version:
+	@echo $(UV_VERSION)
+
+.PHONY: check-locks
+check-locks: ## Assert each inline-metadata script's lock matches its pins (relock with `uv lock --script <script>`)
+	$(LOCK_UV) lock --check --script scripts/test_e2e.py
+	$(LOCK_UV) lock --check --script scripts/test_flows.py
+
 # `slim` and `single` are projections of the Kubernetes tiers of the same name,
 # so both targets need DFE_INFRA_DIR pointed at a dfe-infra checkout. Unset,
 # check-profiles reports SKIPPED rather than passing: it cannot read the master.
@@ -646,19 +660,22 @@ check-proxy: ## Validate the Envoy proxy configs, plain and TLS, on the pinned E
 # The suite starts each test's stack from its own -f list. Under console TLS that stack needs the TLS fragment and the network it pins, so the suite takes every fragment `make ci` chains, the exposure opt-outs and container logs included; with TLS off it keeps its own list. DFE_E2E_COMPOSE_FILES follows either way, and the suite skips the empty entries the colons leave.
 E2E_COMPOSE_FILES = $(if $(filter true,$(strip $(DFE_PROXY_TLS))),$(UI_CHAIN)):$(strip $(DFE_E2E_COMPOSE_FILES))
 
+# The scripts that import packages beyond the standard library declare them inline (PEP 723) and lock them beside the script, so uv installs exactly the hashed set and a host needs only uv.
+UV ?= uv
+
 .PHONY: test-e2e
 test-e2e: ## End-to-end test executor (pass test names via E2E_TESTS)
-	@DFE_E2E_COMPOSE_FILES='$(E2E_COMPOSE_FILES)' python3 ./scripts/test_e2e.py $(E2E_TESTS)
+	@DFE_E2E_COMPOSE_FILES='$(E2E_COMPOSE_FILES)' $(UV) run --locked --script ./scripts/test_e2e.py $(E2E_TESTS)
 
 # Opt-in, never part of test-e2e: each test stops a backing service or a DFE app
 # under load.
 .PHONY: test-resilience
 test-resilience: ## Outage tests: stop a service under load and prove the rest survives (pass test names via E2E_TESTS)
-	@DFE_E2E_COMPOSE_FILES='$(E2E_COMPOSE_FILES)' python3 ./scripts/test_e2e.py --outages $(E2E_TESTS)
+	@DFE_E2E_COMPOSE_FILES='$(E2E_COMPOSE_FILES)' $(UV) run --locked --script ./scripts/test_e2e.py --outages $(E2E_TESTS)
 
 .PHONY: test-flows
 test-flows: ## Flow shapes against a running stack (needs DFE_ENGINE_REPO; FLOW_ARGS passes flags)
-	@python3 ./scripts/test_flows.py $(FLOW_ARGS)
+	@$(UV) run --locked --script ./scripts/test_flows.py $(FLOW_ARGS)
 
 # The post-deploy source test: add a source through the console and prove every
 # hop. The runner is dfe-infra's, the same one `dfe-ops acceptance --suite

docs/developing.md

@@ -27,7 +27,7 @@ exist, and how a profile decides which of them run.
 |---|---|
 | Docker and Docker Compose v2 | everything |
 | Python 3 | the helper scripts under `scripts/` |
-| PyYAML | `make test-e2e` and `make test-resilience` only (`pip install pyyaml`, or `uv run --with pyyaml`) |
+| uv | `make test-e2e`, `make test-resilience` and `make test-flows` only. It installs the PyYAML and Playwright those scripts declare inline, hash-checked against the `<script>.lock` beside each |
 | ruff | `make check-python` only |
 | pytest | `make check-tests` only (`uvx` fetches the pin if you have it) |
 | git credentials for the hyperi-io repos (or `DFE_SRC_ROOT` checkouts) | `make dev` |
@@ -324,6 +324,10 @@ An archiver that subscribes to the test's landing topic and archives to a
 since a baseline taken before the send. The archive volume outlives the stack, so
 files from an earlier run never count.
 
+The data file goes to the ingest edge 16 requests at a time. The receiver answers each request once the loader has flushed it, so one at a time costs a flush per event.
+
+The suite stops whatever stack the project is running, so it reads the project first. On every way out, a failure or a SIGTERM included, it starts each service that was running or had run to completion again, from the compose files it was started from. A last verdict fails if one comes back on a different config hash. Run it as the user that started the stack: `make` writes that user's GID into the engine's config, and under console TLS its UID and GID into both proxies'.
+
 Under [console TLS](configuration.md#console-tls-opt-in) each test's stack chains the fragments `make ci` does, the exposure opt-outs and container logs included. The console checks go to `https://<DFE_EXTERNAL_ORIGIN host>:<DFE_UI_PORT>` and verify its certificate by chain and name, against `DFE_PROXY_CA_BUNDLE` when a private CA signed it. A certificate that does not verify fails the check with the variable to change.
 
 Two things about the runner worth knowing before you debug it:
@@ -355,7 +359,7 @@ The `core` footprint starts `dfe-hunt-runner`, as `make dev` does. On a profile
 ### Search to rule to hunt -- the console path
 
 ```bash
-uv run --with pyyaml --with playwright==1.63.0 python3 scripts/test_e2e.py search-rule-hunt
+make test-e2e E2E_TESTS=search-rule-hunt
 ```
 
 A `detection:` block carrying `from_search` makes its rule the way an analyst does, in a browser. It adds HyperDX to the stack, signs in to the console as the stack's admin and opens Observe search on `main`. The search bar gets the block's `search`, held to the run's marker, and the side panel picks its `filter`. Then Create Rule, and the rule page it opens.
@@ -364,7 +368,7 @@ The stored rule has to scan `dfe.main` with no SQL errors and a WHERE that carri
 
 The side panel only offers values rows already hold, so both event sets are sent once before the search. The hunt's first window can reach those rows, so their near misses are judged too, and their matches are not.
 
-The console holds every page behind its setup wizard, so the test first makes the `e2e` organisation and one user of the deployment's own, as the wizard would. It needs an interpreter with Playwright and Google Chrome (`playwright install chrome` where Chrome is absent). `DFE_E2E_HEADED=1` shows the browser, and `DFE_E2E_SHOTS_DIR` keeps the step screenshots, which otherwise go to a new temporary directory named in the log.
+The console holds every page behind its setup wizard, so the test first makes the `e2e` organisation and one user of the deployment's own, as the wizard would. uv installs Playwright from the script's inline metadata, and the test drives Google Chrome (`uvx playwright install chrome` where it is absent). `DFE_E2E_HEADED=1` shows the browser, and `DFE_E2E_SHOTS_DIR` keeps the step screenshots, which otherwise go to a new temporary directory named in the log.
 
 Under console TLS the suite first verifies the console certificate itself, then pins Chrome to that certificate's public key, because Chrome takes no CA bundle.
 
@@ -431,6 +435,7 @@ that is a finding about the console rather than about the pipeline.
 | `make check-proxy` | `envoy --mode validate` on both proxy configs and their console TLS variants, on the Envoy the stack pins |
 | `make check-python` | `ruff check` and `ruff format --check` on `scripts/` |
 | `make check-tests` | `pytest scripts/tests` -- unit tests over the credential helpers |
+| `make check-locks` | `uv lock --check` on each script that declares packages inline, so a pin moved without `uv lock --script <script>` fails here rather than mid-suite |
 
 `check-compose` is hermetic: it reads the mandatory `${VAR:?}` keys out of the
 compose file and substitutes placeholders, so it needs neither the stack SSoT nor
@@ -443,9 +448,9 @@ for why a lower ceiling takes health endpoints dark.
 external URLs and does not check `#anchors`, so a live file with a stale anchor
 still passes.
 
-`check-dockerfile` pulls the pinned hadolint image and `check-tests` resolves the
-pinned pytest through `uvx`, so both want a network the first time. The rest need
-none.
+`check-dockerfile` pulls the pinned hadolint image, `check-tests` resolves the
+pinned pytest through `uvx` and `check-locks` the pinned uv, so those want a
+network the first time. The rest need none.
 
 `check-proxy` pulls the pinned Envoy image the same way and mints a throwaway P-384 certificate with the host's `openssl` for the TLS variants.
 

docs/observability.md

@@ -209,6 +209,8 @@ Node metrics are still not collected.
   through the API reaches the running receiver, and a record matching its rule
   lands in a table of its own.
 
+Once every claim has run, the three ingest rows are deleted: a lightweight `DELETE`, or a mutation (`ALTER TABLE ... DELETE`) on a table whose projections refuse one. A delete that does not finish within 60s is reported, never failed.
+
 The freshness window is what makes the second claim mean "streaming now" rather
 than "streamed once". The first two are the pair the Kubernetes bootstrap smoke
 asserts as CORE 1 and CORE 2.

docs/troubleshooting.md

@@ -367,16 +367,17 @@ nothing set and requiring that it fails for the right reason. A green run proves
 the stack refuses to resolve with nothing set; it does not prove every single
 image is pinned, because compose aborts on the first missing variable.
 
-### `PyYAML is required and not installed`
+### `PyYAML is not installed in this interpreter`
 
-PyYAML is the e2e harness's only third-party dependency, and it is not in the
-standard library:
+The e2e harness declares PyYAML and Playwright inline (PEP 723), locked in `scripts/test_e2e.py.lock`, and `make test-e2e` runs it through uv, which installs them. A bare `python3 scripts/test_e2e.py` skips that step:
 
 ```bash
-pip install pyyaml
-uv run --with pyyaml python3 scripts/test_e2e.py   # or run it this way
+make test-e2e
+uv run --locked --script scripts/test_e2e.py   # the same, without make
 ```
 
+`The lockfile at uv.lock needs to be updated` from either means a pin in the script moved without its lock: run `uv lock --script scripts/test_e2e.py` and commit both.
+
 The power-on self test (`scripts/post.py`) has no such dependency -- it is
 stdlib-only and works on a bare machine.
 

scripts/_pipeline.py

@@ -26,6 +26,8 @@
 import re
 import time
 import typing
+from concurrent.futures import ThreadPoolExecutor
+from http.client import HTTPException
 from urllib.error import URLError
 from urllib.request import Request, urlopen
 
@@ -169,6 +171,26 @@ def http_post(
     )[0]
 
 
+def post_all(
+    url: str, bodies: typing.Sequence[str], *, workers: int, timeout: int = 10
+) -> list[int]:
+    """POST every body from `workers` concurrent senders, returning each status in order.
+
+    The receiver answers once the next hop holds the record, so one request at a
+    time waits out the loader's flush on every event. A request that got no HTTP
+    answer at all reports status 0.
+    """
+
+    def _send(body: str) -> int:
+        try:
+            return http_post(url, body, timeout=timeout)
+        except (HTTPException, OSError):
+            return 0
+
+    with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
+        return list(pool.map(_send, bodies))
+
+
 def http_delete(url: str, token: str = "", timeout: int = 10) -> int:
     """DELETE a resource and return the HTTP status.
 
@@ -200,6 +222,26 @@ def ch_query(sql: str, *, debug: typing.Callable[[str], None] | None = None) ->
         return ""
 
 
+def ch_execute(sql: str, *, timeout: int = 10) -> tuple[bool, str]:
+    """Run a statement over ClickHouse's HTTP interface, keeping the server's answer.
+
+    Returns (True, body) on success and (False, the error text) otherwise, for a
+    caller that has to act on WHY a statement was refused.
+    """
+    request = Request(clickhouse_url(), data=sql.encode(), method="POST")
+    request.add_header("X-ClickHouse-User", env_or("CLICKHOUSE_USERNAME", "default"))
+    request.add_header("X-ClickHouse-Key", os.environ.get("CLICKHOUSE_PASSWORD", ""))
+    try:
+        with urlopen(request, timeout=timeout) as response:
+            return True, response.read().decode().strip()
+    except URLError as error:
+        if hasattr(error, "read"):
+            return False, error.read().decode(errors="replace").strip()
+        return False, str(error.reason)
+    except OSError as error:
+        return False, str(error)
+
+
 def ch_count(database: str, table: str, *, debug=None) -> int:
     """Return the row count of a table, or 0 if it cannot be read."""
     count = ch_int(f"SELECT count() FROM {database}.{table}", debug=debug)

scripts/_search_rule.py

@@ -59,9 +59,9 @@
 # An option in whichever console select is open.
 OPEN_OPTION = ".ant-select-dropdown:not(.ant-select-dropdown-hidden) .ant-select-item-option-content"
 PLAYWRIGHT_HINT = (
-    "run the suite under an interpreter carrying Playwright, e.g. "
-    "`uv run --with pyyaml --with playwright python3 scripts/test_e2e.py`, and "
-    "`playwright install chrome` where Google Chrome is not installed"
+    "run the suite through uv, which installs the Playwright it locks "
+    "(`make test-e2e`, or `uv run --locked --script scripts/test_e2e.py`), and install "
+    "Google Chrome where it is absent (`uvx playwright install chrome`)"
 )