diff --git a/Makefile b/Makefile index f688c1e..fd04d6b 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,8 @@ THOROUGH ?= false docs \ docs-check \ lint \ + test-render \ + test-gateway-runtime \ create \ delete \ floci \ @@ -110,6 +112,12 @@ lint: ## Lint and template-render the helm chart helm template $(RELEASE) $(CHART) > /dev/null @echo "All helm checks passed." +test-render: ## Run deterministic chart regression tests (Python + PyYAML) + $(or $(PYTHON),python3) helm/scripts/test_render.py + +test-gateway-runtime: ## Exercise Envoy retries against a fault-injecting backend (Docker + PyYAML) + $(or $(PYTHON),python3) helm/scripts/test_gateway_runtime.py + ##@ Local kind cluster (single-node dev) create: ## Create a local kind cluster diff --git a/helm/AGENTS.md b/helm/AGENTS.md index 33632bf..9c41855 100644 --- a/helm/AGENTS.md +++ b/helm/AGENTS.md @@ -62,7 +62,7 @@ The Helm chart itself, packaged and published as `firebolt-instance` to `oci://g - `templates/namespace.yaml` — optional Namespace. - `templates/tests/test-*.yaml` — `helm test` Pod manifests. Each is independent; `helm test` runs them in declaration order. - `files/memlock-setup.sh` — the sidecar script loaded into the engine pod when `engineSpec.memlockSetup: true`. -- `scripts/validate-chart.sh` — full validation pipeline used by CI and runnable locally. +- `scripts/validate-chart.sh` — full validation pipeline used by CI and runnable locally, including `scripts/test_render.py` (Helm + Python/PyYAML). `make test-gateway-runtime` uses Docker to exercise the rendered retry policy against a backend that disconnects after accepting a mutation. - `charts/` — populated by `helm dependency build`; contents are not committed. ## Adding a new value diff --git a/helm/scripts/test_gateway_runtime.py b/helm/scripts/test_gateway_runtime.py new file mode 100644 index 0000000..6f5e472 --- /dev/null +++ b/helm/scripts/test_gateway_runtime.py @@ -0,0 +1,107 @@ +"""Run the chart's Envoy retry policy against a deliberately disconnecting backend. + +Requires Docker, Helm, and PyYAML. Only the created test container is removed. +The fixture substitutes the upstream address and removes routing/discovery filters; +the chart's route and retry policy are exercised unmodified. +""" +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +import yaml +from test_render import envoy_config, render, resource + + +class RetryRuntimeTests(unittest.TestCase): + def test_delivered_reset_is_not_replayed_but_drained_response_is(self): + calls = {"/reset": 0, "/drained": 0} + + class Backend(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", 0))) + calls[self.path] += 1 + if self.path == "/reset": + # The mutation has happened. Close before response headers. + self.close_connection = True + return + drained = calls[self.path] == 1 + self.send_response(503 if drained else 200) + if drained: + self.send_header("X-Firebolt-Drained", "true") + self.send_header("Content-Length", "0") + self.end_headers() + + server = ThreadingHTTPServer(("0.0.0.0", 0), Backend) + threading.Thread(target=server.serve_forever, daemon=True).start() + config = envoy_config() + listener = config["static_resources"]["listeners"][0] + hcm = listener["filter_chains"][0]["filters"][0]["typed_config"] + hcm["http_filters"] = [f for f in hcm["http_filters"] if f["name"] in { + "envoy.filters.http.health_check", "envoy.filters.http.router", + }] + config["static_resources"]["clusters"] = [{ + "name": "dynamic_forward_proxy", "type": "LOGICAL_DNS", "connect_timeout": "1s", + "dns_lookup_family": "V4_ONLY", + "load_assignment": {"cluster_name": "dynamic_forward_proxy", "endpoints": [{ + "lb_endpoints": [{"endpoint": {"address": {"socket_address": { + "address": "host.docker.internal", "port_value": server.server_port, + }}}}], + }]}, + }] + config["static_resources"]["listeners"] = [listener] + image = resource(render(), "Deployment", "audit-gateway")["spec"]["template"]["spec"]["containers"][0]["image"] + container = None + try: + with tempfile.TemporaryDirectory(prefix="firebolt-gateway-test-") as temp: + path = Path(temp) / "envoy.yaml" + path.write_text(yaml.safe_dump(config)) + container = subprocess.check_output([ + "docker", "run", "-d", + *(["--add-host=host.docker.internal:host-gateway"] if sys.platform == "linux" else []), + "-p", "127.0.0.1::8080", "-v", f"{path}:/etc/envoy/envoy.yaml:ro", + image, "envoy", "-c", "/etc/envoy/envoy.yaml", "--concurrency", "1", + ], text=True).strip() + port = int(subprocess.check_output(["docker", "port", container, "8080/tcp"], text=True).strip().rsplit(":", 1)[1]) + + def request(method, path, body=None): + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=20) + try: + conn.request(method, path, body=body) + response = conn.getresponse() + response.read() + return response.status + finally: + conn.close() + + deadline = time.monotonic() + 30 + while True: + try: + if request("GET", "/healthz") == 200: + break + except (OSError, http.client.HTTPException): + pass + if time.monotonic() >= deadline: + self.fail(subprocess.check_output(["docker", "logs", container], text=True, stderr=subprocess.STDOUT)) + time.sleep(0.1) + self.assertEqual(request("POST", "/reset", "mutation"), 503) + self.assertEqual(calls["/reset"], 1, subprocess.check_output(["docker", "logs", container], text=True, stderr=subprocess.STDOUT)) + self.assertEqual(request("POST", "/drained", "mutation"), 200) + self.assertEqual(calls["/drained"], 2, "the pre-work fence should be retried") + finally: + if container: + subprocess.run(["docker", "rm", "-f", container], check=True, stdout=subprocess.DEVNULL) + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + unittest.main() diff --git a/helm/scripts/test_render.py b/helm/scripts/test_render.py new file mode 100644 index 0000000..a2e5ccb --- /dev/null +++ b/helm/scripts/test_render.py @@ -0,0 +1,48 @@ +"""Chart behavior regressions. Requires Helm and PyYAML (also used by yamllint).""" +import json +import os +from pathlib import Path +import subprocess +import unittest + +import yaml + +CHART = Path(__file__).resolve().parents[1] + + +def render(values=None, release="audit", namespace="audit", expect_error=False): + result = subprocess.run( + [os.environ.get("HELM", "helm"), "template", release, str(CHART), + "--namespace", namespace, "-f", "-"], + input=json.dumps(values or {}), text=True, capture_output=True, + ) + if expect_error: + if result.returncode == 0: + raise AssertionError("invalid values rendered successfully") + return result.stderr + if result.returncode: + raise AssertionError(result.stderr) + return [doc for doc in yaml.safe_load_all(result.stdout) if doc] + + +def resource(docs, kind, name): + return next(d for d in docs if d["kind"] == kind and d["metadata"]["name"] == name) + + +def envoy_config(values=None): + return yaml.safe_load(resource(render(values), "ConfigMap", "audit-gateway")["data"]["envoy.yaml"]) + + +class GatewaySafetyTests(unittest.TestCase): + def test_retries_do_not_replay_delivered_requests(self): + config = envoy_config() + hcm = config["static_resources"]["listeners"][0]["filter_chains"][0]["filters"][0]["typed_config"] + policy = hcm["route_config"]["virtual_hosts"][0]["routes"][0]["route"]["retry_policy"] + self.assertEqual(set(policy["retry_on"].split(",")), { + "connect-failure", "refused-stream", "reset-before-request", "retriable-headers", + }) + self.assertEqual(policy["retriable_headers"], [{"name": "X-Firebolt-Drained", "present_match": True}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/helm/scripts/validate-chart.sh b/helm/scripts/validate-chart.sh index 69da655..e1167ee 100755 --- a/helm/scripts/validate-chart.sh +++ b/helm/scripts/validate-chart.sh @@ -153,5 +153,8 @@ done echo "" echo "===================================" + +# PyYAML is installed with yamllint in CI. Override PYTHON for a local venv. +"${PYTHON:-python3}" "$SCRIPT_DIR/test_render.py" echo "✓ All validation checks passed!" echo "===================================" diff --git a/helm/templates/gateway-configmap.yaml b/helm/templates/gateway-configmap.yaml index c8124dc..95307e4 100644 --- a/helm/templates/gateway-configmap.yaml +++ b/helm/templates/gateway-configmap.yaml @@ -133,7 +133,7 @@ data: cluster: dynamic_forward_proxy timeout: 0s # Retry contract: - # - connect-failure / refused-stream / reset: + # - connect-failure / refused-stream / reset-before-request: # transport-level failures where the engine # could not have observed the request, so a # retry cannot duplicate side effects. @@ -148,6 +148,8 @@ data: # between SIGTERM and the next health-check # ejection would propagate a 503 to the # client. + # Never retry a generic reset: the engine may have + # accepted and mutated state before disconnecting. # We deliberately do NOT match bare 5xx: a # 5xx returned by the engine after work has # started may have applied side effects. @@ -165,7 +167,7 @@ data: # short exponential back-off lets a DNS refresh or # health-check ejection take effect between tries. retry_policy: - retry_on: connect-failure,refused-stream,reset,retriable-headers + retry_on: connect-failure,refused-stream,reset-before-request,retriable-headers retriable_headers: - name: X-Firebolt-Drained present_match: true