Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ THOROUGH ?= false
docs \
docs-check \
lint \
test-render \
test-gateway-runtime \
create \
delete \
floci \
Expand Down Expand Up @@ -110,6 +112,12 @@ lint: ## Lint and template-render the helm chart
helm template $(RELEASE) $(CHART) > /dev/null
@echo "All helm checks passed."

test-render: ## Run deterministic chart regression tests (Python + PyYAML)
$(or $(PYTHON),python3) helm/scripts/test_render.py

test-gateway-runtime: ## Exercise Envoy retries against a fault-injecting backend (Docker + PyYAML)
$(or $(PYTHON),python3) helm/scripts/test_gateway_runtime.py

##@ Local kind cluster (single-node dev)

create: ## Create a local kind cluster
Expand Down
2 changes: 1 addition & 1 deletion helm/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ The Helm chart itself, packaged and published as `firebolt-instance` to `oci://g
- `templates/namespace.yaml` — optional Namespace.
- `templates/tests/test-*.yaml` — `helm test` Pod manifests. Each is independent; `helm test` runs them in declaration order.
- `files/memlock-setup.sh` — the sidecar script loaded into the engine pod when `engineSpec.memlockSetup: true`.
- `scripts/validate-chart.sh` — full validation pipeline used by CI and runnable locally.
- `scripts/validate-chart.sh` — full validation pipeline used by CI and runnable locally, including `scripts/test_render.py` (Helm + Python/PyYAML). `make test-gateway-runtime` uses Docker to exercise the rendered retry policy against a backend that disconnects after accepting a mutation.
- `charts/` — populated by `helm dependency build`; contents are not committed.

## Adding a new value
Expand Down
107 changes: 107 additions & 0 deletions helm/scripts/test_gateway_runtime.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
"""Run the chart's Envoy retry policy against a deliberately disconnecting backend.

Requires Docker, Helm, and PyYAML. Only the created test container is removed.
The fixture substitutes the upstream address and removes routing/discovery filters;
the chart's route and retry policy are exercised unmodified.
"""
import http.client
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import unittest

import yaml
from test_render import envoy_config, render, resource


class RetryRuntimeTests(unittest.TestCase):
def test_delivered_reset_is_not_replayed_but_drained_response_is(self):
calls = {"/reset": 0, "/drained": 0}

class Backend(BaseHTTPRequestHandler):
def log_message(self, *_args):
pass

def do_POST(self):
self.rfile.read(int(self.headers.get("Content-Length", 0)))
calls[self.path] += 1
if self.path == "/reset":
# The mutation has happened. Close before response headers.
self.close_connection = True
return
drained = calls[self.path] == 1
self.send_response(503 if drained else 200)
if drained:
self.send_header("X-Firebolt-Drained", "true")
self.send_header("Content-Length", "0")
self.end_headers()

server = ThreadingHTTPServer(("0.0.0.0", 0), Backend)
threading.Thread(target=server.serve_forever, daemon=True).start()
config = envoy_config()
listener = config["static_resources"]["listeners"][0]
hcm = listener["filter_chains"][0]["filters"][0]["typed_config"]
hcm["http_filters"] = [f for f in hcm["http_filters"] if f["name"] in {
"envoy.filters.http.health_check", "envoy.filters.http.router",
}]
config["static_resources"]["clusters"] = [{
"name": "dynamic_forward_proxy", "type": "LOGICAL_DNS", "connect_timeout": "1s",
"dns_lookup_family": "V4_ONLY",
"load_assignment": {"cluster_name": "dynamic_forward_proxy", "endpoints": [{
"lb_endpoints": [{"endpoint": {"address": {"socket_address": {
"address": "host.docker.internal", "port_value": server.server_port,
}}}}],
}]},
}]
config["static_resources"]["listeners"] = [listener]
image = resource(render(), "Deployment", "audit-gateway")["spec"]["template"]["spec"]["containers"][0]["image"]
container = None
try:
with tempfile.TemporaryDirectory(prefix="firebolt-gateway-test-") as temp:
path = Path(temp) / "envoy.yaml"
path.write_text(yaml.safe_dump(config))
container = subprocess.check_output([
"docker", "run", "-d",
*(["--add-host=host.docker.internal:host-gateway"] if sys.platform == "linux" else []),
"-p", "127.0.0.1::8080", "-v", f"{path}:/etc/envoy/envoy.yaml:ro",
image, "envoy", "-c", "/etc/envoy/envoy.yaml", "--concurrency", "1",
], text=True).strip()
port = int(subprocess.check_output(["docker", "port", container, "8080/tcp"], text=True).strip().rsplit(":", 1)[1])

def request(method, path, body=None):
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=20)
try:
conn.request(method, path, body=body)
response = conn.getresponse()
response.read()
return response.status
finally:
conn.close()

deadline = time.monotonic() + 30
while True:
try:
if request("GET", "/healthz") == 200:
break
except (OSError, http.client.HTTPException):
pass
if time.monotonic() >= deadline:
self.fail(subprocess.check_output(["docker", "logs", container], text=True, stderr=subprocess.STDOUT))
time.sleep(0.1)
self.assertEqual(request("POST", "/reset", "mutation"), 503)
self.assertEqual(calls["/reset"], 1, subprocess.check_output(["docker", "logs", container], text=True, stderr=subprocess.STDOUT))
self.assertEqual(request("POST", "/drained", "mutation"), 200)
self.assertEqual(calls["/drained"], 2, "the pre-work fence should be retried")
finally:
if container:
subprocess.run(["docker", "rm", "-f", container], check=True, stdout=subprocess.DEVNULL)
server.shutdown()
server.server_close()


if __name__ == "__main__":
unittest.main()
48 changes: 48 additions & 0 deletions helm/scripts/test_render.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Chart behavior regressions. Requires Helm and PyYAML (also used by yamllint)."""
import json
import os
from pathlib import Path
import subprocess
import unittest

import yaml

CHART = Path(__file__).resolve().parents[1]


def render(values=None, release="audit", namespace="audit", expect_error=False):
result = subprocess.run(
[os.environ.get("HELM", "helm"), "template", release, str(CHART),
"--namespace", namespace, "-f", "-"],
input=json.dumps(values or {}), text=True, capture_output=True,
)
if expect_error:
if result.returncode == 0:
raise AssertionError("invalid values rendered successfully")
return result.stderr
if result.returncode:
raise AssertionError(result.stderr)
return [doc for doc in yaml.safe_load_all(result.stdout) if doc]


def resource(docs, kind, name):
return next(d for d in docs if d["kind"] == kind and d["metadata"]["name"] == name)


def envoy_config(values=None):
return yaml.safe_load(resource(render(values), "ConfigMap", "audit-gateway")["data"]["envoy.yaml"])


class GatewaySafetyTests(unittest.TestCase):
def test_retries_do_not_replay_delivered_requests(self):
config = envoy_config()
hcm = config["static_resources"]["listeners"][0]["filter_chains"][0]["filters"][0]["typed_config"]
policy = hcm["route_config"]["virtual_hosts"][0]["routes"][0]["route"]["retry_policy"]
self.assertEqual(set(policy["retry_on"].split(",")), {
"connect-failure", "refused-stream", "reset-before-request", "retriable-headers",
})
self.assertEqual(policy["retriable_headers"], [{"name": "X-Firebolt-Drained", "present_match": True}])


if __name__ == "__main__":
unittest.main()
3 changes: 3 additions & 0 deletions helm/scripts/validate-chart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -153,5 +153,8 @@ done
echo ""

echo "==================================="

# PyYAML is installed with yamllint in CI. Override PYTHON for a local venv.
"${PYTHON:-python3}" "$SCRIPT_DIR/test_render.py"
echo "✓ All validation checks passed!"
echo "==================================="
6 changes: 4 additions & 2 deletions helm/templates/gateway-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ data:
cluster: dynamic_forward_proxy
timeout: 0s
# Retry contract:
# - connect-failure / refused-stream / reset:
# - connect-failure / refused-stream / reset-before-request:
# transport-level failures where the engine
# could not have observed the request, so a
# retry cannot duplicate side effects.
Expand All @@ -148,6 +148,8 @@ data:
# between SIGTERM and the next health-check
# ejection would propagate a 503 to the
# client.
# Never retry a generic reset: the engine may have
# accepted and mutated state before disconnecting.
# We deliberately do NOT match bare 5xx: a
# 5xx returned by the engine after work has
# started may have applied side effects.
Expand All @@ -165,7 +167,7 @@ data:
# short exponential back-off lets a DNS refresh or
# health-check ejection take effect between tries.
retry_policy:
retry_on: connect-failure,refused-stream,reset,retriable-headers
retry_on: connect-failure,refused-stream,reset-before-request,retriable-headers
retriable_headers:
- name: X-Firebolt-Drained
present_match: true
Expand Down