Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion .github/workflows/core-protocol.yml
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,40 @@ jobs:
MK06_GO_FIXTURE="$RUNNER_TEMP/mk06-go-fixture" python3 contracts/proto/tests/process_recovery.py 2>&1 | tee "$RUNNER_TEMP/events-process.log"
- name: Native protobuf round trips and failure recovery
if: ${{ !cancelled() }}
run: python3 -B scripts/test-core-rpc-docker.py 2>&1 | tee "$RUNNER_TEMP/core-rpc-docker.log"
run: |
# Absolute, symlink-free, and absent: the fixture creates a private empty 0700 root.
evidence="$(realpath -m -- "$RUNNER_TEMP/core-rpc-docker-evidence")"
if [[ -e "$evidence" || -L "$evidence" ]]; then
echo "RPC evidence root must be absent before the fixture creates it" >&2
exit 1
fi
python3 -B scripts/test-core-rpc-docker.py --output-root "$evidence" 2>&1 | tee "$RUNNER_TEMP/core-rpc-docker.log"
- name: Stage readable core RPC evidence
if: ${{ always() }}
run: |
# Secret files are chowned to container uid 65532 and mode 0400.
# upload-artifact fails the whole zip if any selected file is unreadable.
src="$RUNNER_TEMP/core-rpc-docker-evidence"
dest="$RUNNER_TEMP/core-rpc-docker-evidence-readable"
rm -rf -- "$dest"
if [[ ! -d "$src" ]]; then
echo "No core RPC evidence directory to stage"
exit 0
fi
mkdir -- "$dest"
chmod 0700 -- "$dest"
while IFS= read -r -d '' dir; do
rel="${dir#"$src"/}"
mkdir -p -- "$dest/$rel"
chmod 0700 -- "$dest/$rel"
done < <(find "$src" -mindepth 1 -type d -print0)
while IFS= read -r -d '' file; do
rel="${file#"$src"/}"
parent="$dest/$(dirname -- "$rel")"
mkdir -p -- "$parent"
chmod 0700 -- "$parent"
cp -p -- "$file" "$dest/$rel"
done < <(find "$src" -type f -readable -print0)
- name: Service boundary contracts
if: ${{ !cancelled() }}
run: python3 -B scripts/test-core-boundaries.py
Expand All @@ -184,6 +217,7 @@ jobs:
${{ runner.temp }}/core-tests.log
${{ runner.temp }}/extension-tests.jsonl
${{ runner.temp }}/core-rpc-docker.log
${{ runner.temp }}/core-rpc-docker-evidence-readable
${{ runner.temp }}/store-postgres.jsonl
${{ runner.temp }}/events-process.log
if-no-files-found: warn
Expand Down
27 changes: 17 additions & 10 deletions scripts/test-core-rpc-docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,20 @@ def request(url, token=None, method="GET", body=None, user=None):
return error.code, error.read()


def wait_for_http(call, on_ready, *, attempts=100, pause=0.3):
"""Retry until HTTP 200. A live proxy's 502/503 JSON is not readiness."""
for _ in range(attempts):
try:
status, body = call()
if status == 200:
on_ready()
return json.loads(body)
except (OSError, TimeoutError):
pass
time.sleep(pause)
raise AssertionError("test endpoint did not become available")


class FixtureError(RuntimeError):
"""Safe public description; raw Docker/SQL output stays in private files."""

Expand Down Expand Up @@ -1085,16 +1099,9 @@ def rpc(path, user=None, token=ext_token):
)

def wait(call):
for _ in range(100):
try:
status, body = call()
if status == 200:
context.capture_owned("ready", runtime=True)
return json.loads(body)
except (OSError, TimeoutError):
pass
time.sleep(0.3)
raise AssertionError("test endpoint did not become available")
return wait_for_http(
call, lambda: context.capture_owned("ready", runtime=True)
)

def snapshot(service):
data = json.loads(
Expand Down
74 changes: 74 additions & 0 deletions scripts/test_core_rpc_fixture.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,80 @@ def test_failed_or_interrupted_rotation_restores_read_only_mode(self):
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o400)


class HttpReadinessTests(unittest.TestCase):
def test_execute_wait_delegates_to_http_200_helper(self):
source = ast.parse(SOURCE.read_text())
waits = [
node
for node in ast.walk(source)
if isinstance(node, ast.FunctionDef) and node.name == "wait"
]
self.assertEqual(len(waits), 1)
calls = [
node
for node in ast.walk(waits[0])
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "wait_for_http"
]
self.assertEqual(len(calls), 1)

def test_proxy_error_json_is_not_readiness(self):
calls = []

def call():
calls.append(1)
if len(calls) < 3:
return 503, b'{"error":"core_unavailable"}'
return 200, b'{"user_id":"1"}'

ready = []
with mock.patch.object(FIXTURE.time, "sleep") as sleep:
body = FIXTURE.wait_for_http(
call, lambda: ready.append(True), pause=0
)
self.assertEqual(body, {"user_id": "1"})
self.assertEqual(ready, [True])
self.assertEqual(len(calls), 3)
self.assertGreaterEqual(sleep.call_count, 2)

def test_connection_errors_retry_until_http_200(self):
state = {"n": 0}

def call():
state["n"] += 1
if state["n"] < 2:
raise TimeoutError("not listening")
return 200, b'{"ok":true}'

with mock.patch.object(FIXTURE.time, "sleep"):
body = FIXTURE.wait_for_http(call, lambda: None, pause=0)
self.assertEqual(body, {"ok": True})
self.assertEqual(state["n"], 2)

def test_persistent_gateway_error_is_not_success(self):
def call():
return 502, b'{"error":"core_unavailable"}'

with mock.patch.object(FIXTURE.time, "sleep"):
with self.assertRaises(AssertionError):
FIXTURE.wait_for_http(
call, lambda: self.fail("ready"), attempts=3, pause=0
)

def test_readiness_observation_failure_is_not_retried(self):
def call():
return 200, b'{"ok":true}'

def on_ready():
raise FIXTURE.FixtureError("product executable changed across restart")

with mock.patch.object(FIXTURE.time, "sleep") as sleep:
with self.assertRaises(FIXTURE.FixtureError):
FIXTURE.wait_for_http(call, on_ready, pause=0)
sleep.assert_not_called()


# @mark deployment.database-authority role=test
class DurableOutputTests(unittest.TestCase):
def test_cli_requires_explicit_output_before_any_docker_operation(self):
Expand Down
3 changes: 3 additions & 0 deletions scripts/workflow-topology.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,9 @@ test('manual checks reuse the exact core and extension verification pipeline', (
'contracts/proto/tests/process_recovery.py', 'LMM_RELAY_EXTENSION_BIN']) {
assert.ok(source.includes(command), command);
}
assert.match(source, /python3 -B scripts\/test-core-rpc-docker\.py --output-root "\$evidence"/);
assert.match(source, /core-rpc-docker-evidence-readable/);
assert.doesNotMatch(source, /runner\.temp \}\}\/core-rpc-docker-evidence\n/);
assert.match(source, /services:\n postgres:/);
assert.match(source, /DATABASE_URL: postgres:\/\/postgres:postgres@127\.0\.0\.1:5432\/postgres/);
assert.match(source, /working-directory: services\/extensions/);
Expand Down
Loading