diff --git a/.github/workflows/validate-service-cache.yml b/.github/workflows/validate-service-cache.yml new file mode 100644 index 00000000000..30844ab7ecf --- /dev/null +++ b/.github/workflows/validate-service-cache.yml @@ -0,0 +1,288 @@ +name: Validate service cache planner + +on: + push: + branches: + - build/service-cache-20260906 + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + persist-credentials: 'false' + fetch-depth: 0 + + - name: Compile planner + run: python3 -m py_compile self-host/scripts/affected-services.py + + - name: Validate planner against production graph + shell: bash + run: | + set -euo pipefail + tmp_output="$(mktemp)" + python3 self-host/scripts/affected-services.py \ + --all \ + --github-output "$tmp_output" + + python3 - "$tmp_output" <<'PY' + import importlib.util + import json + import pathlib + import sys + + output_path = pathlib.Path(sys.argv[1]) + output = {} + for line in output_path.read_text().splitlines(): + key, value = line.split("=", 1) + output[key] = value + + assert output["run_services"] == "true" + assert output["run_web"] == "true" + assert output["run_workers"] == "true" + targets = json.loads(output["service_targets"]) + assert len(targets) == 12, targets + assert len(set(targets)) == 12, targets + + script = pathlib.Path("self-host/scripts/affected-services.py") + spec = importlib.util.spec_from_file_location("affected_services", script) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + + closures = module.load_closures() + module.validate_config(closures) + + email = module.compute_impact( + ["services/email_service/src/api.rs"], False, closures + ) + assert email["run_services"] is True, email + assert "self-host-email-email-service" in email["service_targets"], email + assert email["run_web"] is False, email + + web = module.compute_impact(["apps/web/src/app.tsx"], False, closures) + assert web["run_web"] is True, web + assert web["run_services"] is False, web + + service_image = module.compute_impact( + ["self-host/images/Dockerfile.services"], False, closures + ) + assert service_image["run_services"] is True, service_image + assert service_image["service_targets"] == [], service_image + + lock = module.compute_impact(["Cargo.lock"], False, closures) + assert lock["run_services"] is True, lock + assert len(lock["service_targets"]) == 12, lock + assert lock["run_workers"] is True, lock + assert lock["run_init"] is True, lock + + websocket = module.compute_impact( + ["services/websocket_service/src/main.rs"], False, closures + ) + assert websocket["run_workers"] is True, websocket + + irrelevant = module.compute_impact(["docs/cache-notes.md"], False, closures) + assert irrelevant["run_services"] is False, irrelevant + assert irrelevant["run_web"] is False, irrelevant + assert irrelevant["run_workers"] is False, irrelevant + assert irrelevant["run_init"] is False, irrelevant + + print("planner assertions passed") + PY + + - name: Check existing self-host invariants + run: python3 self-host/scripts/check-drift.py + + - name: Setup Nix with private cache + uses: './.github/actions/setup-nix' + with: + nix-cache-url: ${{ vars.NIX_CACHE_URL }} + nix-cache-public-key: ${{ vars.NIX_CACHE_PUBLIC_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} + nix-cache-signing-key: ${{ github.event_name != 'pull_request' && secrets.NIX_CACHE_SIGNING_KEY || '' }} + fail-on-cache-error: 'true' + + - name: Reset Nix upload queue + if: github.event_name != 'pull_request' + shell: bash + run: | + set -euo pipefail + queue=/var/lib/nix-cache-upload + for dir in spool inflight failed; do + if [ -d "$queue/$dir" ]; then + sudo find "$queue/$dir" -type f -delete + fi + done + sudo rm -f "$queue/stop" "$queue/done" "$queue/stats" 2>/dev/null || true + + - name: Validate self-host Nix targets without compiling + shell: bash + run: | + set -euo pipefail + # Deliberately validate only the production self-host graph. A full + # `nix flake show` evaluates unrelated desktop/Tauri outputs and can + # fail on their independent fixed-output hashes even though the + # self-host targets remain valid. + nix build --dry-run .#self-host-email-source-check + nix build --dry-run .#self-host-email-binaries + + rm -f /tmp/planner-output + python3 self-host/scripts/affected-services.py \ + --all \ + --github-output /tmp/planner-output \ + >/dev/null + + targets="$(python3 - <<'PY' + import json + + with open('/tmp/planner-output', encoding='utf-8') as fh: + for line in fh: + if line.startswith('service_targets='): + for target in json.loads(line.split('=', 1)[1]): + print(target) + break + else: + raise SystemExit('service_targets output missing') + PY + )" + + while IFS= read -r target; do + [ -n "$target" ] || continue + echo "dry-run .#$target" + nix build --dry-run ".#$target" + done <<< "$targets" + + - name: Build production Email aggregate for real + id: build + if: github.event_name != 'pull_request' + shell: bash + run: | + set -euo pipefail + log=/tmp/nix-email-build.log + started="$(date +%s)" + + echo "Building .#self-host-email-binaries with private S3 cache read/write enabled" + nix build \ + --print-build-logs \ + --max-jobs 2 \ + --cores 4 \ + .#self-host-email-binaries \ + --out-link result-bins \ + 2>&1 | tee "$log" + + finished="$(date +%s)" + echo "duration_seconds=$((finished - started))" >> "$GITHUB_OUTPUT" + + cache_hits="$(grep -Ec "copying path '.*' from 's3" "$log" || true)" + local_builds="$(grep -Ec "building '/nix/store/.*\\.drv'" "$log" || true)" + total=$((cache_hits + local_builds)) + if [ "$total" -gt 0 ]; then + hit_rate="$(awk -v h="$cache_hits" -v t="$total" 'BEGIN { printf "%.1f", (h * 100) / t }')" + else + hit_rate="100.0" + fi + + echo "cache_hits=$cache_hits" >> "$GITHUB_OUTPUT" + echo "local_builds=$local_builds" >> "$GITHUB_OUTPUT" + echo "cache_hit_rate=$hit_rate" >> "$GITHUB_OUTPUT" + + { + echo "### Nix cache validation" + echo "" + echo "- Build duration: $((finished - started))s" + echo "- S3 substituted paths: $cache_hits" + echo "- Locally built derivations: $local_builds" + echo "- Path-level cache hit proxy: ${hit_rate}%" + echo "" + echo "> The hit rate is a path-level proxy from Nix logs, not a byte-weighted cache ratio." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Verify built production binaries + if: github.event_name != 'pull_request' + shell: bash + run: | + set -euo pipefail + test -d result-bins/bin + echo "Built binaries:" + find -L result-bins/bin -maxdepth 1 -type f -printf '%f\n' | sort + echo "Total binaries: $(find -L result-bins/bin -maxdepth 1 -type f | wc -l)" + + - name: Upload newly built Nix paths + id: upload_cache + if: success() && github.event_name != 'pull_request' + shell: bash + env: + NIX_CACHE_URL: ${{ vars.NIX_CACHE_URL }} + NIX_CACHE_SIGNING_KEY: ${{ secrets.NIX_CACHE_SIGNING_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }} + run: | + set -euo pipefail + dest="${NIX_CACHE_URL:-}" + queue=/var/lib/nix-cache-upload + spool="$queue/spool" + + if [ -z "$dest" ]; then + echo "NIX_CACHE_URL is empty; cannot upload cache" >&2 + exit 1 + fi + if [ -z "${NIX_CACHE_SIGNING_KEY:-}" ]; then + echo "NIX_CACHE_SIGNING_KEY is empty; cannot upload cache" >&2 + exit 1 + fi + + paths_file="$(mktemp)" + if [ -d "$spool" ]; then + while IFS= read -r file; do + sudo cat "$file" + done < <(sudo find "$spool" -type f -name '*.path' -print | sort) \ + | awk 'NF' | sort -u > "$paths_file" + fi + + upload_count="$(wc -l < "$paths_file" | tr -d ' ')" + echo "uploaded_paths=$upload_count" >> "$GITHUB_OUTPUT" + if [ "$upload_count" -eq 0 ]; then + echo "No locally built Nix paths to upload; cache was already warm." + echo "- Newly uploaded Nix paths: 0" >> "$GITHUB_STEP_SUMMARY" + rm -f "$paths_file" + exit 0 + fi + + key_file="$RUNNER_TEMP/nix-cache-signing-key" + printf '%s\n' "$NIX_CACHE_SIGNING_KEY" > "$key_file" + chmod 600 "$key_file" + trap 'rm -f "$key_file" "$paths_file"' EXIT + + if [[ "$dest" == *'?'* ]]; then + signed_dest="${dest}&secret-key=${key_file}" + else + signed_dest="${dest}?secret-key=${key_file}" + fi + + copy_flags=() + if nix copy --help 2>&1 | grep -q -- '--no-recursive'; then + copy_flags+=(--no-recursive) + fi + + mapfile -t paths < "$paths_file" + batch_size=48 + for ((i=0; i<${#paths[@]}; i+=batch_size)); do + batch=("${paths[@]:i:batch_size}") + echo "Uploading Nix cache batch $((i / batch_size + 1)) (${#batch[@]} paths)" + nix copy "${copy_flags[@]}" --to "$signed_dest" "${batch[@]}" + done + + sudo find "$spool" -type f -delete + echo "Uploaded $upload_count newly built Nix paths to the private cache." + echo "- Newly uploaded Nix paths: $upload_count" >> "$GITHUB_STEP_SUMMARY" diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix index 4b5c833f376..f2e24131b19 100644 --- a/nix/cloud-storage.nix +++ b/nix/cloud-storage.nix @@ -1302,6 +1302,7 @@ // dopplerConfigBinPackages // deployServiceBinaryPackages // deployLambdaPackages + // selfHostEmailBinaryPackages // pkgs.lib.optionalAttrs isLinux { local-stack-binaries = localStackBinaries; self-host-email-binaries = selfHostEmailBinaries; diff --git a/self-host/scripts/affected-services.py b/self-host/scripts/affected-services.py new file mode 100644 index 00000000000..a3126d3786a --- /dev/null +++ b/self-host/scripts/affected-services.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 +"""Compute the smallest self-host build set for a Git diff. + +The Rust service impact calculation is dependency-aware: a service is rebuilt +only when a changed workspace directory appears in that service package's +transitive workspace closure from .github/workspace-dep-closures.json. + +The script fails closed when its hard-coded production service inventory drifts +from nix/cloud-storage.nix, so a future service addition cannot silently skip +CI rebuilds. + +GitHub Actions output: + run_services=true|false + service_targets=["self-host-email-email-service", ...] + run_web=true|false + run_workers=true|false + run_init=true|false + changed_count=N +""" +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import re +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[2] +CLOSURES_PATH = ROOT / ".github/workspace-dep-closures.json" +NIX_PATH = ROOT / "nix/cloud-storage.nix" + +SERVICE_ROOTS: dict[str, str] = { + "authentication-service": "authentication_service", + "connection-gateway": "connection_gateway", + "contacts-service": "contacts_service", + "document-storage-service": "document_storage_service", + "email-service": "email_service", + "image-proxy-service": "image_proxy_service", + "notification-service": "notification_service", + "static-file-service": "static_file_service", + "unfurl-service": "unfurl_service", + "search-processing-service": "search_processing_service", + "upload-finalizer": "document_upload_finalizer_handler", + "macro-db-migrator": "macro_db_migrator", +} + +# These inputs change the derivation graph/toolchain or files copied into every +# pruned Email source. They intentionally invalidate every Email service. +FULL_SERVICE_EXACT = { + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + "flake.nix", + "flake.lock", + ".github/workspace-dep-closures.json", + ".github/workflows/self-host-images.yml", + "self-host/scripts/affected-services.py", + "nix/cloud-storage.nix", + "nix/systems.nix", +} +FULL_SERVICE_PREFIXES = ( + "nix-support/", + ".cargo/", + ".github/actions/setup-nix/", + ".github/actions/teardown-nix/", + ".sqlx/", + "static_assets/", + "self-host/scripts/", +) + +SERVICE_IMAGE_EXACT = { + "self-host/images/Dockerfile.services", +} +SERVICE_IMAGE_PREFIXES = ( + "self-host/images/services/", +) + +WEB_EXACT = { + "package.json", + "bun.lock", + "bun.lockb", + "flake.nix", + "flake.lock", + "rust-toolchain.toml", + ".github/workflows/self-host-images.yml", + "self-host/scripts/affected-services.py", + "self-host/images/Dockerfile.web", +} +WEB_PREFIXES = ( + "apps/web/", + "packages/", + ".github/actions/setup-nix/", +) + +WORKER_EXACT = { + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + ".github/workflows/self-host-images.yml", + "self-host/scripts/affected-services.py", + "docker/websocket-service.Dockerfile", +} +WORKER_PREFIXES = ( + "services/websocket_service/", +) + +INIT_EXACT = { + ".github/kafka-cluster-topics.json", + "infra/stacks/fusionauth-instance/templates/reconcile_secondary_idp_link.js", +} +INIT_PREFIXES = ( + "self-host/init/", + "self-host/kickstart/", + "infra/stacks/opensearch/helpers/", +) + +ROOT_SHARED_SUFFIXES = { + ".md", + ".html", + ".txt", + ".json", + ".jsonl", + ".toml", + ".canvas", + ".sql", + ".sh", + ".bop", + ".bin", +} + + +def matches(path: str, exact: set[str], prefixes: tuple[str, ...]) -> bool: + return path in exact or any(path.startswith(prefix) for prefix in prefixes) + + +def git_changed_files(base: str, head: str) -> list[str]: + # --no-renames is deliberate. A rename crossing service boundaries must + # expose both the deleted old path and the added new path, otherwise the + # service losing the file could be incorrectly treated as unaffected. + proc = subprocess.run( + ["git", "diff", "--no-renames", "--name-only", base, head], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + return [line.strip() for line in proc.stdout.splitlines() if line.strip()] + + +def path_in_dir(path: str, directory: str) -> bool: + return path == directory or path.startswith(directory + "/") + + +def is_shared_root_dep(path: str) -> bool: + # Mirrors rootDepsSrc in nix/cloud-storage.nix: top-level shared asset files + # become inputs to every pruned Email service source. + if "/" in path or path in {"Cargo.toml", "Cargo.lock"}: + return False + return pathlib.PurePosixPath(path).suffix in ROOT_SHARED_SUFFIXES + + +def load_closures() -> dict[str, list[str]]: + doc = json.loads(CLOSURES_PATH.read_text(encoding="utf-8")) + closures = doc.get("closures") + if not isinstance(closures, dict): + raise RuntimeError(f"{CLOSURES_PATH.relative_to(ROOT)} has no closures object") + return closures + + +def nix_email_definitions() -> dict[str, str]: + """Read serviceName/packageName pairs from the Email production Nix list. + + This is intentionally only a drift guard, not a general Nix parser. Stable + comments delimit the whole definition list so nested `binaries = [ ... ];` + arrays cannot be mistaken for the end of the service inventory. If the Nix + block is refactored enough that either marker disappears, CI fails closed. + """ + text = NIX_PATH.read_text(encoding="utf-8") + start_marker = "selfHostEmailBinaryDefinitions = [" + end_marker = "# Strip --no-default-features" + + if start_marker not in text or end_marker not in text: + raise RuntimeError( + "could not locate the complete selfHostEmailBinaryDefinitions block " + "in nix/cloud-storage.nix" + ) + + tail = text.split(start_marker, 1)[1] + block, separator, _ = tail.partition(end_marker) + if not separator: + raise RuntimeError( + "could not locate the end of selfHostEmailBinaryDefinitions in " + "nix/cloud-storage.nix" + ) + + pairs = re.findall( + r'serviceName\s*=\s*"([^"]+)";\s*\n\s*packageName\s*=\s*"([^"]+)";', + block, + ) + if not pairs: + raise RuntimeError( + "could not parse any Email service definitions from nix/cloud-storage.nix" + ) + + definitions = dict(pairs) + if len(definitions) != len(pairs): + raise RuntimeError( + "duplicate serviceName entries found in selfHostEmailBinaryDefinitions" + ) + return definitions + + +def validate_config(closures: dict[str, list[str]]) -> None: + nix_defs = nix_email_definitions() + if nix_defs != SERVICE_ROOTS: + missing = sorted(set(nix_defs) - set(SERVICE_ROOTS)) + extra = sorted(set(SERVICE_ROOTS) - set(nix_defs)) + mismatched = sorted( + name + for name in set(nix_defs) & set(SERVICE_ROOTS) + if nix_defs[name] != SERVICE_ROOTS[name] + ) + raise RuntimeError( + "affected-services.py service inventory drifted from " + "selfHostEmailBinaryDefinitions; " + f"missing={missing}, extra={extra}, package_mismatch={mismatched}" + ) + + missing_closures = sorted( + package for package in SERVICE_ROOTS.values() if package not in closures + ) + if missing_closures: + raise RuntimeError( + ".github/workspace-dep-closures.json is missing production packages: " + + ", ".join(missing_closures) + ) + + +def changed_path_hits_closure( + path: str, + package_name: str, + closures: dict[str, list[str]], +) -> bool: + package_closure = closures.get(package_name) + if package_closure is None: + return False + return any(path_in_dir(path, directory) for directory in package_closure) + + +def compute_service_targets( + changed: list[str], + force_all: bool, + closures: dict[str, list[str]], +) -> list[str]: + if force_all or any( + matches(path, FULL_SERVICE_EXACT, FULL_SERVICE_PREFIXES) + or is_shared_root_dep(path) + for path in changed + ): + return [f"self-host-email-{name}" for name in SERVICE_ROOTS] + + affected: list[str] = [] + for service_name, package_name in SERVICE_ROOTS.items(): + if any( + changed_path_hits_closure(path, package_name, closures) + for path in changed + ): + affected.append(f"self-host-email-{service_name}") + return affected + + +def worker_is_affected( + changed: list[str], + force_all: bool, + closures: dict[str, list[str]], +) -> bool: + if force_all: + return True + if any(matches(path, WORKER_EXACT, WORKER_PREFIXES) for path in changed): + return True + + websocket_closure = closures.get("websocket_service") + if websocket_closure is not None: + return any( + path_in_dir(path, directory) + for path in changed + for directory in websocket_closure + ) + + # Fail safe if the generated closure does not expose websocket_service. + # This is intentionally conservative rather than risking a stale worker. + return any( + path.startswith("crates/") or path.startswith("services/websocket_service/") + for path in changed + ) + + +def compute_impact( + changed: list[str], + force_all: bool, + closures: dict[str, list[str]], +) -> dict[str, object]: + service_targets = compute_service_targets(changed, force_all, closures) + + run_services = force_all or bool(service_targets) or any( + matches(path, SERVICE_IMAGE_EXACT, SERVICE_IMAGE_PREFIXES) + for path in changed + ) + run_web = force_all or any( + matches(path, WEB_EXACT, WEB_PREFIXES) for path in changed + ) + run_workers = worker_is_affected(changed, force_all, closures) + run_init = ( + force_all + or "self-host-email-macro-db-migrator" in service_targets + or any(matches(path, INIT_EXACT, INIT_PREFIXES) for path in changed) + ) + + return { + "run_services": run_services, + "service_targets": service_targets, + "run_web": run_web, + "run_workers": run_workers, + "run_init": run_init, + "changed_count": len(changed), + } + + +def write_outputs(values: dict[str, object], output_path: str | None) -> None: + rendered = { + key: json.dumps(value, separators=(",", ":")) + if isinstance(value, (list, dict)) + else str(value).lower() + if isinstance(value, bool) + else str(value) + for key, value in values.items() + } + + for key, value in rendered.items(): + print(f"{key}={value}") + + if output_path: + with open(output_path, "a", encoding="utf-8") as fh: + for key, value in rendered.items(): + fh.write(f"{key}={value}\n") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base") + parser.add_argument("--head", default="HEAD") + parser.add_argument("--all", action="store_true", dest="force_all") + parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT")) + args = parser.parse_args() + + closures = load_closures() + validate_config(closures) + + if args.force_all: + changed: list[str] = [] + else: + if not args.base: + parser.error("--base is required unless --all is used") + changed = git_changed_files(args.base, args.head) + + values = compute_impact(changed, args.force_all, closures) + write_outputs(values, args.github_output) + + print("\nChanged files:") + if changed: + for path in changed: + print(f" {path}") + else: + print(" (forced full build)" if args.force_all else " (none)") + + print("\nAffected Email service targets:") + service_targets = values["service_targets"] + if isinstance(service_targets, list) and service_targets: + for target in service_targets: + print(f" .#{target}") + else: + print(" (none)") + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, RuntimeError, json.JSONDecodeError, subprocess.CalledProcessError) as exc: + print(f"affected-services: {exc}", file=sys.stderr) + sys.exit(2)