diff --git a/services/ai-service/eval/online/live_eval.py b/services/ai-service/eval/online/live_eval.py index 2a3d9982..240694d2 100644 --- a/services/ai-service/eval/online/live_eval.py +++ b/services/ai-service/eval/online/live_eval.py @@ -22,6 +22,7 @@ import argparse import json +import os import subprocess import sys import time @@ -42,6 +43,10 @@ get_root_cause, list_fault_specs, ) +from eval.online.safe_live_fault_specs import ( + SAFE_FAULT_SPECS, + list_safe_fault_specs, +) REPORTS_DIR = Path(__file__).resolve().parents[2] / "eval" / "reports" @@ -627,6 +632,283 @@ def run_live( return report +# ───────────────────────────────────────────────────────────────────────────── +# SAFE-LIVE — operations-backend safe-injection API only. +# No kubectl, DB exec, DB scale, direct SQL mutation, or existing-resource patch. +# ───────────────────────────────────────────────────────────────────────────── +def _safe_internal_headers() -> dict[str, str]: + from app.core.config import settings + from app.tools.spring_client import INTERNAL_OPS_TOKEN_HEADER + + headers = { + "Content-Type": "application/json", + "X-Agent-Request-Id": "safeinject-eval", + } + if settings.internal_ops_token: + headers[INTERNAL_OPS_TOKEN_HEADER] = settings.internal_ops_token + return headers + + +def _safe_login(client, *, base_url: str, email: str, password: str) -> tuple[str, str]: + resp = client.post( + f"{base_url}/api/v1/auth/login", + json={"email": email, "password": password}, + headers={"Content-Type": "application/json"}, + ) + resp.raise_for_status() + body = resp.json() + token = body.get("accessToken") + project_id = body.get("workspaceId") + if not token or not project_id: + raise RuntimeError(f"login response missing accessToken/workspaceId: {body}") + return str(token), str(project_id) + + +def _safe_inject_connector( + client, + *, + ops_base_url: str, + project_id: str, + run_id: str, + fault_id: str, +) -> dict: + resp = client.post( + f"{ops_base_url}/internal/ops/projects/{project_id}/safe-injection/connectors", + headers=_safe_internal_headers(), + json={"runId": run_id, "fault": fault_id}, + ) + resp.raise_for_status() + env = resp.json() + if not env.get("ok"): + raise RuntimeError(f"safe injection failed: {env}") + return env["result"] + + +def _safe_cleanup(client, *, ops_base_url: str, project_id: str, run_id: str) -> dict: + resp = client.delete( + f"{ops_base_url}/internal/ops/projects/{project_id}/safe-injection/runs/{run_id}", + headers=_safe_internal_headers(), + ) + resp.raise_for_status() + env = resp.json() + if not env.get("ok"): + raise RuntimeError(f"safe cleanup failed: {env}") + return env["result"] + + +def _run_completed(events: Sequence[dict]) -> bool: + return any(e.get("type") == "run_completed" for e in events) + + +def _extract_candidates(payload: dict) -> tuple[list[str], list[float]]: + candidates = payload.get("root_cause_candidates") + if not candidates and isinstance(payload.get("body"), dict): + candidates = payload["body"].get("root_cause_candidates") + if not candidates and isinstance(payload.get("data"), dict): + return _extract_candidates(payload["data"]) + if not candidates and payload.get("root_cause_id"): + candidates = [{ + "root_cause_id": payload.get("root_cause_id"), + "confidence": payload.get("confidence") or 0.0, + }] + ranking: list[str] = [] + confidences: list[float] = [] + for item in candidates or []: + if not isinstance(item, dict): + continue + root_cause_id = item.get("root_cause_id") + if not root_cause_id: + continue + ranking.append(str(root_cause_id)) + try: + confidences.append(float(item.get("confidence") or 0.0)) + except (TypeError, ValueError): + confidences.append(0.0) + return ranking, confidences + + +def _poll_agent_ranking( + client, + *, + agent_base_url: str, + token: str, + agent_project_id: str, + connector_name: str, + timeout_s: int, + interval_s: float, +) -> tuple[str | None, list[str], list[float], str]: + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + message = ( + f"커넥터 {connector_name} 상태를 조회하고 근본 원인을 분석해줘. " + "필요하면 상태, 로그, 최근 변경, lag 같은 운영 증거를 수집해줘." + ) + resp = client.post( + f"{agent_base_url}/api/v1/agent/runs", + headers=headers, + json={ + "project_id": agent_project_id, + "message": message, + "mode": "incident_analysis", + "stream": False, + }, + ) + resp.raise_for_status() + env = resp.json() + run_id = ((env.get("data") or {}).get("run_id")) + if not run_id: + raise RuntimeError(f"agent run_id missing: {env}") + + deadline = time.monotonic() + timeout_s + events: list[dict] = [] + while time.monotonic() < deadline: + ev = client.get( + f"{agent_base_url}/api/v1/agent/runs/{run_id}/events/history", + headers=headers, + ) + ev.raise_for_status() + events = ((ev.json().get("data") or {}).get("events")) or [] + if _run_completed(events): + break + time.sleep(interval_s) + + # Report snapshot is the preferred actual RCA artifact. Reproducibility is a state-patch fallback. + for path in ( + f"/api/v1/agent/runs/{run_id}/report", + f"/api/v1/agent/runs/{run_id}/reproducibility", + ): + got = client.get(f"{agent_base_url}{path}", headers=headers) + if got.status_code >= 400: + continue + body = got.json() + data = body.get("data") or {} + ranking, confidences = _extract_candidates(data) + if ranking: + return run_id, ranking, confidences, "safe-live agent rca" + + return run_id, [], [], "safe-live agent rca not captured" + + +def run_safe_live( + fault_ids: Sequence[str] | None, + *, + confirm: bool = False, + ops_base_url: str | None = None, + safe_project_id: str | None = None, + agent_base_url: str | None = None, + agent_project_id: str | None = None, + email: str | None = None, + password: str | None = None, + run_id: str | None = None, + poll_timeout_s: int = 180, + poll_interval_s: float = 2.0, +) -> dict: + """Safe live injection using only new labelled test connector resources. + + This mode deliberately avoids the destructive live_fault_specs inject/recover steps. + It requires --confirm because it creates and deletes labelled test resources. + """ + if not confirm: + raise SystemExit("safe-live requires --confirm because it creates labelled test resources.") + import httpx + from app.core.config import settings + + ops_base_url = (ops_base_url or os.getenv("SAFE_INJECT_OPS_BASE_URL") + or settings.spring_ops_base_url).rstrip("/") + agent_base_url = (agent_base_url or os.getenv("BIFROST_BASE_URL") + or "http://localhost:8000").rstrip("/") + safe_project_id = safe_project_id or os.getenv("SAFE_INJECT_PROJECT_ID") + if not safe_project_id: + raise SystemExit("safe-live requires --safe-project-id or SAFE_INJECT_PROJECT_ID") + email = email or os.getenv("BIFROST_EMAIL") + password = password or os.getenv("BIFROST_PASSWORD") + run_id = run_id or "safeinject-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + spec_index = {s.fault_id: s for s in list_safe_fault_specs()} + selected = [spec_index[f] for f in fault_ids if f in spec_index] if fault_ids else list(SAFE_FAULT_SPECS) + if not selected: + raise SystemExit("selected safe-live faults are empty") + + observations: list[FaultObservation] = [] + cleanup_results: list[dict] = [] + with httpx.Client(timeout=30.0) as client: + token: str | None = None + if email and password: + token, resolved_project = _safe_login( + client, base_url=agent_base_url, email=email, password=password + ) + agent_project_id = agent_project_id or os.getenv("BIFROST_PROJECT_ID") or resolved_project + elif not agent_project_id: + agent_project_id = os.getenv("BIFROST_PROJECT_ID") + + try: + for spec in selected: + print(f"\n=== safe-live fault: {spec.fault_id} ===") + injection = _safe_inject_connector( + client, + ops_base_url=ops_base_url, + project_id=safe_project_id, + run_id=run_id, + fault_id=spec.fault_id, + ) + connector_name = str(injection["connector_name"]) + print(f" injected connector={connector_name}") + + if token and agent_project_id: + agent_run, ranking, confidences, note = _poll_agent_ranking( + client, + agent_base_url=agent_base_url, + token=token, + agent_project_id=agent_project_id, + connector_name=connector_name, + timeout_s=poll_timeout_s, + interval_s=poll_interval_s, + ) + print(f" rca run={agent_run} ranking={ranking[:5]}") + observations.append(FaultObservation( + fault_id=spec.fault_id, + expected_root_cause_ids=spec.expected_root_cause_ids, + predicted_ranking=ranking, + predicted_confidences=confidences, + incident_id=agent_run, + grouping_key=connector_name, + captured=bool(ranking), + note=note, + )) + else: + observations.append(FaultObservation( + fault_id=spec.fault_id, + expected_root_cause_ids=spec.expected_root_cause_ids, + predicted_ranking=[], + predicted_confidences=[], + grouping_key=connector_name, + captured=False, + note="safe-live injection completed; agent credentials not supplied", + )) + finally: + cleanup = _safe_cleanup( + client, + ops_base_url=ops_base_url, + project_id=safe_project_id, + run_id=run_id, + ) + cleanup_results.append(cleanup) + if cleanup.get("residual_count") != 0: + raise RuntimeError(f"safe-live cleanup residuals remain: {cleanup}") + + report = score_observations(observations) + report["mode"] = "safe-live" + report["safe_run_id"] = run_id + report["safe_project_id"] = safe_project_id + report["cleanup"] = cleanup_results + report["spec_summary"] = { + "safe": [s.fault_id for s in SAFE_FAULT_SPECS], + "forbidden_surfaces": ["kubectl exec", "scale/patch/delete existing resources", "direct DB SQL"], + } + json_path, md_path = write_reports(report, mode="safe-live") + report["_report_paths"] = {"json": str(json_path), "md": str(md_path)} + return report + + def _print_summary(report: dict) -> None: keys = ["mode", "total_cases", "AC@1", "AC@3", "AC@5", "Avg@5", "ECE", "weak_layers"] summary = {k: report.get(k) for k in keys if k in report} @@ -649,9 +931,14 @@ def main(argv: Sequence[str] | None = None) -> int: action="store_true", help="가드. auto fault 실제 주입/폴링/복구(--confirm 필수).", ) + mode.add_argument( + "--safe-live", + action="store_true", + help="가드. 신규 라벨 테스트 커넥터만 생성/정리하는 안전 주입 모드(--confirm 필수).", + ) parser.add_argument("--confirm", action="store_true", help="live 이중 가드.") parser.add_argument( - "--faults", nargs="*", default=None, help="live 에서 주입할 auto fault id 들(미지정 시 전체)." + "--faults", nargs="*", default=None, help="live/safe-live 에서 주입할 fault id 들(미지정 시 전체)." ) parser.add_argument( "--poll-timeout", type=int, default=300, help="live: RCA 결과 폴링 timeout(초, 기본 300=5분)." @@ -659,12 +946,33 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument( "--poll-interval", type=int, default=15, help="live: 폴링 간격(초, 기본 15)." ) + parser.add_argument("--safe-project-id", default=None, help="safe-live: Spring internal-ops project id/slug.") + parser.add_argument("--agent-project-id", default=None, help="safe-live: FastAPI agent project UUID.") + parser.add_argument("--ops-base-url", default=None, help="safe-live: operations-backend base URL.") + parser.add_argument("--agent-base-url", default=None, help="safe-live: ai-service/API base URL.") + parser.add_argument("--email", default=None, help="safe-live: agent login email.") + parser.add_argument("--password", default=None, help="safe-live: agent login password.") + parser.add_argument("--safe-run-id", default=None, help="safe-live: deterministic cleanup run id.") parser.add_argument( "--no-write", action="store_true", help="dry-run: 리포트 파일 미기록(콘솔만)." ) args = parser.parse_args(argv) - if args.live: + if args.safe_live: + report = run_safe_live( + args.faults, + confirm=args.confirm, + ops_base_url=args.ops_base_url, + safe_project_id=args.safe_project_id, + agent_base_url=args.agent_base_url, + agent_project_id=args.agent_project_id, + email=args.email, + password=args.password, + run_id=args.safe_run_id, + poll_timeout_s=args.poll_timeout, + poll_interval_s=float(args.poll_interval), + ) + elif args.live: report = run_live( args.faults, poll_timeout_s=args.poll_timeout, diff --git a/services/ai-service/eval/online/safe_live_fault_specs.py b/services/ai-service/eval/online/safe_live_fault_specs.py new file mode 100644 index 00000000..7cc63d09 --- /dev/null +++ b/services/ai-service/eval/online/safe_live_fault_specs.py @@ -0,0 +1,68 @@ +"""Safe live fault specs for connector-only RCA eval. + +This catalog is intentionally separate from live_fault_specs.py. It never carries +kubectl, scale, DB exec, or SQL mutation steps. The runner injects by calling the +operations-backend safe-injection API, which creates only labelled test resources. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from app.catalogs.root_causes import ROOT_CAUSE_INDEX + + +@dataclass(frozen=True) +class SafeFaultSpec: + fault_id: str + description: str + expected_root_cause_ids: tuple[str, ...] + + @property + def primary_root_cause_id(self) -> str: + return self.expected_root_cause_ids[0] + + +SAFE_FAULT_SPECS: tuple[SafeFaultSpec, ...] = ( + SafeFaultSpec( + fault_id="auth", + description="new labelled test connector with invalid sink credentials", + expected_root_cause_ids=("SINK_AUTH_EXPIRED", "CREDENTIAL_ROTATION_REGRESSION"), + ), + SafeFaultSpec( + fault_id="schema", + description="new labelled test connector with incompatible converter config", + expected_root_cause_ids=("SCHEMA_MISMATCH", "PIPELINE_CONFIG_INVALID"), + ), + SafeFaultSpec( + fault_id="lag", + description="new labelled paused test connector to exercise lag-style RCA path", + expected_root_cause_ids=("CONSUMER_LAG_SPIKE",), + ), + SafeFaultSpec( + fault_id="sink-fail", + description="new labelled test connector with nonexistent sink target", + expected_root_cause_ids=("SINK_DB_CONNECTION_TIMEOUT", "CONNECTOR_TASK_FAILED"), + ), + SafeFaultSpec( + fault_id="no-fault", + description="new labelled paused control connector without a fault expectation", + expected_root_cause_ids=("UNKNOWN_WITH_EVIDENCE_GAP",), + ), +) + + +def _validate() -> None: + seen: set[str] = set() + for spec in SAFE_FAULT_SPECS: + if spec.fault_id in seen: + raise ValueError(f"duplicate safe fault id: {spec.fault_id}") + seen.add(spec.fault_id) + for root_cause_id in spec.expected_root_cause_ids: + if root_cause_id not in ROOT_CAUSE_INDEX: + raise ValueError(f"{spec.fault_id}: unknown root cause {root_cause_id}") + + +_validate() + +def list_safe_fault_specs() -> tuple[SafeFaultSpec, ...]: + return SAFE_FAULT_SPECS diff --git a/services/ai-service/eval/online/test_live_eval.py b/services/ai-service/eval/online/test_live_eval.py index efd50cc1..37979db1 100644 --- a/services/ai-service/eval/online/test_live_eval.py +++ b/services/ai-service/eval/online/test_live_eval.py @@ -18,12 +18,14 @@ observations_to_cases, run_dry_run, run_live, + run_safe_live, score_observations, ) from eval.online.live_fault_specs import ( FAULT_SPECS, list_fault_specs, ) +from eval.online.safe_live_fault_specs import SAFE_FAULT_SPECS # ── spec 카탈로그 무결성 ────────────────────────────────────────────────────── @@ -52,6 +54,15 @@ def test_auto_faults_present() -> None: assert {s.fault_id for s in auto} == {"sink_db_down", "source_db_down"} +def test_safe_faults_present_and_catalog_backed() -> None: + assert {s.fault_id for s in SAFE_FAULT_SPECS} == { + "auth", "schema", "lag", "sink-fail", "no-fault", + } + for spec in SAFE_FAULT_SPECS: + for root_cause_id in spec.expected_root_cause_ids: + assert is_known_root_cause(root_cause_id) + + # ── 채점 파이프라인(테스트 더블에 안 기댄 순수 계산) ────────────────────────── def test_acceptable_set_top1_counts_as_hit() -> None: # acceptable set 의 두 번째 id 가 top-1 이어도 hit@1 으로 인정돼야 한다. @@ -110,6 +121,11 @@ def test_live_requires_confirm() -> None: run_live(None, confirm=False) +def test_safe_live_requires_confirm() -> None: + with pytest.raises(SystemExit): + run_safe_live(["auth"], confirm=False, safe_project_id="e2e-rca-test") + + def test_live_never_touches_subprocess_without_confirm(monkeypatch) -> None: # confirm 없이는 어떤 subprocess(_run_cmd)도 호출되면 안 된다(SystemExit 가 먼저). def _boom(*a, **k): # pragma: no cover - 호출되면 실패 @@ -203,6 +219,127 @@ def test_live_timeout_yields_uncaptured_miss(monkeypatch, tmp_path) -> None: assert any("scale deploy tenant-mariadb --replicas=1" in c for c in calls) +class _FakeResponse: + def __init__(self, payload: dict, status_code: int = 200): + self._payload = payload + self.status_code = status_code + + def json(self) -> dict: + return self._payload + + def raise_for_status(self) -> None: + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + +class _FakeSafeClient: + calls: list[tuple[str, str, dict | None]] = [] + + def __init__(self, *args, **kwargs): + _ = (args, kwargs) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def post(self, url: str, **kwargs): + self.calls.append(("POST", url, kwargs.get("json"))) + if url.endswith("/api/v1/auth/login"): + return _FakeResponse({ + "accessToken": "token", + "workspaceId": "00000000-0000-0000-0000-000000000001", + }) + if url.endswith("/safe-injection/connectors"): + return _FakeResponse({ + "ok": True, + "result": { + "run_id": kwargs["json"]["runId"], + "fault": kwargs["json"]["fault"], + "expected_root_cause_id": "SINK_AUTH_EXPIRED", + "pipeline_id": "00000000-0000-0000-0000-000000000002", + "datasource_id": "00000000-0000-0000-0000-000000000003", + "connector_name": "safeinject-r123-a1", + "namespace": "platform-kafka", + "labels": {"bifrost.io/safe-inject": "true"}, + "safe_scope": "safeinject:test-connector-only", + }, + }, status_code=201) + if url.endswith("/api/v1/agent/runs"): + return _FakeResponse({"data": {"run_id": "run_safe_001", "status": "running"}}) + raise AssertionError(f"unexpected POST {url}") + + def get(self, url: str, **kwargs): + self.calls.append(("GET", url, None)) + if url.endswith("/events/history"): + return _FakeResponse({"data": {"events": [{"type": "run_completed"}]}}) + if url.endswith("/report"): + return _FakeResponse({"data": {"body": {"root_cause_candidates": [ + {"root_cause_id": "SINK_AUTH_EXPIRED", "confidence": 0.86}, + {"root_cause_id": "CONNECTOR_TASK_FAILED", "confidence": 0.2}, + ]}}}) + if url.endswith("/reproducibility"): + return _FakeResponse({"data": {"root_cause_candidates": []}}) + raise AssertionError(f"unexpected GET {url}") + + def delete(self, url: str, **kwargs): + self.calls.append(("DELETE", url, None)) + if "/safe-injection/runs/" in url: + return _FakeResponse({ + "ok": True, + "result": { + "run_id": "safe-run", + "deleted_k8s_connectors": 1, + "deleted_metadata_rows": 3, + "residual_count": 0, + "residuals": [], + }, + }) + raise AssertionError(f"unexpected DELETE {url}") + + +def test_safe_live_injects_scores_and_cleans_without_destructive_helpers(monkeypatch, tmp_path) -> None: + import httpx + + _FakeSafeClient.calls = [] + monkeypatch.setattr(httpx, "Client", _FakeSafeClient) + monkeypatch.setattr(le, "REPORTS_DIR", tmp_path) + monkeypatch.setattr( + le, "_run_cmd", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("destructive command")), + ) + monkeypatch.setattr( + le, "_kubectl_psql", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("db exec")), + ) + + report = run_safe_live( + ["auth"], + confirm=True, + ops_base_url="http://ops", + safe_project_id="e2e-rca-test", + agent_base_url="http://agent", + email="ta@bifrost.io", + password="pw", + run_id="safe-run", + poll_timeout_s=1, + poll_interval_s=0, + ) + + assert report["mode"] == "safe-live" + assert report["AC@1"] == 1.0 + assert report["cleanup"][0]["residual_count"] == 0 + assert any( + call[0] == "POST" and call[1].endswith("/safe-injection/connectors") + for call in _FakeSafeClient.calls + ) + assert any( + call[0] == "DELETE" and "/safe-injection/runs/safe-run" in call[1] + for call in _FakeSafeClient.calls + ) + + # ── consistency: 채점이 결정적 ─────────────────────────────────────────────── def test_scoring_is_deterministic() -> None: a = score_observations(build_dry_run_observations()) diff --git a/services/operations-backend/src/main/java/com/bifrost/ops/internalops/controller/InternalOpsSafeInjectionController.java b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/controller/InternalOpsSafeInjectionController.java new file mode 100644 index 00000000..fcf49446 --- /dev/null +++ b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/controller/InternalOpsSafeInjectionController.java @@ -0,0 +1,89 @@ +package com.bifrost.ops.internalops.controller; + +import com.bifrost.ops.global.common.error.ApiException; +import com.bifrost.ops.internalops.AgentHeaders; +import com.bifrost.ops.internalops.dto.OpsEnvelope; +import com.bifrost.ops.internalops.safeinject.SafeInjectionCleanupResult; +import com.bifrost.ops.internalops.safeinject.SafeInjectionCreateRequest; +import com.bifrost.ops.internalops.safeinject.SafeInjectionCreateResult; +import com.bifrost.ops.internalops.safeinject.SafeInjectionService; +import jakarta.servlet.http.HttpServletRequest; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/internal/ops/projects/{projectId}/safe-injection") +public class InternalOpsSafeInjectionController { + + private static final String OP_CREATE = "safe_inject_connector"; + private static final String OP_CLEANUP = "safe_inject_cleanup"; + private static final String OP_RESIDUALS = "safe_inject_residuals"; + + private final SafeInjectionService service; + + public InternalOpsSafeInjectionController(SafeInjectionService service) { + this.service = service; + } + + @PostMapping("/connectors") + public ResponseEntity> create( + @PathVariable String projectId, + @RequestBody SafeInjectionCreateRequest body, + HttpServletRequest request) { + String requestId = AgentHeaders.requestId(request); + try { + return ResponseEntity.status(HttpStatus.CREATED) + .body(OpsEnvelope.ok(requestId, OP_CREATE, service.create(projectId, body))); + } catch (IllegalArgumentException e) { + return ResponseEntity.badRequest() + .body(OpsEnvelope.error(requestId, OP_CREATE, "VALIDATION_FAILED", e.getMessage(), false)); + } catch (ApiException e) { + return apiError(requestId, OP_CREATE, e); + } + } + + @DeleteMapping("/runs/{runId}") + public ResponseEntity> cleanup( + @PathVariable String projectId, + @PathVariable String runId, + HttpServletRequest request) { + String requestId = AgentHeaders.requestId(request); + try { + return ResponseEntity.ok(OpsEnvelope.ok(requestId, OP_CLEANUP, service.cleanup(projectId, runId))); + } catch (IllegalArgumentException e) { + return ResponseEntity.badRequest() + .body(OpsEnvelope.error(requestId, OP_CLEANUP, "VALIDATION_FAILED", e.getMessage(), false)); + } catch (ApiException e) { + return apiError(requestId, OP_CLEANUP, e); + } + } + + @GetMapping("/runs/{runId}/residuals") + public ResponseEntity> residuals( + @PathVariable String projectId, + @PathVariable String runId, + HttpServletRequest request) { + String requestId = AgentHeaders.requestId(request); + try { + return ResponseEntity.ok(OpsEnvelope.ok(requestId, OP_RESIDUALS, service.residualsOnly(projectId, runId))); + } catch (IllegalArgumentException e) { + return ResponseEntity.badRequest() + .body(OpsEnvelope.error(requestId, OP_RESIDUALS, "VALIDATION_FAILED", e.getMessage(), false)); + } catch (ApiException e) { + return apiError(requestId, OP_RESIDUALS, e); + } + } + + private static ResponseEntity> apiError(String requestId, String operation, ApiException e) { + return ResponseEntity.status(e.code().status()) + .body(OpsEnvelope.error(requestId, operation, e.code().name(), e.getMessage(), false, + "check_safe_injection_scope")); + } +} diff --git a/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCleanupResult.java b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCleanupResult.java new file mode 100644 index 00000000..3b9144b4 --- /dev/null +++ b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCleanupResult.java @@ -0,0 +1,19 @@ +package com.bifrost.ops.internalops.safeinject; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public record SafeInjectionCleanupResult( + @JsonProperty("run_id") + String runId, + @JsonProperty("deleted_k8s_connectors") + int deletedK8sConnectors, + @JsonProperty("deleted_metadata_rows") + int deletedMetadataRows, + @JsonProperty("residual_count") + int residualCount, + @JsonProperty("residuals") + List residuals +) { +} diff --git a/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCreateRequest.java b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCreateRequest.java new file mode 100644 index 00000000..bbb3f666 --- /dev/null +++ b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCreateRequest.java @@ -0,0 +1,8 @@ +package com.bifrost.ops.internalops.safeinject; + +public record SafeInjectionCreateRequest( + String runId, + String fault, + String connectorName +) { +} diff --git a/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCreateResult.java b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCreateResult.java new file mode 100644 index 00000000..6786feab --- /dev/null +++ b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionCreateResult.java @@ -0,0 +1,27 @@ +package com.bifrost.ops.internalops.safeinject; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; +import java.util.UUID; + +public record SafeInjectionCreateResult( + @JsonProperty("run_id") + String runId, + String fault, + @JsonProperty("expected_root_cause_id") + String expectedRootCauseId, + @JsonProperty("pipeline_id") + UUID pipelineId, + @JsonProperty("datasource_id") + UUID datasourceId, + @JsonProperty("metadata_persisted") + boolean metadataPersisted, + @JsonProperty("connector_name") + String connectorName, + String namespace, + Map labels, + @JsonProperty("safe_scope") + String safeScope +) { +} diff --git a/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionFault.java b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionFault.java new file mode 100644 index 00000000..20f0500f --- /dev/null +++ b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionFault.java @@ -0,0 +1,46 @@ +package com.bifrost.ops.internalops.safeinject; + +import java.util.Locale; + +public enum SafeInjectionFault { + AUTH("auth", "SINK_AUTH_EXPIRED", "a1"), + SCHEMA("schema", "SCHEMA_MISMATCH", "s1"), + LAG("lag", "CONSUMER_LAG_SPIKE", "l1"), + SINK_FAIL("sink-fail", "SINK_DB_CONNECTION_TIMEOUT", "f1"), + NO_FAULT("no-fault", "UNKNOWN_WITH_EVIDENCE_GAP", "n1"); + + private final String wireName; + private final String expectedRootCauseId; + private final String code; + + SafeInjectionFault(String wireName, String expectedRootCauseId, String code) { + this.wireName = wireName; + this.expectedRootCauseId = expectedRootCauseId; + this.code = code; + } + + public String wireName() { + return wireName; + } + + public String expectedRootCauseId() { + return expectedRootCauseId; + } + + public String code() { + return code; + } + + public static SafeInjectionFault parse(String raw) { + if (raw == null || raw.isBlank()) { + throw new IllegalArgumentException("fault must not be blank"); + } + String normalized = raw.trim().toLowerCase(Locale.ROOT).replace('_', '-'); + for (SafeInjectionFault fault : values()) { + if (fault.wireName.equals(normalized)) { + return fault; + } + } + throw new IllegalArgumentException("unsupported safe injection fault: " + raw); + } +} diff --git a/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionService.java b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionService.java new file mode 100644 index 00000000..ff5d6fd8 --- /dev/null +++ b/services/operations-backend/src/main/java/com/bifrost/ops/internalops/safeinject/SafeInjectionService.java @@ -0,0 +1,359 @@ +package com.bifrost.ops.internalops.safeinject; + +import com.bifrost.ops.global.common.error.ApiException; +import com.bifrost.ops.global.common.error.ErrorCode; +import com.bifrost.ops.workspace.persistence.entity.WorkspaceEntity; +import com.bifrost.ops.workspace.persistence.repository.WorkspaceRepository; +import io.fabric8.kubernetes.api.model.GenericKubernetesResource; +import io.fabric8.kubernetes.api.model.GenericKubernetesResourceBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.regex.Pattern; + +@Service +public class SafeInjectionService { + + public static final String LABEL_SAFE = "bifrost.io/safe-inject"; + public static final String LABEL_RUN = "bifrost.io/safe-inject-run"; + public static final String LABEL_FAULT = "bifrost.io/safe-inject-fault"; + public static final String LABEL_PROJECT = "bifrost.io/safe-inject-project"; + public static final String LABEL_CLUSTER = "strimzi.io/cluster"; + + private static final String OP_SCOPE = "safeinject:test-connector-only"; + private static final String CONNECTOR_PREFIX = "safeinject-"; + private static final Pattern DNS_LABEL = Pattern.compile("[a-z0-9]([-a-z0-9]*[a-z0-9])?"); + private static final Pattern RUN_ID = Pattern.compile("[A-Za-z0-9._:-]{1,80}"); + + private final KubernetesClient k8s; + private final WorkspaceRepository workspaceRepository; + private final String namespace; + private final String connectCluster; + private final Set allowedProjects; + + public SafeInjectionService(KubernetesClient k8s, + WorkspaceRepository workspaceRepository, + @Value("${kafka-cluster.namespace:platform-kafka}") String namespace, + @Value("${kafka-connect.cluster:platform-connect}") String connectCluster, + @Value("${safe-inject.allowed-projects:safeinject,e2e-rca-test,e2e-rca-test-0621}") + String allowedProjects) { + this.k8s = k8s; + this.workspaceRepository = workspaceRepository; + this.namespace = nonBlankOrDefault(namespace, "platform-kafka"); + this.connectCluster = nonBlankOrDefault(connectCluster, "platform-connect"); + this.allowedProjects = parseAllowedProjects(allowedProjects); + } + + public SafeInjectionCreateResult create(String projectId, SafeInjectionCreateRequest request) { + SafeInjectionFault fault = SafeInjectionFault.parse(request.fault()); + String runId = requireRunId(request.runId()); + WorkspaceEntity workspace = requireWorkspace(projectId); + requireTestWorkspace(projectId, workspace); + + String runKey = runKey(workspace.getId(), runId); + String connectorName = connectorName(request.connectorName(), runKey, fault); + UUID pipelineId = stableUuid("pipeline", workspace.getId(), runId, fault); + UUID datasourceId = stableUuid("datasource", workspace.getId(), runId, fault); + Map labels = labels(projectId, runKey, fault); + + assertNoExistingConnector(connectorName); + GenericKubernetesResource connector = buildConnector(connectorName, labels, fault, projectId, runKey); + k8s.resource(connector).inNamespace(namespace).create(); + + return new SafeInjectionCreateResult( + runId, + fault.wireName(), + fault.expectedRootCauseId(), + pipelineId, + datasourceId, + false, + connectorName, + namespace, + labels, + OP_SCOPE); + } + + public SafeInjectionCleanupResult cleanup(String projectId, String rawRunId) { + String runId = requireRunId(rawRunId); + WorkspaceEntity workspace = requireWorkspace(projectId); + requireTestWorkspace(projectId, workspace); + String runKey = runKey(workspace.getId(), runId); + + int deletedK8s = deleteK8sConnectors(runKey); + List residuals = residuals(runKey); + return new SafeInjectionCleanupResult(runId, deletedK8s, 0, residuals.size(), residuals); + } + + public SafeInjectionCleanupResult residualsOnly(String projectId, String rawRunId) { + String runId = requireRunId(rawRunId); + WorkspaceEntity workspace = requireWorkspace(projectId); + requireTestWorkspace(projectId, workspace); + String runKey = runKey(workspace.getId(), runId); + List residuals = residuals(runKey); + return new SafeInjectionCleanupResult(runId, 0, 0, residuals.size(), residuals); + } + + private int deleteK8sConnectors(String runKey) { + int deleted = 0; + var list = k8s.genericKubernetesResources("kafka.strimzi.io/v1", "KafkaConnector") + .inNamespace(namespace) + .withLabel(LABEL_SAFE, "true") + .withLabel(LABEL_RUN, runKey) + .list(); + for (GenericKubernetesResource item : list.getItems()) { + String name = item.getMetadata() == null ? null : item.getMetadata().getName(); + if (name == null || !name.startsWith(CONNECTOR_PREFIX) || !isSafeResource(item, runKey)) { + continue; + } + k8s.resource(connectorTemplate(name)).inNamespace(namespace).delete(); + deleted++; + } + return deleted; + } + + private List residuals(String runKey) { + List out = new ArrayList<>(); + var list = k8s.genericKubernetesResources("kafka.strimzi.io/v1", "KafkaConnector") + .inNamespace(namespace) + .withLabel(LABEL_SAFE, "true") + .withLabel(LABEL_RUN, runKey) + .list(); + for (GenericKubernetesResource item : list.getItems()) { + String name = item.getMetadata() == null ? null : item.getMetadata().getName(); + if (name != null && name.startsWith(CONNECTOR_PREFIX) && isSafeResource(item, runKey)) { + out.add("k8s:KafkaConnector/" + name); + } + } + return List.copyOf(out); + } + + private void assertNoExistingConnector(String connectorName) { + GenericKubernetesResource existing = k8s.resource(connectorTemplate(connectorName)).inNamespace(namespace).get(); + if (existing != null) { + throw new IllegalArgumentException("connector name collides with existing resource: " + connectorName); + } + } + + private GenericKubernetesResource buildConnector(String name, + Map labels, + SafeInjectionFault fault, + String projectId, + String runKey) { + Map spec = new LinkedHashMap<>(); + spec.put("class", connectorClass(fault)); + spec.put("tasksMax", 1); + spec.put("config", connectorConfig(fault, projectId, runKey)); + + var builder = new GenericKubernetesResourceBuilder() + .withApiVersion("kafka.strimzi.io/v1") + .withKind("KafkaConnector") + .withNewMetadata() + .withName(name) + .withNamespace(namespace) + .addToLabels(labels) + .endMetadata() + .addToAdditionalProperties("spec", spec); + if (fault == SafeInjectionFault.LAG || fault == SafeInjectionFault.NO_FAULT) { + builder.editMetadata().addToAnnotations("strimzi.io/pause", "true").endMetadata(); + } + return builder.build(); + } + + private Map connectorConfig(SafeInjectionFault fault, String projectId, String runKey) { + Map config = new LinkedHashMap<>(); + config.put("topics", safeTopic(projectId, runKey, fault)); + config.put("key.converter", "org.apache.kafka.connect.storage.StringConverter"); + config.put("value.converter", "org.apache.kafka.connect.json.JsonConverter"); + config.put("value.converter.schemas.enable", "false"); + config.put("errors.tolerance", "none"); + switch (fault) { + case AUTH -> { + config.put("connection.url", "jdbc:postgresql://safeinject.invalid:5432/safeinject"); + config.put("connection.user", "safeinject_invalid_user"); + config.put("connection.password", "safeinject_invalid_password"); + config.put("insert.mode", "insert"); + config.put("auto.create", "false"); + config.put("connection.attempts", "1"); + config.put("retry.backoff.ms", "100"); + } + case SCHEMA -> { + config.put("value.converter", "com.bifrost.safeinject.NoSuchConverter"); + config.put("file", "/tmp/bifrost-safeinject-schema-" + runKey + ".log"); + } + case LAG -> config.put("file", "/tmp/bifrost-safeinject-lag-" + runKey + ".log"); + case SINK_FAIL -> { + config.put("connection.url", "jdbc:postgresql://safeinject-sink.invalid:5432/safeinject"); + config.put("connection.user", "safeinject"); + config.put("connection.password", "safeinject"); + config.put("insert.mode", "insert"); + config.put("auto.create", "false"); + config.put("connection.attempts", "1"); + config.put("retry.backoff.ms", "100"); + } + case NO_FAULT -> config.put("file", "/tmp/bifrost-safeinject-control-" + runKey + ".log"); + } + return config; + } + + private String connectorClass(SafeInjectionFault fault) { + return switch (fault) { + case AUTH, SINK_FAIL -> "io.confluent.connect.jdbc.JdbcSinkConnector"; + case SCHEMA, LAG, NO_FAULT -> "org.apache.kafka.connect.file.FileStreamSinkConnector"; + }; + } + + private Map labels(String projectId, String runKey, SafeInjectionFault fault) { + Map labels = new LinkedHashMap<>(); + labels.put(LABEL_CLUSTER, connectCluster); + labels.put(LABEL_SAFE, "true"); + labels.put(LABEL_RUN, runKey); + labels.put(LABEL_FAULT, fault.wireName()); + labels.put(LABEL_PROJECT, labelValue(projectId)); + return labels; + } + + private GenericKubernetesResource connectorTemplate(String name) { + return new GenericKubernetesResourceBuilder() + .withApiVersion("kafka.strimzi.io/v1") + .withKind("KafkaConnector") + .withNewMetadata() + .withName(name) + .withNamespace(namespace) + .endMetadata() + .build(); + } + + private boolean isSafeResource(GenericKubernetesResource resource, String runKey) { + if (resource.getMetadata() == null || resource.getMetadata().getLabels() == null) { + return false; + } + Map labels = resource.getMetadata().getLabels(); + if (!"true".equals(labels.get(LABEL_SAFE))) { + return false; + } + return runKey == null || runKey.equals(labels.get(LABEL_RUN)); + } + + private WorkspaceEntity requireWorkspace(String projectId) { + return workspaceRepository.findByNamespace(projectId) + .or(() -> parseUuid(projectId).flatMap(workspaceRepository::findById)) + .orElseThrow(() -> new ApiException(ErrorCode.WORKSPACE_NOT_FOUND, + "프로젝트를 찾을 수 없습니다: " + projectId)); + } + + private void requireTestWorkspace(String projectId, WorkspaceEntity workspace) { + TreeSet candidates = new TreeSet<>(); + candidates.add(projectId); + candidates.add(workspace.getId().toString()); + candidates.add(nullToEmpty(workspace.getNamespace())); + candidates.add(nullToEmpty(workspace.getName())); + boolean allowed = candidates.stream() + .map(v -> v.toLowerCase(Locale.ROOT)) + .anyMatch(v -> allowedProjects.contains(v) + || v.startsWith("safeinject") + || v.startsWith("e2e-rca-test")); + if (!allowed) { + throw new ApiException(ErrorCode.WORKSPACE_FORBIDDEN, + "safe injection is allowed only for configured test workspaces"); + } + } + + private String connectorName(String requested, String runKey, SafeInjectionFault fault) { + String generated = CONNECTOR_PREFIX + runKey + "-" + fault.code(); + String name = requested == null || requested.isBlank() ? generated : requested.trim(); + if (!generated.equals(name)) { + throw new IllegalArgumentException("connectorName override is not supported for residual-safe cleanup"); + } + if (!name.startsWith(CONNECTOR_PREFIX)) { + throw new IllegalArgumentException("safe connector name must start with " + CONNECTOR_PREFIX); + } + if (name.length() > 63 || !DNS_LABEL.matcher(name).matches()) { + throw new IllegalArgumentException("safe connector name must be a DNS-1123 label up to 63 chars"); + } + return name; + } + + private String requireRunId(String runId) { + if (runId == null || runId.isBlank() || !RUN_ID.matcher(runId.trim()).matches()) { + throw new IllegalArgumentException("runId must match [A-Za-z0-9._:-]{1,80}"); + } + return runId.trim(); + } + + private UUID stableUuid(String kind, UUID workspaceId, String runId, SafeInjectionFault fault) { + String key = "safeinject:" + kind + ":" + workspaceId + ":" + runId + ":" + fault.wireName(); + return UUID.nameUUIDFromBytes(key.getBytes(StandardCharsets.UTF_8)); + } + + private String runKey(UUID workspaceId, String runId) { + return "r" + sha256(workspaceId + ":" + runId).substring(0, 12); + } + + private String safeTopic(String projectId, String runKey, SafeInjectionFault fault) { + return "safeinject." + labelValue(projectId) + "." + runKey + "." + fault.code(); + } + + private String labelValue(String value) { + String normalized = nullToEmpty(value) + .toLowerCase(Locale.ROOT) + .replaceAll("[^a-z0-9_.-]", "-"); + if (normalized.isBlank()) { + normalized = "unknown"; + } + if (normalized.length() <= 63) { + return normalized; + } + return normalized.substring(0, 50) + "-" + sha256(normalized).substring(0, 12); + } + + private static Optional parseUuid(String value) { + try { + return Optional.of(UUID.fromString(value)); + } catch (RuntimeException ignored) { + return Optional.empty(); + } + } + + private static Set parseAllowedProjects(String raw) { + Set out = new TreeSet<>(); + if (raw != null) { + for (String part : raw.split(",")) { + if (!part.isBlank()) { + out.add(part.trim().toLowerCase(Locale.ROOT)); + } + } + } + return Set.copyOf(out); + } + + private static String sha256(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + + private static String nonBlankOrDefault(String value, String fallback) { + return value == null || value.isBlank() ? fallback : value; + } + + private static String nullToEmpty(String value) { + return value == null ? "" : value; + } +} diff --git a/services/operations-backend/src/main/resources/application.yml b/services/operations-backend/src/main/resources/application.yml index e43bd108..93357155 100644 --- a/services/operations-backend/src/main/resources/application.yml +++ b/services/operations-backend/src/main/resources/application.yml @@ -58,6 +58,10 @@ internal: ops: token: ${INTERNAL_OPS_TOKEN:} +# safe live fault injection 전용 내부 경로. 기본값은 테스트 workspace slug만 허용한다. +safe-inject: + allowed-projects: ${SAFE_INJECT_ALLOWED_PROJECTS:safeinject,e2e-rca-test,e2e-rca-test-0621} + # 라이브 read-tool 스모크 프로젝트 복구 seed(#650). 수동 DB INSERT 대신 앱 부트스트랩에서 # 고정 UUID(11111111-2222-3333-4444-555555555555) 프로젝트와 최소 샘플 데이터를 보장한다. smoke: diff --git a/services/operations-backend/src/test/java/com/bifrost/ops/internalops/safeinject/SafeInjectionServiceTest.java b/services/operations-backend/src/test/java/com/bifrost/ops/internalops/safeinject/SafeInjectionServiceTest.java new file mode 100644 index 00000000..16a84d4c --- /dev/null +++ b/services/operations-backend/src/test/java/com/bifrost/ops/internalops/safeinject/SafeInjectionServiceTest.java @@ -0,0 +1,182 @@ +package com.bifrost.ops.internalops.safeinject; + +import com.bifrost.ops.global.common.error.ApiException; +import com.bifrost.ops.workspace.persistence.entity.WorkspaceEntity; +import com.bifrost.ops.workspace.persistence.repository.WorkspaceRepository; +import io.fabric8.kubernetes.api.model.GenericKubernetesResource; +import io.fabric8.kubernetes.api.model.GenericKubernetesResourceBuilder; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@EnableKubernetesMockClient(crud = true) +class SafeInjectionServiceTest { + + KubernetesClient client; + + private static final String NS = "platform-kafka"; + private static final String CLUSTER = "platform-connect"; + + private WorkspaceRepository workspaceRepository; + private SafeInjectionService service; + + @BeforeEach + void setUp() { + registerKafkaConnectorCrd(); + workspaceRepository = mock(WorkspaceRepository.class); + workspace("e2e-rca-test", UUID.fromString("8898903c-d5db-4a8c-9ff3-104632f4f70f")); + service = new SafeInjectionService( + client, + workspaceRepository, + NS, + CLUSTER, + "e2e-rca-test"); + } + + @Test + void createSchemaConnectorUsesSafeLabelsAndConverterFault() { + SafeInjectionCreateResult result = service.create("e2e-rca-test", + new SafeInjectionCreateRequest("run-001", "schema", null)); + + GenericKubernetesResource cr = client.resource(connectorTemplate(result.connectorName())) + .inNamespace(NS) + .get(); + assertThat(cr).isNotNull(); + assertThat(cr.getMetadata().getLabels()) + .containsEntry(SafeInjectionService.LABEL_SAFE, "true") + .containsEntry(SafeInjectionService.LABEL_RUN, result.labels().get(SafeInjectionService.LABEL_RUN)) + .containsEntry("strimzi.io/cluster", CLUSTER); + + Map spec = asMap(cr.getAdditionalProperties().get("spec")); + Map config = asMap(spec.get("config")); + assertThat(spec.get("class")).isEqualTo("org.apache.kafka.connect.file.FileStreamSinkConnector"); + assertThat(config.get("value.converter")).isEqualTo("com.bifrost.safeinject.NoSuchConverter"); + assertThat(result.pipelineId()).isNotNull(); + assertThat(result.datasourceId()).isNotNull(); + assertThat(result.metadataPersisted()).isFalse(); + } + + @Test + void rejectsWorkspaceOutsideSafeAllowlist() { + workspace("prod-team", UUID.randomUUID()); + + assertThatThrownBy(() -> service.create("prod-team", + new SafeInjectionCreateRequest("run-002", "auth", null))) + .isInstanceOf(ApiException.class) + .hasMessageContaining("configured test workspaces"); + } + + @Test + void rejectsConnectorNameOverrideSoCleanupRemainsDeterministic() { + assertThatThrownBy(() -> service.create("e2e-rca-test", + new SafeInjectionCreateRequest("run-003", "auth", "safeinject-custom"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("override is not supported"); + } + + @Test + void rejectsExistingNonSafeKubernetesConnectorCollision() { + String generated = service.create("e2e-rca-test", + new SafeInjectionCreateRequest("collision-seed", "auth", null)).connectorName(); + service.cleanup("e2e-rca-test", "collision-seed"); + client.resource(new GenericKubernetesResourceBuilder() + .withApiVersion("kafka.strimzi.io/v1") + .withKind("KafkaConnector") + .withNewMetadata() + .withName(generated) + .withNamespace(NS) + .addToLabels("strimzi.io/cluster", CLUSTER) + .endMetadata() + .addToAdditionalProperties("spec", Map.of("class", "example.NonSafe", "tasksMax", 1, "config", Map.of())) + .build()) + .inNamespace(NS) + .create(); + + assertThatThrownBy(() -> service.create("e2e-rca-test", + new SafeInjectionCreateRequest("collision-seed", "auth", null))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("existing resource"); + } + + @Test + void rejectsExistingSafeConnectorInsteadOfUpdatingIt() { + service.create("e2e-rca-test", + new SafeInjectionCreateRequest("same-run", "lag", null)); + + assertThatThrownBy(() -> service.create("e2e-rca-test", + new SafeInjectionCreateRequest("same-run", "lag", null))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("existing resource"); + } + + @Test + void cleanupDeletesSafeResourcesAndReturnsResidualZero() { + SafeInjectionCreateResult result = service.create("e2e-rca-test", + new SafeInjectionCreateRequest("run-clean", "sink-fail", null)); + + SafeInjectionCleanupResult cleanup = service.cleanup("e2e-rca-test", "run-clean"); + + assertThat(cleanup.deletedK8sConnectors()).isEqualTo(1); + assertThat(cleanup.deletedMetadataRows()).isZero(); + assertThat(cleanup.residualCount()).isZero(); + assertThat(client.resource(connectorTemplate(result.connectorName())).inNamespace(NS).get()).isNull(); + } + + private void registerKafkaConnectorCrd() { + client.apiextensions().v1().customResourceDefinitions().resource( + new CustomResourceDefinitionBuilder() + .withNewMetadata().withName("kafkaconnectors.kafka.strimzi.io").endMetadata() + .withNewSpec() + .withGroup("kafka.strimzi.io") + .withScope("Namespaced") + .withNewNames() + .withPlural("kafkaconnectors") + .withSingular("kafkaconnector") + .withKind("KafkaConnector") + .endNames() + .addNewVersion().withName("v1").withServed(true).withStorage(true) + .withNewSchema().withNewOpenAPIV3Schema().withType("object") + .endOpenAPIV3Schema().endSchema() + .endVersion() + .endSpec() + .build() + ).create(); + } + + private void workspace(String namespace, UUID id) { + WorkspaceEntity workspace = new WorkspaceEntity(); + workspace.setId(id); + workspace.setName(namespace); + workspace.setNamespace(namespace); + workspace.setStatus(WorkspaceEntity.Status.ACTIVE); + when(workspaceRepository.findByNamespace(namespace)).thenReturn(Optional.of(workspace)); + when(workspaceRepository.findById(id)).thenReturn(Optional.of(workspace)); + } + + private GenericKubernetesResource connectorTemplate(String name) { + return new GenericKubernetesResourceBuilder() + .withApiVersion("kafka.strimzi.io/v1") + .withKind("KafkaConnector") + .withNewMetadata() + .withName(name) + .withNamespace(NS) + .endMetadata() + .build(); + } + + @SuppressWarnings("unchecked") + private Map asMap(Object value) { + return (Map) value; + } +}