From 211a3dbc48203fd1d22ee748c5d10c47a2067d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=84=B8=EB=B9=88=20=EA=B6=8C?= Date: Tue, 23 Jun 2026 14:29:20 +0900 Subject: [PATCH 1/3] Preserve RCA accuracy on production read-tool payloads Current develop already contains the bulk RCA accuracy implementation from #993, so this change forward-ports only the production-transfer hardening still missing from the c1 local-loop branch: vendor SQL/Connect error signatures, missing-database task-failure evidence, and source/sink side inference from the specific failed structured payload item. Constraint: PR base must remain develop and deploy surface must exclude local C3 measurement harness files Rejected: Replace whole c1 files on current develop | current develop has newer RCA, retrieval, and catalog work that would be regressed Confidence: high Scope-risk: narrow Directive: Keep this extractor emitting catalog phrases only; do not include raw connector names, credentials, or fixture labels in summaries Tested: AI_LLM_API_KEY= OPENAI_API_KEY= uv run --extra dev pytest tests/test_evidence_signals.py -q Tested: AI_LLM_API_KEY= OPENAI_API_KEY= uv run --extra dev pytest -q --- services/ai-service/app/evidence/signals.py | 136 ++++++++++++++++-- .../ai-service/tests/test_evidence_signals.py | 133 +++++++++++++++++ 2 files changed, 257 insertions(+), 12 deletions(-) diff --git a/services/ai-service/app/evidence/signals.py b/services/ai-service/app/evidence/signals.py index 20d89356..7af9f832 100644 --- a/services/ai-service/app/evidence/signals.py +++ b/services/ai-service/app/evidence/signals.py @@ -16,35 +16,52 @@ _TASK_FAILURE_RE = re.compile(r"\bFAILED\b|failed|failure|exception|오류|에러|실패", re.IGNORECASE) _TRACE_RE = re.compile(r"trace|stack|exception|worker log|stacktrace|로그|트레이스", re.IGNORECASE) _AUTH_RE = re.compile( - r"access\s*denied|permission\s*denied|unauthori[sz]ed|authentication|" + r"\b1045\b|\b28000\b|access\s*denied|permission\s*denied|unauthori[sz]ed|authentication|" r"\bauth\b|auth(?:entication)?\s*error|token\s*expired|credential\s*expired|" r"password authentication failed|invalid password|invalid credential|" - r"credential invalid|credential expired|sasl|인증\s*실패|권한\s*거부|" + r"credential invalid|credential expired|sasl|using password|인증\s*실패|권한\s*거부|" r"credential\s*만료|credential\s*미적용|토큰\s*만료", re.IGNORECASE, ) _SCHEMA_RE = re.compile( - r"schema|serialization|deserialization|incompatible|compatibility|converter|" - r"avro|serde|type mismatch|schema registry|스키마|역직렬화|호환성", + r"\b1366\b|\b22007\b|schema|serialization|deserialization|deserialize|" + r"incompatible|compatibility|converter|avro|serde|type mismatch|schema registry|" + r"incorrect\s+\w+\s+value|incorrect .* value|field type|dataexception|" + r"invalid input syntax|date/time field value out of range|invalid datetime format|" + r"(?:date|time|timestamp|datetime)\s+(?:parse|parsing)\s+(?:error|fail)|" + r"parse\s+(?:error|fail).*(?:date|time|timestamp|datetime)|" + r"cannot parse .*?(?:date|time|timestamp|datetime)|스키마|역직렬화|호환성", re.IGNORECASE, ) _CONFIG_RE = re.compile( r"invalid config|unknown config|config validation|configuration error|" - r"invalid option|required configuration|bad config|config diff|" + r"configuration value is invalid|" + r"(?:invalid|bad) value for configuration|value is invalid|integer is expected|" + r"invalid option|required configuration|missing required config(?:uration)?|" + r"required config(?:uration)?.*(?:not present|missing)|" + r"configuration .*?(?:not present|missing|required)|" + r"invalid/empty configuration value|empty configuration value|" + r"bad config|config diff|transform|class not found|classnotfound|noclassdeffound|" + r"failed to find any class|" r"config\s*변경|설정\s*오류|설정\s*오타|잘못된\s*설정", re.IGNORECASE, ) _CONSTRAINT_RE = re.compile( - r"duplicate key|unique constraint|constraint violation|foreign key|not null|" - r"data integrity|sqlintegrity|record rejected|duplicate records?|" + r"\b4025\b|\b23000\b|duplicate key|unique constraint|constraint violation|foreign key|not null|" + r"check constraint|constraint .* failed|data integrity|sqlintegrity|record rejected|duplicate records?|" r"중복\s*레코드|제약\s*위반|" r"(?:record|row|sink|data)\s+validation fail", re.IGNORECASE, ) -_TIMEOUT_RE = re.compile(r"timeout|timed out|read timed out|write timed out|타임아웃", re.IGNORECASE) +_TIMEOUT_RE = re.compile( + r"\b2002\b|timeout|timed out|timeout expired|connection timed out|" + r"read timed out|write timed out|can't connect to server|could not connect|타임아웃", + re.IGNORECASE, +) _NETWORK_RE = re.compile( r"dns|unknownhost|no route|network unreachable|connection refused|" r"connect exception|tcp connect|endpoint unreachable|host unreachable|i/o error|" + r"can't connect to server|could not connect|name or service not known|temporary failure in name resolution|" r"네트워크\s*단절|연결\s*실패|호스트.*포트", re.IGNORECASE, ) @@ -99,8 +116,19 @@ r"|상태\s*[:=]?\s*(?:FAILED|실패|오류|에러)", re.IGNORECASE, ) +_DATABASE_DOES_NOT_EXIST_RE = re.compile( + r"database\s+[\"'][^\"']+[\"']\s+does not exist|database\s+\S+\s+does not exist", + re.IGNORECASE, +) _SCHEMA_VERSION_RE = re.compile(r"schema version|subject version|schema registry", re.IGNORECASE) -_SCHEMA_STRUCTURE_RE = re.compile(r"type mismatch|field type|필드 타입", re.IGNORECASE) +_SCHEMA_STRUCTURE_RE = re.compile( + r"\b1366\b|\b22007\b|incorrect\s+\w+\s+value|incorrect .* value|" + r"type mismatch|field type|invalid input syntax|date/time field value out of range|" + r"invalid datetime format|(?:date|time|timestamp|datetime)\s+(?:parse|parsing)\s+(?:error|fail)|" + r"parse\s+(?:error|fail).*(?:date|time|timestamp|datetime)|" + r"cannot parse .*?(?:date|time|timestamp|datetime)|필드 타입", + re.IGNORECASE, +) _CONFIG_CHANGE_RE = re.compile(r"config change|config diff|config 변경", re.IGNORECASE) _DUPLICATE_COUNT_RE = re.compile(r"duplicate count|중복\s*레코드", re.IGNORECASE) _READ_LATENCY_RE = re.compile( @@ -279,8 +307,11 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: tags: list[str] = [] lower = text.casefold() - side = _side_hint(lower, tool_name) + structured_side = _structured_fault_side(raw_payload) + side = structured_side or _side_hint(lower, tool_name) auth_side = _auth_side_hint(pieces, tool_name) + if auth_side == "unknown" and structured_side in {"source", "sink"}: + auth_side = structured_side failed = _has_signal(pieces, _TASK_FAILURE_RE, "failure") connector_failed = _has_signal(pieces, _CONNECTOR_FAILED_RE, "connector") or ( @@ -293,6 +324,7 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: constraint = _has_signal(pieces, _CONSTRAINT_RE, "duplicate") timeout = _has_signal(pieces, _TIMEOUT_RE, "timeout") network = _has_signal(pieces, _NETWORK_RE, "network") + missing_database = _has_signal(pieces, _DATABASE_DOES_NOT_EXIST_RE, "failure") lag_spike = _has_signal(pieces, _LAG_SPIKE_RE, "lag") offset_slow = _has_signal(pieces, _OFFSET_PROGRESS_RE, "lag") deployment = _has_signal(pieces, _DEPLOYMENT_RE, "deployment") @@ -300,9 +332,9 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: retry = _has_signal(pieces, _RETRY_RE, "retry") idempotency = _has_signal(pieces, _IDEMPOTENCY_RE, "idempotency") - if connector_failed: + if connector_failed or missing_database: _add(tags, "connector task status FAILED") - if trace and failed: + if (trace and failed) or missing_database: _add(tags, "task trace 또는 worker log") if auth: @@ -413,6 +445,72 @@ def _flatten_text(value: Any, context: str = "") -> Iterable[str]: yield from _flatten_text(item, context) +def _structured_fault_side(value: Any) -> str | None: + sides: list[str] = [] + for item in _walk(value): + if not isinstance(item, dict) or not _has_structured_fault_indicator(item): + continue + side = _side_from_mapping(item) + if side and side not in sides: + sides.append(side) + return sides[0] if len(sides) == 1 else None + + +def _has_structured_fault_indicator(item: dict[Any, Any]) -> bool: + for key, raw in item.items(): + key_text = _key_name(key) + raw_text = str(raw).casefold() + if key_text in {"state", "connectorstate", "taskstate", "status"} and raw_text == "failed": + return True + if key_text == "connectionstatus" and raw_text in {"down", "failed"}: + return True + if key_text in {"failedtasks", "tasksfailed", "errorcount", "matchcount"} and _positive_number(raw): + return True + if key_text == "exitcode" and _nonzero_number(raw): + return True + if key_text in {"trace", "stack", "stacktrace"} and str(raw or "").strip(): + return True + + text = " ".join(_flatten_text(item)) + if _is_normal_only(text): + return False + return bool( + _TASK_FAILURE_RE.search(text) + or _AUTH_RE.search(text) + or _SCHEMA_RE.search(text) + or _CONFIG_RE.search(text) + or _CONSTRAINT_RE.search(text) + or _TIMEOUT_RE.search(text) + or _NETWORK_RE.search(text) + or _RETRY_RE.search(text) + or _DATABASE_DOES_NOT_EXIST_RE.search(text) + ) + + +def _side_from_mapping(item: dict[Any, Any]) -> str | None: + for key, raw in item.items(): + if _key_name(key) not in {"datasourcerole", "role", "type", "kind"}: + continue + value = str(raw or "").casefold() + if value in {"source", "sink"}: + return value + return None + + +def _walk(value: Any) -> Iterable[Any]: + yield value + if isinstance(value, dict): + for item in value.values(): + yield from _walk(item) + elif isinstance(value, (list, tuple, set)): + for item in value: + yield from _walk(item) + + +def _key_name(value: Any) -> str: + return str(value).casefold().replace("_", "").replace("-", "") + + def _side_hint(text: str, tool_name: str) -> str: has_source = bool(_SOURCE_HINT_RE.search(text)) has_sink = bool(_SINK_HINT_RE.search(text)) @@ -512,3 +610,17 @@ def _has_fault_negation(text: str, fault: str) -> bool: def _add(tags: list[str], tag: str) -> None: if tag not in tags: tags.append(tag) + + +def _positive_number(value: Any) -> bool: + try: + return float(value) > 0 + except (TypeError, ValueError): + return False + + +def _nonzero_number(value: Any) -> bool: + try: + return float(value) != 0 + except (TypeError, ValueError): + return False diff --git a/services/ai-service/tests/test_evidence_signals.py b/services/ai-service/tests/test_evidence_signals.py index f7ff6ec4..95da87dc 100644 --- a/services/ai-service/tests/test_evidence_signals.py +++ b/services/ai-service/tests/test_evidence_signals.py @@ -167,3 +167,136 @@ def test_signal_summary_keeps_fault_when_negative_and_fault_fragments_are_distin summary = evidence_signal_summary("search_logs", payload) assert "serialization/deserialization/schema error" in summary + + +def test_vendor_auth_code_uses_datasource_role_for_sink_auth() -> None: + payload = { + "logs": [ + { + "datasourceRole": "sink", + "engine": "mariadb", + "exit_code": 1, + "stderr": "ERROR 1045 (28000): Access denied for user '[USER]'@'localhost' (using password: YES)", + } + ] + } + + summary = evidence_signal_summary("search_logs", payload) + + assert "sink auth/permission error log" in summary + assert "source auth/permission error log" not in summary + + +def test_vendor_timeout_codes_use_failed_item_side_for_source_and_sink() -> None: + source = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "datasourceRole": "source", + "engine": "postgresql", + "exit_code": 2, + "stderr": 'psql: error: connection to server at "10.255.255.1", port 5432 failed: timeout expired', + } + ] + }, + ) + sink = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "datasourceRole": "sink", + "engine": "mariadb", + "exit_code": 1, + "stderr": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + } + ] + }, + ) + + assert "source connection timeout 증가" in source + assert "pipeline extract/read 단계 timeout log" in source + assert "sink write timeout 증가" in sink + assert "sink dependency 연결 실패 또는 connection timeout" in sink + + +def test_mixed_datasource_payload_uses_failed_item_side_for_sink_timeout() -> None: + summary = evidence_signal_summary( + "list_datasources", + { + "datasources": [ + { + "id": "source-db", + "role": "source", + "connectionStatus": "UP", + }, + { + "id": "sink-db", + "role": "sink", + "connectionStatus": "DOWN", + "lastError": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + }, + ] + }, + ) + + assert "sink write timeout 증가" in summary + assert "sink dependency 연결 실패 또는 connection timeout" in summary + assert "Bifrost에서 source endpoint reachability 실패" not in summary + assert "source connection timeout 증가" not in summary + + +def test_missing_database_validation_emits_connector_failure_signal() -> None: + summary = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "datasourceRole": "source", + "engine": "postgresql", + "exit_code": 2, + "stderr": 'psql: error: FATAL: database "missingdb" does not exist', + } + ] + }, + ) + + assert "connector task status FAILED" in summary + assert "task trace 또는 worker log" in summary + assert "config validation error 또는 invalid option log" not in summary + + +def test_vendor_schema_and_constraint_codes_emit_specific_catalog_signals() -> None: + schema = evidence_signal_summary( + "search_logs", + {"logs": [{"stderr": "ERROR 1366 (22007): Incorrect integer value: 'not-a-number' for column amount"}]}, + ) + constraint = evidence_signal_summary( + "search_logs", + {"logs": [{"stderr": "ERROR 4025 (23000): CONSTRAINT `chk_orders` failed for `testdb`.`orders`"}]}, + ) + + assert "serialization/deserialization/schema error" in schema + assert "데이터 샘플 구조 변화" in schema + assert "sink constraint 또는 duplicate key error" in constraint + assert "동일 record 반복 실패" in constraint + + +def test_vendor_datetime_parse_failure_emits_schema_signal() -> None: + summary = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "engine": "postgresql", + "exit_code": 1, + "stderr": 'ERROR: invalid input syntax for type timestamp: "not-a-timestamp"', + } + ] + }, + ) + + assert "serialization/deserialization/schema error" in summary + assert "데이터 샘플 구조 변화" in summary + assert "config validation error 또는 invalid option log" not in summary From 8281722fca5e7322fd98ef15dc90d4dad343b4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=84=B8=EB=B9=88=20=EA=B6=8C?= Date: Tue, 23 Jun 2026 15:14:11 +0900 Subject: [PATCH 2/3] =?UTF-8?q?prod=20payload=20=EC=88=AB=EC=9E=90=20?= =?UTF-8?q?=EC=98=A4=EB=B0=9C=ED=99=94=EC=99=80=20side=20=EA=B3=BC?= =?UTF-8?q?=EC=9E=89=EB=8B=A8=EC=A0=95=EC=9D=84=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 배포 게이트는 read-tool payload 전체 flatten 과정에서 bare vendor code가 port/value/count metadata와 충돌하고, role이 없거나 dual-role인 connection failure를 source/extract evidence로 승격한다고 판정했다. Vendor code는 ERROR/SQLSTATE 같은 live error anchor가 있을 때만 잡도록 좁히고, side가 unknown인 network/timeout은 source/sink를 단정하지 않는 neutral evidence로만 emit한다. Constraint: PR #1008 deploy gate requires normal numeric metadata and ambiguous datasource role payloads to avoid side-specific fault evidence Rejected: Drop all numeric scalar flattening | broader behavior change and unrelated evidence loss risk Rejected: Suppress all unknown-side connectivity evidence | would lose recall for real unscoped connection failures Confidence: high Scope-risk: narrow Directive: Do not restore bare numeric vendor-code regexes without metadata false-positive probes Tested: uv run pytest tests/test_evidence_signals.py Tested: AI_LLM_API_KEY= OPENAI_API_KEY= uv run --extra dev pytest -q Tested: gate before/after probe removed numeric false positives and source-biased ambiguous-side evidence Not-tested: Branch-local c3 scripts are absent on feat/rca-accuracy-prod; external borrowed c3 harnesses were run but are not clean branch-local regression evidence --- services/ai-service/app/evidence/signals.py | 23 ++++++--- .../ai-service/tests/test_evidence_signals.py | 50 +++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/services/ai-service/app/evidence/signals.py b/services/ai-service/app/evidence/signals.py index 7af9f832..e798d42b 100644 --- a/services/ai-service/app/evidence/signals.py +++ b/services/ai-service/app/evidence/signals.py @@ -15,8 +15,13 @@ _FAILED_RE = re.compile(r"\bFAILED\b|failed|failure|exception|\berror\b|오류|에러|실패", re.IGNORECASE) _TASK_FAILURE_RE = re.compile(r"\bFAILED\b|failed|failure|exception|오류|에러|실패", re.IGNORECASE) _TRACE_RE = re.compile(r"trace|stack|exception|worker log|stacktrace|로그|트레이스", re.IGNORECASE) +_AUTH_VENDOR_CODE = r"(?:\bERROR\s+1045\b|\b1045\s*\(\s*28000\s*\)|\bSQLSTATE\s*\[?\s*28000\s*\]?)" +_SCHEMA_VENDOR_CODE = r"(?:\bERROR\s+1366\b|\b1366\s*\(\s*22007\s*\)|\bSQLSTATE\s*\[?\s*22007\s*\]?)" +_CONSTRAINT_VENDOR_CODE = r"(?:\bERROR\s+4025\b|\b4025\s*\(\s*23000\s*\)|\bSQLSTATE\s*\[?\s*23000\s*\]?)" +_TIMEOUT_VENDOR_CODE = r"(?:\bERROR\s+2002\b|\b2002\s*\(\s*HY000\s*\))" _AUTH_RE = re.compile( - r"\b1045\b|\b28000\b|access\s*denied|permission\s*denied|unauthori[sz]ed|authentication|" + _AUTH_VENDOR_CODE + + r"|access\s*denied|permission\s*denied|unauthori[sz]ed|authentication|" r"\bauth\b|auth(?:entication)?\s*error|token\s*expired|credential\s*expired|" r"password authentication failed|invalid password|invalid credential|" r"credential invalid|credential expired|sasl|using password|인증\s*실패|권한\s*거부|" @@ -24,7 +29,8 @@ re.IGNORECASE, ) _SCHEMA_RE = re.compile( - r"\b1366\b|\b22007\b|schema|serialization|deserialization|deserialize|" + _SCHEMA_VENDOR_CODE + + r"|schema|serialization|deserialization|deserialize|" r"incompatible|compatibility|converter|avro|serde|type mismatch|schema registry|" r"incorrect\s+\w+\s+value|incorrect .* value|field type|dataexception|" r"invalid input syntax|date/time field value out of range|invalid datetime format|" @@ -47,14 +53,16 @@ re.IGNORECASE, ) _CONSTRAINT_RE = re.compile( - r"\b4025\b|\b23000\b|duplicate key|unique constraint|constraint violation|foreign key|not null|" + _CONSTRAINT_VENDOR_CODE + + r"|duplicate key|unique constraint|constraint violation|foreign key|not null|" r"check constraint|constraint .* failed|data integrity|sqlintegrity|record rejected|duplicate records?|" r"중복\s*레코드|제약\s*위반|" r"(?:record|row|sink|data)\s+validation fail", re.IGNORECASE, ) _TIMEOUT_RE = re.compile( - r"\b2002\b|timeout|timed out|timeout expired|connection timed out|" + _TIMEOUT_VENDOR_CODE + + r"|timeout|timed out|timeout expired|connection timed out|" r"read timed out|write timed out|can't connect to server|could not connect|타임아웃", re.IGNORECASE, ) @@ -122,7 +130,8 @@ ) _SCHEMA_VERSION_RE = re.compile(r"schema version|subject version|schema registry", re.IGNORECASE) _SCHEMA_STRUCTURE_RE = re.compile( - r"\b1366\b|\b22007\b|incorrect\s+\w+\s+value|incorrect .* value|" + _SCHEMA_VENDOR_CODE + + r"|incorrect\s+\w+\s+value|incorrect .* value|" r"type mismatch|field type|invalid input syntax|date/time field value out of range|" r"invalid datetime format|(?:date|time|timestamp|datetime)\s+(?:parse|parsing)\s+(?:error|fail)|" r"parse\s+(?:error|fail).*(?:date|time|timestamp|datetime)|" @@ -381,7 +390,7 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: elif side == "sink": _add(tags, "sink dependency 연결 실패 또는 connection timeout") else: - _add(tags, "Bifrost에서 source endpoint reachability 실패") + _add(tags, "endpoint reachability failure log") if timeout: if side == "source": @@ -391,7 +400,7 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: _add(tags, "sink write timeout 증가") _add(tags, "sink dependency 연결 실패 또는 connection timeout") else: - _add(tags, "pipeline extract/read 단계 timeout log") + _add(tags, "connection timeout log") if _has_signal(pieces, _READ_LATENCY_RE, "latency"): _add(tags, "source read latency 증가") diff --git a/services/ai-service/tests/test_evidence_signals.py b/services/ai-service/tests/test_evidence_signals.py index 95da87dc..eda2f690 100644 --- a/services/ai-service/tests/test_evidence_signals.py +++ b/services/ai-service/tests/test_evidence_signals.py @@ -169,6 +169,56 @@ def test_signal_summary_keeps_fault_when_negative_and_fault_fragments_are_distin assert "serialization/deserialization/schema error" in summary +@pytest.mark.parametrize( + "payload", + [ + {"datasource": {"role": "source", "connectionStatus": "UP", "port": 2002}}, + {"metric": {"name": "records_processed", "value": 2002}}, + {"searchResult": {"matchCount": 1045, "logs": []}}, + {"summary": {"total": 23000, "taskId": 4025, "count": 1366, "year": 22007}}, + {"message": "codes observed in inventory text: 1045 2002 1366 22007 4025 23000"}, + ], +) +def test_vendor_numeric_codes_in_metadata_or_code_only_text_do_not_emit_signals(payload: dict) -> None: + summary = evidence_signal_summary("list_datasources", payload) + + assert summary == "" + + +@pytest.mark.parametrize( + "payload", + [ + { + "datasources": [ + { + "connectionStatus": "DOWN", + "lastError": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + } + ] + }, + { + "datasources": [ + { + "role": "source+sink", + "connectionStatus": "DOWN", + "lastError": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + } + ] + }, + ], +) +def test_unknown_or_dual_role_timeout_evidence_stays_side_neutral(payload: dict) -> None: + summary = evidence_signal_summary("list_datasources", payload) + + assert "connection timeout log" in summary + assert "endpoint reachability failure log" in summary + assert "Bifrost에서 source endpoint reachability 실패" not in summary + assert "pipeline extract/read 단계 timeout log" not in summary + assert "source connection timeout 증가" not in summary + assert "sink write timeout 증가" not in summary + assert "sink dependency 연결 실패 또는 connection timeout" not in summary + + def test_vendor_auth_code_uses_datasource_role_for_sink_auth() -> None: payload = { "logs": [ From cffee3df17108e52a60e6c2fa77f54939f755189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=84=B8=EB=B9=88=20=EA=B6=8C?= Date: Tue, 23 Jun 2026 15:39:49 +0900 Subject: [PATCH 3/3] =?UTF-8?q?prod=20=EC=88=AB=EC=9E=90=ED=98=95=20lag=20?= =?UTF-8?q?payload=EC=9D=98=20CONSUMER=5FLAG=5FSPIKE=20=EB=88=84=EB=9D=BD?= =?UTF-8?q?=20=EB=B3=B5=EA=B5=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_consumer_lag/get_consumer_groups의 prod DTO는 totalLag와 partitions[].lag, group lag를 숫자로만 전달할 수 있어 기존 텍스트 anchor 중심 evidence_signal_summary가 consumer lag 급증 증거를 만들지 못했다. 명시적 lag 필드만 구조적으로 판독해 consumer lag evidence를 추가하고, offset progression 둔화는 기존 텍스트/추세 근거 경로에 그대로 남겼다. Constraint: #957 trend 엄격성 유지 - bare lag 수치만으로 offset progression 둔화를 emit하지 않음 Rejected: 텍스트 lag anchor 완화 | 숫자 vendor code와 side-neutral false-positive guardrail을 되돌릴 위험 Rejected: 모든 lag-like 숫자 필드 일반화 | totalLag, partitions[].lag, group lag 명시 필드만 요구 Confidence: high Scope-risk: narrow Directive: lag recall을 넓힐 때도 consumer lag read-tool의 명시적 lag 필드와 offset progression 근거를 분리할 것 Tested: uv run pytest tests/test_evidence_signals.py -q Tested: uv run pytest tests/test_rca.py::test_consumer_lag_spike_partial_evidence tests/test_rca.py::test_consumer_lag_snapshot_does_not_satisfy_trend_evidence tests/test_rca.py::test_consumer_lag_spike_accepts_live_metric_trend_evidence -q Tested: uv run pytest tests/test_rca_classification_accuracy.py::test_consumer_lag_spike_from_realistic_incident_evidence tests/test_rca_classification_accuracy.py::test_consumer_lag_snapshot_without_trend_stays_unknown -q Tested: uv run pytest tests/test_verifier.py::test_incident_verifier_accepts_live_metric_trend_evidence tests/test_verifier.py::test_incident_verifier_rejects_single_consumer_lag_snapshot_as_trend_evidence -q Tested: uv run pytest -q Tested: uv run python -m compileall -q app tests Tested: git diff --check Not-tested: dedicated lint/typecheck not configured under services/ai-service --- services/ai-service/app/evidence/signals.py | 46 +++++++++- .../ai-service/tests/test_evidence_signals.py | 86 +++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/services/ai-service/app/evidence/signals.py b/services/ai-service/app/evidence/signals.py index e798d42b..00844559 100644 --- a/services/ai-service/app/evidence/signals.py +++ b/services/ai-service/app/evidence/signals.py @@ -151,6 +151,7 @@ ) _OFFSET_PROGRESS_RE = re.compile(r"offset progression|commit rate|offset_progression", re.IGNORECASE) _INGRESS_RE = re.compile(r"topic ingress|incoming messages|bytes-in", re.IGNORECASE) +_CONSUMER_LAG_TOOLS = {"get_consumer_lag", "get_consumer_groups"} _NEGATED_SIGNAL_PATTERNS = { "global": ( @@ -334,7 +335,9 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: timeout = _has_signal(pieces, _TIMEOUT_RE, "timeout") network = _has_signal(pieces, _NETWORK_RE, "network") missing_database = _has_signal(pieces, _DATABASE_DOES_NOT_EXIST_RE, "failure") - lag_spike = _has_signal(pieces, _LAG_SPIKE_RE, "lag") + lag_spike = _has_signal(pieces, _LAG_SPIKE_RE, "lag") or _has_structured_consumer_lag_signal( + tool_name, raw_payload + ) offset_slow = _has_signal(pieces, _OFFSET_PROGRESS_RE, "lag") deployment = _has_signal(pieces, _DEPLOYMENT_RE, "deployment") degradation = _has_signal(pieces, _DEGRADATION_RE, "failure") @@ -454,6 +457,47 @@ def _flatten_text(value: Any, context: str = "") -> Iterable[str]: yield from _flatten_text(item, context) +def _has_structured_consumer_lag_signal(tool_name: str, value: Any) -> bool: + if tool_name not in _CONSUMER_LAG_TOOLS: + return False + + for item in _walk(value): + if not isinstance(item, dict): + continue + + if any(_positive_number(raw) for raw in _values_for_key(item, {"totallag"})): + return True + + if tool_name == "get_consumer_groups" and any( + _positive_number(raw) for raw in _values_for_key(item, {"lag"}) + ): + return True + + if tool_name == "get_consumer_lag" and any( + _contains_positive_lag_value(raw) for raw in _values_for_key(item, {"partitions"}) + ): + return True + + return False + + +def _contains_positive_lag_value(value: Any) -> bool: + if isinstance(value, dict): + return any( + (_key_name(key) == "lag" and _positive_number(raw)) or _contains_positive_lag_value(raw) + for key, raw in value.items() + ) + if isinstance(value, (list, tuple, set)): + return any(_contains_positive_lag_value(item) for item in value) + return False + + +def _values_for_key(item: dict[Any, Any], names: set[str]) -> Iterable[Any]: + for key, raw in item.items(): + if _key_name(key) in names: + yield raw + + def _structured_fault_side(value: Any) -> str | None: sides: list[str] = [] for item in _walk(value): diff --git a/services/ai-service/tests/test_evidence_signals.py b/services/ai-service/tests/test_evidence_signals.py index eda2f690..6d02a554 100644 --- a/services/ai-service/tests/test_evidence_signals.py +++ b/services/ai-service/tests/test_evidence_signals.py @@ -169,6 +169,92 @@ def test_signal_summary_keeps_fault_when_negative_and_fault_fragments_are_distin assert "serialization/deserialization/schema error" in summary +@pytest.mark.parametrize( + ("tool_name", "payload"), + [ + ( + "get_consumer_lag", + { + "operation": "get_consumer_lag", + "result": { + "consumerGroup": "connect-orders-sink", + "totalLag": 9000, + "partitions": [{"partition": 0, "lag": 9000}], + }, + }, + ), + ( + "get_consumer_lag", + { + "result": { + "consumerGroup": "connect-orders-sink", + "totalLag": 0, + "partitions": [{"partition": 0, "lag": 42}], + }, + }, + ), + ( + "get_consumer_groups", + { + "operation": "get_consumer_groups", + "result": { + "groups": [{"groupId": "connect-orders-sink", "lag": 9000}], + }, + }, + ), + ], +) +def test_numeric_consumer_lag_payload_emits_lag_signal(tool_name: str, payload: dict) -> None: + summary = evidence_signal_summary(tool_name, payload) + + assert "consumer lag 급증" in summary + assert "offset progression 둔화" not in summary + + +@pytest.mark.parametrize( + ("tool_name", "payload"), + [ + ( + "get_consumer_lag", + { + "operation": "get_consumer_lag", + "result": { + "consumerGroup": "connect-orders-sink", + "totalLag": 0, + "partitions": [{"partition": 0, "lag": 0}], + }, + }, + ), + ( + "get_consumer_groups", + { + "operation": "get_consumer_groups", + "result": { + "groups": [{"groupId": "connect-orders-sink", "lag": 0}], + }, + }, + ), + ( + "get_consumer_lag", + { + "operation": "get_consumer_lag", + "result": { + "port": 2002, + "metric": {"value": 1045}, + "summary": {"total": 23000, "taskId": 4025, "count": 1366, "year": 22007}, + }, + }, + ), + ], +) +def test_numeric_consumer_lag_payload_ignores_zero_and_unrelated_numbers( + tool_name: str, payload: dict +) -> None: + summary = evidence_signal_summary(tool_name, payload) + + assert summary == "" + + @pytest.mark.parametrize( "payload", [