diff --git a/services/ai-service/app/evidence/signals.py b/services/ai-service/app/evidence/signals.py index 20d89356..00844559 100644 --- a/services/ai-service/app/evidence/signals.py +++ b/services/ai-service/app/evidence/signals.py @@ -15,36 +15,61 @@ _FAILED_RE = re.compile(r"\bFAILED\b|failed|failure|exception|\berror\b|오류|에러|실패", re.IGNORECASE) _TASK_FAILURE_RE = re.compile(r"\bFAILED\b|failed|failure|exception|오류|에러|실패", re.IGNORECASE) _TRACE_RE = re.compile(r"trace|stack|exception|worker log|stacktrace|로그|트레이스", re.IGNORECASE) +_AUTH_VENDOR_CODE = r"(?:\bERROR\s+1045\b|\b1045\s*\(\s*28000\s*\)|\bSQLSTATE\s*\[?\s*28000\s*\]?)" +_SCHEMA_VENDOR_CODE = r"(?:\bERROR\s+1366\b|\b1366\s*\(\s*22007\s*\)|\bSQLSTATE\s*\[?\s*22007\s*\]?)" +_CONSTRAINT_VENDOR_CODE = r"(?:\bERROR\s+4025\b|\b4025\s*\(\s*23000\s*\)|\bSQLSTATE\s*\[?\s*23000\s*\]?)" +_TIMEOUT_VENDOR_CODE = r"(?:\bERROR\s+2002\b|\b2002\s*\(\s*HY000\s*\))" _AUTH_RE = re.compile( - r"access\s*denied|permission\s*denied|unauthori[sz]ed|authentication|" + _AUTH_VENDOR_CODE + + r"|access\s*denied|permission\s*denied|unauthori[sz]ed|authentication|" r"\bauth\b|auth(?:entication)?\s*error|token\s*expired|credential\s*expired|" r"password authentication failed|invalid password|invalid credential|" - r"credential invalid|credential expired|sasl|인증\s*실패|권한\s*거부|" + r"credential invalid|credential expired|sasl|using password|인증\s*실패|권한\s*거부|" r"credential\s*만료|credential\s*미적용|토큰\s*만료", re.IGNORECASE, ) _SCHEMA_RE = re.compile( - r"schema|serialization|deserialization|incompatible|compatibility|converter|" - r"avro|serde|type mismatch|schema registry|스키마|역직렬화|호환성", + _SCHEMA_VENDOR_CODE + + r"|schema|serialization|deserialization|deserialize|" + r"incompatible|compatibility|converter|avro|serde|type mismatch|schema registry|" + r"incorrect\s+\w+\s+value|incorrect .* value|field type|dataexception|" + r"invalid input syntax|date/time field value out of range|invalid datetime format|" + r"(?:date|time|timestamp|datetime)\s+(?:parse|parsing)\s+(?:error|fail)|" + r"parse\s+(?:error|fail).*(?:date|time|timestamp|datetime)|" + r"cannot parse .*?(?:date|time|timestamp|datetime)|스키마|역직렬화|호환성", re.IGNORECASE, ) _CONFIG_RE = re.compile( r"invalid config|unknown config|config validation|configuration error|" - r"invalid option|required configuration|bad config|config diff|" + r"configuration value is invalid|" + r"(?:invalid|bad) value for configuration|value is invalid|integer is expected|" + r"invalid option|required configuration|missing required config(?:uration)?|" + r"required config(?:uration)?.*(?:not present|missing)|" + r"configuration .*?(?:not present|missing|required)|" + r"invalid/empty configuration value|empty configuration value|" + r"bad config|config diff|transform|class not found|classnotfound|noclassdeffound|" + r"failed to find any class|" r"config\s*변경|설정\s*오류|설정\s*오타|잘못된\s*설정", re.IGNORECASE, ) _CONSTRAINT_RE = re.compile( - r"duplicate key|unique constraint|constraint violation|foreign key|not null|" - r"data integrity|sqlintegrity|record rejected|duplicate records?|" + _CONSTRAINT_VENDOR_CODE + + r"|duplicate key|unique constraint|constraint violation|foreign key|not null|" + r"check constraint|constraint .* failed|data integrity|sqlintegrity|record rejected|duplicate records?|" r"중복\s*레코드|제약\s*위반|" r"(?:record|row|sink|data)\s+validation fail", re.IGNORECASE, ) -_TIMEOUT_RE = re.compile(r"timeout|timed out|read timed out|write timed out|타임아웃", re.IGNORECASE) +_TIMEOUT_RE = re.compile( + _TIMEOUT_VENDOR_CODE + + r"|timeout|timed out|timeout expired|connection timed out|" + r"read timed out|write timed out|can't connect to server|could not connect|타임아웃", + re.IGNORECASE, +) _NETWORK_RE = re.compile( r"dns|unknownhost|no route|network unreachable|connection refused|" r"connect exception|tcp connect|endpoint unreachable|host unreachable|i/o error|" + r"can't connect to server|could not connect|name or service not known|temporary failure in name resolution|" r"네트워크\s*단절|연결\s*실패|호스트.*포트", re.IGNORECASE, ) @@ -99,8 +124,20 @@ r"|상태\s*[:=]?\s*(?:FAILED|실패|오류|에러)", re.IGNORECASE, ) +_DATABASE_DOES_NOT_EXIST_RE = re.compile( + r"database\s+[\"'][^\"']+[\"']\s+does not exist|database\s+\S+\s+does not exist", + re.IGNORECASE, +) _SCHEMA_VERSION_RE = re.compile(r"schema version|subject version|schema registry", re.IGNORECASE) -_SCHEMA_STRUCTURE_RE = re.compile(r"type mismatch|field type|필드 타입", re.IGNORECASE) +_SCHEMA_STRUCTURE_RE = re.compile( + _SCHEMA_VENDOR_CODE + + r"|incorrect\s+\w+\s+value|incorrect .* value|" + r"type mismatch|field type|invalid input syntax|date/time field value out of range|" + r"invalid datetime format|(?:date|time|timestamp|datetime)\s+(?:parse|parsing)\s+(?:error|fail)|" + r"parse\s+(?:error|fail).*(?:date|time|timestamp|datetime)|" + r"cannot parse .*?(?:date|time|timestamp|datetime)|필드 타입", + re.IGNORECASE, +) _CONFIG_CHANGE_RE = re.compile(r"config change|config diff|config 변경", re.IGNORECASE) _DUPLICATE_COUNT_RE = re.compile(r"duplicate count|중복\s*레코드", re.IGNORECASE) _READ_LATENCY_RE = re.compile( @@ -114,6 +151,7 @@ ) _OFFSET_PROGRESS_RE = re.compile(r"offset progression|commit rate|offset_progression", re.IGNORECASE) _INGRESS_RE = re.compile(r"topic ingress|incoming messages|bytes-in", re.IGNORECASE) +_CONSUMER_LAG_TOOLS = {"get_consumer_lag", "get_consumer_groups"} _NEGATED_SIGNAL_PATTERNS = { "global": ( @@ -279,8 +317,11 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: tags: list[str] = [] lower = text.casefold() - side = _side_hint(lower, tool_name) + structured_side = _structured_fault_side(raw_payload) + side = structured_side or _side_hint(lower, tool_name) auth_side = _auth_side_hint(pieces, tool_name) + if auth_side == "unknown" and structured_side in {"source", "sink"}: + auth_side = structured_side failed = _has_signal(pieces, _TASK_FAILURE_RE, "failure") connector_failed = _has_signal(pieces, _CONNECTOR_FAILED_RE, "connector") or ( @@ -293,16 +334,19 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: constraint = _has_signal(pieces, _CONSTRAINT_RE, "duplicate") timeout = _has_signal(pieces, _TIMEOUT_RE, "timeout") network = _has_signal(pieces, _NETWORK_RE, "network") - lag_spike = _has_signal(pieces, _LAG_SPIKE_RE, "lag") + missing_database = _has_signal(pieces, _DATABASE_DOES_NOT_EXIST_RE, "failure") + lag_spike = _has_signal(pieces, _LAG_SPIKE_RE, "lag") or _has_structured_consumer_lag_signal( + tool_name, raw_payload + ) offset_slow = _has_signal(pieces, _OFFSET_PROGRESS_RE, "lag") deployment = _has_signal(pieces, _DEPLOYMENT_RE, "deployment") degradation = _has_signal(pieces, _DEGRADATION_RE, "failure") retry = _has_signal(pieces, _RETRY_RE, "retry") idempotency = _has_signal(pieces, _IDEMPOTENCY_RE, "idempotency") - if connector_failed: + if connector_failed or missing_database: _add(tags, "connector task status FAILED") - if trace and failed: + if (trace and failed) or missing_database: _add(tags, "task trace 또는 worker log") if auth: @@ -349,7 +393,7 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: elif side == "sink": _add(tags, "sink dependency 연결 실패 또는 connection timeout") else: - _add(tags, "Bifrost에서 source endpoint reachability 실패") + _add(tags, "endpoint reachability failure log") if timeout: if side == "source": @@ -359,7 +403,7 @@ def evidence_signal_summary(tool_name: str, raw_payload: Any) -> str: _add(tags, "sink write timeout 증가") _add(tags, "sink dependency 연결 실패 또는 connection timeout") else: - _add(tags, "pipeline extract/read 단계 timeout log") + _add(tags, "connection timeout log") if _has_signal(pieces, _READ_LATENCY_RE, "latency"): _add(tags, "source read latency 증가") @@ -413,6 +457,113 @@ def _flatten_text(value: Any, context: str = "") -> Iterable[str]: yield from _flatten_text(item, context) +def _has_structured_consumer_lag_signal(tool_name: str, value: Any) -> bool: + if tool_name not in _CONSUMER_LAG_TOOLS: + return False + + for item in _walk(value): + if not isinstance(item, dict): + continue + + if any(_positive_number(raw) for raw in _values_for_key(item, {"totallag"})): + return True + + if tool_name == "get_consumer_groups" and any( + _positive_number(raw) for raw in _values_for_key(item, {"lag"}) + ): + return True + + if tool_name == "get_consumer_lag" and any( + _contains_positive_lag_value(raw) for raw in _values_for_key(item, {"partitions"}) + ): + return True + + return False + + +def _contains_positive_lag_value(value: Any) -> bool: + if isinstance(value, dict): + return any( + (_key_name(key) == "lag" and _positive_number(raw)) or _contains_positive_lag_value(raw) + for key, raw in value.items() + ) + if isinstance(value, (list, tuple, set)): + return any(_contains_positive_lag_value(item) for item in value) + return False + + +def _values_for_key(item: dict[Any, Any], names: set[str]) -> Iterable[Any]: + for key, raw in item.items(): + if _key_name(key) in names: + yield raw + + +def _structured_fault_side(value: Any) -> str | None: + sides: list[str] = [] + for item in _walk(value): + if not isinstance(item, dict) or not _has_structured_fault_indicator(item): + continue + side = _side_from_mapping(item) + if side and side not in sides: + sides.append(side) + return sides[0] if len(sides) == 1 else None + + +def _has_structured_fault_indicator(item: dict[Any, Any]) -> bool: + for key, raw in item.items(): + key_text = _key_name(key) + raw_text = str(raw).casefold() + if key_text in {"state", "connectorstate", "taskstate", "status"} and raw_text == "failed": + return True + if key_text == "connectionstatus" and raw_text in {"down", "failed"}: + return True + if key_text in {"failedtasks", "tasksfailed", "errorcount", "matchcount"} and _positive_number(raw): + return True + if key_text == "exitcode" and _nonzero_number(raw): + return True + if key_text in {"trace", "stack", "stacktrace"} and str(raw or "").strip(): + return True + + text = " ".join(_flatten_text(item)) + if _is_normal_only(text): + return False + return bool( + _TASK_FAILURE_RE.search(text) + or _AUTH_RE.search(text) + or _SCHEMA_RE.search(text) + or _CONFIG_RE.search(text) + or _CONSTRAINT_RE.search(text) + or _TIMEOUT_RE.search(text) + or _NETWORK_RE.search(text) + or _RETRY_RE.search(text) + or _DATABASE_DOES_NOT_EXIST_RE.search(text) + ) + + +def _side_from_mapping(item: dict[Any, Any]) -> str | None: + for key, raw in item.items(): + if _key_name(key) not in {"datasourcerole", "role", "type", "kind"}: + continue + value = str(raw or "").casefold() + if value in {"source", "sink"}: + return value + return None + + +def _walk(value: Any) -> Iterable[Any]: + yield value + if isinstance(value, dict): + for item in value.values(): + yield from _walk(item) + elif isinstance(value, (list, tuple, set)): + for item in value: + yield from _walk(item) + + +def _key_name(value: Any) -> str: + return str(value).casefold().replace("_", "").replace("-", "") + + def _side_hint(text: str, tool_name: str) -> str: has_source = bool(_SOURCE_HINT_RE.search(text)) has_sink = bool(_SINK_HINT_RE.search(text)) @@ -512,3 +663,17 @@ def _has_fault_negation(text: str, fault: str) -> bool: def _add(tags: list[str], tag: str) -> None: if tag not in tags: tags.append(tag) + + +def _positive_number(value: Any) -> bool: + try: + return float(value) > 0 + except (TypeError, ValueError): + return False + + +def _nonzero_number(value: Any) -> bool: + try: + return float(value) != 0 + except (TypeError, ValueError): + return False diff --git a/services/ai-service/tests/test_evidence_signals.py b/services/ai-service/tests/test_evidence_signals.py index f7ff6ec4..6d02a554 100644 --- a/services/ai-service/tests/test_evidence_signals.py +++ b/services/ai-service/tests/test_evidence_signals.py @@ -167,3 +167,272 @@ def test_signal_summary_keeps_fault_when_negative_and_fault_fragments_are_distin summary = evidence_signal_summary("search_logs", payload) assert "serialization/deserialization/schema error" in summary + + +@pytest.mark.parametrize( + ("tool_name", "payload"), + [ + ( + "get_consumer_lag", + { + "operation": "get_consumer_lag", + "result": { + "consumerGroup": "connect-orders-sink", + "totalLag": 9000, + "partitions": [{"partition": 0, "lag": 9000}], + }, + }, + ), + ( + "get_consumer_lag", + { + "result": { + "consumerGroup": "connect-orders-sink", + "totalLag": 0, + "partitions": [{"partition": 0, "lag": 42}], + }, + }, + ), + ( + "get_consumer_groups", + { + "operation": "get_consumer_groups", + "result": { + "groups": [{"groupId": "connect-orders-sink", "lag": 9000}], + }, + }, + ), + ], +) +def test_numeric_consumer_lag_payload_emits_lag_signal(tool_name: str, payload: dict) -> None: + summary = evidence_signal_summary(tool_name, payload) + + assert "consumer lag 급증" in summary + assert "offset progression 둔화" not in summary + + +@pytest.mark.parametrize( + ("tool_name", "payload"), + [ + ( + "get_consumer_lag", + { + "operation": "get_consumer_lag", + "result": { + "consumerGroup": "connect-orders-sink", + "totalLag": 0, + "partitions": [{"partition": 0, "lag": 0}], + }, + }, + ), + ( + "get_consumer_groups", + { + "operation": "get_consumer_groups", + "result": { + "groups": [{"groupId": "connect-orders-sink", "lag": 0}], + }, + }, + ), + ( + "get_consumer_lag", + { + "operation": "get_consumer_lag", + "result": { + "port": 2002, + "metric": {"value": 1045}, + "summary": {"total": 23000, "taskId": 4025, "count": 1366, "year": 22007}, + }, + }, + ), + ], +) +def test_numeric_consumer_lag_payload_ignores_zero_and_unrelated_numbers( + tool_name: str, payload: dict +) -> None: + summary = evidence_signal_summary(tool_name, payload) + + assert summary == "" + + +@pytest.mark.parametrize( + "payload", + [ + {"datasource": {"role": "source", "connectionStatus": "UP", "port": 2002}}, + {"metric": {"name": "records_processed", "value": 2002}}, + {"searchResult": {"matchCount": 1045, "logs": []}}, + {"summary": {"total": 23000, "taskId": 4025, "count": 1366, "year": 22007}}, + {"message": "codes observed in inventory text: 1045 2002 1366 22007 4025 23000"}, + ], +) +def test_vendor_numeric_codes_in_metadata_or_code_only_text_do_not_emit_signals(payload: dict) -> None: + summary = evidence_signal_summary("list_datasources", payload) + + assert summary == "" + + +@pytest.mark.parametrize( + "payload", + [ + { + "datasources": [ + { + "connectionStatus": "DOWN", + "lastError": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + } + ] + }, + { + "datasources": [ + { + "role": "source+sink", + "connectionStatus": "DOWN", + "lastError": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + } + ] + }, + ], +) +def test_unknown_or_dual_role_timeout_evidence_stays_side_neutral(payload: dict) -> None: + summary = evidence_signal_summary("list_datasources", payload) + + assert "connection timeout log" in summary + assert "endpoint reachability failure log" in summary + assert "Bifrost에서 source endpoint reachability 실패" not in summary + assert "pipeline extract/read 단계 timeout log" not in summary + assert "source connection timeout 증가" not in summary + assert "sink write timeout 증가" not in summary + assert "sink dependency 연결 실패 또는 connection timeout" not in summary + + +def test_vendor_auth_code_uses_datasource_role_for_sink_auth() -> None: + payload = { + "logs": [ + { + "datasourceRole": "sink", + "engine": "mariadb", + "exit_code": 1, + "stderr": "ERROR 1045 (28000): Access denied for user '[USER]'@'localhost' (using password: YES)", + } + ] + } + + summary = evidence_signal_summary("search_logs", payload) + + assert "sink auth/permission error log" in summary + assert "source auth/permission error log" not in summary + + +def test_vendor_timeout_codes_use_failed_item_side_for_source_and_sink() -> None: + source = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "datasourceRole": "source", + "engine": "postgresql", + "exit_code": 2, + "stderr": 'psql: error: connection to server at "10.255.255.1", port 5432 failed: timeout expired', + } + ] + }, + ) + sink = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "datasourceRole": "sink", + "engine": "mariadb", + "exit_code": 1, + "stderr": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + } + ] + }, + ) + + assert "source connection timeout 증가" in source + assert "pipeline extract/read 단계 timeout log" in source + assert "sink write timeout 증가" in sink + assert "sink dependency 연결 실패 또는 connection timeout" in sink + + +def test_mixed_datasource_payload_uses_failed_item_side_for_sink_timeout() -> None: + summary = evidence_signal_summary( + "list_datasources", + { + "datasources": [ + { + "id": "source-db", + "role": "source", + "connectionStatus": "UP", + }, + { + "id": "sink-db", + "role": "sink", + "connectionStatus": "DOWN", + "lastError": "ERROR 2002 (HY000): Can't connect to server on '10.255.255.1' (110)", + }, + ] + }, + ) + + assert "sink write timeout 증가" in summary + assert "sink dependency 연결 실패 또는 connection timeout" in summary + assert "Bifrost에서 source endpoint reachability 실패" not in summary + assert "source connection timeout 증가" not in summary + + +def test_missing_database_validation_emits_connector_failure_signal() -> None: + summary = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "datasourceRole": "source", + "engine": "postgresql", + "exit_code": 2, + "stderr": 'psql: error: FATAL: database "missingdb" does not exist', + } + ] + }, + ) + + assert "connector task status FAILED" in summary + assert "task trace 또는 worker log" in summary + assert "config validation error 또는 invalid option log" not in summary + + +def test_vendor_schema_and_constraint_codes_emit_specific_catalog_signals() -> None: + schema = evidence_signal_summary( + "search_logs", + {"logs": [{"stderr": "ERROR 1366 (22007): Incorrect integer value: 'not-a-number' for column amount"}]}, + ) + constraint = evidence_signal_summary( + "search_logs", + {"logs": [{"stderr": "ERROR 4025 (23000): CONSTRAINT `chk_orders` failed for `testdb`.`orders`"}]}, + ) + + assert "serialization/deserialization/schema error" in schema + assert "데이터 샘플 구조 변화" in schema + assert "sink constraint 또는 duplicate key error" in constraint + assert "동일 record 반복 실패" in constraint + + +def test_vendor_datetime_parse_failure_emits_schema_signal() -> None: + summary = evidence_signal_summary( + "search_logs", + { + "logs": [ + { + "engine": "postgresql", + "exit_code": 1, + "stderr": 'ERROR: invalid input syntax for type timestamp: "not-a-timestamp"', + } + ] + }, + ) + + assert "serialization/deserialization/schema error" in summary + assert "데이터 샘플 구조 변화" in summary + assert "config validation error 또는 invalid option log" not in summary