diff --git a/ai/.env.example b/ai/.env.example index 59d079e4..708db850 100644 --- a/ai/.env.example +++ b/ai/.env.example @@ -18,3 +18,29 @@ LLM_API_KEY= LLM_BASE_URL=https://factchat-cloud.mindlogic.ai/v1/gateway LLM_PRO_MODEL=gemini-3.1-pro-preview LLM_PRO_TEMPERATURE=0.2 + +# Github 관련 +CORE_INTERNAL_BASE_URL=http://localhost:38010 +CORE_INTERNAL_API_KEY= +CORE_INTERNAL_TIMEOUT_SEC=10 + +GITHUB_FALLBACK_TOKEN= +GITHUB_API_BASE_URL=https://api.github.com +REPO_MAX_SOURCE_FILES=8 +REPO_MAX_SOURCE_FILE_BYTES=50000 +REPO_FETCH_TIMEOUT_SEC=30 + +# 웹 이력서 관련 +WEB_FETCH_TIMEOUT_SEC=20 +WEB_MAX_HTML_BYTES=2000000 + +# 개발 및 테스트는 mock +# 운영은 gemini +EMBEDDING_PROVIDER=gemini +EMBEDDING_MODEL=gemini-embedding-001 +EMBEDDING_DIM=1536 +EMBEDDING_CHUNK_SIZE=1000 +EMBEDDING_CHUNK_OVERLAP=200 +EMBEDDING_BATCH_SIZE=32 + +GEMINI_API_KEY= diff --git a/ai/CLAUDE.md b/ai/CLAUDE.md index fb5b3fbc..68c09f19 100644 --- a/ai/CLAUDE.md +++ b/ai/CLAUDE.md @@ -88,11 +88,13 @@ ai/ 본 서버는 RabbitMQ **consumer**로 작동. -| Queue | Bind | -|-------|------| -| `q.ai.resume` | `ai.request.resume.*` | -| `q.ai.repo` | `ai.request.repo.*` | -| `q.ai.session` | `ai.request.session.*` | +| Queue | Routing key | 상태 | +|-------|-------------|------| +| `ai.analyze.resume` | `analyze.resume` | 본 구현 (PDF → MD) | +| `ai.analyze.repository` | `analyze.repository` | 본 구현 (GitHub README + tree + 소스 sampling) | +| `ai.analyze.web` | `analyze.web` | 본 구현 (URL → trafilatura) | +| `ai.generate.questions` | `generate.questions` | 큐만, 코드 미구현 | +| `ai.generate.followup` | `generate.followup` | 큐만, 코드 미구현 | 콜백 발행: `ai.callback.{type}` 익스체인지. 상세 envelope/스키마/재시도: [`/docs/messaging.md`](../docs/messaging.md). @@ -162,11 +164,14 @@ chain = prompt | llm | PydanticOutputParser(pydantic_object=...) ## 7. RAG 파이프라인 -### 7.1 인제스트 -1. 마크다운 입력 -2. 청킹 (LangChain `RecursiveCharacterTextSplitter`, chunk_size=1000, overlap=200) -3. 임베딩 생성 (Gemini `text-embedding-004` 또는 OpenAI `text-embedding-3-small`) -4. Core API 호출 → pgvector INSERT +### 7.1 인제스트 (본 구현) +1. 마크다운 입력 (`analyzer/_embedding_step.chunk_embed_and_upsert`) +2. 청킹 — `rag/chunker.MarkdownChunker` (`RecursiveCharacterTextSplitter`, 기본 1000/200) +3. 임베딩 생성 — `rag/embedder.EmbeddingProvider`. 현재 구현체: + - `MockEmbeddingProvider` (default) — 차원 결정 보류, e2e 흐름 검증용 + - `openai` / `ollama` 구현체는 후속 PR +4. Core API 호출 — `CoreClient.upsert_embeddings(document_id, model, dim, chunks)` → + `PUT /api/internal/documents/{id}/embeddings` (idempotent upsert) ### 7.2 검색 1. 쿼리 텍스트 → 임베딩 diff --git a/ai/pyproject.toml b/ai/pyproject.toml index 380d3396..543cfeba 100644 --- a/ai/pyproject.toml +++ b/ai/pyproject.toml @@ -16,6 +16,9 @@ dependencies = [ "boto3>=1.42.77", "aiofiles>=24.1.0", "pypdf>=5.1.0", + "trafilatura>=2.0.0", + "langchain-text-splitters>=0.3.0", + "google-genai>=1.0.0", "langchain>=1.2.13", "langchain-core>=1.2.22", "langchain-community>=0.4.1", diff --git a/ai/src/ai_server/analyzer/__init__.py b/ai/src/ai_server/analyzer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/src/ai_server/analyzer/_embedding_step.py b/ai/src/ai_server/analyzer/_embedding_step.py new file mode 100644 index 00000000..757585d9 --- /dev/null +++ b/ai/src/ai_server/analyzer/_embedding_step.py @@ -0,0 +1,86 @@ +# 공통 임베딩 모듈 +from __future__ import annotations + +import structlog + +from ai_server.core.client import ( + CoreClient, + CoreEmbeddingUpsertError, + EmbeddingChunkPayload, +) +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import EmbeddingError, EmbeddingProvider + +log = structlog.get_logger(__name__) + + +class EmbeddingStepError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +async def chunk_embed_and_upsert( + *, + document_id: int, + markdown: str, + chunker: MarkdownChunker, + embedder: EmbeddingProvider, + core_client: CoreClient, + log_prefix: str = "analyze", +) -> int: + chunks = chunker.split(markdown) + log.info( + f"{log_prefix}.chunk.done", + document_id=document_id, + chunk_count=len(chunks), + ) + + if not chunks: + return 0 + + try: + vectors = await embedder.embed([c.text for c in chunks]) + except EmbeddingError as err: + raise EmbeddingStepError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + if len(vectors) != len(chunks): + raise EmbeddingStepError( + code="EMBED_COUNT_MISMATCH", + message=(f"embedder가 chunk {len(chunks)}개 중 {len(vectors)}개만 반환"), + retriable=True, + ) + + payloads = [ + EmbeddingChunkPayload( + chunk_index=chunks[i].index, + chunk_text=chunks[i].text, + embedding=vectors[i], + ) + for i in range(len(chunks)) + ] + + try: + upserted = await core_client.upsert_embeddings( + document_id=document_id, + model=embedder.model, + dim=embedder.dim, + chunks=payloads, + ) + except CoreEmbeddingUpsertError as err: + raise EmbeddingStepError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + log.info( + f"{log_prefix}.embed.upserted", + document_id=document_id, + chunk_count=upserted, + model=embedder.model, + dim=embedder.dim, + ) + return upserted diff --git a/ai/src/ai_server/analyzer/repository_analyzer.py b/ai/src/ai_server/analyzer/repository_analyzer.py new file mode 100644 index 00000000..f02b9c27 --- /dev/null +++ b/ai/src/ai_server/analyzer/repository_analyzer.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import structlog + +from ai_server.analyzer._embedding_step import ( + EmbeddingStepError, + chunk_embed_and_upsert, +) +from ai_server.analyzer.sources.github_repo import ( + GitHubRepoSourceExtractor, + RepositoryFetchError, +) +from ai_server.chain.document_analysis_chain import DocumentAnalyzer +from ai_server.core.client import CoreClient, CoreTokenError +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import EmbeddingProvider +from ai_server.storage.base import ObjectStorage + +log = structlog.get_logger(__name__) + + +class RepositoryAnalyzeError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class RepositoryAnalysisResult: + summary: str + tech_stack: list[str] + document_path: str + embedding_chunk_count: int + + +# Core에서 사용자별 GitHub token 수령 → 레포 fetch → LLM 분석 → 마크다운 저장 → 청킹·임베딩 +class RepositoryAnalyzer: + def __init__( + self, + *, + extractor: GitHubRepoSourceExtractor, + core_client: CoreClient, + chain: DocumentAnalyzer, + storage: ObjectStorage, + chunker: MarkdownChunker, + embedder: EmbeddingProvider, + analyzed_key_template: str, + ) -> None: + self._extractor = extractor + self._core_client = core_client + self._chain = chain + self._storage = storage + self._chunker = chunker + self._embedder = embedder + self._analyzed_key_template = analyzed_key_template + + async def analyze( + self, + *, + repository_id: int, + repo_full_name: str, + default_branch: str = "main", + user_id: int | None, + analyzed_document_id: int, + ) -> RepositoryAnalysisResult: + if user_id is None: + raise RepositoryAnalyzeError( + code="MISSING_USER_ID", + message="envelope.context.userId 없이는 GitHub token을 가져올 수 없음", + retriable=False, + ) + + log.info( + "repository.token.fetch", + user_id=user_id, + repository_id=repository_id, + ) + try: + access_token = await self._core_client.fetch_github_token(user_id) + except CoreTokenError as err: + raise RepositoryAnalyzeError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + log.info( + "repository.extract.start", + repository_id=repository_id, + repo_full_name=repo_full_name, + default_branch=default_branch, + ) + try: + source = await self._extractor.extract( + repo_full_name, + access_token=access_token, + ) + except RepositoryFetchError as err: + raise RepositoryAnalyzeError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + if not source.text.strip(): + raise RepositoryAnalyzeError( + code="EMPTY_REPO_CONTENT", + message="레포에서 추출된 텍스트가 비어 있음", + retriable=False, + ) + + log.info( + "repository.llm.start", + repository_id=repository_id, + text_chars=len(source.text), + ) + analysis = await self._chain.analyze( + text=source.text, + source_type=source.source_type, + ) + + out_key = self._analyzed_key_template.format(repository_id=repository_id) + await self._storage.put_text(out_key, analysis.markdown) + log.info( + "repository.markdown.saved", + repository_id=repository_id, + key=out_key, + md_chars=len(analysis.markdown), + ) + + try: + chunk_count = await chunk_embed_and_upsert( + document_id=analyzed_document_id, + markdown=analysis.markdown, + chunker=self._chunker, + embedder=self._embedder, + core_client=self._core_client, + log_prefix="repository", + ) + except EmbeddingStepError as err: + raise RepositoryAnalyzeError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + return RepositoryAnalysisResult( + summary=analysis.summary, + tech_stack=list(analysis.tech_stack), + document_path=out_key, + embedding_chunk_count=chunk_count, + ) diff --git a/ai/src/ai_server/analyzer/resume_analyzer.py b/ai/src/ai_server/analyzer/resume_analyzer.py new file mode 100644 index 00000000..c82f0309 --- /dev/null +++ b/ai/src/ai_server/analyzer/resume_analyzer.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import structlog + +from ai_server.analyzer._embedding_step import ( + EmbeddingStepError, + chunk_embed_and_upsert, +) +from ai_server.analyzer.sources.base import SourceExtractor +from ai_server.chain.document_analysis_chain import DocumentAnalyzer +from ai_server.core.client import CoreClient +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import EmbeddingProvider +from ai_server.storage.base import ObjectStorage + +log = structlog.get_logger(__name__) + + +class ResumeAnalyzeError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class ResumeAnalysisResult: + summary: str + tech_stack: list[str] + document_path: str + embedding_chunk_count: int + + +# 스토리지에서 가져오고 LLM을 통해 분석 후 청킹·임베딩까지 처리 +class ResumeAnalyzer: + def __init__( + self, + *, + extractor: SourceExtractor, + chain: DocumentAnalyzer, + storage: ObjectStorage, + chunker: MarkdownChunker, + embedder: EmbeddingProvider, + core_client: CoreClient, + analyzed_key_template: str, + ) -> None: + self._extractor = extractor + self._chain = chain + self._storage = storage + self._chunker = chunker + self._embedder = embedder + self._core_client = core_client + self._analyzed_key_template = analyzed_key_template + + async def analyze( + self, + *, + resume_id: int, + file_path: str, + analyzed_document_id: int, + ) -> ResumeAnalysisResult: + log.info( + "resume.extract.start", + resume_id=resume_id, + file_path=file_path, + ) + source = await self._extractor.extract(file_path) + + if not source.text.strip(): + raise ResumeAnalyzeError( + code="EMPTY_PDF_TEXT", + message="추출된 텍스트가 비어있음", + retriable=False, + ) + + log.info( + "resume.llm.start", + resume_id=resume_id, + text_chars=len(source.text), + source_type=source.source_type, + ) + analysis = await self._chain.analyze( + text=source.text, + source_type=source.source_type, + ) + + out_key = self._analyzed_key_template.format(resume_id=resume_id) + await self._storage.put_text(out_key, analysis.markdown) + log.info( + "resume.markdown.saved", + resume_id=resume_id, + key=out_key, + md_chars=len(analysis.markdown), + ) + + try: + chunk_count = await chunk_embed_and_upsert( + document_id=analyzed_document_id, + markdown=analysis.markdown, + chunker=self._chunker, + embedder=self._embedder, + core_client=self._core_client, + log_prefix="resume", + ) + except EmbeddingStepError as err: + raise ResumeAnalyzeError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + return ResumeAnalysisResult( + summary=analysis.summary, + tech_stack=list(analysis.tech_stack), + document_path=out_key, + embedding_chunk_count=chunk_count, + ) diff --git a/ai/src/ai_server/analyzer/sources/__init__.py b/ai/src/ai_server/analyzer/sources/__init__.py new file mode 100644 index 00000000..53344b6e --- /dev/null +++ b/ai/src/ai_server/analyzer/sources/__init__.py @@ -0,0 +1,22 @@ +from ai_server.analyzer.sources.base import ( + ExtractedSource, + SourceExtractor, + SourceType, +) +from ai_server.analyzer.sources.github_repo import ( + GitHubRepoSourceExtractor, + RepositoryFetchError, +) +from ai_server.analyzer.sources.pdf import PdfSourceExtractor +from ai_server.analyzer.sources.web import WebFetchError, WebSourceExtractor + +__all__ = [ + "ExtractedSource", + "SourceExtractor", + "SourceType", + "PdfSourceExtractor", + "GitHubRepoSourceExtractor", + "RepositoryFetchError", + "WebSourceExtractor", + "WebFetchError", +] diff --git a/ai/src/ai_server/analyzer/sources/base.py b/ai/src/ai_server/analyzer/sources/base.py new file mode 100644 index 00000000..b8a4e0cb --- /dev/null +++ b/ai/src/ai_server/analyzer/sources/base.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Literal + +from pydantic import BaseModel, Field + +SourceType = Literal["PDF", "REPOSITORY", "WEB"] + + +# 모든 Source Extractor가 공통으로 반환하는 결과 모델 +class ExtractedSource(BaseModel): + text: str + source_type: SourceType + metadata: dict[str, Any] = Field(default_factory=dict) + + +# 단일 입력 형태에 대한 Adapter +class SourceExtractor(ABC): + + @abstractmethod + async def extract(self, locator: str) -> ExtractedSource: ... diff --git a/ai/src/ai_server/analyzer/sources/github_repo.py b/ai/src/ai_server/analyzer/sources/github_repo.py new file mode 100644 index 00000000..70ea5e47 --- /dev/null +++ b/ai/src/ai_server/analyzer/sources/github_repo.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import base64 +from dataclasses import dataclass + +import httpx +import structlog + +from ai_server.analyzer.sources.base import ExtractedSource, SourceExtractor + +log = structlog.get_logger(__name__) + + +# 프로젝트 설정 파일을 먼저 읽는다 +_PRIORITY_FILES: tuple[str, ...] = ( + "package.json", + "pyproject.toml", + "requirements.txt", + "go.mod", + "build.gradle", + "build.gradle.kts", + "pom.xml", + "Cargo.toml", + "composer.json", + "Gemfile", + "Dockerfile", +) + +_TEXT_EXTENSIONS: frozenset[str] = frozenset( + { + ".py", + ".ts", + ".tsx", + ".js", + ".jsx", + ".go", + ".java", + ".kt", + ".rs", + ".rb", + ".php", + ".cs", + ".swift", + ".md", + ".yaml", + ".yml", + ".toml", + ".json", + ".sh", + } +) + + +class RepositoryFetchError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class _RepoConfig: + api_base_url: str + fallback_token: str + max_files: int + max_file_bytes: int + timeout_sec: float + + +# 리드미, 주요 소스를 읽는다 +class GitHubRepoSourceExtractor(SourceExtractor): + def __init__( + self, + *, + api_base_url: str, + fallback_token: str = "", + max_files: int = 8, + max_file_bytes: int = 50_000, + timeout_sec: float = 30.0, + client: httpx.AsyncClient | None = None, + ) -> None: + self._cfg = _RepoConfig( + api_base_url=api_base_url.rstrip("/"), + fallback_token=fallback_token, + max_files=max_files, + max_file_bytes=max_file_bytes, + timeout_sec=timeout_sec, + ) + self._client = client + + async def extract( + self, + locator: str, + *, + access_token: str | None = None, + ) -> ExtractedSource: + """`locator`는 "owner/repo" 형태의 full name. + + `access_token`이 주어지면 그 token으로 호출(권장 — Core에서 받은 값). + None이면 fallback_token(public repo용)으로 호출. + """ + owner_repo = locator.strip().strip("/") + if "/" not in owner_repo: + raise RepositoryFetchError( + code="INVALID_REPO_LOCATOR", + message=f"locator must be 'owner/repo', got: {locator!r}", + retriable=False, + ) + + effective_token = access_token or self._cfg.fallback_token + + if self._client is not None: + return await self._extract_with_client(self._client, owner_repo) + + async with self._build_client(effective_token) as client: + return await self._extract_with_client(client, owner_repo) + + def _build_client(self, token: str) -> httpx.AsyncClient: + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + return httpx.AsyncClient( + base_url=self._cfg.api_base_url, + headers=headers, + timeout=self._cfg.timeout_sec, + ) + + async def _extract_with_client( + self, + client: httpx.AsyncClient, + owner_repo: str, + ) -> ExtractedSource: + repo_info = await self._fetch_json(client, f"/repos/{owner_repo}") + default_branch = repo_info.get("default_branch") or "main" + + readme_text = await self._fetch_readme(client, owner_repo) + tree_paths, truncated = await self._fetch_tree( + client, owner_repo, default_branch + ) + picked = _select_files(tree_paths, self._cfg.max_files) + snippets = await self._fetch_snippets( + client, owner_repo, default_branch, picked + ) + + parts: list[str] = [] + parts.append(f"# {owner_repo} (branch: {default_branch})") + if desc := repo_info.get("description"): + parts.append(f"\n> {desc}") + + if readme_text: + parts.append("\n## README\n") + parts.append(readme_text) + + parts.append("\n## File tree (truncated)\n") + parts.append("\n".join(f"- {p}" for p in tree_paths[:200])) + if truncated: + parts.append("\n(tree truncated by GitHub API)") + + if snippets: + parts.append("\n## Sampled sources\n") + for path, body in snippets: + parts.append(f"\n### `{path}`\n```\n{body}\n```") + + text = "\n".join(parts).strip() + return ExtractedSource( + text=text, + source_type="REPOSITORY", + metadata={ + "locator": owner_repo, + "default_branch": default_branch, + "tree_size": len(tree_paths), + "tree_truncated": truncated, + "sampled_files": [p for p, _ in snippets], + }, + ) + + async def _fetch_json(self, client: httpx.AsyncClient, path: str) -> dict: + resp = await client.get(path) + if resp.status_code == 404: + raise RepositoryFetchError( + code="REPO_NOT_FOUND", + message=f"GitHub returned 404 for {path}", + retriable=False, + ) + if resp.status_code in (401, 403): + raise RepositoryFetchError( + code="REPO_AUTH_OR_RATE", + message=f"GitHub auth/rate-limit: {resp.status_code} {resp.text[:200]}", + retriable=True, + ) + resp.raise_for_status() + return resp.json() + + async def _fetch_readme( + self, + client: httpx.AsyncClient, + owner_repo: str, + ) -> str: + try: + data = await self._fetch_json(client, f"/repos/{owner_repo}/readme") + except RepositoryFetchError as err: + if err.code == "REPO_NOT_FOUND": + return "" # README 없는 레포는 정상 + raise + if data.get("encoding") != "base64": + return "" + try: + return base64.b64decode(data.get("content", "")).decode( + "utf-8", errors="replace" + ) + except Exception: + return "" + + async def _fetch_tree( + self, + client: httpx.AsyncClient, + owner_repo: str, + branch: str, + ) -> tuple[list[str], bool]: + data = await self._fetch_json( + client, + f"/repos/{owner_repo}/git/trees/{branch}?recursive=1", + ) + tree = data.get("tree", []) + paths = [ + item["path"] + for item in tree + if item.get("type") == "blob" and item.get("path") + ] + return paths, bool(data.get("truncated")) + + async def _fetch_snippets( + self, + client: httpx.AsyncClient, + owner_repo: str, + branch: str, + picked: list[str], + ) -> list[tuple[str, str]]: + out: list[tuple[str, str]] = [] + for path in picked: + try: + data = await self._fetch_json( + client, f"/repos/{owner_repo}/contents/{path}?ref={branch}" + ) + except RepositoryFetchError as err: + log.warning( + "repo.snippet.skip", + path=path, + code=err.code, + ) + continue + if isinstance(data, list) or data.get("encoding") != "base64": + continue + try: + body = base64.b64decode(data.get("content", "")).decode( + "utf-8", errors="replace" + ) + except Exception: # noqa: BLE001 + continue + if len(body.encode("utf-8")) > self._cfg.max_file_bytes: + body = body[: self._cfg.max_file_bytes] + "\n... (truncated)" + out.append((path, body)) + return out + + +def _select_files(paths: list[str], cap: int) -> list[str]: + """우선순위 → 디렉토리당 대표 텍스트 파일 1개씩, 합쳐서 cap까지.""" + chosen: list[str] = [] + seen: set[str] = set() + + for needle in _PRIORITY_FILES: + for p in paths: + if p == needle or p.endswith("/" + needle): + if p not in seen: + chosen.append(p) + seen.add(p) + break + if len(chosen) >= cap: + return chosen + + by_dir: dict[str, str] = {} + for p in paths: + if p in seen: + continue + if "." not in p.rsplit("/", 1)[-1]: + continue + ext = "." + p.rsplit(".", 1)[-1].lower() + if ext not in _TEXT_EXTENSIONS: + continue + d = p.rsplit("/", 1)[0] if "/" in p else "" + if d not in by_dir: + by_dir[d] = p + for p in by_dir.values(): + if p not in seen: + chosen.append(p) + seen.add(p) + if len(chosen) >= cap: + break + return chosen diff --git a/ai/src/ai_server/analyzer/sources/pdf.py b/ai/src/ai_server/analyzer/sources/pdf.py new file mode 100644 index 00000000..8dcf38b3 --- /dev/null +++ b/ai/src/ai_server/analyzer/sources/pdf.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import asyncio +import io + +from pypdf import PdfReader + +from ai_server.analyzer.sources.base import ExtractedSource, SourceExtractor +from ai_server.storage.base import ObjectStorage + + +def _extract_pdf_text(data: bytes) -> str: + reader = PdfReader(io.BytesIO(data)) + parts: list[str] = [] + for page in reader.pages: + parts.append(page.extract_text() or "") + return "\n\n".join(p for p in parts if p).strip() + + +# PDF 를 읽어 페이지 텍스트를 이어붙인다. +class PdfSourceExtractor(SourceExtractor): + + def __init__(self, storage: ObjectStorage) -> None: + self._storage = storage + + async def extract(self, locator: str) -> ExtractedSource: + data = await self._storage.get_bytes(locator) + text = await asyncio.to_thread(_extract_pdf_text, data) + return ExtractedSource( + text=text, + source_type="PDF", + metadata={"locator": locator, "bytes": len(data)}, + ) diff --git a/ai/src/ai_server/analyzer/sources/web.py b/ai/src/ai_server/analyzer/sources/web.py new file mode 100644 index 00000000..ca44f9c5 --- /dev/null +++ b/ai/src/ai_server/analyzer/sources/web.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import asyncio + +import httpx +import structlog +import trafilatura + +from ai_server.analyzer.sources.base import ExtractedSource, SourceExtractor + +log = structlog.get_logger(__name__) + + +class WebFetchError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +# 라이브러리로 본문 추출 +class WebSourceExtractor(SourceExtractor): + def __init__( + self, + *, + timeout_sec: float = 20.0, + max_html_bytes: int = 2_000_000, + client: httpx.AsyncClient | None = None, + ) -> None: + self._timeout_sec = timeout_sec + self._max_html_bytes = max_html_bytes + self._client = client + + async def extract(self, locator: str) -> ExtractedSource: + url = locator.strip() + if not (url.startswith("http://") or url.startswith("https://")): + raise WebFetchError( + code="INVALID_WEB_URL", + message=f"locator must be http(s) URL, got: {locator!r}", + retriable=False, + ) + + html, final_url, content_type = await self._fetch_html(url) + text = await asyncio.to_thread(_extract_main_text, html, final_url) + + if not text.strip(): + raise WebFetchError( + code="EMPTY_WEB_BODY", + message="추출된 본문이 비어 있음 — JS 렌더링 페이지이거나 비어있는 문서일 가능성", + retriable=False, + ) + + return ExtractedSource( + text=text, + source_type="WEB", + metadata={ + "locator": url, + "final_url": final_url, + "content_type": content_type, + "html_bytes": len(html.encode("utf-8")), + "text_chars": len(text), + }, + ) + + async def _fetch_html(self, url: str) -> tuple[str, str, str]: + if self._client is not None: + return await self._do_fetch(self._client, url) + async with httpx.AsyncClient( + timeout=self._timeout_sec, + follow_redirects=True, + headers={ + "User-Agent": "StackUp-AI/1.0 (+resume web extractor)", + "Accept": "text/html,application/xhtml+xml", + }, + ) as client: + return await self._do_fetch(client, url) + + async def _do_fetch( + self, + client: httpx.AsyncClient, + url: str, + ) -> tuple[str, str, str]: + try: + resp = await client.get(url) + except httpx.HTTPError as exc: + raise WebFetchError( + code="WEB_FETCH_FAILED", + message=f"HTTP 요청 실패: {exc}", + retriable=True, + ) from exc + + if resp.status_code >= 400: + raise WebFetchError( + code="WEB_HTTP_STATUS", + message=f"HTTP {resp.status_code}", + retriable=resp.status_code >= 500, + ) + + content_type = resp.headers.get("content-type", "") + if "html" not in content_type.lower(): + raise WebFetchError( + code="WEB_NOT_HTML", + message=f"HTML이 아님 (content-type={content_type})", + retriable=False, + ) + + raw = resp.content + if len(raw) > self._max_html_bytes: + raise WebFetchError( + code="WEB_HTML_TOO_LARGE", + message=f"HTML이 한도({self._max_html_bytes}B)를 초과: {len(raw)}B", + retriable=False, + ) + + html = resp.text + return html, str(resp.url), content_type + + +def _extract_main_text(html: str, url: str) -> str: + result = trafilatura.extract( + html, + url=url, + include_comments=False, + include_tables=True, + favor_recall=True, + ) + return (result or "").strip() diff --git a/ai/src/ai_server/analyzer/web_resume_analyzer.py b/ai/src/ai_server/analyzer/web_resume_analyzer.py new file mode 100644 index 00000000..b5da258d --- /dev/null +++ b/ai/src/ai_server/analyzer/web_resume_analyzer.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import structlog + +from ai_server.analyzer._embedding_step import ( + EmbeddingStepError, + chunk_embed_and_upsert, +) +from ai_server.analyzer.sources.base import SourceExtractor +from ai_server.analyzer.sources.web import WebFetchError +from ai_server.chain.document_analysis_chain import DocumentAnalyzer +from ai_server.core.client import CoreClient +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import EmbeddingProvider +from ai_server.storage.base import ObjectStorage + +log = structlog.get_logger(__name__) + + +class WebResumeAnalyzeError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class WebResumeAnalysisResult: + summary: str + tech_stack: list[str] + document_path: str + embedding_chunk_count: int + + +# 웹포폴 -> 라이브러리로 텍스트만 -> LLM -> 마크다운 -> 청킹·임베딩·pgvector upsert +class WebResumeAnalyzer: + def __init__( + self, + *, + extractor: SourceExtractor, + chain: DocumentAnalyzer, + storage: ObjectStorage, + chunker: MarkdownChunker, + embedder: EmbeddingProvider, + core_client: CoreClient, + analyzed_key_template: str, + ) -> None: + self._extractor = extractor + self._chain = chain + self._storage = storage + self._chunker = chunker + self._embedder = embedder + self._core_client = core_client + self._analyzed_key_template = analyzed_key_template + + async def analyze( + self, + *, + resume_id: int, + url: str, + analyzed_document_id: int, + ) -> WebResumeAnalysisResult: + log.info("web_resume.extract.start", resume_id=resume_id, url=url) + try: + source = await self._extractor.extract(url) + except WebFetchError as err: + raise WebResumeAnalyzeError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + log.info( + "web_resume.llm.start", + resume_id=resume_id, + text_chars=len(source.text), + ) + analysis = await self._chain.analyze( + text=source.text, source_type=source.source_type + ) + + out_key = self._analyzed_key_template.format(resume_id=resume_id) + await self._storage.put_text(out_key, analysis.markdown) + log.info( + "web_resume.markdown.saved", + resume_id=resume_id, + key=out_key, + md_chars=len(analysis.markdown), + ) + + try: + chunk_count = await chunk_embed_and_upsert( + document_id=analyzed_document_id, + markdown=analysis.markdown, + chunker=self._chunker, + embedder=self._embedder, + core_client=self._core_client, + log_prefix="web_resume", + ) + except EmbeddingStepError as err: + raise WebResumeAnalyzeError( + code=err.code, message=err.message, retriable=err.retriable + ) from err + + return WebResumeAnalysisResult( + summary=analysis.summary, + tech_stack=list(analysis.tech_stack), + document_path=out_key, + embedding_chunk_count=chunk_count, + ) diff --git a/ai/src/ai_server/chain/__init__.py b/ai/src/ai_server/chain/__init__.py new file mode 100644 index 00000000..d0f5e24a --- /dev/null +++ b/ai/src/ai_server/chain/__init__.py @@ -0,0 +1,13 @@ +from ai_server.chain.document_analysis_chain import ( + DocumentAnalysisResult, + DocumentAnalyzer, + LlmDocumentAnalyzer, + build_document_analysis_chain, +) + +__all__ = [ + "DocumentAnalysisResult", + "DocumentAnalyzer", + "LlmDocumentAnalyzer", + "build_document_analysis_chain", +] diff --git a/ai/src/ai_server/chain/document_analysis_chain.py b/ai/src/ai_server/chain/document_analysis_chain.py new file mode 100644 index 00000000..b212e9c3 --- /dev/null +++ b/ai/src/ai_server/chain/document_analysis_chain.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from typing import Protocol + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import Runnable +from pydantic import BaseModel, Field + +from ai_server.analyzer.sources.base import SourceType +from ai_server.chain.prompts.document_analysis import HUMAN_PROMPT, SYSTEM_PROMPT +from ai_server.config.settings import Settings + + +class DocumentAnalysisResult(BaseModel): + summary: str = Field(..., description="2~4 sentence Korean summary") + tech_stack: list[str] = Field(default_factory=list) + markdown: str = Field(..., description="Interviewer-facing markdown") + + +class DocumentAnalyzer(Protocol): + async def analyze( + self, *, text: str, source_type: SourceType + ) -> DocumentAnalysisResult: ... + + +# 랭체인 파이프라인 호출을 감싼다 +class LlmDocumentAnalyzer: + + def __init__(self, chain: Runnable) -> None: + self._chain = chain + + async def analyze( + self, *, text: str, source_type: SourceType + ) -> DocumentAnalysisResult: + result = await self._chain.ainvoke({"text": text, "source_type": source_type}) + if not isinstance(result, DocumentAnalysisResult): + raise TypeError( + f"chain returned {type(result).__name__}, expected DocumentAnalysisResult" + ) + return result + + +# 프롬프트 -> LLM -> 파서 하나로 묶어서 처리함 +def build_document_analysis_chain(settings: Settings) -> Runnable: + from langchain_openai import ChatOpenAI + + parser = PydanticOutputParser(pydantic_object=DocumentAnalysisResult) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", SYSTEM_PROMPT), + ("human", HUMAN_PROMPT), + ] + ).partial(format_instructions=parser.get_format_instructions()) + + llm = ChatOpenAI( + model=settings.llm_pro_model, + temperature=settings.llm_pro_temperature, + api_key=settings.llm_api_key or None, + base_url=settings.llm_base_url, + ) + return prompt | llm | parser diff --git a/ai/src/ai_server/chain/prompts/__init__.py b/ai/src/ai_server/chain/prompts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/src/ai_server/chain/prompts/document_analysis.py b/ai/src/ai_server/chain/prompts/document_analysis.py new file mode 100644 index 00000000..6cc9784c --- /dev/null +++ b/ai/src/ai_server/chain/prompts/document_analysis.py @@ -0,0 +1,23 @@ +# 문서 분석 프롬프트 템플릿 +# source_type 으로 PDF, 웹, 리포지토리 구분함 +SYSTEM_PROMPT = ( + "당신은 IT 직군 채용을 진행하는 시니어 면접관입니다. " + "지원자 자료(이력서·레포 README·기술 블로그 등)를 분석하여 면접 준비에 사용할 " + "구조화된 결과를 산출하세요.\n" + "- 사실에 근거해서 작성하고, 자료에 없는 내용을 추측해 채우지 마세요.\n" + "- 한국어로 작성하되 기술 용어는 영문 원어를 그대로 둡니다.\n" + "- 응답은 반드시 지정된 JSON 스키마를 따릅니다." +) + +HUMAN_PROMPT = ( + "다음은 지원자 자료입니다 (출처 유형: {source_type}).\n" + "---\n" + "{text}\n" + "---\n\n" + "요구 사항:\n" + "1. summary: 2~4문장 한국어 요약.\n" + "2. tech_stack: 자료에 명시된 핵심 기술 5~15개. 영문 표기.\n" + "3. markdown: 면접관이 훑어볼 한국어 마크다운. 섹션 구조는 " + "'## 개요', '## 주요 경험', '## 기술', '## 추가 메모' 사용.\n\n" + "{format_instructions}" +) diff --git a/ai/src/ai_server/config/settings.py b/ai/src/ai_server/config/settings.py index 0a8530af..3d1a90f0 100644 --- a/ai/src/ai_server/config/settings.py +++ b/ai/src/ai_server/config/settings.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic_settings import BaseSettings, SettingsConfigDict @@ -14,25 +16,67 @@ class Settings(BaseSettings): debug: bool = False # RabbitMQ - rabbitmq_url: str = "amqp://stackup:stackup@localhost:5672/" + rabbitmq_url: str = "amqp://stackup:stackup@localhost:38050/" - # AI consumer + # AI consumer (큐 이름, prefetch, 콜백 라우팅 등) ai_queue_resume: str = "ai.analyze.resume" + ai_queue_repository: str = "ai.analyze.repository" + ai_queue_web: str = "ai.analyze.web" ai_queue_prefetch: int = 10 ai_callback_exchange: str = "stackup.ai-to-core" ai_callback_routing_analysis: str = "callback.analysis" ai_publisher_name: str = "ai-server" ai_idempotency_lru_size: int = 1024 - # S3 / MinIO - s3_endpoint_url: str = "http://localhost:9000" + storage_backend: Literal["local", "s3"] = "s3" + storage_local_root: str = "./var/storage" + + s3_endpoint_url: str = "http://localhost:38060" s3_access_key: str = "" s3_secret_key: str = "" s3_bucket_name: str = "stackup" + s3_region: str = "us-east-1" + + # 일단 충대 API 키 사용 + llm_api_key: str = "" + llm_base_url: str = "https://factchat-cloud.mindlogic.ai/v1/gateway" + llm_pro_model: str = "gemini-3.1-pro-preview" + llm_pro_temperature: float = 0.2 + + analyzed_resume_md_key_template: str = "analyzed/resume/{resume_id}/summary.md" + analyzed_repository_md_key_template: str = ( + "analyzed/repository/{repository_id}/summary.md" + ) + analyzed_web_resume_md_key_template: str = ( + "analyzed/web-resume/{resume_id}/summary.md" + ) + + # Core 서버 internal API (사용자별 GitHub access_token 조회 등) + core_internal_base_url: str = "http://localhost:38010" + core_internal_api_key: str = "" + core_internal_timeout_sec: float = 10.0 + + # GitHub repo 분석용 + github_api_base_url: str = "https://api.github.com" + # 사용자별 token은 Core에서 받음. 아래 token은 fallback (public repo 한정). + github_fallback_token: str = "" + repo_max_source_files: int = 8 + repo_max_source_file_bytes: int = 50_000 + repo_fetch_timeout_sec: float = 30.0 + + # 웹 이력서 fetch + web_fetch_timeout_sec: float = 20.0 + web_max_html_bytes: int = 2_000_000 # 2MB 상한 + + # 임베딩 관련 + embedding_provider: Literal["mock", "gemini", "openai", "ollama"] = "mock" + embedding_model: str = "gemini-embedding-001" + embedding_dim: int = 1536 + embedding_chunk_size: int = 1000 + embedding_chunk_overlap: int = 200 + embedding_batch_size: int = 32 - # LLM API Keys - openai_api_key: str = "" - google_api_key: str = "" + gemini_api_key: str = "" def get_settings() -> Settings: diff --git a/ai/src/ai_server/core/__init__.py b/ai/src/ai_server/core/__init__.py new file mode 100644 index 00000000..7dbd25bb --- /dev/null +++ b/ai/src/ai_server/core/__init__.py @@ -0,0 +1,15 @@ +from ai_server.core.client import ( + CoreClient, + CoreEmbeddingUpsertError, + CoreTokenError, + EmbeddingChunkPayload, + HttpCoreClient, +) + +__all__ = [ + "CoreClient", + "CoreTokenError", + "CoreEmbeddingUpsertError", + "EmbeddingChunkPayload", + "HttpCoreClient", +] diff --git a/ai/src/ai_server/core/client.py b/ai/src/ai_server/core/client.py new file mode 100644 index 00000000..bfc3028e --- /dev/null +++ b/ai/src/ai_server/core/client.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +import httpx +import structlog + +log = structlog.get_logger(__name__) + + +class CoreTokenError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +class CoreEmbeddingUpsertError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class EmbeddingChunkPayload: + chunk_index: int + chunk_text: str + embedding: list[float] + + +# 코어 서버 API 호출용 +class CoreClient(Protocol): + async def fetch_github_token(self, user_id: int) -> str: ... + + async def upsert_embeddings( + self, + *, + document_id: int, + model: str, + dim: int, + chunks: list[EmbeddingChunkPayload], + ) -> int: ... + + +class HttpCoreClient: + def __init__( + self, + *, + base_url: str, + api_key: str, + timeout_sec: float = 10.0, + client: httpx.AsyncClient | None = None, + ) -> None: + if not base_url: + raise ValueError("Core internal base_url 누락") + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._timeout_sec = timeout_sec + self._client = client + + async def fetch_github_token(self, user_id: int) -> str: + path = f"/api/internal/users/{user_id}/github-token" + if self._client is not None: + return await self._do_fetch(self._client, path) + async with self._build_client() as client: + return await self._do_fetch(client, path) + + def _build_client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + base_url=self._base_url, + headers={"X-Internal-API-Key": self._api_key}, + timeout=self._timeout_sec, + ) + + async def _do_fetch(self, client: httpx.AsyncClient, path: str) -> str: + try: + resp = await client.get(path) + except httpx.HTTPError as exc: + raise CoreTokenError( + code="CORE_UNAVAILABLE", + message=f"Core API 호출 실패: {exc}", + retriable=True, + ) from exc + + status = resp.status_code + if status == 404: + raise CoreTokenError( + code="USER_NOT_FOUND", + message=f"Core에 사용자 또는 token 없음: {path}", + retriable=False, + ) + if status in (401, 403): + raise CoreTokenError( + code="CORE_AUTH_FAILED", + message=f"Core internal 인증 실패: {status}", + retriable=False, + ) + if status >= 500: + raise CoreTokenError( + code="CORE_UNAVAILABLE", + message=f"Core 5xx: {status}", + retriable=True, + ) + if status >= 400: + raise CoreTokenError( + code="CORE_BAD_REQUEST", + message=f"Core {status}: {resp.text[:200]}", + retriable=False, + ) + + try: + data = resp.json() + except ValueError as exc: + raise CoreTokenError( + code="CORE_BAD_RESPONSE", + message=f"Core 응답 JSON 파싱 실패: {exc}", + retriable=True, + ) from exc + + token = data.get("accessToken") or data.get("access_token") + if not token or not isinstance(token, str): + raise CoreTokenError( + code="CORE_TOKEN_EMPTY", + message="Core 응답에 accessToken 없음", + retriable=True, + ) + return token + + async def upsert_embeddings( + self, + *, + document_id: int, + model: str, + dim: int, + chunks: list[EmbeddingChunkPayload], + ) -> int: + """`analyzed_documents.id` 한 건에 대한 chunk + embedding을 Core가 + pgvector(document_embeddings)에 idempotent upsert. + + body: + { "model": "...", "dim": 1536, + "chunks": [{ "chunkIndex": 0, "chunkText": "...", "embedding": [...] }, ...] } + + 반환: upsert된 chunk 수. + """ + body = { + "model": model, + "dim": dim, + "chunks": [ + { + "chunkIndex": c.chunk_index, + "chunkText": c.chunk_text, + "embedding": c.embedding, + } + for c in chunks + ], + } + path = f"/api/internal/documents/{document_id}/embeddings" + if self._client is not None: + return await self._do_upsert(self._client, path, body) + async with self._build_client() as client: + return await self._do_upsert(client, path, body) + + async def _do_upsert( + self, + client: httpx.AsyncClient, + path: str, + body: dict, + ) -> int: + try: + resp = await client.put(path, json=body) + except httpx.HTTPError as exc: + raise CoreEmbeddingUpsertError( + code="CORE_UNAVAILABLE", + message=f"Core API 호출 실패: {exc}", + retriable=True, + ) from exc + + status = resp.status_code + if status == 404: + raise CoreEmbeddingUpsertError( + code="DOCUMENT_NOT_FOUND", + message=f"Core에 document 없음: {path}", + retriable=False, + ) + if status in (401, 403): + raise CoreEmbeddingUpsertError( + code="CORE_AUTH_FAILED", + message=f"Core internal 인증 실패: {status}", + retriable=False, + ) + if status >= 500: + raise CoreEmbeddingUpsertError( + code="CORE_UNAVAILABLE", + message=f"Core 5xx: {status}", + retriable=True, + ) + if status >= 400: + raise CoreEmbeddingUpsertError( + code="CORE_BAD_REQUEST", + message=f"Core {status}: {resp.text[:200]}", + retriable=False, + ) + + try: + data = resp.json() + except ValueError as exc: + raise CoreEmbeddingUpsertError( + code="CORE_BAD_RESPONSE", + message=f"Core 응답 JSON 파싱 실패: {exc}", + retriable=True, + ) from exc + + count = data.get("upserted") if isinstance(data, dict) else None + if not isinstance(count, int): + return len(body["chunks"]) + return count diff --git a/ai/src/ai_server/messaging/consumers/repository_consumer.py b/ai/src/ai_server/messaging/consumers/repository_consumer.py new file mode 100644 index 00000000..97f50ce2 --- /dev/null +++ b/ai/src/ai_server/messaging/consumers/repository_consumer.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import structlog +from aio_pika.abc import AbstractIncomingMessage + +from ai_server.analyzer.repository_analyzer import ( + RepositoryAnalyzeError, + RepositoryAnalyzer, +) +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.messaging.publisher import CallbackPublisher +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + RepositoryAnalyzeRequest, +) + +log = structlog.get_logger(__name__) + + +class RepositoryConsumer: + def __init__( + self, + *, + analyzer: RepositoryAnalyzer, + publisher: CallbackPublisher, + idempotency: LruIdempotencyStore, + callback_routing_key: str, + ) -> None: + self._analyzer = analyzer + self._publisher = publisher + self._idempotency = idempotency + self._callback_routing_key = callback_routing_key + + async def handle(self, message: AbstractIncomingMessage) -> None: + async with message.process(requeue=False): + try: + envelope = Envelope[RepositoryAnalyzeRequest].model_validate_json( + message.body + ) + except Exception as exc: + log.error( + "repository.parse.failed", + error=str(exc), + delivery_tag=message.delivery_tag, + ) + raise + + if self._idempotency.is_seen_then_mark(envelope.message_id): + log.info( + "repository.idempotent.skip", + message_id=envelope.message_id, + trace_id=envelope.trace_id, + ) + return + + req = envelope.payload + user_id = envelope.context.user_id + log.info( + "repository.analyze.start", + message_id=envelope.message_id, + repository_id=req.repository_id, + repo_full_name=req.repo_full_name, + user_id=user_id, + trace_id=envelope.trace_id, + ) + + payload = await self._run_and_build_payload( + req, user_id=user_id, trace_id=envelope.trace_id + ) + + await self._publisher.publish( + routing_key=self._callback_routing_key, + message_type="callback.analysis", + payload=payload, + trace_id=envelope.trace_id, + correlation_id=envelope.message_id, + context=envelope.context, + ) + log.info( + "repository.analyze.done", + message_id=envelope.message_id, + repository_id=req.repository_id, + status=payload.status, + trace_id=envelope.trace_id, + ) + + async def _run_and_build_payload( + self, + req: RepositoryAnalyzeRequest, + *, + user_id: int | None, + trace_id: str, + ) -> AnalysisCallbackPayload: + try: + result = await self._analyzer.analyze( + repository_id=req.repository_id, + repo_full_name=req.repo_full_name, + default_branch=req.default_branch, + user_id=user_id, + analyzed_document_id=req.analyzed_document_id, + ) + except RepositoryAnalyzeError as err: + log.warning( + "repository.analyze.domain_failed", + repository_id=req.repository_id, + code=err.code, + retriable=err.retriable, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="REPOSITORY", + target_id=req.repository_id, + status="FAILED", + error_code=err.code, + error_message=err.message, + retriable=err.retriable, + ) + except Exception as exc: + log.exception( + "repository.analyze.unexpected_failed", + repository_id=req.repository_id, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="REPOSITORY", + target_id=req.repository_id, + status="FAILED", + error_code="UNEXPECTED", + error_message=str(exc), + retriable=True, + ) + + return AnalysisCallbackPayload( + target_type="REPOSITORY", + target_id=req.repository_id, + status="ANALYZED", + summary=result.summary, + tech_stack=result.tech_stack, + document_path=result.document_path, + embedding_chunk_count=result.embedding_chunk_count, + ) diff --git a/ai/src/ai_server/messaging/consumers/resume_consumer.py b/ai/src/ai_server/messaging/consumers/resume_consumer.py index 6142359e..8aa90944 100644 --- a/ai/src/ai_server/messaging/consumers/resume_consumer.py +++ b/ai/src/ai_server/messaging/consumers/resume_consumer.py @@ -3,6 +3,7 @@ import structlog from aio_pika.abc import AbstractIncomingMessage +from ai_server.analyzer.resume_analyzer import ResumeAnalyzeError, ResumeAnalyzer from ai_server.messaging.idempotency import LruIdempotencyStore from ai_server.messaging.publisher import CallbackPublisher from ai_server.model.envelope import Envelope @@ -14,29 +15,16 @@ log = structlog.get_logger(__name__) -def build_echo_callback( - envelope: Envelope[ResumeAnalyzeRequest], -) -> AnalysisCallbackPayload: - rid = envelope.payload.resume_id - return AnalysisCallbackPayload( - target_type="RESUME", - target_id=rid, - status="ANALYZED", - summary="[ECHO] not yet analyzed", - tech_stack=[], - document_s3_key=f"echo/resume/{rid}/echo.md", - embedding_chunk_count=0, - ) - - class ResumeConsumer: def __init__( self, *, + analyzer: ResumeAnalyzer, publisher: CallbackPublisher, idempotency: LruIdempotencyStore, callback_routing_key: str, ) -> None: + self._analyzer = analyzer self._publisher = publisher self._idempotency = idempotency self._callback_routing_key = callback_routing_key @@ -63,24 +51,80 @@ async def handle(self, message: AbstractIncomingMessage) -> None: ) return + req = envelope.payload log.info( - "resume.echo.start", + "resume.analyze.start", message_id=envelope.message_id, - resume_id=envelope.payload.resume_id, + resume_id=req.resume_id, trace_id=envelope.trace_id, ) - callback = build_echo_callback(envelope) + + payload = await self._run_and_build_payload(req, envelope.trace_id) + await self._publisher.publish( routing_key=self._callback_routing_key, message_type="callback.analysis", - payload=callback, + payload=payload, trace_id=envelope.trace_id, correlation_id=envelope.message_id, context=envelope.context, ) log.info( - "resume.echo.done", + "resume.analyze.done", message_id=envelope.message_id, - resume_id=envelope.payload.resume_id, + resume_id=req.resume_id, + status=payload.status, trace_id=envelope.trace_id, ) + + async def _run_and_build_payload( + self, + req: ResumeAnalyzeRequest, + trace_id: str, + ) -> AnalysisCallbackPayload: + try: + result = await self._analyzer.analyze( + resume_id=req.resume_id, + file_path=req.file_path, + analyzed_document_id=req.analyzed_document_id, + ) + except ResumeAnalyzeError as err: + log.warning( + "resume.analyze.domain_failed", + resume_id=req.resume_id, + code=err.code, + retriable=err.retriable, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="RESUME", + target_id=req.resume_id, + status="FAILED", + error_code=err.code, + error_message=err.message, + retriable=err.retriable, + ) + except Exception as exc: + log.exception( + "resume.analyze.unexpected_failed", + resume_id=req.resume_id, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="RESUME", + target_id=req.resume_id, + status="FAILED", + error_code="UNEXPECTED", + error_message=str(exc), + retriable=True, + ) + + return AnalysisCallbackPayload( + target_type="RESUME", + target_id=req.resume_id, + status="ANALYZED", + summary=result.summary, + tech_stack=result.tech_stack, + document_path=result.document_path, + embedding_chunk_count=result.embedding_chunk_count, + ) diff --git a/ai/src/ai_server/messaging/consumers/web_consumer.py b/ai/src/ai_server/messaging/consumers/web_consumer.py new file mode 100644 index 00000000..830b4a89 --- /dev/null +++ b/ai/src/ai_server/messaging/consumers/web_consumer.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import structlog +from aio_pika.abc import AbstractIncomingMessage + +from ai_server.analyzer.web_resume_analyzer import ( + WebResumeAnalyzeError, + WebResumeAnalyzer, +) +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.messaging.publisher import CallbackPublisher +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + WebResumeAnalyzeRequest, +) + +log = structlog.get_logger(__name__) + + +class WebResumeConsumer: + def __init__( + self, + *, + analyzer: WebResumeAnalyzer, + publisher: CallbackPublisher, + idempotency: LruIdempotencyStore, + callback_routing_key: str, + ) -> None: + self._analyzer = analyzer + self._publisher = publisher + self._idempotency = idempotency + self._callback_routing_key = callback_routing_key + + async def handle(self, message: AbstractIncomingMessage) -> None: + async with message.process(requeue=False): + try: + envelope = Envelope[WebResumeAnalyzeRequest].model_validate_json( + message.body + ) + except Exception as exc: + log.error( + "web_resume.parse.failed", + error=str(exc), + delivery_tag=message.delivery_tag, + ) + raise + + if self._idempotency.is_seen_then_mark(envelope.message_id): + log.info( + "web_resume.idempotent.skip", + message_id=envelope.message_id, + trace_id=envelope.trace_id, + ) + return + + req = envelope.payload + log.info( + "web_resume.analyze.start", + message_id=envelope.message_id, + resume_id=req.resume_id, + url=req.url, + trace_id=envelope.trace_id, + ) + + payload = await self._run_and_build_payload(req, envelope.trace_id) + + await self._publisher.publish( + routing_key=self._callback_routing_key, + message_type="callback.analysis", + payload=payload, + trace_id=envelope.trace_id, + correlation_id=envelope.message_id, + context=envelope.context, + ) + log.info( + "web_resume.analyze.done", + message_id=envelope.message_id, + resume_id=req.resume_id, + status=payload.status, + trace_id=envelope.trace_id, + ) + + async def _run_and_build_payload( + self, + req: WebResumeAnalyzeRequest, + trace_id: str, + ) -> AnalysisCallbackPayload: + try: + result = await self._analyzer.analyze( + resume_id=req.resume_id, + url=req.url, + analyzed_document_id=req.analyzed_document_id, + ) + except WebResumeAnalyzeError as err: + log.warning( + "web_resume.analyze.domain_failed", + resume_id=req.resume_id, + code=err.code, + retriable=err.retriable, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="WEB", + target_id=req.resume_id, + status="FAILED", + error_code=err.code, + error_message=err.message, + retriable=err.retriable, + ) + except Exception as exc: + log.exception( + "web_resume.analyze.unexpected_failed", + resume_id=req.resume_id, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="WEB", + target_id=req.resume_id, + status="FAILED", + error_code="UNEXPECTED", + error_message=str(exc), + retriable=True, + ) + + return AnalysisCallbackPayload( + target_type="WEB", + target_id=req.resume_id, + status="ANALYZED", + summary=result.summary, + tech_stack=result.tech_stack, + document_path=result.document_path, + embedding_chunk_count=result.embedding_chunk_count, + ) diff --git a/ai/src/ai_server/messaging/runner.py b/ai/src/ai_server/messaging/runner.py index fdded71b..3b437b94 100644 --- a/ai/src/ai_server/messaging/runner.py +++ b/ai/src/ai_server/messaging/runner.py @@ -3,17 +3,33 @@ import structlog from aio_pika.abc import AbstractRobustQueue +from ai_server.analyzer.repository_analyzer import RepositoryAnalyzer +from ai_server.analyzer.resume_analyzer import ResumeAnalyzer +from ai_server.analyzer.sources.github_repo import GitHubRepoSourceExtractor +from ai_server.analyzer.sources.pdf import PdfSourceExtractor +from ai_server.analyzer.sources.web import WebSourceExtractor +from ai_server.analyzer.web_resume_analyzer import WebResumeAnalyzer +from ai_server.chain.document_analysis_chain import ( + LlmDocumentAnalyzer, + build_document_analysis_chain, +) from ai_server.config.settings import Settings +from ai_server.core.client import HttpCoreClient from ai_server.messaging.connection import RabbitConnection +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import build_embedding_provider +from ai_server.messaging.consumers.repository_consumer import RepositoryConsumer from ai_server.messaging.consumers.resume_consumer import ResumeConsumer +from ai_server.messaging.consumers.web_consumer import WebResumeConsumer from ai_server.messaging.idempotency import LruIdempotencyStore from ai_server.messaging.publisher import CallbackPublisher +from ai_server.storage.factory import build_storage log = structlog.get_logger(__name__) +# FastAPI 서버가 켜져있는 동안 메세지를 보관함 class MessagingRuntime: - """Holds long-lived messaging components for FastAPI lifespan.""" def __init__(self, settings: Settings) -> None: self._settings = settings @@ -29,34 +45,128 @@ def __init__(self, settings: Settings) -> None: self._idempotency = LruIdempotencyStore( max_size=settings.ai_idempotency_lru_size, ) + + storage = build_storage(settings) + chain = build_document_analysis_chain(settings) + chain_analyzer = LlmDocumentAnalyzer(chain) + + chunker = MarkdownChunker( + chunk_size=settings.embedding_chunk_size, + chunk_overlap=settings.embedding_chunk_overlap, + ) + embedder = build_embedding_provider( + provider=settings.embedding_provider, + dim=settings.embedding_dim, + model=settings.embedding_model, + gemini_api_key=settings.gemini_api_key, + ) + + core_client = HttpCoreClient( + base_url=settings.core_internal_base_url, + api_key=settings.core_internal_api_key, + timeout_sec=settings.core_internal_timeout_sec, + ) + + # 이력서 PDF + resume_analyzer = ResumeAnalyzer( + extractor=PdfSourceExtractor(storage=storage), + chain=chain_analyzer, + storage=storage, + chunker=chunker, + embedder=embedder, + core_client=core_client, + analyzed_key_template=settings.analyzed_resume_md_key_template, + ) + + # 리포지토리 + repo_analyzer = RepositoryAnalyzer( + extractor=GitHubRepoSourceExtractor( + api_base_url=settings.github_api_base_url, + fallback_token=settings.github_fallback_token, + max_files=settings.repo_max_source_files, + max_file_bytes=settings.repo_max_source_file_bytes, + timeout_sec=settings.repo_fetch_timeout_sec, + ), + core_client=core_client, + chain=chain_analyzer, + storage=storage, + chunker=chunker, + embedder=embedder, + analyzed_key_template=settings.analyzed_repository_md_key_template, + ) + + # 웹 이력서 + web_analyzer = WebResumeAnalyzer( + extractor=WebSourceExtractor( + timeout_sec=settings.web_fetch_timeout_sec, + max_html_bytes=settings.web_max_html_bytes, + ), + chain=chain_analyzer, + storage=storage, + chunker=chunker, + embedder=embedder, + core_client=core_client, + analyzed_key_template=settings.analyzed_web_resume_md_key_template, + ) + self._resume_consumer = ResumeConsumer( + analyzer=resume_analyzer, + publisher=self._publisher, + idempotency=self._idempotency, + callback_routing_key=settings.ai_callback_routing_analysis, + ) + self._repository_consumer = RepositoryConsumer( + analyzer=repo_analyzer, + publisher=self._publisher, + idempotency=self._idempotency, + callback_routing_key=settings.ai_callback_routing_analysis, + ) + self._web_consumer = WebResumeConsumer( + analyzer=web_analyzer, publisher=self._publisher, idempotency=self._idempotency, callback_routing_key=settings.ai_callback_routing_analysis, ) - self._resume_queue: AbstractRobustQueue | None = None - self._consumer_tag: str | None = None + + self._consumers: list[tuple[AbstractRobustQueue, str]] = [] async def start(self) -> None: await self._connection.open() await self._publisher.open() channel = self._connection.channel + + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_resume, + handler=self._resume_consumer.handle, + ) + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_repository, + handler=self._repository_consumer.handle, + ) + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_web, + handler=self._web_consumer.handle, + ) + + async def _start_consumer(self, channel, *, queue_name, handler) -> None: queue = await channel.declare_queue( - self._settings.ai_queue_resume, + queue_name, durable=True, - passive=True, # 정의 파일이 이미 선언함 - ) - self._resume_queue = queue - self._consumer_tag = await queue.consume(self._resume_consumer.handle) - log.info( - "ai.consumer.started", - queue=self._settings.ai_queue_resume, - consumer_tag=self._consumer_tag, + passive=True, ) + tag = await queue.consume(handler) + self._consumers.append((queue, tag)) + log.info("ai.consumer.started", queue=queue_name, consumer_tag=tag) async def stop(self) -> None: - if self._resume_queue is not None and self._consumer_tag is not None: - await self._resume_queue.cancel(self._consumer_tag) - log.info("ai.consumer.stopped", consumer_tag=self._consumer_tag) + for queue, tag in self._consumers: + try: + await queue.cancel(tag) + log.info("ai.consumer.stopped", consumer_tag=tag) + except Exception: + log.exception("ai.consumer.cancel_failed", consumer_tag=tag) await self._connection.close() diff --git a/ai/src/ai_server/model/envelope.py b/ai/src/ai_server/model/envelope.py index 7a5af61c..8d28f112 100644 --- a/ai/src/ai_server/model/envelope.py +++ b/ai/src/ai_server/model/envelope.py @@ -13,6 +13,8 @@ class MessageContext(BaseModel): user_id: int | None = None session_id: int | None = None + document_id: int | None = None + repository_id: int | None = None class Envelope(BaseModel, Generic[PayloadT]): diff --git a/ai/src/ai_server/model/messages/analyze.py b/ai/src/ai_server/model/messages/analyze.py index a9a5f473..aff39b25 100644 --- a/ai/src/ai_server/model/messages/analyze.py +++ b/ai/src/ai_server/model/messages/analyze.py @@ -4,7 +4,7 @@ from ai_server.model._config import camel_config -TargetType = Literal["RESUME", "REPOSITORY"] +TargetType = Literal["RESUME", "REPOSITORY", "WEB"] AnalysisStatus = Literal["ANALYZED", "FAILED"] @@ -12,7 +12,25 @@ class ResumeAnalyzeRequest(BaseModel): model_config = camel_config() resume_id: int - s3_key: str + file_path: str + analyzed_document_id: int + + +class RepositoryAnalyzeRequest(BaseModel): + model_config = camel_config() + + repository_id: int + repo_full_name: str + default_branch: str = "main" + analyzed_document_id: int + + +class WebResumeAnalyzeRequest(BaseModel): + model_config = camel_config() + + resume_id: int + url: str + analyzed_document_id: int class AnalysisCallbackPayload(BaseModel): @@ -23,7 +41,7 @@ class AnalysisCallbackPayload(BaseModel): status: AnalysisStatus summary: str | None = None tech_stack: list[str] = [] - document_s3_key: str | None = None + document_path: str | None = None embedding_chunk_count: int = 0 error_code: str | None = None error_message: str | None = None diff --git a/ai/src/ai_server/rag/__init__.py b/ai/src/ai_server/rag/__init__.py new file mode 100644 index 00000000..865cb7eb --- /dev/null +++ b/ai/src/ai_server/rag/__init__.py @@ -0,0 +1,18 @@ +from ai_server.rag.chunker import Chunk, MarkdownChunker +from ai_server.rag.embedder import ( + EmbeddingError, + EmbeddingProvider, + GeminiEmbeddingProvider, + MockEmbeddingProvider, + build_embedding_provider, +) + +__all__ = [ + "Chunk", + "MarkdownChunker", + "EmbeddingError", + "EmbeddingProvider", + "MockEmbeddingProvider", + "GeminiEmbeddingProvider", + "build_embedding_provider", +] diff --git a/ai/src/ai_server/rag/chunker.py b/ai/src/ai_server/rag/chunker.py new file mode 100644 index 00000000..8c6a6ed4 --- /dev/null +++ b/ai/src/ai_server/rag/chunker.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from langchain_text_splitters import RecursiveCharacterTextSplitter + + +@dataclass(frozen=True) +class Chunk: + index: int + text: str + + +# md를 잘라냄. size와 overlap은 설정에서 가져다 씀 +class MarkdownChunker: + def __init__(self, *, chunk_size: int, chunk_overlap: int) -> None: + if chunk_size <= 0: + raise ValueError(f"chunk_size must be > 0, got {chunk_size}") + if chunk_overlap < 0 or chunk_overlap >= chunk_size: + raise ValueError( + f"chunk_overlap must be in [0, chunk_size), got {chunk_overlap}" + ) + self._splitter = RecursiveCharacterTextSplitter( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + length_function=len, + ) + + def split(self, markdown: str) -> list[Chunk]: + if not markdown or not markdown.strip(): + return [] + parts = self._splitter.split_text(markdown) + return [Chunk(index=i, text=text) for i, text in enumerate(parts)] diff --git a/ai/src/ai_server/rag/embedder.py b/ai/src/ai_server/rag/embedder.py new file mode 100644 index 00000000..d6239758 --- /dev/null +++ b/ai/src/ai_server/rag/embedder.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import hashlib +import struct +from typing import Protocol + + +class EmbeddingError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +# 구현체는 바꿔서 사용할 수 있음 +class EmbeddingProvider(Protocol): + @property + def dim(self) -> int: ... + + @property + def model(self) -> str: ... + + async def embed(self, texts: list[str]) -> list[list[float]]: ... + + +# 우선 mock 구현체 +class MockEmbeddingProvider: + def __init__(self, *, dim: int = 1536, model: str = "mock") -> None: + if dim <= 0: + raise ValueError(f"dim must be > 0, got {dim}") + self._dim = dim + self._model = model + + @property + def dim(self) -> int: + return self._dim + + @property + def model(self) -> str: + return self._model + + async def embed(self, texts: list[str]) -> list[list[float]]: + return [self._embed_one(t) for t in texts] + + def _embed_one(self, text: str) -> list[float]: + # [-1, 1] 범위로 매핑 진행 + digest = hashlib.sha256(text.encode("utf-8")).digest() + repeats = (self._dim * 4 + len(digest) - 1) // len(digest) + blob = (digest * repeats)[: self._dim * 4] + ints = struct.unpack(f">{self._dim}I", blob) + scale = 2.0 / 0xFFFFFFFF + return [v * scale - 1.0 for v in ints] + + +# Gemini Embedding 을 사용합니다. +# 이건 충대키로 안되니 키 발급 필요함 +class GeminiEmbeddingProvider: + def __init__(self, *, api_key: str, model: str, dim: int) -> None: + if not api_key: + raise ValueError("GEMINI_API_KEY 누락 — provider=gemini 사용 불가") + if dim <= 0: + raise ValueError(f"dim must be > 0, got {dim}") + from google import genai + + self._client = genai.Client(api_key=api_key) + self._model = model + self._dim = dim + + @property + def dim(self) -> int: + return self._dim + + @property + def model(self) -> str: + return self._model + + async def embed(self, texts: list[str]) -> list[list[float]]: + if not texts: + return [] + from google.genai import types as genai_types + + try: + resp = await self._client.aio.models.embed_content( + model=self._model, + contents=texts, + config=genai_types.EmbedContentConfig( + task_type="RETRIEVAL_DOCUMENT", + output_dimensionality=self._dim, + ), + ) + except Exception as exc: + raise EmbeddingError( + code="GEMINI_FAILED", + message=f"Gemini embedding 호출 실패: {exc}", + retriable=True, + ) from exc + + return [list(e.values) for e in resp.embeddings] + + +def build_embedding_provider( + *, + provider: str, + dim: int, + model: str, + gemini_api_key: str = "", +) -> EmbeddingProvider: + if provider == "mock": + return MockEmbeddingProvider(dim=dim, model=model) + if provider == "gemini": + return GeminiEmbeddingProvider(api_key=gemini_api_key, model=model, dim=dim) + if provider == "openai": + raise NotImplementedError("openai embedding provider 미구현 — 후속 PR에서 추가") + if provider == "ollama": + raise NotImplementedError("ollama embedding provider 미구현 — 후속 PR에서 추가") + raise ValueError(f"Unsupported EMBEDDING_PROVIDER={provider!r}") diff --git a/ai/src/ai_server/storage/__init__.py b/ai/src/ai_server/storage/__init__.py new file mode 100644 index 00000000..d4b17699 --- /dev/null +++ b/ai/src/ai_server/storage/__init__.py @@ -0,0 +1,12 @@ +from ai_server.storage.base import ObjectStorage, StorageKeyError +from ai_server.storage.factory import build_storage +from ai_server.storage.local_fs import LocalFilesystemStorage +from ai_server.storage.s3 import S3Storage + +__all__ = [ + "ObjectStorage", + "StorageKeyError", + "LocalFilesystemStorage", + "S3Storage", + "build_storage", +] diff --git a/ai/src/ai_server/storage/base.py b/ai/src/ai_server/storage/base.py new file mode 100644 index 00000000..b9b95744 --- /dev/null +++ b/ai/src/ai_server/storage/base.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod + + +# 스토리지 키 전용 에러 +class StorageKeyError(ValueError): + pass + + +class ObjectStorage(ABC): + + @abstractmethod + async def get_bytes(self, key: str) -> bytes: ... + + @abstractmethod + async def put_bytes( + self, + key: str, + data: bytes, + *, + content_type: str | None = None, + ) -> None: ... + + @abstractmethod + async def put_text( + self, + key: str, + text: str, + *, + content_type: str = "text/markdown; charset=utf-8", + ) -> None: ... + + @abstractmethod + async def exists(self, key: str) -> bool: ... diff --git a/ai/src/ai_server/storage/factory.py b/ai/src/ai_server/storage/factory.py new file mode 100644 index 00000000..7587d6f2 --- /dev/null +++ b/ai/src/ai_server/storage/factory.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from ai_server.config.settings import Settings +from ai_server.storage.base import ObjectStorage +from ai_server.storage.local_fs import LocalFilesystemStorage +from ai_server.storage.s3 import S3Storage + + +# 설정에 명시된 스토리지를 생성함 +def build_storage(settings: Settings) -> ObjectStorage: + backend = settings.storage_backend + if backend == "local": + return LocalFilesystemStorage(root_dir=settings.storage_local_root) + if backend == "s3": + return S3Storage( + bucket=settings.s3_bucket_name, + endpoint_url=settings.s3_endpoint_url or None, + access_key=settings.s3_access_key or None, + secret_key=settings.s3_secret_key or None, + region=settings.s3_region, + ) + raise ValueError(f"Unsupported STORAGE_BACKEND={backend!r}") diff --git a/ai/src/ai_server/storage/local_fs.py b/ai/src/ai_server/storage/local_fs.py new file mode 100644 index 00000000..2d2fc8b1 --- /dev/null +++ b/ai/src/ai_server/storage/local_fs.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import aiofiles +import aiofiles.os + +from ai_server.storage.base import ObjectStorage, StorageKeyError + + +# local 일 때 사용되는 스토리지 시스템 +class LocalFilesystemStorage(ObjectStorage): + + def __init__(self, root_dir: str) -> None: + self._root = Path(root_dir).resolve() + + @property + def root(self) -> Path: + return self._root + + def _resolve(self, key: str) -> Path: + if not key or key.startswith("/") or "\x00" in key: + raise StorageKeyError(f"invalid storage key: {key!r}") + candidate = (self._root / key).resolve() + root_str = str(self._root) + cand_str = str(candidate) + if cand_str != root_str and not cand_str.startswith(root_str + os.sep): + raise StorageKeyError(f"key escapes storage root: {key!r}") + return candidate + + async def get_bytes(self, key: str) -> bytes: + path = self._resolve(key) + async with aiofiles.open(path, "rb") as f: + return await f.read() + + async def put_bytes( + self, + key: str, + data: bytes, + *, + content_type: str | None = None, + ) -> None: + path = self._resolve(key) + await aiofiles.os.makedirs(path.parent, exist_ok=True) + async with aiofiles.open(path, "wb") as f: + await f.write(data) + + async def put_text( + self, + key: str, + text: str, + *, + content_type: str = "text/markdown; charset=utf-8", + ) -> None: + await self.put_bytes(key, text.encode("utf-8"), content_type=content_type) + + async def exists(self, key: str) -> bool: + try: + path = self._resolve(key) + except StorageKeyError: + return False + return await aiofiles.os.path.exists(path) diff --git a/ai/src/ai_server/storage/s3.py b/ai/src/ai_server/storage/s3.py new file mode 100644 index 00000000..24ebe37b --- /dev/null +++ b/ai/src/ai_server/storage/s3.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import boto3 +from botocore.client import BaseClient +from botocore.exceptions import ClientError + +from ai_server.storage.base import ObjectStorage, StorageKeyError + + +# S3 호환 스토리지 (MinIO 포함) +class S3Storage(ObjectStorage): + + def __init__( + self, + *, + bucket: str, + endpoint_url: str | None = None, + access_key: str | None = None, + secret_key: str | None = None, + region: str = "us-east-1", + client: BaseClient | None = None, + ) -> None: + if not bucket: + raise ValueError("bucket name required") + self._bucket = bucket + if client is not None: + self._client = client + else: + self._client = boto3.client( + "s3", + endpoint_url=endpoint_url or None, + aws_access_key_id=access_key or None, + aws_secret_access_key=secret_key or None, + region_name=region, + ) + + @property + def bucket(self) -> str: + return self._bucket + + @property + def client(self) -> BaseClient: + return self._client + + @staticmethod + def _validate_key(key: str) -> None: + if not key or key.startswith("/") or "\x00" in key: + raise StorageKeyError(f"invalid storage key: {key!r}") + + async def get_bytes(self, key: str) -> bytes: + self._validate_key(key) + resp = await asyncio.to_thread( + self._client.get_object, Bucket=self._bucket, Key=key + ) + body = resp["Body"] + return await asyncio.to_thread(body.read) + + async def put_bytes( + self, + key: str, + data: bytes, + *, + content_type: str | None = None, + ) -> None: + self._validate_key(key) + kwargs: dict[str, Any] = { + "Bucket": self._bucket, + "Key": key, + "Body": data, + } + if content_type: + kwargs["ContentType"] = content_type + await asyncio.to_thread(self._client.put_object, **kwargs) + + async def put_text( + self, + key: str, + text: str, + *, + content_type: str = "text/markdown; charset=utf-8", + ) -> None: + await self.put_bytes(key, text.encode("utf-8"), content_type=content_type) + + async def exists(self, key: str) -> bool: + try: + self._validate_key(key) + except StorageKeyError: + return False + try: + await asyncio.to_thread( + self._client.head_object, Bucket=self._bucket, Key=key + ) + return True + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + + if code in ("404", "NoSuchKey", "NotFound"): + return False + raise diff --git a/ai/tests/test_chunker.py b/ai/tests/test_chunker.py new file mode 100644 index 00000000..de6328c1 --- /dev/null +++ b/ai/tests/test_chunker.py @@ -0,0 +1,48 @@ +import pytest + +from ai_server.rag.chunker import MarkdownChunker + + +def test_split_short_text_into_single_chunk() -> None: + chunker = MarkdownChunker(chunk_size=200, chunk_overlap=20) + chunks = chunker.split("# 짧은 문서\n한 문장.") + assert len(chunks) == 1 + assert chunks[0].index == 0 + assert "짧은 문서" in chunks[0].text + + +def test_split_long_text_into_multiple_chunks_with_overlap() -> None: + # 충분히 길어서 chunk_size 초과 + body = "## 개요\n" + ("문장. " * 200) + chunker = MarkdownChunker(chunk_size=200, chunk_overlap=50) + chunks = chunker.split(body) + assert len(chunks) >= 2 + assert chunks[0].index == 0 + assert chunks[-1].index == len(chunks) - 1 + for c in chunks: + assert len(c.text) <= 200 + 50 # splitter 특성상 약간의 여유 + + +def test_empty_text_returns_no_chunks() -> None: + chunker = MarkdownChunker(chunk_size=100, chunk_overlap=10) + assert chunker.split("") == [] + assert chunker.split(" \n\t ") == [] + + +def test_invalid_chunk_size_raises() -> None: + with pytest.raises(ValueError): + MarkdownChunker(chunk_size=0, chunk_overlap=0) + + +def test_overlap_must_be_less_than_chunk_size() -> None: + with pytest.raises(ValueError): + MarkdownChunker(chunk_size=100, chunk_overlap=100) + with pytest.raises(ValueError): + MarkdownChunker(chunk_size=100, chunk_overlap=-1) + + +def test_chunk_indices_are_sequential() -> None: + chunker = MarkdownChunker(chunk_size=50, chunk_overlap=10) + chunks = chunker.split("abc " * 200) + indices = [c.index for c in chunks] + assert indices == list(range(len(chunks))) diff --git a/ai/tests/test_core_client.py b/ai/tests/test_core_client.py new file mode 100644 index 00000000..9f71227e --- /dev/null +++ b/ai/tests/test_core_client.py @@ -0,0 +1,243 @@ +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from ai_server.core.client import ( + CoreEmbeddingUpsertError, + CoreTokenError, + EmbeddingChunkPayload, + HttpCoreClient, +) + + +def _make_client( + *, + status: int = 200, + json_body: dict | None = None, + text: str = "", + raise_exc: Exception | None = None, +) -> MagicMock: + client = MagicMock() + resp = MagicMock(spec=httpx.Response) + resp.status_code = status + resp.text = text + if json_body is None: + resp.json = MagicMock(side_effect=ValueError("no json")) + else: + resp.json = MagicMock(return_value=json_body) + if raise_exc is not None: + client.get = AsyncMock(side_effect=raise_exc) + else: + client.get = AsyncMock(return_value=resp) + return client + + +def _build(client: MagicMock) -> HttpCoreClient: + return HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + + +@pytest.mark.asyncio +async def test_returns_access_token_on_200_camel() -> None: + client = _make_client(json_body={"accessToken": "tok-1"}) + core = _build(client) + token = await core.fetch_github_token(user_id=42) + assert token == "tok-1" + client.get.assert_awaited_once_with("/api/internal/users/42/github-token") + + +@pytest.mark.asyncio +async def test_returns_access_token_on_snake_case_response() -> None: + client = _make_client(json_body={"access_token": "tok-s"}) + core = _build(client) + assert await core.fetch_github_token(user_id=1) == "tok-s" + + +@pytest.mark.asyncio +async def test_404_translates_to_user_not_found() -> None: + client = _make_client(status=404) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "USER_NOT_FOUND" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_401_translates_to_core_auth_failed() -> None: + client = _make_client(status=401) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_AUTH_FAILED" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_5xx_translates_to_retriable_unavailable() -> None: + client = _make_client(status=503) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_UNAVAILABLE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_other_4xx_non_retriable() -> None: + client = _make_client(status=400, text="bad") + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_BAD_REQUEST" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_httpx_error_translates_to_retriable() -> None: + client = _make_client(raise_exc=httpx.ConnectError("dns fail")) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_UNAVAILABLE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_empty_token_field_treated_as_retriable() -> None: + client = _make_client(json_body={"accessToken": ""}) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_TOKEN_EMPTY" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_invalid_json_translates_to_bad_response() -> None: + client = _make_client(status=200, json_body=None) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_BAD_RESPONSE" + + +def test_constructor_requires_base_url() -> None: + with pytest.raises(ValueError): + HttpCoreClient(base_url="", api_key="x") + + +# ----------- upsert_embeddings ----------- + + +def _make_put_client( + *, + status: int = 200, + json_body: dict | None = None, + raise_exc: Exception | None = None, +) -> MagicMock: + client = MagicMock() + resp = MagicMock(spec=httpx.Response) + resp.status_code = status + resp.text = "" + resp.json = ( + MagicMock(return_value=json_body) + if json_body is not None + else MagicMock(side_effect=ValueError("no json")) + ) + if raise_exc is not None: + client.put = AsyncMock(side_effect=raise_exc) + else: + client.put = AsyncMock(return_value=resp) + return client + + +def _payloads(n: int = 2, dim: int = 4) -> list[EmbeddingChunkPayload]: + return [ + EmbeddingChunkPayload( + chunk_index=i, chunk_text=f"chunk {i}", embedding=[0.1] * dim + ) + for i in range(n) + ] + + +@pytest.mark.asyncio +async def test_upsert_embeddings_happy_path_returns_count() -> None: + client = _make_put_client(status=200, json_body={"upserted": 2}) + core = HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + n = await core.upsert_embeddings( + document_id=88, + model="mock", + dim=4, + chunks=_payloads(2, 4), + ) + assert n == 2 + client.put.assert_awaited_once() + args, kwargs = client.put.call_args + assert args[0] == "/api/internal/documents/88/embeddings" + body = kwargs["json"] + assert body["dim"] == 4 + assert body["model"] == "mock" + assert len(body["chunks"]) == 2 + assert body["chunks"][0] == { + "chunkIndex": 0, + "chunkText": "chunk 0", + "embedding": [0.1, 0.1, 0.1, 0.1], + } + + +@pytest.mark.asyncio +async def test_upsert_embeddings_falls_back_to_len_when_no_upserted_field() -> None: + client = _make_put_client(status=200, json_body={}) + core = HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + n = await core.upsert_embeddings( + document_id=1, model="m", dim=4, chunks=_payloads(3, 4) + ) + assert n == 3 # 응답에 카운트 없으면 보낸 만큼 적용 가정 + + +@pytest.mark.asyncio +async def test_upsert_embeddings_404_translates_to_document_not_found() -> None: + client = _make_put_client(status=404) + core = HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + with pytest.raises(CoreEmbeddingUpsertError) as exc_info: + await core.upsert_embeddings( + document_id=99, model="m", dim=4, chunks=_payloads() + ) + assert exc_info.value.code == "DOCUMENT_NOT_FOUND" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_upsert_embeddings_5xx_retriable() -> None: + client = _make_put_client(status=503) + core = HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + with pytest.raises(CoreEmbeddingUpsertError) as exc_info: + await core.upsert_embeddings( + document_id=1, model="m", dim=4, chunks=_payloads() + ) + assert exc_info.value.code == "CORE_UNAVAILABLE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_upsert_embeddings_401_non_retriable_auth() -> None: + client = _make_put_client(status=401) + core = HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + with pytest.raises(CoreEmbeddingUpsertError) as exc_info: + await core.upsert_embeddings( + document_id=1, model="m", dim=4, chunks=_payloads() + ) + assert exc_info.value.code == "CORE_AUTH_FAILED" + + +@pytest.mark.asyncio +async def test_upsert_embeddings_httpx_error_retriable() -> None: + client = _make_put_client(raise_exc=httpx.ConnectError("dns fail")) + core = HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + with pytest.raises(CoreEmbeddingUpsertError) as exc_info: + await core.upsert_embeddings( + document_id=1, model="m", dim=4, chunks=_payloads() + ) + assert exc_info.value.code == "CORE_UNAVAILABLE" + assert exc_info.value.retriable is True diff --git a/ai/tests/test_document_chain.py b/ai/tests/test_document_chain.py new file mode 100644 index 00000000..32317641 --- /dev/null +++ b/ai/tests/test_document_chain.py @@ -0,0 +1,35 @@ +import pytest +from langchain_core.runnables import RunnableLambda + +from ai_server.chain.document_analysis_chain import ( + DocumentAnalysisResult, + LlmDocumentAnalyzer, +) + + +@pytest.mark.asyncio +async def test_llm_document_analyzer_invokes_chain_with_inputs() -> None: + fake_result = DocumentAnalysisResult( + summary="요약", tech_stack=["Python"], markdown="## 개요\n..." + ) + captured: dict = {} + + async def fake_invoke(inputs: dict) -> DocumentAnalysisResult: + captured.update(inputs) + return fake_result + + analyzer = LlmDocumentAnalyzer(RunnableLambda(fake_invoke)) + out = await analyzer.analyze(text="raw text", source_type="PDF") + + assert captured == {"text": "raw text", "source_type": "PDF"} + assert out is fake_result + + +@pytest.mark.asyncio +async def test_llm_document_analyzer_rejects_wrong_type() -> None: + async def fake_invoke(_: dict) -> dict: + return {"summary": "not a model"} + + analyzer = LlmDocumentAnalyzer(RunnableLambda(fake_invoke)) + with pytest.raises(TypeError): + await analyzer.analyze(text="x", source_type="PDF") diff --git a/ai/tests/test_embedder.py b/ai/tests/test_embedder.py new file mode 100644 index 00000000..bdb67160 --- /dev/null +++ b/ai/tests/test_embedder.py @@ -0,0 +1,156 @@ +import pytest + +from ai_server.rag.embedder import MockEmbeddingProvider, build_embedding_provider + + +@pytest.mark.asyncio +async def test_mock_embed_returns_fixed_dim_vectors() -> None: + emb = MockEmbeddingProvider(dim=8) + vectors = await emb.embed(["a", "b", "ccc"]) + assert len(vectors) == 3 + for v in vectors: + assert len(v) == 8 + for x in v: + assert -1.0 <= x <= 1.0 + + +@pytest.mark.asyncio +async def test_mock_embed_is_deterministic() -> None: + emb = MockEmbeddingProvider(dim=16) + a = await emb.embed(["같은 입력"]) + b = await emb.embed(["같은 입력"]) + assert a == b + + +@pytest.mark.asyncio +async def test_mock_embed_different_input_yields_different_vector() -> None: + emb = MockEmbeddingProvider(dim=16) + a = await emb.embed(["A"]) + b = await emb.embed(["B"]) + assert a[0] != b[0] + + +def test_mock_provider_exposes_dim_and_model() -> None: + emb = MockEmbeddingProvider(dim=1536, model="mock-test") + assert emb.dim == 1536 + assert emb.model == "mock-test" + + +def test_constructor_rejects_non_positive_dim() -> None: + with pytest.raises(ValueError): + MockEmbeddingProvider(dim=0) + with pytest.raises(ValueError): + MockEmbeddingProvider(dim=-1) + + +def test_factory_returns_mock_for_mock_provider() -> None: + emb = build_embedding_provider(provider="mock", dim=64, model="any") + assert isinstance(emb, MockEmbeddingProvider) + assert emb.dim == 64 + + +def test_factory_raises_for_unimplemented_providers() -> None: + with pytest.raises(NotImplementedError): + build_embedding_provider(provider="openai", dim=1536, model="x") + with pytest.raises(NotImplementedError): + build_embedding_provider(provider="ollama", dim=768, model="x") + + +def test_factory_rejects_unknown_provider() -> None: + with pytest.raises(ValueError): + build_embedding_provider(provider="unknown", dim=64, model="x") + + +# --- Gemini provider --- + + +def test_factory_rejects_gemini_without_api_key() -> None: + with pytest.raises(ValueError): + build_embedding_provider( + provider="gemini", + dim=1536, + model="gemini-embedding-001", + gemini_api_key="", + ) + + +def test_factory_builds_gemini_provider_with_api_key() -> None: + from ai_server.rag.embedder import GeminiEmbeddingProvider + + emb = build_embedding_provider( + provider="gemini", + dim=1536, + model="gemini-embedding-001", + gemini_api_key="fake-key", + ) + assert isinstance(emb, GeminiEmbeddingProvider) + assert emb.dim == 1536 + assert emb.model == "gemini-embedding-001" + + +@pytest.mark.asyncio +async def test_gemini_embed_returns_vector_list_from_sdk_response() -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock, MagicMock, patch + + from ai_server.rag.embedder import GeminiEmbeddingProvider + + # SDK 응답 형태 모사: resp.embeddings[i].values + fake_resp = SimpleNamespace( + embeddings=[ + SimpleNamespace(values=[0.1, 0.2, 0.3]), + SimpleNamespace(values=[0.4, 0.5, 0.6]), + ] + ) + fake_aio = MagicMock() + fake_aio.models.embed_content = AsyncMock(return_value=fake_resp) + fake_client = MagicMock() + fake_client.aio = fake_aio + + with patch("google.genai.Client", return_value=fake_client): + emb = GeminiEmbeddingProvider( + api_key="fake", model="gemini-embedding-001", dim=3 + ) + out = await emb.embed(["chunk A", "chunk B"]) + assert out == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]] + fake_aio.models.embed_content.assert_awaited_once() + kwargs = fake_aio.models.embed_content.await_args.kwargs + assert kwargs["model"] == "gemini-embedding-001" + assert kwargs["contents"] == ["chunk A", "chunk B"] + + +@pytest.mark.asyncio +async def test_gemini_embed_empty_input_returns_empty_without_sdk_call() -> None: + from unittest.mock import MagicMock, patch + + from ai_server.rag.embedder import GeminiEmbeddingProvider + + fake_client = MagicMock() + with patch("google.genai.Client", return_value=fake_client): + emb = GeminiEmbeddingProvider( + api_key="fake", model="gemini-embedding-001", dim=8 + ) + assert await emb.embed([]) == [] + fake_client.aio.models.embed_content.assert_not_called() + + +@pytest.mark.asyncio +async def test_gemini_embed_wraps_sdk_exception_as_retriable() -> None: + from unittest.mock import AsyncMock, MagicMock, patch + + from ai_server.rag.embedder import EmbeddingError, GeminiEmbeddingProvider + + fake_aio = MagicMock() + fake_aio.models.embed_content = AsyncMock(side_effect=RuntimeError("rate limit")) + fake_client = MagicMock() + fake_client.aio = fake_aio + + with patch("google.genai.Client", return_value=fake_client): + emb = GeminiEmbeddingProvider( + api_key="fake", model="gemini-embedding-001", dim=4 + ) + + with pytest.raises(EmbeddingError) as exc_info: + await emb.embed(["x"]) + assert exc_info.value.code == "GEMINI_FAILED" + assert exc_info.value.retriable is True diff --git a/ai/tests/test_github_repo_extractor.py b/ai/tests/test_github_repo_extractor.py new file mode 100644 index 00000000..9fa165ab --- /dev/null +++ b/ai/tests/test_github_repo_extractor.py @@ -0,0 +1,191 @@ +import base64 +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from ai_server.analyzer.sources.github_repo import ( + GitHubRepoSourceExtractor, + RepositoryFetchError, + _select_files, +) + + +def _make_client(routes: dict[str, dict | int]) -> tuple[MagicMock, AsyncMock]: + """`routes`: path → response dict (JSON) or int (HTTP status).""" + client = MagicMock() + + async def _get(path: str) -> MagicMock: + body = routes.get(path) + resp = MagicMock(spec=httpx.Response) + if isinstance(body, int): + resp.status_code = body + resp.text = "" + resp.raise_for_status = MagicMock() + return resp + resp.status_code = 200 + resp.json = MagicMock(return_value=body) + resp.raise_for_status = MagicMock() + return resp + + client.get = AsyncMock(side_effect=_get) + return client, client.get + + +def _b64(text: str) -> str: + return base64.b64encode(text.encode("utf-8")).decode("ascii") + + +@pytest.mark.asyncio +async def test_extract_assembles_readme_tree_and_snippets() -> None: + routes = { + "/repos/user/repo": { + "default_branch": "dev", + "description": "demo project", + }, + "/repos/user/repo/readme": { + "encoding": "base64", + "content": _b64("# Hello\nReadme body"), + }, + "/repos/user/repo/git/trees/dev?recursive=1": { + "tree": [ + {"path": "pyproject.toml", "type": "blob"}, + {"path": "src/app.py", "type": "blob"}, + {"path": "tests/test_app.py", "type": "blob"}, + {"path": "docs", "type": "tree"}, + ], + "truncated": False, + }, + "/repos/user/repo/contents/pyproject.toml?ref=dev": { + "encoding": "base64", + "content": _b64('[project]\nname = "app"'), + }, + "/repos/user/repo/contents/src/app.py?ref=dev": { + "encoding": "base64", + "content": _b64("def main():\n return 1"), + }, + "/repos/user/repo/contents/tests/test_app.py?ref=dev": { + "encoding": "base64", + "content": _b64("def test(): pass"), + }, + } + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + result = await extractor.extract("user/repo") + + assert result.source_type == "REPOSITORY" + assert "user/repo" in result.text + assert "Readme body" in result.text + assert "pyproject.toml" in result.text + assert "def main()" in result.text + assert result.metadata["default_branch"] == "dev" + assert result.metadata["tree_size"] == 3 + assert "pyproject.toml" in result.metadata["sampled_files"] + + +@pytest.mark.asyncio +async def test_extract_handles_missing_readme_gracefully() -> None: + routes = { + "/repos/user/repo": {"default_branch": "main"}, + "/repos/user/repo/readme": 404, + "/repos/user/repo/git/trees/main?recursive=1": {"tree": [], "truncated": False}, + } + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + result = await extractor.extract("user/repo") + assert "user/repo" in result.text + assert result.metadata["tree_size"] == 0 + + +@pytest.mark.asyncio +async def test_extract_raises_on_repo_not_found() -> None: + routes = {"/repos/user/repo": 404} + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + with pytest.raises(RepositoryFetchError) as exc_info: + await extractor.extract("user/repo") + assert exc_info.value.code == "REPO_NOT_FOUND" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_extract_raises_on_rate_limit_as_retriable() -> None: + routes = {"/repos/user/repo": 403} + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + with pytest.raises(RepositoryFetchError) as exc_info: + await extractor.extract("user/repo") + assert exc_info.value.code == "REPO_AUTH_OR_RATE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_extract_rejects_malformed_locator() -> None: + client, _ = _make_client({}) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + with pytest.raises(RepositoryFetchError) as exc_info: + await extractor.extract("not-a-full-name") + assert exc_info.value.code == "INVALID_REPO_LOCATOR" + + +def test_select_files_prioritizes_manifests_then_one_per_dir() -> None: + paths = [ + "package.json", + "src/index.ts", + "src/util/format.ts", + "tests/index.test.ts", + "README.md", + ] + picked = _select_files(paths, cap=4) + # manifest 우선 + assert "package.json" in picked + # 디렉토리당 1개씩 sampling (src/, tests/, root) + assert any(p.startswith("src/") for p in picked) + assert any(p.startswith("tests/") for p in picked) + assert len(picked) <= 4 + + +def test_select_files_respects_cap() -> None: + paths = [f"src/a{i}.py" for i in range(20)] + picked = _select_files(paths, cap=3) + assert len(picked) <= 3 + + +def test_build_client_includes_authorization_with_user_token() -> None: + extractor = GitHubRepoSourceExtractor(api_base_url="https://api.github.com") + client = extractor._build_client("user-token-xyz") + try: + assert client.headers.get("Authorization") == "Bearer user-token-xyz" + finally: + # 비동기 cleanup 없이 닫기 OK — 실제 연결은 안 함 + pass + + +def test_build_client_omits_authorization_when_token_empty() -> None: + extractor = GitHubRepoSourceExtractor(api_base_url="https://api.github.com") + client = extractor._build_client("") + assert "Authorization" not in client.headers + + +def test_fallback_token_used_when_no_per_call_token() -> None: + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", + fallback_token="fallback-tok", + ) + # access_token=None → fallback 사용 + client = extractor._build_client(extractor._cfg.fallback_token) + assert client.headers.get("Authorization") == "Bearer fallback-tok" diff --git a/ai/tests/test_messages_analyze.py b/ai/tests/test_messages_analyze.py index 49a6d3ab..1f200fbe 100644 --- a/ai/tests/test_messages_analyze.py +++ b/ai/tests/test_messages_analyze.py @@ -3,16 +3,23 @@ from ai_server.model.messages.analyze import ( AnalysisCallbackPayload, + RepositoryAnalyzeRequest, ResumeAnalyzeRequest, + WebResumeAnalyzeRequest, ) def test_resume_request_parses() -> None: req = ResumeAnalyzeRequest.model_validate( - {"resumeId": 42, "s3Key": "resumes/raw/123/abc.pdf"} + { + "resumeId": 42, + "filePath": "resumes/raw/123/abc.pdf", + "analyzedDocumentId": 77, + } ) assert req.resume_id == 42 - assert req.s3_key == "resumes/raw/123/abc.pdf" + assert req.file_path == "resumes/raw/123/abc.pdf" + assert req.analyzed_document_id == 77 def test_resume_request_requires_fields() -> None: @@ -20,25 +27,51 @@ def test_resume_request_requires_fields() -> None: ResumeAnalyzeRequest.model_validate({"resumeId": 42}) +def test_resume_request_serializes_camel_case() -> None: + req = ResumeAnalyzeRequest(resume_id=7, file_path="r/7.pdf", analyzed_document_id=1) + dumped = req.model_dump(by_alias=True) + assert dumped == { + "resumeId": 7, + "filePath": "r/7.pdf", + "analyzedDocumentId": 1, + } + + def test_callback_success_serializes_camel_case() -> None: cb = AnalysisCallbackPayload( target_type="RESUME", target_id=42, status="ANALYZED", - summary="[ECHO] not yet analyzed", - tech_stack=[], - document_s3_key="echo/resume/42/echo.md", + summary="요약", + tech_stack=["Python", "FastAPI"], + document_path="analyzed/resume/42/summary.md", embedding_chunk_count=0, ) dumped = cb.model_dump(by_alias=True) assert dumped["targetType"] == "RESUME" assert dumped["targetId"] == 42 assert dumped["status"] == "ANALYZED" - assert dumped["techStack"] == [] - assert dumped["documentS3Key"] == "echo/resume/42/echo.md" + assert dumped["techStack"] == ["Python", "FastAPI"] + assert dumped["documentPath"] == "analyzed/resume/42/summary.md" assert dumped["embeddingChunkCount"] == 0 +def test_callback_failure_carries_error_fields() -> None: + cb = AnalysisCallbackPayload( + target_type="RESUME", + target_id=42, + status="FAILED", + error_code="EMPTY_PDF_TEXT", + error_message="no text", + retriable=False, + ) + dumped = cb.model_dump(by_alias=True) + assert dumped["status"] == "FAILED" + assert dumped["errorCode"] == "EMPTY_PDF_TEXT" + assert dumped["errorMessage"] == "no text" + assert dumped["retriable"] is False + + def test_callback_target_type_must_be_known() -> None: with pytest.raises(ValidationError): AnalysisCallbackPayload( @@ -46,3 +79,59 @@ def test_callback_target_type_must_be_known() -> None: target_id=1, status="ANALYZED", ) + + +def test_repository_request_parses_with_default_branch() -> None: + req = RepositoryAnalyzeRequest.model_validate( + { + "repositoryId": 5, + "repoFullName": "user/repo", + "analyzedDocumentId": 88, + } + ) + assert req.repository_id == 5 + assert req.repo_full_name == "user/repo" + assert req.default_branch == "main" + assert req.analyzed_document_id == 88 + + +def test_repository_request_serializes_camel_case() -> None: + req = RepositoryAnalyzeRequest( + repository_id=5, + repo_full_name="user/repo", + default_branch="dev", + analyzed_document_id=1, + ) + dumped = req.model_dump(by_alias=True) + assert dumped == { + "repositoryId": 5, + "repoFullName": "user/repo", + "defaultBranch": "dev", + "analyzedDocumentId": 1, + } + + +def test_web_resume_request_parses() -> None: + req = WebResumeAnalyzeRequest.model_validate( + { + "resumeId": 9, + "url": "https://example.com/me", + "analyzedDocumentId": 12, + } + ) + assert req.resume_id == 9 + assert req.url == "https://example.com/me" + assert req.analyzed_document_id == 12 + + +def test_callback_accepts_web_target_type() -> None: + cb = AnalysisCallbackPayload( + target_type="WEB", + target_id=9, + status="ANALYZED", + summary="ok", + document_path="analyzed/web-resume/9/summary.md", + ) + dumped = cb.model_dump(by_alias=True) + assert dumped["targetType"] == "WEB" + assert dumped["documentPath"] == "analyzed/web-resume/9/summary.md" diff --git a/ai/tests/test_pdf_extractor.py b/ai/tests/test_pdf_extractor.py new file mode 100644 index 00000000..1d4bb988 --- /dev/null +++ b/ai/tests/test_pdf_extractor.py @@ -0,0 +1,66 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from ai_server.analyzer.sources.pdf import PdfSourceExtractor + + +def _fake_pdf_reader(page_texts: list[str]) -> MagicMock: + reader = MagicMock() + pages = [] + for t in page_texts: + page = MagicMock() + page.extract_text.return_value = t + pages.append(page) + reader.pages = pages + return reader + + +@pytest.mark.asyncio +async def test_extract_reads_from_storage_and_joins_pages() -> None: + storage = AsyncMock() + storage.get_bytes.return_value = b"PDF-BYTES" + + fake_reader = _fake_pdf_reader(["page one", "page two"]) + with patch( + "ai_server.analyzer.sources.pdf.PdfReader", return_value=fake_reader + ) as reader_cls: + extractor = PdfSourceExtractor(storage=storage) + result = await extractor.extract("resumes/raw/1/x.pdf") + + storage.get_bytes.assert_awaited_once_with("resumes/raw/1/x.pdf") + reader_cls.assert_called_once() + assert "page one" in result.text + assert "page two" in result.text + assert result.source_type == "PDF" + assert result.metadata["bytes"] == len(b"PDF-BYTES") + assert result.metadata["locator"] == "resumes/raw/1/x.pdf" + + +@pytest.mark.asyncio +async def test_extract_skips_none_page_text() -> None: + storage = AsyncMock() + storage.get_bytes.return_value = b"PDF-BYTES" + + fake_reader = _fake_pdf_reader(["", "real content"]) + fake_reader.pages[0].extract_text.return_value = None # pypdf may return None + + with patch("ai_server.analyzer.sources.pdf.PdfReader", return_value=fake_reader): + extractor = PdfSourceExtractor(storage=storage) + result = await extractor.extract("a.pdf") + + assert result.text == "real content" + + +@pytest.mark.asyncio +async def test_extract_returns_empty_on_empty_pdf() -> None: + storage = AsyncMock() + storage.get_bytes.return_value = b"PDF-BYTES" + + fake_reader = _fake_pdf_reader([]) + with patch("ai_server.analyzer.sources.pdf.PdfReader", return_value=fake_reader): + extractor = PdfSourceExtractor(storage=storage) + result = await extractor.extract("empty.pdf") + + assert result.text == "" + assert result.source_type == "PDF" diff --git a/ai/tests/test_repository_analyzer.py b/ai/tests/test_repository_analyzer.py new file mode 100644 index 00000000..d1a7633e --- /dev/null +++ b/ai/tests/test_repository_analyzer.py @@ -0,0 +1,151 @@ +from unittest.mock import AsyncMock + +import pytest + +from ai_server.analyzer.repository_analyzer import ( + RepositoryAnalyzeError, + RepositoryAnalyzer, +) +from ai_server.analyzer.sources.base import ExtractedSource +from ai_server.analyzer.sources.github_repo import RepositoryFetchError +from ai_server.chain.document_analysis_chain import DocumentAnalysisResult +from ai_server.core.client import CoreTokenError +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import MockEmbeddingProvider + + +def _make_analyzer( + *, + extract_result: ExtractedSource | Exception | None = None, + analysis: DocumentAnalysisResult | None = None, + token_result: str | Exception = "user-token", + upsert_result: int = 2, +) -> tuple[RepositoryAnalyzer, AsyncMock, AsyncMock, AsyncMock, AsyncMock]: + extractor = AsyncMock() + if isinstance(extract_result, Exception): + extractor.extract = AsyncMock(side_effect=extract_result) + else: + extractor.extract = AsyncMock(return_value=extract_result) + + chain = AsyncMock() + if analysis is not None: + chain.analyze = AsyncMock(return_value=analysis) + + storage = AsyncMock() + + core_client = AsyncMock() + if isinstance(token_result, Exception): + core_client.fetch_github_token = AsyncMock(side_effect=token_result) + else: + core_client.fetch_github_token = AsyncMock(return_value=token_result) + core_client.upsert_embeddings = AsyncMock(return_value=upsert_result) + + analyzer = RepositoryAnalyzer( + extractor=extractor, + core_client=core_client, + chain=chain, + storage=storage, + chunker=MarkdownChunker(chunk_size=200, chunk_overlap=50), + embedder=MockEmbeddingProvider(dim=16), + analyzed_key_template="analyzed/repository/{repository_id}/summary.md", + ) + return analyzer, extractor, chain, storage, core_client + + +@pytest.mark.asyncio +async def test_happy_path_fetches_token_extracts_analyzes_and_upserts() -> None: + analyzer, extractor, chain, storage, core_client = _make_analyzer( + extract_result=ExtractedSource( + text="README + tree" * 30, source_type="REPOSITORY" + ), + analysis=DocumentAnalysisResult( + summary="요약", tech_stack=["Go"], markdown="## 개요\n" + "x" * 300 + ), + token_result="user-abc", + upsert_result=2, + ) + result = await analyzer.analyze( + repository_id=7, + repo_full_name="user/repo", + default_branch="main", + user_id=42, + analyzed_document_id=88, + ) + + core_client.fetch_github_token.assert_awaited_once_with(42) + extractor.extract.assert_awaited_once_with("user/repo", access_token="user-abc") + chain.analyze.assert_awaited_once() + storage.put_text.assert_awaited_once() + core_client.upsert_embeddings.assert_awaited_once() + kwargs = core_client.upsert_embeddings.await_args.kwargs + assert kwargs["document_id"] == 88 + assert kwargs["dim"] == 16 + assert result.embedding_chunk_count == 2 + assert result.document_path == "analyzed/repository/7/summary.md" + + +@pytest.mark.asyncio +async def test_missing_user_id_raises_before_token_fetch() -> None: + analyzer, _, _, _, core_client = _make_analyzer() + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze( + repository_id=1, + repo_full_name="user/repo", + user_id=None, + analyzed_document_id=1, + ) + assert exc_info.value.code == "MISSING_USER_ID" + core_client.fetch_github_token.assert_not_called() + + +@pytest.mark.asyncio +async def test_token_fetch_error_propagates() -> None: + analyzer, _, _, _, _ = _make_analyzer( + token_result=CoreTokenError( + code="USER_NOT_FOUND", message="404", retriable=False + ), + ) + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze( + repository_id=1, + repo_full_name="user/repo", + user_id=42, + analyzed_document_id=1, + ) + assert exc_info.value.code == "USER_NOT_FOUND" + + +@pytest.mark.asyncio +async def test_fetch_error_translates() -> None: + analyzer, _, _, storage, core_client = _make_analyzer( + extract_result=RepositoryFetchError( + code="REPO_NOT_FOUND", message="not found", retriable=False + ), + ) + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze( + repository_id=1, + repo_full_name="user/repo", + user_id=42, + analyzed_document_id=1, + ) + assert exc_info.value.code == "REPO_NOT_FOUND" + storage.put_text.assert_not_called() + core_client.upsert_embeddings.assert_not_called() + + +@pytest.mark.asyncio +async def test_empty_text_raises_empty_repo_content() -> None: + analyzer, _, _, storage, core_client = _make_analyzer( + extract_result=ExtractedSource(text=" ", source_type="REPOSITORY"), + ) + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze( + repository_id=1, + repo_full_name="user/repo", + user_id=42, + analyzed_document_id=1, + ) + assert exc_info.value.code == "EMPTY_REPO_CONTENT" + storage.put_text.assert_not_called() + core_client.upsert_embeddings.assert_not_called() diff --git a/ai/tests/test_repository_consumer.py b/ai/tests/test_repository_consumer.py new file mode 100644 index 00000000..909085b8 --- /dev/null +++ b/ai/tests/test_repository_consumer.py @@ -0,0 +1,152 @@ +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ai_server.analyzer.repository_analyzer import ( + RepositoryAnalysisResult, + RepositoryAnalyzeError, +) +from ai_server.messaging.consumers.repository_consumer import RepositoryConsumer +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + RepositoryAnalyzeRequest, +) + + +def _request_envelope(message_id: str = "req-1", repository_id: int = 7) -> bytes: + env = Envelope[RepositoryAnalyzeRequest].model_validate( + { + "messageId": message_id, + "messageType": "analyze.repository", + "version": "v1", + "traceId": "trace-r", + "publishedAt": datetime(2026, 5, 14, tzinfo=timezone.utc), + "publisher": "core-server", + "payload": { + "repositoryId": repository_id, + "repoFullName": "user/repo", + "defaultBranch": "main", + "analyzedDocumentId": 88, + }, + "context": {"userId": 1}, + } + ) + return env.model_dump_json(by_alias=True).encode("utf-8") + + +def _incoming_message(body: bytes) -> MagicMock: + msg = MagicMock() + msg.body = body + msg.delivery_tag = 1 + process_ctx = AsyncMock() + process_ctx.__aenter__.return_value = None + process_ctx.__aexit__.return_value = False + msg.process = MagicMock(return_value=process_ctx) + return msg + + +def _make_consumer( + analyzer: AsyncMock, +) -> tuple[RepositoryConsumer, AsyncMock]: + publisher = AsyncMock() + consumer = RepositoryConsumer( + analyzer=analyzer, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=16), + callback_routing_key="callback.analysis", + ) + return consumer, publisher + + +def _captured_payload(publisher: AsyncMock) -> AnalysisCallbackPayload: + publisher.publish.assert_awaited_once() + kwargs = publisher.publish.await_args.kwargs + assert kwargs["message_type"] == "callback.analysis" + payload = kwargs["payload"] + assert isinstance(payload, AnalysisCallbackPayload) + return payload + + +@pytest.mark.asyncio +async def test_happy_path_publishes_analyzed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=RepositoryAnalysisResult( + summary="요약", + tech_stack=["Go"], + document_path="analyzed/repository/7/summary.md", + embedding_chunk_count=3, + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + analyzer.analyze.assert_awaited_once_with( + repository_id=7, + repo_full_name="user/repo", + default_branch="main", + user_id=1, + analyzed_document_id=88, + ) + payload = _captured_payload(publisher) + assert payload.status == "ANALYZED" + assert payload.target_type == "REPOSITORY" + assert payload.target_id == 7 + assert payload.summary == "요약" + assert payload.tech_stack == ["Go"] + assert payload.document_path == "analyzed/repository/7/summary.md" + assert payload.embedding_chunk_count == 3 + + +@pytest.mark.asyncio +async def test_domain_error_publishes_failed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + side_effect=RepositoryAnalyzeError( + code="REPO_NOT_FOUND", message="404", retriable=False + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "REPO_NOT_FOUND" + assert payload.retriable is False + assert payload.target_type == "REPOSITORY" + + +@pytest.mark.asyncio +async def test_unexpected_error_publishes_retriable_failure() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock(side_effect=RuntimeError("boom")) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "UNEXPECTED" + assert payload.retriable is True + + +@pytest.mark.asyncio +async def test_idempotent_duplicate_skipped() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=RepositoryAnalysisResult( + summary="x", + tech_stack=[], + document_path="analyzed/repository/7/summary.md", + embedding_chunk_count=0, + ) + ) + consumer, publisher = _make_consumer(analyzer) + body = _request_envelope(message_id="dup-r") + await consumer.handle(_incoming_message(body)) + await consumer.handle(_incoming_message(body)) + + assert analyzer.analyze.await_count == 1 + assert publisher.publish.await_count == 1 diff --git a/ai/tests/test_resume_analyzer.py b/ai/tests/test_resume_analyzer.py new file mode 100644 index 00000000..e4ad0e71 --- /dev/null +++ b/ai/tests/test_resume_analyzer.py @@ -0,0 +1,108 @@ +from unittest.mock import AsyncMock + +import pytest + +from ai_server.analyzer.resume_analyzer import ResumeAnalyzeError, ResumeAnalyzer +from ai_server.analyzer.sources.base import ExtractedSource +from ai_server.chain.document_analysis_chain import DocumentAnalysisResult +from ai_server.core.client import CoreEmbeddingUpsertError +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import MockEmbeddingProvider + + +def _make_analyzer( + *, + extracted: ExtractedSource, + analysis: DocumentAnalysisResult | None = None, + upsert_result: int | Exception = 3, +) -> tuple[ResumeAnalyzer, AsyncMock, AsyncMock, AsyncMock, AsyncMock]: + extractor = AsyncMock() + extractor.extract = AsyncMock(return_value=extracted) + chain = AsyncMock() + if analysis is not None: + chain.analyze = AsyncMock(return_value=analysis) + storage = AsyncMock() + + core_client = AsyncMock() + if isinstance(upsert_result, Exception): + core_client.upsert_embeddings = AsyncMock(side_effect=upsert_result) + else: + core_client.upsert_embeddings = AsyncMock(return_value=upsert_result) + + analyzer = ResumeAnalyzer( + extractor=extractor, + chain=chain, + storage=storage, + chunker=MarkdownChunker(chunk_size=200, chunk_overlap=50), + embedder=MockEmbeddingProvider(dim=16), + core_client=core_client, + analyzed_key_template="analyzed/resume/{resume_id}/summary.md", + ) + return analyzer, extractor, chain, storage, core_client + + +@pytest.mark.asyncio +async def test_happy_path_extracts_analyzes_saves_and_upserts_embeddings() -> None: + analyzer, extractor, chain, storage, core_client = _make_analyzer( + extracted=ExtractedSource(text="hello", source_type="PDF"), + analysis=DocumentAnalysisResult( + summary="요약", tech_stack=["Go"], markdown="## 개요\n" + "x" * 500 + ), + upsert_result=3, + ) + result = await analyzer.analyze( + resume_id=42, file_path="r/raw/1/a.pdf", analyzed_document_id=77 + ) + + extractor.extract.assert_awaited_once_with("r/raw/1/a.pdf") + chain.analyze.assert_awaited_once() + storage.put_text.assert_awaited_once() + + core_client.upsert_embeddings.assert_awaited_once() + kwargs = core_client.upsert_embeddings.await_args.kwargs + assert kwargs["document_id"] == 77 + assert kwargs["dim"] == 16 + assert kwargs["model"] == "mock" + assert len(kwargs["chunks"]) > 0 # 마크다운이 chunk_size 넘으니 여러 chunk + + assert result.summary == "요약" + assert result.tech_stack == ["Go"] + assert result.document_path == "analyzed/resume/42/summary.md" + assert result.embedding_chunk_count == 3 + + +@pytest.mark.asyncio +async def test_empty_text_raises_before_llm_or_embedding() -> None: + analyzer, _, chain, storage, core_client = _make_analyzer( + extracted=ExtractedSource(text=" \n\t ", source_type="PDF"), + ) + with pytest.raises(ResumeAnalyzeError) as exc_info: + await analyzer.analyze( + resume_id=1, file_path="empty.pdf", analyzed_document_id=9 + ) + + assert exc_info.value.code == "EMPTY_PDF_TEXT" + assert exc_info.value.retriable is False + chain.analyze.assert_not_called() + storage.put_text.assert_not_called() + core_client.upsert_embeddings.assert_not_called() + + +@pytest.mark.asyncio +async def test_embedding_upsert_failure_translates_to_domain_error() -> None: + analyzer, _, _, storage, core_client = _make_analyzer( + extracted=ExtractedSource(text="hello", source_type="PDF"), + analysis=DocumentAnalysisResult( + summary="x", tech_stack=[], markdown="md content" + ), + upsert_result=CoreEmbeddingUpsertError( + code="DOCUMENT_NOT_FOUND", message="404", retriable=False + ), + ) + with pytest.raises(ResumeAnalyzeError) as exc_info: + await analyzer.analyze(resume_id=1, file_path="x.pdf", analyzed_document_id=9) + assert exc_info.value.code == "DOCUMENT_NOT_FOUND" + assert exc_info.value.retriable is False + # 마크다운 자체는 저장됐을 수 있음 — embedding만 실패 + storage.put_text.assert_awaited_once() + core_client.upsert_embeddings.assert_awaited_once() diff --git a/ai/tests/test_resume_consumer.py b/ai/tests/test_resume_consumer.py index ccf410da..1f00f471 100644 --- a/ai/tests/test_resume_consumer.py +++ b/ai/tests/test_resume_consumer.py @@ -1,56 +1,173 @@ from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock -from ai_server.messaging.consumers.resume_consumer import ( - build_echo_callback, +import pytest + +from ai_server.analyzer.resume_analyzer import ( + ResumeAnalyzeError, + ResumeAnalysisResult, +) +from ai_server.messaging.consumers.resume_consumer import ResumeConsumer +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + ResumeAnalyzeRequest, ) -from ai_server.model.envelope import Envelope, MessageContext -from ai_server.model.messages.analyze import ResumeAnalyzeRequest -def _request_envelope() -> Envelope[ResumeAnalyzeRequest]: - return Envelope[ResumeAnalyzeRequest].model_validate( +def _request_envelope(message_id: str = "req-1", resume_id: int = 42) -> bytes: + env = Envelope[ResumeAnalyzeRequest].model_validate( { - "messageId": "req-1", + "messageId": message_id, "messageType": "analyze.resume", "version": "v1", "traceId": "trace-x", "publishedAt": datetime(2026, 5, 8, tzinfo=timezone.utc), "publisher": "core-server", - "payload": {"resumeId": 42, "s3Key": "resumes/raw/123/abc.pdf"}, + "payload": { + "resumeId": resume_id, + "filePath": "resumes/raw/123/abc.pdf", + "analyzedDocumentId": 77, + }, "context": {"userId": 123}, } ) + return env.model_dump_json(by_alias=True).encode("utf-8") + + +def _incoming_message(body: bytes) -> MagicMock: + msg = MagicMock() + msg.body = body + msg.delivery_tag = 1 + # `async with message.process(...)` 형태 호출을 위해 async context manager 반환. + process_ctx = AsyncMock() + process_ctx.__aenter__.return_value = None + process_ctx.__aexit__.return_value = False + msg.process = MagicMock(return_value=process_ctx) + return msg + + +def _make_consumer( + analyzer: AsyncMock, + *, + idempotency: LruIdempotencyStore | None = None, +) -> tuple[ResumeConsumer, AsyncMock]: + publisher = AsyncMock() + consumer = ResumeConsumer( + analyzer=analyzer, + publisher=publisher, + idempotency=idempotency or LruIdempotencyStore(max_size=16), + callback_routing_key="callback.analysis", + ) + return consumer, publisher -def test_echo_callback_target_type_resume() -> None: - cb = build_echo_callback(_request_envelope()) - assert cb.target_type == "RESUME" - assert cb.target_id == 42 - assert cb.status == "ANALYZED" +def _captured_payload(publisher: AsyncMock) -> AnalysisCallbackPayload: + publisher.publish.assert_awaited_once() + kwargs = publisher.publish.await_args.kwargs + assert kwargs["message_type"] == "callback.analysis" + assert kwargs["routing_key"] == "callback.analysis" + payload = kwargs["payload"] + assert isinstance(payload, AnalysisCallbackPayload) + return payload + + +@pytest.mark.asyncio +async def test_happy_path_publishes_analyzed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=ResumeAnalysisResult( + summary="요약", + tech_stack=["Python", "FastAPI"], + document_path="analyzed/resume/42/summary.md", + embedding_chunk_count=4, + ) + ) + consumer, publisher = _make_consumer(analyzer) + + await consumer.handle(_incoming_message(_request_envelope())) + + analyzer.analyze.assert_awaited_once_with( + resume_id=42, file_path="resumes/raw/123/abc.pdf", analyzed_document_id=77 + ) + payload = _captured_payload(publisher) + assert payload.status == "ANALYZED" + assert payload.target_type == "RESUME" + assert payload.target_id == 42 + assert payload.summary == "요약" + assert payload.tech_stack == ["Python", "FastAPI"] + assert payload.document_path == "analyzed/resume/42/summary.md" + assert payload.embedding_chunk_count == 4 + + +@pytest.mark.asyncio +async def test_domain_error_publishes_failed_callback_with_code() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + side_effect=ResumeAnalyzeError( + code="EMPTY_PDF_TEXT", + message="no text", + retriable=False, + ) + ) + consumer, publisher = _make_consumer(analyzer) + + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "EMPTY_PDF_TEXT" + assert payload.error_message == "no text" + assert payload.retriable is False + assert payload.summary is None + assert payload.document_path is None + + +@pytest.mark.asyncio +async def test_unexpected_error_publishes_failed_callback_as_retriable() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock(side_effect=RuntimeError("boom")) + consumer, publisher = _make_consumer(analyzer) + + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "UNEXPECTED" + assert payload.retriable is True + + +@pytest.mark.asyncio +async def test_idempotent_message_skips_analyzer_and_publish() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=ResumeAnalysisResult( + summary="요약", + tech_stack=[], + document_path="analyzed/resume/42/summary.md", + embedding_chunk_count=0, + ) + ) + idem = LruIdempotencyStore(max_size=16) + consumer, publisher = _make_consumer(analyzer, idempotency=idem) + body = _request_envelope(message_id="dup-1") + await consumer.handle(_incoming_message(body)) + await consumer.handle(_incoming_message(body)) # 중복 메시지 -def test_echo_callback_includes_marker_summary() -> None: - cb = build_echo_callback(_request_envelope()) - assert cb.summary is not None - assert "ECHO" in cb.summary - assert cb.tech_stack == [] - assert cb.document_s3_key == "echo/resume/42/echo.md" - assert cb.embedding_chunk_count == 0 + assert analyzer.analyze.await_count == 1 + assert publisher.publish.await_count == 1 -def test_echo_callback_preserves_resume_id_in_s3_key() -> None: - env = _request_envelope() - env_dict = env.model_dump(by_alias=True) - env_dict["payload"]["resumeId"] = 7 - other = Envelope[ResumeAnalyzeRequest].model_validate(env_dict) - cb = build_echo_callback(other) - assert cb.target_id == 7 - assert cb.document_s3_key == "echo/resume/7/echo.md" +@pytest.mark.asyncio +async def test_parse_failure_raises_for_dlq() -> None: + analyzer = AsyncMock() + consumer, publisher = _make_consumer(analyzer) + bad = _incoming_message(b"not-json") + with pytest.raises(Exception): + await consumer.handle(bad) -def test_extract_context_passes_user_id() -> None: - env = _request_envelope() - cb = build_echo_callback(env) - # context 검증은 consumer entry에서 하지만 핸들러에 의존하지 않음. - # 여기서는 envelope.context가 그대로 보존되는지만 확인. - assert env.context.user_id == 123 + analyzer.analyze.assert_not_called() + publisher.publish.assert_not_called() diff --git a/ai/tests/test_settings.py b/ai/tests/test_settings.py index 98818e43..6baef0c6 100644 --- a/ai/tests/test_settings.py +++ b/ai/tests/test_settings.py @@ -20,7 +20,7 @@ def test_settings_defaults_for_messaging(monkeypatch: pytest.MonkeyPatch) -> Non s = Settings(_env_file=None) # type: ignore[call-arg] - assert s.rabbitmq_url == "amqp://stackup:stackup@localhost:5672/" + assert s.rabbitmq_url == "amqp://stackup:stackup@localhost:38050/" assert s.ai_queue_resume == "ai.analyze.resume" assert s.ai_queue_prefetch == 10 assert s.ai_callback_exchange == "stackup.ai-to-core" diff --git a/ai/tests/test_storage_factory.py b/ai/tests/test_storage_factory.py new file mode 100644 index 00000000..4369bdcc --- /dev/null +++ b/ai/tests/test_storage_factory.py @@ -0,0 +1,46 @@ +from unittest.mock import MagicMock, patch + +from ai_server.config.settings import Settings +from ai_server.storage.factory import build_storage +from ai_server.storage.local_fs import LocalFilesystemStorage +from ai_server.storage.s3 import S3Storage + + +def test_local_backend_returns_local_fs(tmp_path) -> None: + s = Settings( + _env_file=None, # type: ignore[call-arg] + storage_backend="local", + storage_local_root=str(tmp_path), + ) + storage = build_storage(s) + assert isinstance(storage, LocalFilesystemStorage) + assert storage.root == tmp_path.resolve() + + +def test_s3_backend_builds_with_endpoint_and_credentials() -> None: + s = Settings( + _env_file=None, # type: ignore[call-arg] + storage_backend="s3", + s3_bucket_name="stackup", + s3_endpoint_url="http://minio:9000", + s3_access_key="ak", + s3_secret_key="sk", + s3_region="us-east-1", + ) + with ( + patch("ai_server.storage.factory.boto3", create=True), + patch( + "ai_server.storage.s3.boto3.client", return_value=MagicMock() + ) as boto3_client, + ): + storage = build_storage(s) + + assert isinstance(storage, S3Storage) + assert storage.bucket == "stackup" + boto3_client.assert_called_once_with( + "s3", + endpoint_url="http://minio:9000", + aws_access_key_id="ak", + aws_secret_access_key="sk", + region_name="us-east-1", + ) diff --git a/ai/tests/test_storage_local_fs.py b/ai/tests/test_storage_local_fs.py new file mode 100644 index 00000000..9b2557fd --- /dev/null +++ b/ai/tests/test_storage_local_fs.py @@ -0,0 +1,60 @@ +from pathlib import Path + +import pytest + +from ai_server.storage.base import StorageKeyError +from ai_server.storage.local_fs import LocalFilesystemStorage + + +@pytest.mark.asyncio +async def test_put_then_get_roundtrip(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + await storage.put_bytes("nested/dir/file.bin", b"hello world") + assert await storage.get_bytes("nested/dir/file.bin") == b"hello world" + + +@pytest.mark.asyncio +async def test_put_text_writes_utf8(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + await storage.put_text("note.md", "# 한글") + assert await storage.get_bytes("note.md") == "# 한글".encode("utf-8") + + +@pytest.mark.asyncio +async def test_put_creates_parent_dirs(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + await storage.put_text("a/b/c/d.md", "x") + assert (tmp_path / "a" / "b" / "c" / "d.md").exists() + + +@pytest.mark.asyncio +async def test_exists_returns_false_for_missing(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + assert await storage.exists("never/written.md") is False + + +@pytest.mark.asyncio +async def test_rejects_absolute_key(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + with pytest.raises(StorageKeyError): + await storage.put_bytes("/etc/passwd", b"x") + + +@pytest.mark.asyncio +async def test_rejects_parent_traversal(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + with pytest.raises(StorageKeyError): + await storage.get_bytes("../outside.txt") + + +@pytest.mark.asyncio +async def test_rejects_empty_key(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + with pytest.raises(StorageKeyError): + await storage.put_bytes("", b"x") + + +@pytest.mark.asyncio +async def test_exists_returns_false_on_invalid_key(tmp_path: Path) -> None: + storage = LocalFilesystemStorage(root_dir=str(tmp_path)) + assert await storage.exists("../escape.md") is False diff --git a/ai/tests/test_storage_s3.py b/ai/tests/test_storage_s3.py new file mode 100644 index 00000000..11e94522 --- /dev/null +++ b/ai/tests/test_storage_s3.py @@ -0,0 +1,115 @@ +from unittest.mock import MagicMock + +import pytest +from botocore.exceptions import ClientError + +from ai_server.storage.base import StorageKeyError +from ai_server.storage.s3 import S3Storage + + +def _make_storage() -> tuple[S3Storage, MagicMock]: + client = MagicMock() + storage = S3Storage(bucket="test-bucket", client=client) + return storage, client + + +@pytest.mark.asyncio +async def test_get_bytes_calls_s3_get_object() -> None: + storage, client = _make_storage() + body = MagicMock() + body.read.return_value = b"hello" + client.get_object.return_value = {"Body": body} + + out = await storage.get_bytes("a/b/c.pdf") + + client.get_object.assert_called_once_with(Bucket="test-bucket", Key="a/b/c.pdf") + body.read.assert_called_once() + assert out == b"hello" + + +@pytest.mark.asyncio +async def test_put_bytes_passes_content_type() -> None: + storage, client = _make_storage() + await storage.put_bytes("k.bin", b"x", content_type="application/octet-stream") + + client.put_object.assert_called_once() + kwargs = client.put_object.call_args.kwargs + assert kwargs["Bucket"] == "test-bucket" + assert kwargs["Key"] == "k.bin" + assert kwargs["Body"] == b"x" + assert kwargs["ContentType"] == "application/octet-stream" + + +@pytest.mark.asyncio +async def test_put_bytes_omits_content_type_when_none() -> None: + storage, client = _make_storage() + await storage.put_bytes("k.bin", b"x") + + kwargs = client.put_object.call_args.kwargs + assert "ContentType" not in kwargs + + +@pytest.mark.asyncio +async def test_put_text_encodes_utf8_with_markdown_content_type() -> None: + storage, client = _make_storage() + await storage.put_text("doc.md", "# 한글") + + kwargs = client.put_object.call_args.kwargs + assert kwargs["Body"] == "# 한글".encode("utf-8") + assert "markdown" in kwargs["ContentType"] + + +@pytest.mark.asyncio +async def test_exists_returns_true_when_head_succeeds() -> None: + storage, client = _make_storage() + client.head_object.return_value = {} + assert await storage.exists("doc.md") is True + client.head_object.assert_called_once_with(Bucket="test-bucket", Key="doc.md") + + +@pytest.mark.asyncio +async def test_exists_returns_false_on_404() -> None: + storage, client = _make_storage() + client.head_object.side_effect = ClientError( + {"Error": {"Code": "404", "Message": "Not Found"}}, + "HeadObject", + ) + assert await storage.exists("missing.md") is False + + +@pytest.mark.asyncio +async def test_exists_reraises_unexpected_client_error() -> None: + storage, client = _make_storage() + client.head_object.side_effect = ClientError( + {"Error": {"Code": "AccessDenied", "Message": "nope"}}, + "HeadObject", + ) + with pytest.raises(ClientError): + await storage.exists("forbidden.md") + + +@pytest.mark.asyncio +async def test_rejects_absolute_key() -> None: + storage, client = _make_storage() + with pytest.raises(StorageKeyError): + await storage.put_bytes("/etc/passwd", b"x") + client.put_object.assert_not_called() + + +@pytest.mark.asyncio +async def test_rejects_empty_key() -> None: + storage, _ = _make_storage() + with pytest.raises(StorageKeyError): + await storage.get_bytes("") + + +@pytest.mark.asyncio +async def test_exists_false_on_invalid_key() -> None: + storage, client = _make_storage() + assert await storage.exists("/abs") is False + client.head_object.assert_not_called() + + +def test_constructor_requires_bucket() -> None: + with pytest.raises(ValueError): + S3Storage(bucket="", client=MagicMock()) diff --git a/ai/tests/test_web_consumer.py b/ai/tests/test_web_consumer.py new file mode 100644 index 00000000..880f2f27 --- /dev/null +++ b/ai/tests/test_web_consumer.py @@ -0,0 +1,108 @@ +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ai_server.analyzer.web_resume_analyzer import ( + WebResumeAnalysisResult, + WebResumeAnalyzeError, +) +from ai_server.messaging.consumers.web_consumer import WebResumeConsumer +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + WebResumeAnalyzeRequest, +) + + +def _request_envelope(message_id: str = "req-1", resume_id: int = 11) -> bytes: + env = Envelope[WebResumeAnalyzeRequest].model_validate( + { + "messageId": message_id, + "messageType": "analyze.web", + "version": "v1", + "traceId": "trace-w", + "publishedAt": datetime(2026, 5, 14, tzinfo=timezone.utc), + "publisher": "core-server", + "payload": { + "resumeId": resume_id, + "url": "https://example.com/me", + "analyzedDocumentId": 99, + }, + "context": {"userId": 1}, + } + ) + return env.model_dump_json(by_alias=True).encode("utf-8") + + +def _incoming_message(body: bytes) -> MagicMock: + msg = MagicMock() + msg.body = body + msg.delivery_tag = 1 + process_ctx = AsyncMock() + process_ctx.__aenter__.return_value = None + process_ctx.__aexit__.return_value = False + msg.process = MagicMock(return_value=process_ctx) + return msg + + +def _make_consumer(analyzer: AsyncMock) -> tuple[WebResumeConsumer, AsyncMock]: + publisher = AsyncMock() + consumer = WebResumeConsumer( + analyzer=analyzer, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=16), + callback_routing_key="callback.analysis", + ) + return consumer, publisher + + +def _captured_payload(publisher: AsyncMock) -> AnalysisCallbackPayload: + publisher.publish.assert_awaited_once() + payload = publisher.publish.await_args.kwargs["payload"] + assert isinstance(payload, AnalysisCallbackPayload) + return payload + + +@pytest.mark.asyncio +async def test_happy_path_publishes_analyzed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=WebResumeAnalysisResult( + summary="요약", + tech_stack=["React"], + document_path="analyzed/web-resume/11/summary.md", + embedding_chunk_count=2, + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + analyzer.analyze.assert_awaited_once_with( + resume_id=11, url="https://example.com/me", analyzed_document_id=99 + ) + payload = _captured_payload(publisher) + assert payload.status == "ANALYZED" + assert payload.target_type == "WEB" + assert payload.target_id == 11 + assert payload.summary == "요약" + assert payload.document_path == "analyzed/web-resume/11/summary.md" + + +@pytest.mark.asyncio +async def test_domain_error_publishes_failed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + side_effect=WebResumeAnalyzeError( + code="WEB_HTTP_STATUS", message="404", retriable=False + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "WEB_HTTP_STATUS" + assert payload.retriable is False + assert payload.target_type == "WEB" diff --git a/ai/tests/test_web_extractor.py b/ai/tests/test_web_extractor.py new file mode 100644 index 00000000..3bd75c66 --- /dev/null +++ b/ai/tests/test_web_extractor.py @@ -0,0 +1,117 @@ +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from ai_server.analyzer.sources.web import WebFetchError, WebSourceExtractor + + +def _make_client( + *, + status: int = 200, + content_type: str = "text/html; charset=utf-8", + body: ( + str | bytes + ) = "

주요 내용

본문 텍스트입니다.

", + final_url: str = "https://example.com/r", + raise_exc: Exception | None = None, +) -> MagicMock: + client = MagicMock() + resp = MagicMock(spec=httpx.Response) + resp.status_code = status + resp.headers = {"content-type": content_type} + resp.text = body if isinstance(body, str) else body.decode("utf-8", errors="ignore") + resp.content = body.encode("utf-8") if isinstance(body, str) else body + resp.url = final_url + if raise_exc is not None: + client.get = AsyncMock(side_effect=raise_exc) + else: + client.get = AsyncMock(return_value=resp) + return client + + +@pytest.mark.asyncio +async def test_extract_returns_main_body_text() -> None: + html = ( + "이력서" + "

김OO

" + "

백엔드 개발자. Spring Boot 3년차.

" + "" + ) + client = _make_client(body=html) + extractor = WebSourceExtractor(client=client) + result = await extractor.extract("https://example.com/r") + + assert result.source_type == "WEB" + assert "백엔드 개발자" in result.text + assert "김OO" in result.text + assert result.metadata["final_url"] == "https://example.com/r" + assert result.metadata["text_chars"] > 0 + + +@pytest.mark.asyncio +async def test_rejects_non_http_locator() -> None: + extractor = WebSourceExtractor(client=_make_client()) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("ftp://example.com/x") + assert exc_info.value.code == "INVALID_WEB_URL" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_raises_on_http_error_status() -> None: + client = _make_client(status=503) + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_HTTP_STATUS" + assert exc_info.value.retriable is True # 5xx → retriable + + +@pytest.mark.asyncio +async def test_raises_on_4xx_as_non_retriable() -> None: + client = _make_client(status=404) + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_HTTP_STATUS" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_rejects_non_html_content_type() -> None: + client = _make_client(content_type="application/pdf") + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_NOT_HTML" + + +@pytest.mark.asyncio +async def test_rejects_oversized_html() -> None: + big = b"" + b"a" * 1024 + b"" + client = _make_client(body=big) + extractor = WebSourceExtractor(client=client, max_html_bytes=512) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_HTML_TOO_LARGE" + + +@pytest.mark.asyncio +async def test_raises_on_empty_body() -> None: + client = _make_client(body="") + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "EMPTY_WEB_BODY" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_raises_on_httpx_error_as_retriable() -> None: + client = _make_client(raise_exc=httpx.ConnectError("dns fail")) + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_FETCH_FAILED" + assert exc_info.value.retriable is True diff --git a/ai/tests/test_web_resume_analyzer.py b/ai/tests/test_web_resume_analyzer.py new file mode 100644 index 00000000..57b13d0a --- /dev/null +++ b/ai/tests/test_web_resume_analyzer.py @@ -0,0 +1,84 @@ +from unittest.mock import AsyncMock + +import pytest + +from ai_server.analyzer.sources.base import ExtractedSource +from ai_server.analyzer.sources.web import WebFetchError +from ai_server.analyzer.web_resume_analyzer import ( + WebResumeAnalyzeError, + WebResumeAnalyzer, +) +from ai_server.chain.document_analysis_chain import DocumentAnalysisResult +from ai_server.rag.chunker import MarkdownChunker +from ai_server.rag.embedder import MockEmbeddingProvider + + +def _make_analyzer( + *, + extract_result: ExtractedSource | Exception | None = None, + analysis: DocumentAnalysisResult | None = None, + upsert_result: int = 2, +) -> tuple[WebResumeAnalyzer, AsyncMock, AsyncMock, AsyncMock, AsyncMock]: + extractor = AsyncMock() + if isinstance(extract_result, Exception): + extractor.extract = AsyncMock(side_effect=extract_result) + else: + extractor.extract = AsyncMock(return_value=extract_result) + chain = AsyncMock() + if analysis is not None: + chain.analyze = AsyncMock(return_value=analysis) + storage = AsyncMock() + + core_client = AsyncMock() + core_client.upsert_embeddings = AsyncMock(return_value=upsert_result) + + analyzer = WebResumeAnalyzer( + extractor=extractor, + chain=chain, + storage=storage, + chunker=MarkdownChunker(chunk_size=200, chunk_overlap=50), + embedder=MockEmbeddingProvider(dim=16), + core_client=core_client, + analyzed_key_template="analyzed/web-resume/{resume_id}/summary.md", + ) + return analyzer, extractor, chain, storage, core_client + + +@pytest.mark.asyncio +async def test_happy_path() -> None: + analyzer, extractor, chain, storage, core_client = _make_analyzer( + extract_result=ExtractedSource(text="본문" * 30, source_type="WEB"), + analysis=DocumentAnalysisResult( + summary="요약", tech_stack=["TypeScript"], markdown="## 개요\nweb" + ), + upsert_result=2, + ) + result = await analyzer.analyze( + resume_id=11, url="https://example.com/me", analyzed_document_id=99 + ) + + extractor.extract.assert_awaited_once_with("https://example.com/me") + chain.analyze.assert_awaited_once() + storage.put_text.assert_awaited_once() + core_client.upsert_embeddings.assert_awaited_once() + assert core_client.upsert_embeddings.await_args.kwargs["document_id"] == 99 + assert result.summary == "요약" + assert result.embedding_chunk_count == 2 + assert result.document_path == "analyzed/web-resume/11/summary.md" + + +@pytest.mark.asyncio +async def test_fetch_error_translates_to_domain_error() -> None: + analyzer, _, chain, storage, core_client = _make_analyzer( + extract_result=WebFetchError( + code="WEB_HTTP_STATUS", message="404", retriable=False + ), + ) + with pytest.raises(WebResumeAnalyzeError) as exc_info: + await analyzer.analyze( + resume_id=1, url="https://example.com/x", analyzed_document_id=1 + ) + assert exc_info.value.code == "WEB_HTTP_STATUS" + chain.analyze.assert_not_called() + storage.put_text.assert_not_called() + core_client.upsert_embeddings.assert_not_called() diff --git a/ai/uv.lock b/ai/uv.lock index f2fca866..bc87a385 100644 --- a/ai/uv.lock +++ b/ai/uv.lock @@ -174,6 +174,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + [[package]] name = "black" version = "26.3.1" @@ -238,6 +247,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, ] +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.6" @@ -316,6 +370,73 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "courlan" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "tld" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/54/6d6ceeff4bed42e7a10d6064d35ee43a810e7b3e8beb4abeae8cff4713ae/courlan-1.3.2.tar.gz", hash = "sha256:0b66f4db3a9c39a6e22dd247c72cfaa57d68ea660e94bb2c84ec7db8712af190", size = 206382, upload-time = "2024-10-29T16:40:20.994Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/ca/6a667ccbe649856dcd3458bab80b016681b274399d6211187c6ab969fc50/courlan-1.3.2-py3-none-any.whl", hash = "sha256:d0dab52cf5b5b1000ee2839fbc2837e93b2514d3cb5bb61ae158a55b7a04c6be", size = 33848, upload-time = "2024-10-29T16:40:18.325Z" }, +] + +[[package]] +name = "cryptography" +version = "48.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, + { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556, upload-time = "2026-05-04T22:58:11.172Z" }, + { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776, upload-time = "2026-05-04T22:58:13.712Z" }, + { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121, upload-time = "2026-05-04T22:58:16.448Z" }, + { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042, upload-time = "2026-05-04T22:58:18.544Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526, upload-time = "2026-05-04T22:58:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116, upload-time = "2026-05-04T22:58:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030, upload-time = "2026-05-04T22:58:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640, upload-time = "2026-05-04T22:58:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657, upload-time = "2026-05-04T22:58:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362, upload-time = "2026-05-04T22:58:32.009Z" }, + { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580, upload-time = "2026-05-04T22:58:34.254Z" }, + { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283, upload-time = "2026-05-04T22:58:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954, upload-time = "2026-05-04T22:58:38.791Z" }, + { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313, upload-time = "2026-05-04T22:58:40.746Z" }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, +] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -329,6 +450,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, ] +[[package]] +name = "dateparser" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "regex" }, + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/2d/a0ccdb78788064fa0dc901b8524e50615c42be1d78b78d646d0b28d09180/dateparser-1.4.0.tar.gz", hash = "sha256:97a21840d5ecdf7630c584f673338a5afac5dfe84f647baf4d7e8df98f9354a4", size = 321512, upload-time = "2026-03-26T09:56:10.292Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/0b/3c3bb7cbe757279e693a0be6049048012f794d01f81099609ecd53b899f0/dateparser-1.4.0-py3-none-any.whl", hash = "sha256:7902b8e85d603494bf70a5a0b1decdddb2270b9c6e6b2bc8a57b93476c0df378", size = 300379, upload-time = "2026-03-26T09:56:08.409Z" }, +] + [[package]] name = "dill" version = "0.4.1" @@ -450,6 +586,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "google-auth" +version = "2.52.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/f8/80d2493cbedece1c623dc3e3cb1883300871af0dcdae254409522985ac23/google_auth-2.52.0.tar.gz", hash = "sha256:01f30e1a9e3638698d89464f5e603ce29d18e1c0e63ec31ac570aba4e164aaf5", size = 335027, upload-time = "2026-05-07T19:45:24.033Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/fc/2cdc74252746f547f81ff3f02d4d4234a3f411b5de5b61af97e633a060b9/google_auth-2.52.0-py3-none-any.whl", hash = "sha256:aee92803ba0ff93a70a3b8a35c7b4797837751cd6380b63ff38372b98f3ed627", size = 245614, upload-time = "2026-05-07T19:45:21.914Z" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + +[[package]] +name = "google-genai" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "google-auth", extra = ["requests"] }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "sniffio" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/8e/dfa4b34dd4c0baffccf6466fc68d6d35011662d43e7d79accb902320db74/google_genai-2.3.0.tar.gz", hash = "sha256:e877c750a4ccacdd9928fc3aa8ca8820ce85cade0ca51bd83feceacf5959b579", size = 546930, upload-time = "2026-05-15T06:22:36.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/6e/aa6b30b09f58b946750fc4089c5248fbd3576f746e0e818d88633559dc84/google_genai-2.3.0-py3-none-any.whl", hash = "sha256:89d3c71c9f5f5b931b405b88a5837aea2bd4d27ed90323b9599f5760bbb91d92", size = 805484, upload-time = "2026-05-15T06:22:34.247Z" }, +] + [[package]] name = "greenlet" version = "3.3.2" @@ -490,6 +665,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "htmldate" +version = "1.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "dateparser" }, + { name = "lxml" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/10/ead9dabc999f353c3aa5d0dc0835b1e355215a5ecb489a7f4ef2ddad5e33/htmldate-1.9.4.tar.gz", hash = "sha256:1129063e02dd0354b74264de71e950c0c3fcee191178321418ccad2074cc8ed0", size = 44690, upload-time = "2025-11-04T17:46:44.983Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/bd/adfcdaaad5805c0c5156aeefd64c1e868c05e9c1cd6fd21751f168cd88c7/htmldate-1.9.4-py3-none-any.whl", hash = "sha256:1b94bcc4e08232a5b692159903acf95548b6a7492dddca5bb123d89d6325921c", size = 31558, upload-time = "2025-11-04T17:46:43.258Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -660,6 +851,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, ] +[[package]] +name = "justext" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml", extra = ["html-clean"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/f3/45890c1b314f0d04e19c1c83d534e611513150939a7cf039664d9ab1e649/justext-3.0.2.tar.gz", hash = "sha256:13496a450c44c4cd5b5a75a5efcd9996066d2a189794ea99a49949685a0beb05", size = 828521, upload-time = "2025-02-25T20:21:49.934Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/ac/52f4e86d1924a7fc05af3aeb34488570eccc39b4af90530dd6acecdf16b5/justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7", size = 837940, upload-time = "2025-02-25T20:21:44.179Z" }, +] + [[package]] name = "langchain" version = "1.2.13" @@ -849,6 +1052,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/94/1f5d72655ab6534129540843776c40eff757387b88e798d8b3bf7e313fd4/langsmith-0.7.22-py3-none-any.whl", hash = "sha256:6e9d5148314d74e86748cb9d3898632cad0320c9323d95f70f969e5bc078eee4", size = 359927, upload-time = "2026-03-19T22:45:21.603Z" }, ] +[[package]] +name = "lxml" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/28/30/9abc9e34c657c33834eaf6cd02124c61bdf5944d802aa48e69be8da3585d/lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13", size = 4197006, upload-time = "2026-04-18T04:32:51.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/03/69347590f1cf4a6d5a4944bb6099e6d37f334784f16062234e1f892fdb1d/lxml-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45", size = 8559689, upload-time = "2026-04-18T04:31:57.785Z" }, + { url = "https://files.pythonhosted.org/packages/3f/58/25e00bb40b185c974cfe156c110474d9a8a8390d5f7c92a4e328189bb60e/lxml-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d", size = 4617892, upload-time = "2026-04-18T04:32:01.78Z" }, + { url = "https://files.pythonhosted.org/packages/f5/54/92ad98a94ac318dc4f97aaac22ff8d1b94212b2ae8af5b6e9b354bf825f7/lxml-6.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2", size = 4923489, upload-time = "2026-04-18T04:33:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/15/3b/a20aecfab42bdf4f9b390590d345857ad3ffd7c51988d1c89c53a0c73faf/lxml-6.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491", size = 5082162, upload-time = "2026-04-18T04:33:34.262Z" }, + { url = "https://files.pythonhosted.org/packages/45/26/2cdb3d281ac1bd175603e290cbe4bad6eff127c0f8de90bafd6f8548f0fd/lxml-6.1.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc", size = 4993247, upload-time = "2026-04-18T04:33:36.674Z" }, + { url = "https://files.pythonhosted.org/packages/f6/05/d735aef963740022a08185c84821f689fc903acb3d50326e6b1e9886cc22/lxml-6.1.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e", size = 5613042, upload-time = "2026-04-18T04:33:39.205Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b8/ead7c10efff731738c72e59ed6eb5791854879fbed7ae98781a12006263a/lxml-6.1.0-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2", size = 5228304, upload-time = "2026-04-18T04:33:41.647Z" }, + { url = "https://files.pythonhosted.org/packages/6b/10/e9842d2ec322ea65f0a7270aa0315a53abed06058b88ef1b027f620e7a5f/lxml-6.1.0-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9", size = 5341578, upload-time = "2026-04-18T04:33:44.596Z" }, + { url = "https://files.pythonhosted.org/packages/89/54/40d9403d7c2775fa7301d3ddd3464689bfe9ba71acc17dfff777071b4fdc/lxml-6.1.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe", size = 4700209, upload-time = "2026-04-18T04:33:47.552Z" }, + { url = "https://files.pythonhosted.org/packages/85/b2/bbdcc2cf45dfc7dfffef4fd97e5c47b15919b6a365247d95d6f684ef5e82/lxml-6.1.0-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88", size = 5232365, upload-time = "2026-04-18T04:33:50.249Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/b06875665e53aaba7127611a7bed3b7b9658e20b22bc2dd217a0b7ab0091/lxml-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181", size = 5043654, upload-time = "2026-04-18T04:33:52.71Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9c/e71a069d09641c1a7abeb30e693f828c7c90a41cbe3d650b2d734d876f85/lxml-6.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24", size = 4769326, upload-time = "2026-04-18T04:33:55.244Z" }, + { url = "https://files.pythonhosted.org/packages/cc/06/7a9cd84b3d4ed79adf35f874750abb697dec0b4a81a836037b36e47c091a/lxml-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e", size = 5635879, upload-time = "2026-04-18T04:33:58.509Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f0/9d57916befc1e54c451712c7ee48e9e74e80ae4d03bdce49914e0aee42cd/lxml-6.1.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495", size = 5224048, upload-time = "2026-04-18T04:34:00.943Z" }, + { url = "https://files.pythonhosted.org/packages/99/75/90c4eefda0c08c92221fe0753db2d6699a4c628f76ff4465ec20dea84cc1/lxml-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33", size = 5250241, upload-time = "2026-04-18T04:34:03.365Z" }, + { url = "https://files.pythonhosted.org/packages/5e/73/16596f7e4e38fa33084b9ccbccc22a15f82a290a055126f2c1541236d2ff/lxml-6.1.0-cp313-cp313-win32.whl", hash = "sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62", size = 3596938, upload-time = "2026-04-18T04:31:56.206Z" }, + { url = "https://files.pythonhosted.org/packages/8e/63/981401c5680c1eb30893f00a19641ac80db5d1e7086c62cb4b13ed813038/lxml-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16", size = 3995728, upload-time = "2026-04-18T04:31:58.763Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e8/c358a38ac3e541d16a1b527e4e9cb78c0419b0506a070ace11777e5e8404/lxml-6.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d", size = 3658372, upload-time = "2026-04-18T04:32:03.629Z" }, + { url = "https://files.pythonhosted.org/packages/eb/45/cee4cf203ef0bab5c52afc118da61d6b460c928f2893d40023cfa27e0b80/lxml-6.1.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8", size = 8576713, upload-time = "2026-04-18T04:32:06.831Z" }, + { url = "https://files.pythonhosted.org/packages/8a/a7/eda05babeb7e046839204eaf254cd4d7c9130ce2bbf0d9e90ea41af5654d/lxml-6.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9", size = 4623874, upload-time = "2026-04-18T04:32:10.755Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e9/db5846de9b436b91890a62f29d80cd849ea17948a49bf532d5278ee69a9e/lxml-6.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03", size = 4949535, upload-time = "2026-04-18T04:34:06.657Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ba/0d3593373dcae1d68f40dc3c41a5a92f2544e68115eb2f62319a4c2a6500/lxml-6.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb", size = 5086881, upload-time = "2026-04-18T04:34:09.556Z" }, + { url = "https://files.pythonhosted.org/packages/43/76/759a7484539ad1af0d125a9afe9c3fb5f82a8779fd1f5f56319d9e4ea2fd/lxml-6.1.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c", size = 5031305, upload-time = "2026-04-18T04:34:12.336Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b9/c1f0daf981a11e47636126901fd4ab82429e18c57aeb0fc3ad2940b42d8b/lxml-6.1.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28", size = 5647522, upload-time = "2026-04-18T04:34:14.89Z" }, + { url = "https://files.pythonhosted.org/packages/31/e6/1f533dcd205275363d9ba3511bcec52fa2df86abf8abe6a5f2c599f0dc31/lxml-6.1.0-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086", size = 5239310, upload-time = "2026-04-18T04:34:17.652Z" }, + { url = "https://files.pythonhosted.org/packages/c3/8c/4175fb709c78a6e315ed814ed33be3defd8b8721067e70419a6cf6f971da/lxml-6.1.0-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f", size = 5350799, upload-time = "2026-04-18T04:34:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/6ffdebc5994975f0dde4acb59761902bd9d9bb84422b9a0bd239a7da9ca8/lxml-6.1.0-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292", size = 4697693, upload-time = "2026-04-18T04:34:23.541Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f1/565f36bd5c73294602d48e04d23f81ff4c8736be6ba5e1d1ec670ac9be80/lxml-6.1.0-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb", size = 5250708, upload-time = "2026-04-18T04:34:26.001Z" }, + { url = "https://files.pythonhosted.org/packages/5a/11/a68ab9dd18c5c499404deb4005f4bc4e0e88e5b72cd755ad96efec81d18d/lxml-6.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad", size = 5084737, upload-time = "2026-04-18T04:34:28.32Z" }, + { url = "https://files.pythonhosted.org/packages/ab/78/e8f41e2c74f4af564e6a0348aea69fb6daaefa64bc071ef469823d22cc18/lxml-6.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb", size = 4737817, upload-time = "2026-04-18T04:34:30.784Z" }, + { url = "https://files.pythonhosted.org/packages/06/2d/aa4e117aa2ce2f3b35d9ff246be74a2f8e853baba5d2a92c64744474603a/lxml-6.1.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f", size = 5670753, upload-time = "2026-04-18T04:34:33.675Z" }, + { url = "https://files.pythonhosted.org/packages/08/f5/dd745d50c0409031dbfcc4881740542a01e54d6f0110bd420fa7782110b8/lxml-6.1.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43", size = 5238071, upload-time = "2026-04-18T04:34:36.12Z" }, + { url = "https://files.pythonhosted.org/packages/3e/74/ad424f36d0340a904665867dab310a3f1f4c96ff4039698de83b77f44c1f/lxml-6.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585", size = 5264319, upload-time = "2026-04-18T04:34:39.035Z" }, + { url = "https://files.pythonhosted.org/packages/53/36/a15d8b3514ec889bfd6aa3609107fcb6c9189f8dc347f1c0b81eded8d87c/lxml-6.1.0-cp314-cp314-win32.whl", hash = "sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f", size = 3657139, upload-time = "2026-04-18T04:32:20.006Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/263ebb0710851a3c6c937180a9a86df1206fdfe53cc43005aa2237fd7736/lxml-6.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120", size = 4064195, upload-time = "2026-04-18T04:32:23.876Z" }, + { url = "https://files.pythonhosted.org/packages/80/68/2000f29d323b6c286de077ad20b429fc52272e44eae6d295467043e56012/lxml-6.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946", size = 3741870, upload-time = "2026-04-18T04:32:27.922Z" }, + { url = "https://files.pythonhosted.org/packages/30/e9/21383c7c8d43799f0da90224c0d7c921870d476ec9b3e01e1b2c0b8237c5/lxml-6.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c", size = 8827548, upload-time = "2026-04-18T04:32:15.094Z" }, + { url = "https://files.pythonhosted.org/packages/a5/01/c6bc11cd587030dd4f719f65c5657960649fe3e19196c844c75bf32cd0d6/lxml-6.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d", size = 4735866, upload-time = "2026-04-18T04:32:18.924Z" }, + { url = "https://files.pythonhosted.org/packages/f3/01/757132fff5f4acf25463b5298f1a46099f3a94480b806547b29ce5e385de/lxml-6.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9", size = 4969476, upload-time = "2026-04-18T04:34:41.889Z" }, + { url = "https://files.pythonhosted.org/packages/fd/fb/1bc8b9d27ed64be7c8903db6c89e74dc8c2cd9ec630a7462e4654316dc5b/lxml-6.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9", size = 5103719, upload-time = "2026-04-18T04:34:44.797Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e7/5bf82fa28133536a54601aae633b14988e89ed61d4c1eb6b899b023233aa/lxml-6.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7", size = 5027890, upload-time = "2026-04-18T04:34:47.634Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/e048db5d4b4ea0366648aa595f26bb764b2670903fc585b87436d0a5032c/lxml-6.1.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86", size = 5596008, upload-time = "2026-04-18T04:34:51.503Z" }, + { url = "https://files.pythonhosted.org/packages/9a/c2/d10807bc8da4824b39e5bd01b5d05c077b6fd01bd91584167edf6b269d22/lxml-6.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb", size = 5224451, upload-time = "2026-04-18T04:34:54.263Z" }, + { url = "https://files.pythonhosted.org/packages/3c/15/2ebea45bea427e7f0057e9ce7b2d62c5aba20c6b001cca89ed0aadb3ad41/lxml-6.1.0-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c", size = 5312135, upload-time = "2026-04-18T04:34:56.818Z" }, + { url = "https://files.pythonhosted.org/packages/31/e2/87eeae151b0be2a308d49a7ec444ff3eb192b14251e62addb29d0bf3778f/lxml-6.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f", size = 4639126, upload-time = "2026-04-18T04:34:59.704Z" }, + { url = "https://files.pythonhosted.org/packages/a3/51/8a3f6a20902ad604dd746ec7b4000311b240d389dac5e9d95adefd349e0c/lxml-6.1.0-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773", size = 5232579, upload-time = "2026-04-18T04:35:02.658Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d2/650d619bdbe048d2c3f2c31edb00e35670a5e2d65b4fe3b61bce37b19121/lxml-6.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b", size = 5084206, upload-time = "2026-04-18T04:35:05.175Z" }, + { url = "https://files.pythonhosted.org/packages/dd/8a/672ca1a3cbeabd1f511ca275a916c0514b747f4b85bdaae103b8fa92f307/lxml-6.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405", size = 4758906, upload-time = "2026-04-18T04:35:08.098Z" }, + { url = "https://files.pythonhosted.org/packages/be/f1/ef4b691da85c916cb2feb1eec7414f678162798ac85e042fa164419ac05c/lxml-6.1.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690", size = 5620553, upload-time = "2026-04-18T04:35:11.23Z" }, + { url = "https://files.pythonhosted.org/packages/59/17/94e81def74107809755ac2782fdad4404420f1c92ca83433d117a6d5acf0/lxml-6.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd", size = 5229458, upload-time = "2026-04-18T04:35:14.254Z" }, + { url = "https://files.pythonhosted.org/packages/21/55/c4be91b0f830a871fc1b0d730943d56013b683d4671d5198260e2eae722b/lxml-6.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180", size = 5247861, upload-time = "2026-04-18T04:35:17.006Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ca/77123e4d77df3cb1e968ade7b1f808f5d3a5c1c96b18a33895397de292c1/lxml-6.1.0-cp314-cp314t-win32.whl", hash = "sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2", size = 3897377, upload-time = "2026-04-18T04:32:07.656Z" }, + { url = "https://files.pythonhosted.org/packages/64/ce/3554833989d074267c063209bae8b09815e5656456a2d332b947806b05ff/lxml-6.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5", size = 4392701, upload-time = "2026-04-18T04:32:12.113Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a0/9b916c68c0e57752c07f8f64b30138d9d4059dbeb27b90274dedbea128ff/lxml-6.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac", size = 3817120, upload-time = "2026-04-18T04:32:15.803Z" }, +] + +[package.optional-dependencies] +html-clean = [ + { name = "lxml-html-clean" }, +] + +[[package]] +name = "lxml-html-clean" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/a4/5c62acfacd69ff4f5db395100f5cfb9b54e7ac8c69a235e4e939fd13f021/lxml_html_clean-0.4.4.tar.gz", hash = "sha256:58f39a9d632711202ed1d6d0b9b47a904e306c85de5761543b90e3e3f736acfb", size = 23899, upload-time = "2026-02-27T09:35:52.911Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/76/7ffc1d3005cf7749123bc47cb3ea343cd97b0ac2211bab40f57283577d0e/lxml_html_clean-0.4.4-py3-none-any.whl", hash = "sha256:ce2ef506614ecb85ee1c5fe0a2aa45b06a19514ec7949e9c8f34f06925cfabcb", size = 14565, upload-time = "2026-02-27T09:35:51.86Z" }, +] + [[package]] name = "marshmallow" version = "3.26.2" @@ -1211,6 +1493,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycodestyle" version = "2.14.0" @@ -1220,6 +1523,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -1420,6 +1732,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] +[[package]] +name = "pytz" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1633,15 +1954,18 @@ dependencies = [ { name = "aiofiles" }, { name = "boto3" }, { name = "fastapi" }, + { name = "google-genai" }, { name = "httpx" }, { name = "langchain" }, { name = "langchain-community" }, { name = "langchain-core" }, { name = "langchain-openai" }, + { name = "langchain-text-splitters" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pypdf" }, { name = "structlog" }, + { name = "trafilatura" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -1662,11 +1986,13 @@ requires-dist = [ { name = "boto3", specifier = ">=1.42.77" }, { name = "fastapi", specifier = ">=0.135.2" }, { name = "flake8", marker = "extra == 'dev'", specifier = ">=7.3.0" }, + { name = "google-genai", specifier = ">=1.0.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "langchain", specifier = ">=1.2.13" }, { name = "langchain-community", specifier = ">=0.4.1" }, { name = "langchain-core", specifier = ">=1.2.22" }, { name = "langchain-openai", specifier = ">=0.3.0" }, + { name = "langchain-text-splitters", specifier = ">=0.3.0" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.13.1" }, { name = "pylint", marker = "extra == 'dev'", specifier = ">=4.0.5" }, @@ -1674,6 +2000,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, { name = "structlog", specifier = ">=25.5.0" }, + { name = "trafilatura", specifier = ">=2.0.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.42.0" }, ] provides-extras = ["dev"] @@ -1748,6 +2075,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, ] +[[package]] +name = "tld" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5d/76b4383ac4e5b5e254e50c09807b3e13820bed6d6c11cd540264988d6802/tld-0.13.2.tar.gz", hash = "sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345", size = 467175, upload-time = "2026-03-06T23:50:34.498Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/90/39a85a4b63c84213e78b3c17d22e1bf45328acf8ebb33ef93be30d0a3911/tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c", size = 296743, upload-time = "2026-03-06T23:50:32.465Z" }, +] + [[package]] name = "tomlkit" version = "0.14.0" @@ -1769,6 +2105,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "trafilatura" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "courlan" }, + { name = "htmldate" }, + { name = "justext" }, + { name = "lxml" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/25/e3ebeefdebfdfae8c4a4396f5a6ea51fc6fa0831d63ce338e5090a8003dc/trafilatura-2.0.0.tar.gz", hash = "sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247", size = 253404, upload-time = "2024-12-03T15:23:24.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/b6/097367f180b6383a3581ca1b86fcae284e52075fa941d1232df35293363c/trafilatura-2.0.0-py3-none-any.whl", hash = "sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d", size = 132557, upload-time = "2024-12-03T15:23:21.41Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -1803,6 +2157,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "tzdata" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "urllib3" version = "2.6.3" diff --git a/backend/gradlew b/backend/gradlew old mode 100644 new mode 100755 diff --git a/backend/src/main/java/com/stackup/stackup/auth/application/GithubUserService.java b/backend/src/main/java/com/stackup/stackup/auth/application/GithubUserService.java index cae6a99c..b8bfa9e4 100644 --- a/backend/src/main/java/com/stackup/stackup/auth/application/GithubUserService.java +++ b/backend/src/main/java/com/stackup/stackup/auth/application/GithubUserService.java @@ -8,8 +8,16 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +// record 는 implicit final 이라 Spring AOP(@Transactional) CGLIB 프록시 생성 실패. +// 일반 class + 생성자 주입으로 전환. @Service -public record GithubUserService(UserRepository userRepository) { +public class GithubUserService { + + private final UserRepository userRepository; + + public GithubUserService(UserRepository userRepository) { + this.userRepository = userRepository; + } @Transactional public GithubUserUpsertResult upsertGithubUser( diff --git a/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java b/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java index f82438d2..59a75be8 100644 --- a/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java +++ b/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java @@ -43,7 +43,8 @@ public record Exchanges( public record Names( @NotBlank String coreToAi, - @NotBlank String aiToCore + @NotBlank String aiToCore, + @NotBlank String realtime ) { } } @@ -71,7 +72,8 @@ public record RoutingKeyProperties( @NotBlank String generateQuestions, @NotBlank String generateFollowup, @NotBlank String callbackAnalysis, - @NotBlank String callbackQuestions + @NotBlank String callbackQuestions, + @NotBlank String realtimeSessionNotify ) { } } diff --git a/backend/src/main/java/com/stackup/stackup/common/config/properties/SecurityProperties.java b/backend/src/main/java/com/stackup/stackup/common/config/properties/SecurityProperties.java index 36d0734a..a6c64204 100644 --- a/backend/src/main/java/com/stackup/stackup/common/config/properties/SecurityProperties.java +++ b/backend/src/main/java/com/stackup/stackup/common/config/properties/SecurityProperties.java @@ -23,6 +23,7 @@ public record SecurityProperties( @NotBlank String refreshTokenCookiePath, @NotBlank String refreshTokenCookieSameSite, boolean refreshTokenCookieSecure, - boolean refreshTokenCookieHttpOnly + boolean refreshTokenCookieHttpOnly, + String internalApiKey ) { } diff --git a/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java b/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java index e95296ea..f21e728a 100644 --- a/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java +++ b/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java @@ -26,6 +26,8 @@ public enum ApiErrorCode { DOC_NOT_ANALYZED(HttpStatus.UNPROCESSABLE_CONTENT, "아직 분석되지 않은 문서입니다."), DOC_ANALYSIS_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, "문서 분석에 실패했습니다."), DOC_NOT_FOUND(HttpStatus.NOT_FOUND, "분석 문서를 찾을 수 없습니다."), + INTERNAL_AUTH_FAILED(HttpStatus.UNAUTHORIZED, "내부 API 인증에 실패했습니다."), + EMBEDDING_BAD_REQUEST(HttpStatus.BAD_REQUEST, "임베딩 요청 페이로드가 올바르지 않습니다."), SESSION_INVALID_STATE(HttpStatus.UNPROCESSABLE_CONTENT, "세션 상태가 올바르지 않습니다."), SESSION_MAX_REACHED(HttpStatus.UNPROCESSABLE_CONTENT, "세션 제한에 도달했습니다."), diff --git a/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java b/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java index 9481d827..497da3b4 100644 --- a/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java +++ b/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java @@ -39,6 +39,15 @@ public TopicExchange aiToCoreExchange() { ); } + @Bean + public TopicExchange realtimeExchange() { + return new TopicExchange( + properties.exchanges().names().realtime(), + properties.exchanges().durable(), + properties.exchanges().autoDelete() + ); + } + @Bean public Queue aiAnalyzeResumeQueue() { return new Queue(properties.queues().names().aiAnalyzeResume(), properties.queues().durable()); @@ -73,6 +82,7 @@ public Queue coreCallbackQuestionsQueue() { public Declarables rabbitDeclarables( TopicExchange coreToAiExchange, TopicExchange aiToCoreExchange, + TopicExchange realtimeExchange, Queue aiAnalyzeResumeQueue, Queue aiAnalyzeRepositoryQueue, Queue aiGenerateQuestionsQueue, @@ -83,6 +93,7 @@ public Declarables rabbitDeclarables( return new Declarables( coreToAiExchange, aiToCoreExchange, + realtimeExchange, aiAnalyzeResumeQueue, aiAnalyzeRepositoryQueue, aiGenerateQuestionsQueue, @@ -95,6 +106,8 @@ public Declarables rabbitDeclarables( BindingBuilder.bind(aiGenerateFollowupQueue).to(coreToAiExchange).with(properties.routingKeys().generateFollowup()), BindingBuilder.bind(coreCallbackAnalysisQueue).to(aiToCoreExchange).with(properties.routingKeys().callbackAnalysis()), BindingBuilder.bind(coreCallbackQuestionsQueue).to(aiToCoreExchange).with(properties.routingKeys().callbackQuestions()) + // 주의: q.realtime.session.notify 큐 + binding 은 RealTime 서버가 자체 declare + // (또는 infra/rabbitmq/definitions.json). Core 는 exchange 만 declare. ); } diff --git a/backend/src/main/java/com/stackup/stackup/common/messaging/RealtimeNotifyPublisher.java b/backend/src/main/java/com/stackup/stackup/common/messaging/RealtimeNotifyPublisher.java new file mode 100644 index 00000000..51558f19 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/common/messaging/RealtimeNotifyPublisher.java @@ -0,0 +1,64 @@ +package com.stackup.stackup.common.messaging; + +import com.stackup.stackup.common.config.properties.RabbitMqProperties; +import com.stackup.stackup.common.trace.TraceContext; +import java.time.Instant; +import java.util.UUID; +import org.springframework.amqp.core.MessagePostProcessor; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Component; + +// Core -> 실시간 서버 알림 +// 실시간 서버는 SSE로 구독한 컴포넌트에게 알려줌 +@Component +public class RealtimeNotifyPublisher { + + public record RealtimeNotifyPayload(String eventType, Object data) { + } + + private final RabbitMqProperties properties; + private final RabbitTemplate rabbitTemplate; + + public RealtimeNotifyPublisher(RabbitMqProperties properties, RabbitTemplate rabbitTemplate) { + this.properties = properties; + this.rabbitTemplate = rabbitTemplate; + } + + public MessageEnvelope publishSessionNotify( + long sessionId, + Long userId, + String eventType, + Object data + ) { + var payload = new RealtimeNotifyPayload(eventType, data); + var context = new MessageContext(userId, sessionId, null, null); + var envelope = new MessageEnvelope<>( + UUID.randomUUID().toString(), + "realtime.session.notify", + properties.version(), + TraceContext.getTraceId(), + Instant.now(), + properties.publisher(), + payload, + context + ); + + rabbitTemplate.convertAndSend( + properties.exchanges().names().realtime(), + properties.routingKeys().realtimeSessionNotify(), + envelope, + withEnvelopeHeaders(envelope) + ); + return envelope; + } + + private MessagePostProcessor withEnvelopeHeaders(MessageEnvelope envelope) { + return message -> { + message.getMessageProperties().setContentType(properties.message().contentType()); + message.getMessageProperties().setContentEncoding(properties.message().contentEncoding()); + message.getMessageProperties().setMessageId(envelope.messageId()); + message.getMessageProperties().setHeader(properties.message().traceIdHeader(), envelope.traceId()); + return message; + }; + } +} diff --git a/backend/src/main/java/com/stackup/stackup/common/messaging/domain/ProcessedMessage.java b/backend/src/main/java/com/stackup/stackup/common/messaging/domain/ProcessedMessage.java new file mode 100644 index 00000000..bf224336 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/common/messaging/domain/ProcessedMessage.java @@ -0,0 +1,43 @@ +package com.stackup.stackup.common.messaging.domain; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.Table; +import java.time.Instant; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Getter +@Entity +@Table( + name = "processed_messages", + indexes = { + @Index(name = "idx_processed_messages_processed_at", columnList = "processed_at") + } +) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class ProcessedMessage { + + @Id + @Column(name = "message_id", length = 64) + private String messageId; + + @Column(nullable = false, length = 100) + private String consumer; + + @Column(name = "processed_at", nullable = false) + private Instant processedAt; + + private ProcessedMessage(String messageId, String consumer, Instant processedAt) { + this.messageId = messageId; + this.consumer = consumer; + this.processedAt = processedAt; + } + + public static ProcessedMessage of(String messageId, String consumer) { + return new ProcessedMessage(messageId, consumer, Instant.now()); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/common/messaging/domain/ProcessedMessageRepository.java b/backend/src/main/java/com/stackup/stackup/common/messaging/domain/ProcessedMessageRepository.java new file mode 100644 index 00000000..9b2ad3ed --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/common/messaging/domain/ProcessedMessageRepository.java @@ -0,0 +1,8 @@ +package com.stackup.stackup.common.messaging.domain; + +import org.springframework.data.jpa.repository.JpaRepository; + +public interface ProcessedMessageRepository extends JpaRepository { + + boolean existsById(String messageId); +} diff --git a/backend/src/main/java/com/stackup/stackup/common/security/InternalApiKeyAuthenticationFilter.java b/backend/src/main/java/com/stackup/stackup/common/security/InternalApiKeyAuthenticationFilter.java new file mode 100644 index 00000000..badcb5ba --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/common/security/InternalApiKeyAuthenticationFilter.java @@ -0,0 +1,70 @@ +package com.stackup.stackup.common.security; + +import com.stackup.stackup.common.config.properties.SecurityProperties; +import com.stackup.stackup.common.exception.ApiErrorCode; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +// 유효한 경우 INTERNAL 권한을 가진 인증객체를 SecurityContext에 주입함 +// 그거 아니면 패스 - jwt가 알아서한다 +@Component +public class InternalApiKeyAuthenticationFilter extends OncePerRequestFilter { + + public static final String HEADER = "X-Internal-API-Key"; + public static final String PRINCIPAL = "internal"; + public static final String AUTHORITY = "INTERNAL"; + + private final SecurityProperties properties; + + public InternalApiKeyAuthenticationFilter(SecurityProperties properties) { + this.properties = properties; + } + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain chain) + throws ServletException, IOException { + if (!request.getRequestURI().startsWith("/api/internal/")) { + chain.doFilter(request, response); + return; + } + + String provided = request.getHeader(HEADER); + String expected = properties.internalApiKey(); + if (expected == null || expected.isBlank() || provided == null || !constantTimeEquals(provided, expected)) { + // 인증 실패 — 권한 부여 X. SecurityConfig 에서 hasAuthority("INTERNAL") 로 차단 후 401. + request.setAttribute(JwtAuthenticationFilter.AUTH_ERROR_CODE_ATTRIBUTE, ApiErrorCode.INTERNAL_AUTH_FAILED); + chain.doFilter(request, response); + return; + } + + UsernamePasswordAuthenticationToken auth = new UsernamePasswordAuthenticationToken( + PRINCIPAL, + null, + List.of(new SimpleGrantedAuthority(AUTHORITY)) + ); + SecurityContextHolder.getContext().setAuthentication(auth); + chain.doFilter(request, response); + } + + private static boolean constantTimeEquals(String a, String b) { + byte[] x = a.getBytes(); + byte[] y = b.getBytes(); + if (x.length != y.length) { + return false; + } + int diff = 0; + for (int i = 0; i < x.length; i++) { + diff |= x[i] ^ y[i]; + } + return diff == 0; + } +} diff --git a/backend/src/main/java/com/stackup/stackup/common/security/SecurityConfig.java b/backend/src/main/java/com/stackup/stackup/common/security/SecurityConfig.java index 2995f5f2..7bb5619a 100644 --- a/backend/src/main/java/com/stackup/stackup/common/security/SecurityConfig.java +++ b/backend/src/main/java/com/stackup/stackup/common/security/SecurityConfig.java @@ -29,13 +29,16 @@ public class SecurityConfig { private static final Logger log = LoggerFactory.getLogger(SecurityConfig.class); private final JwtAuthenticationFilter jwtAuthenticationFilter; + private final InternalApiKeyAuthenticationFilter internalApiKeyAuthenticationFilter; private final CorsProperties corsProperties; public SecurityConfig( JwtAuthenticationFilter jwtAuthenticationFilter, + InternalApiKeyAuthenticationFilter internalApiKeyAuthenticationFilter, CorsProperties corsProperties ) { this.jwtAuthenticationFilter = jwtAuthenticationFilter; + this.internalApiKeyAuthenticationFilter = internalApiKeyAuthenticationFilter; this.corsProperties = corsProperties; } @@ -63,9 +66,11 @@ public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Excepti "/api/swagger-ui.html", "/actuator/**" ).permitAll() + .requestMatchers("/api/internal/**").hasAuthority(InternalApiKeyAuthenticationFilter.AUTHORITY) .requestMatchers("/api/**").authenticated() .anyRequest().permitAll() ) + .addFilterBefore(internalApiKeyAuthenticationFilter, UsernamePasswordAuthenticationFilter.class) .addFilterBefore(jwtAuthenticationFilter, UsernamePasswordAuthenticationFilter.class) .build(); } diff --git a/backend/src/main/java/com/stackup/stackup/document/application/AnalysisCallbackService.java b/backend/src/main/java/com/stackup/stackup/document/application/AnalysisCallbackService.java new file mode 100644 index 00000000..b7717705 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/AnalysisCallbackService.java @@ -0,0 +1,139 @@ +package com.stackup.stackup.document.application; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.stackup.stackup.common.messaging.domain.ProcessedMessage; +import com.stackup.stackup.common.messaging.domain.ProcessedMessageRepository; +import com.stackup.stackup.common.sse.SseEventPublisher; +import com.stackup.stackup.common.sse.SseEventType; +import com.stackup.stackup.document.application.dto.AnalysisCallbackEnvelope; +import com.stackup.stackup.document.application.dto.AnalysisCallbackPayload; +import com.stackup.stackup.document.domain.AnalyzedDocument; +import com.stackup.stackup.document.domain.AnalyzedDocumentRepository; +import com.stackup.stackup.github.domain.GithubRepository; +import com.stackup.stackup.resume.domain.Resume; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class AnalysisCallbackService { + + private static final Logger log = LoggerFactory.getLogger(AnalysisCallbackService.class); + private static final String CONSUMER_NAME = "core.callback.analysis"; + + private static final JsonMapper JSON = JsonMapper.builder().build(); + + private final AnalyzedDocumentRepository documentRepository; + private final ProcessedMessageRepository processedMessageRepository; + private final SseEventPublisher sseEventPublisher; + + @Transactional + public void apply(AnalysisCallbackEnvelope envelope) { + if (envelope == null || envelope.payload() == null) { + log.warn("callback.analysis envelope or payload is null — skip"); + return; + } + AnalysisCallbackPayload payload = envelope.payload(); + if (isProcessed(envelope.messageId())) { + log.info("callback.analysis duplicate, skip. messageId={}", envelope.messageId()); + return; + } + log.info("callback.analysis received. messageId={}, targetType={}, targetId={}, status={}, traceId={}", + envelope.messageId(), payload.targetType(), payload.targetId(), payload.status(), envelope.traceId()); + + Long documentId = envelope.context() == null ? null : envelope.context().documentId(); + if (documentId == null) { + log.warn("callback.analysis missing documentId in context. messageId={}", envelope.messageId()); + markProcessed(envelope.messageId()); + return; + } + + AnalyzedDocument doc = documentRepository.findById(documentId).orElse(null); + if (doc == null) { + log.warn("callback.analysis document not found. id={}, messageId={}", documentId, envelope.messageId()); + markProcessed(envelope.messageId()); + return; + } + + if (payload.isAnalyzed()) { + applyAnalyzed(doc, payload); + } else if (payload.isFailed()) { + applyFailed(doc, payload); + } else { + log.warn("callback.analysis unknown status={}, treat as FAILED. messageId={}", + payload.status(), envelope.messageId()); + applyFailed(doc, payload); + } + publishSse(doc, payload); + markProcessed(envelope.messageId()); + } + + private void applyAnalyzed(AnalyzedDocument doc, AnalysisCallbackPayload payload) { + String techStackJson = serializeTechStack(payload.techStack()); + int chunkCount = payload.embeddingChunkCount() == null ? 0 : payload.embeddingChunkCount(); + doc.markAnalyzed(payload.documentPath(), payload.summary(), techStackJson, chunkCount); + + Resume resume = doc.getResume(); + if (resume != null) { + resume.markAnalyzed(); + } + GithubRepository repo = doc.getRepository(); + if (repo != null) { + repo.markAnalyzed(); + } + } + + private void applyFailed(AnalyzedDocument doc, AnalysisCallbackPayload payload) { + doc.markFailed(payload.errorCode(), payload.errorMessage()); + Resume resume = doc.getResume(); + if (resume != null) { + resume.markFailed(); + } + GithubRepository repo = doc.getRepository(); + if (repo != null) { + repo.markFailed(); + } + } + + private void publishSse(AnalyzedDocument doc, AnalysisCallbackPayload payload) { + SseEventType type = doc.getRepository() != null ? SseEventType.REPO_STATE : SseEventType.DOC_STATE; + sseEventPublisher.publishToDocument(doc.getId(), type, payload); + } + + private boolean isProcessed(String messageId) { + if (messageId == null || messageId.isBlank()) { + return false; + } + return processedMessageRepository.existsById(messageId); + } + + private void markProcessed(String messageId) { + if (messageId == null || messageId.isBlank()) { + return; + } + try { + processedMessageRepository.save(ProcessedMessage.of(messageId, CONSUMER_NAME)); + } catch (DataIntegrityViolationException ignored) { + // race: 다른 worker 가 동시에 INSERT. 둘 다 같은 작업 처리 → 무시. + } + } + + private String serializeTechStack(List techStack) { + if (techStack == null) { + return null; + } + try { + return JSON.writeValueAsString(techStack); + } catch (JsonProcessingException e) { + log.warn("techStack serialize failed, store as empty array", e); + return "[]"; + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/application/AnalysisRequestService.java b/backend/src/main/java/com/stackup/stackup/document/application/AnalysisRequestService.java new file mode 100644 index 00000000..b7043bcc --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/AnalysisRequestService.java @@ -0,0 +1,105 @@ +package com.stackup.stackup.document.application; + +import com.stackup.stackup.common.config.properties.RabbitMqProperties; +import com.stackup.stackup.common.messaging.MessageContext; +import com.stackup.stackup.common.messaging.RabbitMessagePublisher; +import com.stackup.stackup.document.application.dto.AnalyzeRepositoryPayload; +import com.stackup.stackup.document.application.dto.AnalyzeResumePayload; +import com.stackup.stackup.document.domain.AnalyzedDocument; +import com.stackup.stackup.document.domain.AnalyzedDocumentRepository; +import com.stackup.stackup.github.domain.GithubRepository; +import com.stackup.stackup.github.domain.GithubRepositoryRepository; +import com.stackup.stackup.resume.domain.Resume; +import com.stackup.stackup.resume.domain.ResumeRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; + +// 이력서 리포 분석 트리거 +// CRUD 및 상태관리 수행 +// 클래스 레벨 @Transactional 은 두지 않음 — @TransactionalEventListener 메서드와 충돌 (Spring 7 제약). +@Service +@RequiredArgsConstructor +public class AnalysisRequestService { + + private final ResumeRepository resumeRepository; + private final GithubRepositoryRepository githubRepositoryRepository; + private final AnalyzedDocumentRepository analyzedDocumentRepository; + private final RabbitMessagePublisher publisher; + private final RabbitMqProperties properties; + private final ApplicationEventPublisher events; + + @Transactional + public AnalysisHandle requestResumeAnalysis(Long userId, Long resumeId) { + Resume resume = resumeRepository.findById(resumeId) + .orElseThrow(() -> new IllegalArgumentException("resume not found: " + resumeId)); + if (!resume.getUser().getId().equals(userId)) { + throw new IllegalArgumentException("resume does not belong to user"); + } + AnalyzedDocument doc = analyzedDocumentRepository.save(AnalyzedDocument.forResume(resume)); + resume.markAnalyzing(); + + events.publishEvent(new ResumeAnalysisRequestedEvent( + userId, + doc.getId(), + new AnalyzeResumePayload(resume.getId(), resume.getFilePath(), doc.getId()) + )); + return new AnalysisHandle(doc.getId(), resume.getId(), null); + } + + @Transactional + public AnalysisHandle requestRepositoryAnalysis(Long userId, Long repositoryId) { + GithubRepository repo = githubRepositoryRepository.findById(repositoryId) + .orElseThrow(() -> new IllegalArgumentException("repository not found: " + repositoryId)); + if (!repo.getUser().getId().equals(userId)) { + throw new IllegalArgumentException("repository does not belong to user"); + } + AnalyzedDocument doc = analyzedDocumentRepository.save(AnalyzedDocument.forRepository(repo)); + repo.markAnalyzing(); + + events.publishEvent(new RepositoryAnalysisRequestedEvent( + userId, + doc.getId(), + new AnalyzeRepositoryPayload( + repo.getId(), + repo.getRepoFullName(), + repo.getDefaultBranch() == null ? "main" : repo.getDefaultBranch(), + doc.getId() + ) + )); + return new AnalysisHandle(doc.getId(), null, repo.getId()); + } + + @Transactional(propagation = Propagation.NOT_SUPPORTED) + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) + public void onResumeAnalysisRequested(ResumeAnalysisRequestedEvent event) { + publisher.publishToAi( + properties.routingKeys().analyzeResume(), + event.payload(), + new MessageContext(event.userId(), null, event.documentId(), null) + ); + } + + @Transactional(propagation = Propagation.NOT_SUPPORTED) + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) + public void onRepositoryAnalysisRequested(RepositoryAnalysisRequestedEvent event) { + publisher.publishToAi( + properties.routingKeys().analyzeRepository(), + event.payload(), + new MessageContext(event.userId(), null, event.documentId(), event.payload().repositoryId()) + ); + } + + public record AnalysisHandle(Long analyzedDocumentId, Long resumeId, Long repositoryId) { + } + + record ResumeAnalysisRequestedEvent(Long userId, Long documentId, AnalyzeResumePayload payload) { + } + + record RepositoryAnalysisRequestedEvent(Long userId, Long documentId, AnalyzeRepositoryPayload payload) { + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/application/DocumentEmbeddingService.java b/backend/src/main/java/com/stackup/stackup/document/application/DocumentEmbeddingService.java new file mode 100644 index 00000000..4057dc4a --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/DocumentEmbeddingService.java @@ -0,0 +1,32 @@ +package com.stackup.stackup.document.application; + +import com.stackup.stackup.common.exception.ApiErrorCode; +import com.stackup.stackup.common.exception.DomainException; +import com.stackup.stackup.document.application.dto.EmbeddingUpsertCommand; +import com.stackup.stackup.document.domain.AnalyzedDocumentRepository; +import com.stackup.stackup.document.domain.DocumentEmbeddingRepository; +import com.stackup.stackup.document.domain.DocumentEmbeddingRepository.EmbeddingChunk; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class DocumentEmbeddingService { + + private final AnalyzedDocumentRepository documentRepository; + private final DocumentEmbeddingRepository embeddingRepository; + + @Transactional + public int upsert(EmbeddingUpsertCommand command) { + if (!documentRepository.existsById(command.documentId())) { + throw new DomainException(ApiErrorCode.DOC_NOT_FOUND); + } + List mapped = command.chunks().stream() + .map(c -> new EmbeddingChunk(c.chunkIndex(), c.chunkText(), c.embedding())) + .toList(); + return embeddingRepository.upsertAll(command.documentId(), command.model(), mapped); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalysisCallbackEnvelope.java b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalysisCallbackEnvelope.java new file mode 100644 index 00000000..ef8687f6 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalysisCallbackEnvelope.java @@ -0,0 +1,16 @@ +package com.stackup.stackup.document.application.dto; + +import com.stackup.stackup.common.messaging.MessageContext; +import java.time.Instant; + +public record AnalysisCallbackEnvelope( + String messageId, + String messageType, + String version, + String traceId, + Instant publishedAt, + String publisher, + AnalysisCallbackPayload payload, + MessageContext context +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalysisCallbackPayload.java b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalysisCallbackPayload.java new file mode 100644 index 00000000..ec03a8da --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalysisCallbackPayload.java @@ -0,0 +1,26 @@ +package com.stackup.stackup.document.application.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record AnalysisCallbackPayload( + String targetType, + Long targetId, + String status, + String summary, + List techStack, + String documentPath, + Integer embeddingChunkCount, + String errorCode, + String errorMessage, + Boolean retriable +) { + public boolean isAnalyzed() { + return "ANALYZED".equals(status); + } + + public boolean isFailed() { + return "FAILED".equals(status); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalyzeRepositoryPayload.java b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalyzeRepositoryPayload.java new file mode 100644 index 00000000..f6184a17 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalyzeRepositoryPayload.java @@ -0,0 +1,9 @@ +package com.stackup.stackup.document.application.dto; + +public record AnalyzeRepositoryPayload( + Long repositoryId, + String repoFullName, + String defaultBranch, + Long analyzedDocumentId +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalyzeResumePayload.java b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalyzeResumePayload.java new file mode 100644 index 00000000..ea832950 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/dto/AnalyzeResumePayload.java @@ -0,0 +1,8 @@ +package com.stackup.stackup.document.application.dto; + +public record AnalyzeResumePayload( + Long resumeId, + String filePath, + Long analyzedDocumentId +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/document/application/dto/EmbeddingUpsertCommand.java b/backend/src/main/java/com/stackup/stackup/document/application/dto/EmbeddingUpsertCommand.java new file mode 100644 index 00000000..1fc03a9f --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/application/dto/EmbeddingUpsertCommand.java @@ -0,0 +1,13 @@ +package com.stackup.stackup.document.application.dto; + +import java.util.List; + +public record EmbeddingUpsertCommand( + long documentId, + String model, + int dim, + List chunks +) { + public record Chunk(int chunkIndex, String chunkText, float[] embedding) { + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/domain/AnalysisStatus.java b/backend/src/main/java/com/stackup/stackup/document/domain/AnalysisStatus.java new file mode 100644 index 00000000..7f7688f5 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/domain/AnalysisStatus.java @@ -0,0 +1,7 @@ +package com.stackup.stackup.document.domain; + +public enum AnalysisStatus { + PROCESSING, + ANALYZED, + FAILED +} diff --git a/backend/src/main/java/com/stackup/stackup/document/domain/AnalyzedDocument.java b/backend/src/main/java/com/stackup/stackup/document/domain/AnalyzedDocument.java index 10cdbd11..6fefdd59 100644 --- a/backend/src/main/java/com/stackup/stackup/document/domain/AnalyzedDocument.java +++ b/backend/src/main/java/com/stackup/stackup/document/domain/AnalyzedDocument.java @@ -18,6 +18,8 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.NoArgsConstructor; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; @Getter @Entity @@ -43,16 +45,65 @@ public class AnalyzedDocument extends BaseSoftDeleteEntity { @JoinColumn(name = "repository_id") private GithubRepository repository; - @Column(name = "document_path", nullable = false, length = 1000) + @Column(name = "document_path", length = 1000) private String documentPath; @Column(length = 2000) private String summary; @Column(name = "tech_stack", columnDefinition = "jsonb") + @JdbcTypeCode(SqlTypes.JSON) private String techStack; @Column(nullable = false, length = 20) @Enumerated(EnumType.STRING) private DocumentStatus status = DocumentStatus.ACTIVE; + + @Column(name = "analysis_status", nullable = false, length = 20) + @Enumerated(EnumType.STRING) + private AnalysisStatus analysisStatus = AnalysisStatus.PROCESSING; + + @Column(name = "error_code", length = 50) + private String errorCode; + + @Column(name = "error_message", length = 1000) + private String errorMessage; + + @Column(name = "embedding_chunk_count", nullable = false) + private int embeddingChunkCount = 0; + + private AnalyzedDocument(Resume resume, GithubRepository repository) { + this.resume = resume; + this.repository = repository; + } + + public static AnalyzedDocument forResume(Resume resume) { + if (resume == null) { + throw new IllegalArgumentException("resume must not be null"); + } + return new AnalyzedDocument(resume, null); + } + + public static AnalyzedDocument forRepository(GithubRepository repository) { + if (repository == null) { + throw new IllegalArgumentException("repository must not be null"); + } + return new AnalyzedDocument(null, repository); + } + + public void markAnalyzed(String documentPath, String summary, String techStackJson, int embeddingChunkCount) { + this.analysisStatus = AnalysisStatus.ANALYZED; + this.documentPath = documentPath; + this.summary = summary; + this.techStack = techStackJson; + this.embeddingChunkCount = embeddingChunkCount; + this.errorCode = null; + this.errorMessage = null; + } + + public void markFailed(String errorCode, String errorMessage) { + this.analysisStatus = AnalysisStatus.FAILED; + this.errorCode = errorCode; + this.errorMessage = errorMessage; + } } diff --git a/backend/src/main/java/com/stackup/stackup/document/domain/DocumentEmbeddingRepository.java b/backend/src/main/java/com/stackup/stackup/document/domain/DocumentEmbeddingRepository.java new file mode 100644 index 00000000..fbb91d98 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/domain/DocumentEmbeddingRepository.java @@ -0,0 +1,13 @@ +package com.stackup.stackup.document.domain; + +import java.util.List; + +public interface DocumentEmbeddingRepository { + + int upsertAll(long documentId, String model, List chunks); + + int countByDocumentId(long documentId); + + record EmbeddingChunk(int chunkIndex, String chunkText, float[] embedding) { + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/infrastructure/AnalysisCallbackHandler.java b/backend/src/main/java/com/stackup/stackup/document/infrastructure/AnalysisCallbackHandler.java new file mode 100644 index 00000000..d6c97cf9 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/infrastructure/AnalysisCallbackHandler.java @@ -0,0 +1,30 @@ +package com.stackup.stackup.document.infrastructure; + +import com.stackup.stackup.document.application.AnalysisCallbackService; +import com.stackup.stackup.document.application.dto.AnalysisCallbackEnvelope; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Component; + +// 도메인 갱신은 application 서비스가 처리 +@Component +@RequiredArgsConstructor +public class AnalysisCallbackHandler { + + private static final Logger log = LoggerFactory.getLogger(AnalysisCallbackHandler.class); + + private final AnalysisCallbackService callbackService; + + @RabbitListener(queues = "${app.messaging.rabbitmq.queues.names.core-callback-analysis}") + public void handle(AnalysisCallbackEnvelope envelope) { + try { + callbackService.apply(envelope); + } catch (RuntimeException e) { + log.error("callback.analysis processing failed. messageId={}", + envelope == null ? null : envelope.messageId(), e); + throw e; + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/infrastructure/JdbcDocumentEmbeddingRepository.java b/backend/src/main/java/com/stackup/stackup/document/infrastructure/JdbcDocumentEmbeddingRepository.java new file mode 100644 index 00000000..e299ab06 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/infrastructure/JdbcDocumentEmbeddingRepository.java @@ -0,0 +1,74 @@ +package com.stackup.stackup.document.infrastructure; + +import com.stackup.stackup.document.domain.DocumentEmbeddingRepository; +import java.sql.PreparedStatement; +import java.util.List; +import javax.sql.DataSource; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + + +// 네이티브 쿼리가 더 쉬워서 JPA 대신 씀 +@Repository +public class JdbcDocumentEmbeddingRepository implements DocumentEmbeddingRepository { + + private static final String UPSERT_SQL = """ + INSERT INTO document_embeddings (document_id, chunk_index, chunk_text, embedding, model) + VALUES (?, ?, ?, ?::vector, ?) + ON CONFLICT (document_id, chunk_index) + DO UPDATE SET + chunk_text = EXCLUDED.chunk_text, + embedding = EXCLUDED.embedding, + model = EXCLUDED.model + """; + + private static final String COUNT_SQL = + "SELECT count(*) FROM document_embeddings WHERE document_id = ?"; + + private final JdbcTemplate jdbc; + + public JdbcDocumentEmbeddingRepository(DataSource dataSource) { + this.jdbc = new JdbcTemplate(dataSource); + } + + @Override + public int upsertAll(long documentId, String model, List chunks) { + if (chunks == null || chunks.isEmpty()) { + return 0; + } + int[][] perBatch = jdbc.batchUpdate(UPSERT_SQL, chunks, chunks.size(), + (PreparedStatement ps, EmbeddingChunk c) -> { + ps.setLong(1, documentId); + ps.setInt(2, c.chunkIndex()); + ps.setString(3, c.chunkText()); + ps.setString(4, toVectorLiteral(c.embedding())); + ps.setString(5, model); + }); + int total = 0; + for (int[] batch : perBatch) { + for (int n : batch) { + total += Math.max(n, 0); + } + } + return total == 0 ? chunks.size() : total; + } + + @Override + public int countByDocumentId(long documentId) { + Integer n = jdbc.queryForObject(COUNT_SQL, Integer.class, documentId); + return n == null ? 0 : n; + } + + private static String toVectorLiteral(float[] embedding) { + StringBuilder sb = new StringBuilder(embedding.length * 8 + 2); + sb.append('['); + for (int i = 0; i < embedding.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(embedding[i]); + } + sb.append(']'); + return sb.toString(); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/presentation/InternalAnalysisTriggerController.java b/backend/src/main/java/com/stackup/stackup/document/presentation/InternalAnalysisTriggerController.java new file mode 100644 index 00000000..d13efedc --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/presentation/InternalAnalysisTriggerController.java @@ -0,0 +1,39 @@ +package com.stackup.stackup.document.presentation; + +import com.stackup.stackup.document.application.AnalysisRequestService; +import com.stackup.stackup.document.application.AnalysisRequestService.AnalysisHandle; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +// 운영환경에서는 AnalysisRequestService 직접 호출 +@RestController +@RequestMapping("/api/internal/analyses") +@RequiredArgsConstructor +public class InternalAnalysisTriggerController { + + private final AnalysisRequestService service; + + @PostMapping("/resume/{resumeId}") + public AnalysisHandle triggerResume( + @PathVariable Long resumeId, + @RequestBody TriggerRequest request + ) { + return service.requestResumeAnalysis(request.userId(), resumeId); + } + + @PostMapping("/repository/{repositoryId}") + public AnalysisHandle triggerRepository( + @PathVariable Long repositoryId, + @RequestBody TriggerRequest request + ) { + return service.requestRepositoryAnalysis(request.userId(), repositoryId); + } + + public record TriggerRequest(@NotNull Long userId) { + } +} diff --git a/backend/src/main/java/com/stackup/stackup/document/presentation/InternalDocumentController.java b/backend/src/main/java/com/stackup/stackup/document/presentation/InternalDocumentController.java new file mode 100644 index 00000000..45eeac62 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/presentation/InternalDocumentController.java @@ -0,0 +1,66 @@ +package com.stackup.stackup.document.presentation; + +import com.stackup.stackup.common.exception.ApiErrorCode; +import com.stackup.stackup.common.exception.DomainException; +import com.stackup.stackup.document.application.DocumentEmbeddingService; +import com.stackup.stackup.document.application.dto.EmbeddingUpsertCommand; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Positive; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * AI 서버 전용. /api/internal/documents/{documentId}/embeddings. + * 청크 + 임베딩을 pgvector 에 idempotent upsert. + */ +@RestController +@RequestMapping("/api/internal/documents") +@RequiredArgsConstructor +public class InternalDocumentController { + + private final DocumentEmbeddingService embeddingService; + + @PutMapping("/{documentId}/embeddings") + public UpsertResponse upsert( + @PathVariable Long documentId, + @Valid @RequestBody UpsertRequest request + ) { + for (ChunkRequest c : request.chunks()) { + if (c.embedding() == null || c.embedding().length != request.dim()) { + throw new DomainException(ApiErrorCode.EMBEDDING_BAD_REQUEST); + } + } + + List mapped = request.chunks().stream() + .map(c -> new EmbeddingUpsertCommand.Chunk(c.chunkIndex(), c.chunkText(), c.embedding())) + .toList(); + int upserted = embeddingService.upsert( + new EmbeddingUpsertCommand(documentId, request.model(), request.dim(), mapped) + ); + return new UpsertResponse(upserted); + } + + public record UpsertRequest( + String model, + @Positive int dim, + @NotNull @NotEmpty List<@Valid ChunkRequest> chunks + ) { + } + + public record ChunkRequest( + int chunkIndex, + String chunkText, + float[] embedding + ) { + } + + public record UpsertResponse(int upserted) { + } +} diff --git a/backend/src/main/java/com/stackup/stackup/github/application/InternalGithubTokenService.java b/backend/src/main/java/com/stackup/stackup/github/application/InternalGithubTokenService.java new file mode 100644 index 00000000..4597ef5e --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/github/application/InternalGithubTokenService.java @@ -0,0 +1,25 @@ +package com.stackup.stackup.github.application; + +import com.stackup.stackup.common.exception.ApiErrorCode; +import com.stackup.stackup.common.exception.DomainException; +import com.stackup.stackup.github.infrastructure.GithubTokenCipher; +import com.stackup.stackup.user.domain.User; +import com.stackup.stackup.user.domain.UserRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class InternalGithubTokenService { + + private final UserRepository userRepository; + private final GithubTokenCipher tokenCipher; + + public String fetchPlainAccessToken(Long userId) { + User user = userRepository.findById(userId) + .orElseThrow(() -> new DomainException(ApiErrorCode.USER_NOT_FOUND)); + return tokenCipher.decrypt(user.getEncryptedGithubAccessToken()); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/github/domain/GithubRepository.java b/backend/src/main/java/com/stackup/stackup/github/domain/GithubRepository.java index 0e1e0b61..c589806d 100644 --- a/backend/src/main/java/com/stackup/stackup/github/domain/GithubRepository.java +++ b/backend/src/main/java/com/stackup/stackup/github/domain/GithubRepository.java @@ -64,4 +64,17 @@ public class GithubRepository extends BaseSoftDeleteEntity { @Column(name = "last_synced_at") private Instant lastSyncedAt; + + public void markAnalyzing() { + this.status = RepositoryStatus.ANALYZING; + } + + public void markAnalyzed() { + this.status = RepositoryStatus.ANALYZED; + this.lastSyncedAt = Instant.now(); + } + + public void markFailed() { + this.status = RepositoryStatus.FAILED; + } } diff --git a/backend/src/main/java/com/stackup/stackup/github/presentation/InternalGithubTokenController.java b/backend/src/main/java/com/stackup/stackup/github/presentation/InternalGithubTokenController.java new file mode 100644 index 00000000..a82450d3 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/github/presentation/InternalGithubTokenController.java @@ -0,0 +1,25 @@ +package com.stackup.stackup.github.presentation; + +import com.stackup.stackup.github.application.InternalGithubTokenService; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +// AI 서버 전용 +@RestController +@RequestMapping("/api/internal/users") +@RequiredArgsConstructor +public class InternalGithubTokenController { + + private final InternalGithubTokenService tokenService; + + @GetMapping("/{userId}/github-token") + public GithubTokenResponse fetchGithubToken(@PathVariable Long userId) { + return new GithubTokenResponse(tokenService.fetchPlainAccessToken(userId)); + } + + public record GithubTokenResponse(String accessToken) { + } +} diff --git a/backend/src/main/java/com/stackup/stackup/resume/domain/Resume.java b/backend/src/main/java/com/stackup/stackup/resume/domain/Resume.java index bd07ad0a..d86197b8 100644 --- a/backend/src/main/java/com/stackup/stackup/resume/domain/Resume.java +++ b/backend/src/main/java/com/stackup/stackup/resume/domain/Resume.java @@ -53,4 +53,16 @@ public class Resume extends BaseSoftDeleteEntity { @Column(nullable = false, length = 20) @Enumerated(EnumType.STRING) private ResumeStatus status = ResumeStatus.PENDING; + + public void markAnalyzing() { + this.status = ResumeStatus.ANALYZING; + } + + public void markAnalyzed() { + this.status = ResumeStatus.ANALYZED; + } + + public void markFailed() { + this.status = ResumeStatus.FAILED; + } } diff --git a/backend/src/main/java/com/stackup/stackup/user/application/UserService.java b/backend/src/main/java/com/stackup/stackup/user/application/UserService.java index d42925d3..78d80ba5 100644 --- a/backend/src/main/java/com/stackup/stackup/user/application/UserService.java +++ b/backend/src/main/java/com/stackup/stackup/user/application/UserService.java @@ -8,8 +8,15 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +// record 는 final → Spring AOP CGLIB 프록시 실패. class 로 전환. @Service -public record UserService(UserRepository userRepository) { +public class UserService { + + private final UserRepository userRepository; + + public UserService(UserRepository userRepository) { + this.userRepository = userRepository; + } @Transactional(readOnly = true) public UserProfileResult getCurrentUser(Long userId) { diff --git a/backend/src/main/resources/application-local.yml b/backend/src/main/resources/application-local.yml index 7653cec3..f01e43a5 100644 --- a/backend/src/main/resources/application-local.yml +++ b/backend/src/main/resources/application-local.yml @@ -13,6 +13,7 @@ app: security: jwt-secret: ${JWT_SECRET:local-development-jwt-secret-must-be-replaced} encryption-key: ${ENCRYPTION_KEY:local-development-encryption-key-must-be-replaced} + internal-api-key: ${CORE_INTERNAL_API_KEY:local-development-internal-api-key} github: client-id: ${GITHUB_OAUTH_CLIENT_ID:local-github-client-id} client-secret: ${GITHUB_OAUTH_CLIENT_SECRET:local-github-client-secret} diff --git a/backend/src/main/resources/application-test.yml b/backend/src/main/resources/application-test.yml index 5fdfeacf..2124d230 100644 --- a/backend/src/main/resources/application-test.yml +++ b/backend/src/main/resources/application-test.yml @@ -12,6 +12,7 @@ app: security: jwt-secret: test-jwt-secret encryption-key: test-encryption-key + internal-api-key: test-internal-api-key github: client-id: test-github-client-id client-secret: test-github-client-secret diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index 3ce77217..a6a78e60 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -50,6 +50,7 @@ app: refresh-token-cookie-same-site: ${REFRESH_TOKEN_COOKIE_SAME_SITE:Strict} refresh-token-cookie-secure: ${REFRESH_TOKEN_COOKIE_SECURE:true} refresh-token-cookie-http-only: ${REFRESH_TOKEN_COOKIE_HTTP_ONLY:true} + internal-api-key: ${CORE_INTERNAL_API_KEY:} github: client-id: ${GITHUB_OAUTH_CLIENT_ID:} client-secret: ${GITHUB_OAUTH_CLIENT_SECRET:} @@ -85,6 +86,7 @@ app: names: core-to-ai: stackup.core-to-ai ai-to-core: stackup.ai-to-core + realtime: stackup.realtime queues: durable: true names: @@ -101,6 +103,7 @@ app: generate-followup: generate.followup callback-analysis: callback.analysis callback-questions: callback.questions + realtime-session-notify: realtime.session.notify sse: timeout: 30m keep-alive-interval: 30s diff --git a/backend/src/main/resources/db/migration/V4__analysis_pipeline.sql b/backend/src/main/resources/db/migration/V4__analysis_pipeline.sql new file mode 100644 index 00000000..74d5724d --- /dev/null +++ b/backend/src/main/resources/db/migration/V4__analysis_pipeline.sql @@ -0,0 +1,61 @@ +-- pgvector 확장은 infra/postgres/init.sql 에서 보장 (Docker 이미지 init) +-- 미보장 환경에서도 안전하도록 IF NOT EXISTS +CREATE EXTENSION IF NOT EXISTS vector; + +-- ============================================================================= +-- analyzed_documents: 분석 라이프사이클 컬럼 추가 +-- ============================================================================= +-- document_path 는 publish 시점에 비어있고 callback ANALYZED 시 채워짐 +ALTER TABLE analyzed_documents + ALTER COLUMN document_path DROP NOT NULL; + +ALTER TABLE analyzed_documents + ADD COLUMN analysis_status VARCHAR(20) NOT NULL DEFAULT 'PROCESSING', + ADD COLUMN error_code VARCHAR(50), + ADD COLUMN error_message VARCHAR(1000), + ADD COLUMN embedding_chunk_count INT NOT NULL DEFAULT 0; + +ALTER TABLE analyzed_documents + ADD CONSTRAINT chk_analyzed_documents_analysis_status + CHECK (analysis_status IN ('PROCESSING', 'ANALYZED', 'FAILED')); + +CREATE INDEX idx_analyzed_documents_analysis_status + ON analyzed_documents (analysis_status); + +-- ============================================================================= +-- document_embeddings: 청크 + pgvector 임베딩 +-- ============================================================================= +CREATE TABLE document_embeddings ( + id BIGSERIAL PRIMARY KEY, + document_id BIGINT NOT NULL REFERENCES analyzed_documents (id) ON DELETE CASCADE, + chunk_index INT NOT NULL, + chunk_text TEXT NOT NULL, + embedding vector(1536) NOT NULL, + model VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_document_embeddings_document_chunk + UNIQUE (document_id, chunk_index) +); + +CREATE INDEX idx_document_embeddings_document_id + ON document_embeddings (document_id); + +-- ANN 인덱스 (ivfflat). lists 는 데이터 양에 따라 튜닝. +-- 초기에는 lists=100 (수만 row 가정). 추후 ANALYZE 후 재생성. +CREATE INDEX idx_document_embeddings_ann + ON document_embeddings USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100); + +-- ============================================================================= +-- processed_messages: 비동기 consumer 멱등성 (RabbitMQ messageId 기록) +-- ============================================================================= +CREATE TABLE processed_messages ( + message_id VARCHAR(64) PRIMARY KEY, + consumer VARCHAR(100) NOT NULL, + processed_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_processed_messages_processed_at + ON processed_messages (processed_at); + +-- 24h 이상 된 row 는 cron 으로 정리 (운영 가이드: docs/messaging.md §6) diff --git a/backend/src/test/java/com/stackup/stackup/StackupApplicationTests.java b/backend/src/test/java/com/stackup/stackup/StackupApplicationTests.java index e1eea7e1..bbe8606e 100644 --- a/backend/src/test/java/com/stackup/stackup/StackupApplicationTests.java +++ b/backend/src/test/java/com/stackup/stackup/StackupApplicationTests.java @@ -2,6 +2,11 @@ import com.stackup.stackup.auth.domain.OAuthStateRepository; import com.stackup.stackup.auth.domain.RefreshTokenRepository; +import com.stackup.stackup.common.messaging.domain.ProcessedMessageRepository; +import com.stackup.stackup.document.domain.AnalyzedDocumentRepository; +import com.stackup.stackup.document.domain.DocumentEmbeddingRepository; +import com.stackup.stackup.github.domain.GithubRepositoryRepository; +import com.stackup.stackup.resume.domain.ResumeRepository; import com.stackup.stackup.user.domain.UserRepository; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; @@ -19,6 +24,21 @@ class StackupApplicationTests { @MockitoBean private UserRepository userRepository; + @MockitoBean + private ResumeRepository resumeRepository; + + @MockitoBean + private GithubRepositoryRepository githubRepositoryRepository; + + @MockitoBean + private AnalyzedDocumentRepository analyzedDocumentRepository; + + @MockitoBean + private ProcessedMessageRepository processedMessageRepository; + + @MockitoBean + private DocumentEmbeddingRepository documentEmbeddingRepository; + @Test void contextLoads() { } diff --git a/backend/src/test/java/com/stackup/stackup/auth/application/OAuthStateServiceTest.java b/backend/src/test/java/com/stackup/stackup/auth/application/OAuthStateServiceTest.java index 79aeaeb4..126fbf72 100644 --- a/backend/src/test/java/com/stackup/stackup/auth/application/OAuthStateServiceTest.java +++ b/backend/src/test/java/com/stackup/stackup/auth/application/OAuthStateServiceTest.java @@ -131,7 +131,8 @@ private static SecurityProperties securityProperties() { "/api/auth", "Strict", true, - true + true, + "test-internal-api-key" ); } } diff --git a/backend/src/test/java/com/stackup/stackup/auth/application/RefreshTokenServiceTest.java b/backend/src/test/java/com/stackup/stackup/auth/application/RefreshTokenServiceTest.java index b2d0de35..74f2d3dd 100644 --- a/backend/src/test/java/com/stackup/stackup/auth/application/RefreshTokenServiceTest.java +++ b/backend/src/test/java/com/stackup/stackup/auth/application/RefreshTokenServiceTest.java @@ -123,7 +123,8 @@ private RefreshTokenService service() { "/api/auth", "Strict", true, - true + true, + "test-internal-api-key" ), new SecureRandom(), Clock.fixed(NOW, ZoneOffset.UTC) diff --git a/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java b/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java index 8d602d07..29fc4a33 100644 --- a/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java +++ b/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java @@ -82,7 +82,7 @@ private RabbitMqProperties rabbitMqProperties() { new RabbitMqProperties.Exchanges( true, false, - new RabbitMqProperties.Exchanges.Names("stackup.core-to-ai", "stackup.ai-to-core") + new RabbitMqProperties.Exchanges.Names("stackup.core-to-ai", "stackup.ai-to-core", "stackup.realtime") ), new RabbitMqProperties.Queues( true, @@ -101,7 +101,8 @@ private RabbitMqProperties rabbitMqProperties() { "generate.questions", "generate.followup", "callback.analysis", - "callback.questions" + "callback.questions", + "realtime.session.notify" ) ); } diff --git a/backend/src/test/java/com/stackup/stackup/github/infrastructure/GithubTokenCipherTest.java b/backend/src/test/java/com/stackup/stackup/github/infrastructure/GithubTokenCipherTest.java index 720b560c..fb4dfdc8 100644 --- a/backend/src/test/java/com/stackup/stackup/github/infrastructure/GithubTokenCipherTest.java +++ b/backend/src/test/java/com/stackup/stackup/github/infrastructure/GithubTokenCipherTest.java @@ -49,7 +49,8 @@ private static SecurityProperties securityProperties(String encryptionKey) { "/api/auth", "Strict", true, - true + true, + "test-internal-api-key" ); } diff --git a/docker-compose.yml b/docker-compose.yml index aa52d5ca..a0c82547 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,6 +87,16 @@ services: LLM_API_KEY: ${LLM_API_KEY:-} LLM_BASE_URL: ${LLM_BASE_URL:-https://factchat-cloud.mindlogic.ai/v1/gateway} LLM_PRO_MODEL: ${LLM_PRO_MODEL:-gemini-3.1-pro-preview} + # AI → Core 내부 API + CORE_INTERNAL_BASE_URL: ${CORE_INTERNAL_BASE_URL:-http://host.docker.internal:38010} + CORE_INTERNAL_API_KEY: ${CORE_INTERNAL_API_KEY:-local-development-internal-api-key} + # 임베딩 + EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-mock} + EMBEDDING_MODEL: ${EMBEDDING_MODEL:-gemini-embedding-001} + EMBEDDING_DIM: ${EMBEDDING_DIM:-1536} + GEMINI_API_KEY: ${GEMINI_API_KEY:-} + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: rabbitmq: condition: service_healthy diff --git a/docs/api-conventions.md b/docs/api-conventions.md index 2a81e0b2..48b83a9b 100644 --- a/docs/api-conventions.md +++ b/docs/api-conventions.md @@ -91,6 +91,17 @@ GET /api/system/health 헬스체크 (PG/MQ/S3/AI 상태) GET /api/system/version 버전 정보 ``` +### 2.6 내부 API (서비스 간 호출 전용) + +`/api/internal/*` prefix. **외부에서 호출 금지** — Spring Security에서 외부 패킷을 차단하고 `X-Internal-API-Key` 헤더로 검증. AI Server와 RealTime Server만 호출자. + +``` +GET /api/internal/users/{userId}/github-token AI → Core +PUT /api/internal/documents/{documentId}/embeddings AI → Core +``` + +자세한 요청·응답 스키마와 인증 규약은 §10 내부 API 부록 참조. + --- ## 3. 요청 규약 @@ -301,3 +312,76 @@ HTTP/1.1 202 Accepted 2. SSE 끊기면 `statusUrl` 폴링 fallback (5초 간격) 상세: [`event-stream.md`](./event-stream.md) + +--- + +## 10. 내부 API 부록 (서비스 간 호출) + +`/api/internal/*` endpoint는 다음 규약을 따른다. + +### 10.1 인증 +- 모든 요청에 `X-Internal-API-Key: {key}` 헤더 필수 +- 키는 환경변수 `CORE_INTERNAL_API_KEY` (Core/AI/RealTime 동일 값) +- JWT 사용자 인증 불필요. 단 호출자가 신뢰 영역 안에 있어야 함 (Docker 내부망, K8s ClusterIP 등) +- 키 누락/불일치 → `401 Unauthorized`, body: `{ "code": "INTERNAL_AUTH_FAILED" }` + +### 10.2 `GET /api/internal/users/{userId}/github-token` + +분석 시점에 사용자 GitHub access token을 짧게 위임. AES-256으로 저장된 평문 token을 복호화해 반환 (메모리/로그/응답 body에만 노출, DB·이벤트엔벨로프에는 절대 동봉하지 않음). + +**Path parameter** +| 이름 | 타입 | 설명 | +|------|------|------| +| `userId` | long | `users.id` | + +**Response 200** +```json +{ "accessToken": "ghp_xxx..." } +``` + +**Response 404** — 사용자 없음 또는 token 누락 (`USER_NOT_FOUND`) +**Response 401/403** — 인증 실패 (`INTERNAL_AUTH_FAILED`) +**Response 5xx** — Core 일시 장애 (호출자는 retriable로 간주) + +### 10.3 `PUT /api/internal/documents/{documentId}/embeddings` + +`analyzed_documents` 한 건에 대한 chunk + embedding을 `document_embeddings` 테이블에 idempotent upsert. AI가 분석 결과를 callback 발행 직전에 호출한다. + +**Path parameter** +| 이름 | 타입 | 설명 | +|------|------|------| +| `documentId` | long | `analyzed_documents.id` (Core가 publish 직전 미리 생성) | + +**Request body** +```json +{ + "model": "gemini-embedding-001", + "dim": 1536, + "chunks": [ + { "chunkIndex": 0, "chunkText": "...", "embedding": [0.012, -0.003, ...] }, + { "chunkIndex": 1, "chunkText": "...", "embedding": [...] } + ] +} +``` + +| 필드 | 타입 | 비고 | +|------|------|------| +| `model` | string | 임베딩 모델 ID (감사용) | +| `dim` | int | 임베딩 차원. DB 컬럼 차원과 일치해야 함 (불일치 → 400) | +| `chunks[].chunkIndex` | int | 0-base, 같은 document 내 unique | +| `chunks[].chunkText` | string | 원문 청크 (≤ N KB, 컬럼 길이 제한) | +| `chunks[].embedding` | float[] | 길이 == `dim` | + +**Response 200** +```json +{ "upserted": 18 } +``` + +**Response 400** — `dim` 불일치, payload 검증 실패 (`EMBEDDING_BAD_REQUEST`) +**Response 404** — `analyzed_documents.id` 없음 (`DOCUMENT_NOT_FOUND`) +**Response 401/403** — `INTERNAL_AUTH_FAILED` +**Response 5xx** — Core 일시 장애 (retriable) + +**Idempotency** +- 같은 `(documentId, chunkIndex)` 조합 재호출 시 row를 덮어씀 (INSERT ... ON CONFLICT UPDATE) +- 부분 재시도 시 누락 청크가 발생하지 않도록 AI는 전체 청크를 한 번에 보내는 것을 권장 diff --git a/docs/environment.md b/docs/environment.md index f28568cb..6c5966c0 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -92,18 +92,55 @@ AI_SERVER_BASE_URL=http://ai:8000 AI_SERVER_PORT=8000 AI_LOG_LEVEL=INFO -# ===== LLM ===== +# ===== LLM (Mindlogic CNU AC Gateway 단일 경로) ===== +LLM_API_KEY= # 학교 발급 키 +LLM_BASE_URL=https://factchat-cloud.mindlogic.ai/v1/gateway +LLM_PRO_MODEL=gemini-3.1-pro-preview +LLM_PRO_TEMPERATURE=0.2 + +# (외부 직접 호출용, fallback) GEMINI_API_KEY= OPENAI_API_KEY= # Whisper STT에도 사용 -LLM_DEFAULT_PROVIDER=gemini # gemini | openai -LLM_PRO_MODEL=gemini-3.1-pro -LLM_FLASH_MODEL=gemini-3.1-flash -# ===== Embedding ===== -EMBEDDING_MODEL=text-embedding-004 -EMBEDDING_DIM=768 # 모델별 차원수 (DB 스키마와 일치 필수) +# ===== Object Storage ===== +STORAGE_BACKEND=s3 # s3 | local +STORAGE_LOCAL_ROOT=./var/storage # backend=local일 때만 +S3_ENDPOINT_URL=http://localhost:38060 +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +S3_BUCKET_NAME=stackup +S3_REGION=us-east-1 -# ===== STT/TTS ===== +# ===== Core 내부 API (github-token, embeddings upsert) ===== +CORE_INTERNAL_BASE_URL=http://localhost:38010 +CORE_INTERNAL_API_KEY= +CORE_INTERNAL_TIMEOUT_SEC=10 + +# ===== GitHub Repo 분석 ===== +GITHUB_API_BASE_URL=https://api.github.com +GITHUB_FALLBACK_TOKEN= # 사용자별 토큰 미발급 시 public 한정 호출 (옵션) +REPO_MAX_SOURCE_FILES=8 +REPO_MAX_SOURCE_FILE_BYTES=50000 +REPO_FETCH_TIMEOUT_SEC=30 + +# ===== Web 이력서 fetch ===== +WEB_FETCH_TIMEOUT_SEC=20 +WEB_MAX_HTML_BYTES=2000000 + +# ===== Embedding ===== +EMBEDDING_PROVIDER=gemini # mock | gemini | openai | ollama (개발/테스트는 mock 권장) +EMBEDDING_MODEL=gemini-embedding-001 +EMBEDDING_DIM=1536 # DB 컬럼 차원과 일치 필수 +EMBEDDING_CHUNK_SIZE=1000 +EMBEDDING_CHUNK_OVERLAP=200 +EMBEDDING_BATCH_SIZE=32 + +# ===== Markdown 산출물 키 템플릿 ===== +ANALYZED_RESUME_MD_KEY_TEMPLATE=analyzed/resume/{resume_id}/summary.md +ANALYZED_REPOSITORY_MD_KEY_TEMPLATE=analyzed/repository/{repository_id}/summary.md +ANALYZED_WEB_RESUME_MD_KEY_TEMPLATE=analyzed/web-resume/{resume_id}/summary.md + +# ===== STT/TTS (Phase 2) ===== STT_PROVIDER=whisper-api # whisper-api | whisper-self-hosted WHISPER_MODEL=whisper-1 # OpenAI Whisper API 모델 TTS_PROVIDER= # 결정 시 추가 @@ -148,7 +185,9 @@ VITE_SENTRY_DSN= # 옵션 | 변수 | 기본값 | 용도 | |------|--------|------| | `RABBITMQ_URL` | `amqp://stackup:stackup@localhost:5672/` | AMQP 연결 | -| `AI_QUEUE_RESUME` | `ai.analyze.resume` | 이력서 분석 큐 | +| `AI_QUEUE_RESUME` | `ai.analyze.resume` | 이력서(PDF) 분석 큐 | +| `AI_QUEUE_REPOSITORY` | `ai.analyze.repository` | 레포 분석 큐 | +| `AI_QUEUE_WEB` | `ai.analyze.web` | 웹 이력서(URL) 분석 큐 | | `AI_QUEUE_PREFETCH` | `10` | consumer prefetch | | `AI_CALLBACK_EXCHANGE` | `stackup.ai-to-core` | 콜백 발행 exchange | | `AI_CALLBACK_ROUTING_ANALYSIS` | `callback.analysis` | 콜백 routing key | diff --git a/docs/messaging.md b/docs/messaging.md index 9a64eb7f..f882608d 100644 --- a/docs/messaging.md +++ b/docs/messaging.md @@ -20,6 +20,7 @@ |-------|----------|-------------|----------| | `ai.analyze.repository` | `stackup.core-to-ai` | `analyze.repository` | AI Server | | `ai.analyze.resume` | `stackup.core-to-ai` | `analyze.resume` | AI Server | +| `ai.analyze.web` | `stackup.core-to-ai` | `analyze.web` | AI Server | | `ai.generate.questions` | `stackup.core-to-ai` | `generate.questions` | AI Server | | `ai.generate.followup` | `stackup.core-to-ai` | `generate.followup` | AI Server | | `core.callback.analysis` | `stackup.ai-to-core` | `callback.analysis` | Core Server | @@ -43,7 +44,7 @@ ``` `action` ∈ `analyze | generate | callback | realtime` -`aggregate` ∈ `resume | repository | questions | followup | analysis | feedback | session` +`aggregate` ∈ `resume | repository | web | questions | followup | analysis | feedback | session` 새 routing key 추가 시 본 패턴 유지. @@ -99,14 +100,15 @@ | Use Case | Request RK | Callback RK | Callback Queue | |----------|------------|-------------|-----------------| -| 이력서 분석 (US-09) | `analyze.resume` | `callback.analysis` | `core.callback.analysis` | +| 이력서(PDF) 분석 (US-09) | `analyze.resume` | `callback.analysis` | `core.callback.analysis` | +| 웹 이력서(URL) 분석 (US-09) | `analyze.web` | `callback.analysis` | `core.callback.analysis` | | 레포 분석 (US-10) | `analyze.repository` | `callback.analysis` | `core.callback.analysis` | | 질문 풀 생성 (US-18) | `generate.questions` | `callback.questions` | `core.callback.questions` | | 꼬리질문 생성 (US-19) | `generate.followup` | `callback.questions` | `core.callback.questions` | | 피드백 생성 (US-24) | `generate.feedback` *(예정)* | `callback.feedback` *(예정)* | `core.callback.feedback` *(예정)* | | 세션 알림 (RT2 SSE) | `realtime.session.notify` | (없음 — 단방향 push) | `q.realtime.session.notify` | -> `callback.analysis` 큐는 resume/repo 두 use case가 공유. consumer는 `payload.targetType` 으로 분기한다. +> `callback.analysis` 큐는 resume/web/repo 세 use case가 공유. consumer는 `payload.targetType` 으로 분기한다. --- @@ -118,45 +120,67 @@ "messageType": "analyze.resume", "payload": { "resumeId": 42, - "s3Key": "resumes/raw/123/abc.pdf" + "filePath": "resumes/raw/123/abc.pdf", + "analyzedDocumentId": 101 }, "context": { "userId": 123 } } ``` +| 필드 | 설명 | +|------|------| +| `resumeId` | `resumes.id` | +| `filePath` | 객체 스토리지 키 (Core가 업로드 시 저장) | +| `analyzedDocumentId` | Core가 publish 직전 `analyzed_documents`에 PROCESSING row를 미리 생성하고 그 id를 전달 — AI는 embedding upsert 시 이 id를 사용 | + ### 5.2 `analyze.repository` ```json { "messageType": "analyze.repository", "payload": { "repositoryId": 7, - "githubFullName": "octocat/hello-world", + "repoFullName": "octocat/hello-world", "defaultBranch": "main", - "githubAccessTokenEncrypted": "..." + "analyzedDocumentId": 102 }, "context": { "userId": 123 } } ``` -> `githubAccessTokenEncrypted`는 Core가 Cipher로 한 번 더 sealing해서 보냄. AI Server는 같은 키로 복호화 (또는 GitHub access token 위임을 위한 별도 short-lived token 발급 검토). +> 구 스펙(`githubAccessTokenEncrypted` envelope 동봉)은 폐기. AI Server는 분석 시점에 Core 내부 API `GET /api/internal/users/{userId}/github-token` 으로 평문 토큰을 짧게 위임받아 사용. envelope에는 비밀이 절대 포함되지 않는다. -### 5.3 `callback.analysis` (성공) +### 5.3 `analyze.web` +```json +{ + "messageType": "analyze.web", + "payload": { + "resumeId": 42, + "url": "https://example.com/me", + "analyzedDocumentId": 103 + }, + "context": { "userId": 123 } +} +``` + +> 웹 이력서(URL)는 AI Server가 trafilatura로 본문을 추출 → 동일 분석 체인으로 처리. resume 도메인을 재사용하므로 callback의 `targetType` 은 `WEB`. + +### 5.4 `callback.analysis` (성공) ```json { "messageType": "callback.analysis", "payload": { "targetType": "RESUME", "targetId": 42, + "status": "ANALYZED", "summary": "Java/Spring 3년차, 결제 시스템 개발...", "techStack": ["Java", "Spring Boot", "PostgreSQL"], - "documentS3Key": "analyzed/resume/42/summary.md", - "embeddingChunkCount": 18, - "status": "ANALYZED" + "documentPath": "analyzed/resume/42/summary.md", + "embeddingChunkCount": 18 } } ``` -### 5.4 `callback.analysis` (실패) +### 5.5 `callback.analysis` (실패) ```json { "messageType": "callback.analysis", @@ -171,9 +195,10 @@ } ``` -→ `targetType` ∈ `RESUME | REPOSITORY` +→ `targetType` ∈ `RESUME | REPOSITORY | WEB` +→ `documentPath` 는 객체 스토리지 키 (bucket 제외). Core는 같은 storage 추상화로 fetch. -### 5.5 `generate.questions` +### 5.6 `generate.questions` ```json { "messageType": "generate.questions", @@ -188,7 +213,7 @@ } ``` -### 5.6 `callback.questions` (질문 풀) +### 5.7 `callback.questions` (질문 풀) ```json { "messageType": "callback.questions", @@ -203,7 +228,7 @@ } ``` -### 5.7 `generate.followup` +### 5.8 `generate.followup` ```json { "messageType": "generate.followup", @@ -217,7 +242,7 @@ } ``` -### 5.8 `callback.questions` (꼬리질문) +### 5.9 `callback.questions` (꼬리질문) ```json { "messageType": "callback.questions", @@ -242,7 +267,7 @@ > `callback.questions` 큐는 두 종류(`POOL`, `FOLLOWUP`)를 받으므로 consumer는 `payload.kind`로 분기. -### 5.9 `generate.feedback` *(예정)* +### 5.10 `generate.feedback` *(예정)* ```json { "messageType": "generate.feedback", @@ -250,7 +275,7 @@ } ``` -### 5.10 `callback.feedback` *(예정)* +### 5.11 `callback.feedback` *(예정)* ```json { "messageType": "callback.feedback", @@ -268,7 +293,7 @@ } ``` -### 5.11 `realtime.session.notify` +### 5.12 `realtime.session.notify` ```json { "messageType": "realtime.session.notify", @@ -350,9 +375,22 @@ docker exec stackup-rabbitmq rabbitmqadmin \ -u stackup -p stackup \ publish exchange=stackup.core-to-ai \ routing_key=analyze.resume \ - payload='{"messageId":"smoke-1","messageType":"analyze.resume","version":"v1","traceId":"local-test","publishedAt":"2026-04-27T15:00:00Z","publisher":"manual","payload":{"resumeId":1,"s3Key":"resumes/raw/1/test.pdf"},"context":{"userId":1}}' + payload='{"messageId":"smoke-1","messageType":"analyze.resume","version":"v1","traceId":"local-test","publishedAt":"2026-04-27T15:00:00Z","publisher":"manual","payload":{"resumeId":1,"filePath":"resumes/raw/1/test.pdf","analyzedDocumentId":1},"context":{"userId":1}}' ``` +--- + +## 10. AI ↔ Core 내부 API (RabbitMQ 외) + +분석 파이프라인 일부는 동기적 데이터 위임이 필요해 Core가 내부 전용 REST endpoint를 노출한다. 모두 `X-Internal-API-Key` 헤더 검증. + +| Method | Path | 호출자 | 용도 | +|--------|------|--------|------| +| `GET` | `/api/internal/users/{userId}/github-token` | AI | 사용자별 GitHub access token을 분석 시점에 짧게 위임 (envelope에 비밀 미동봉) | +| `PUT` | `/api/internal/documents/{documentId}/embeddings` | AI | 청크 + 임베딩을 `document_embeddings`에 idempotent upsert | + +요청·응답 스키마 및 인증 규약은 [`/docs/api-conventions.md §10`](./api-conventions.md) 참조. + 큐 상태 확인: ```bash docker exec stackup-rabbitmq rabbitmqctl list_queues -q name messages consumers diff --git a/infra/CLAUDE.md b/infra/CLAUDE.md index 056e6053..f1e2b37f 100644 --- a/infra/CLAUDE.md +++ b/infra/CLAUDE.md @@ -108,6 +108,7 @@ docker exec -i stackup-postgres psql -U stackup stackup < backup.sql |-------|----------|-------------|----------| | `ai.analyze.repository` | `stackup.core-to-ai` | `analyze.repository` | AI Server | | `ai.analyze.resume` | `stackup.core-to-ai` | `analyze.resume` | AI Server | +| `ai.analyze.web` | `stackup.core-to-ai` | `analyze.web` | AI Server | | `ai.generate.questions` | `stackup.core-to-ai` | `generate.questions` | AI Server | | `ai.generate.followup` | `stackup.core-to-ai` | `generate.followup` | AI Server | | `core.callback.analysis` | `stackup.ai-to-core` | `callback.analysis` | Core Server | diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json index 88e92eb7..31972b86 100644 --- a/infra/rabbitmq/definitions.json +++ b/infra/rabbitmq/definitions.json @@ -57,6 +57,13 @@ "auto_delete": false, "arguments": {} }, + { + "name": "ai.analyze.web", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, { "name": "ai.generate.questions", "vhost": "/", @@ -108,6 +115,13 @@ "destination_type": "queue", "routing_key": "analyze.resume" }, + { + "source": "stackup.core-to-ai", + "vhost": "/", + "destination": "ai.analyze.web", + "destination_type": "queue", + "routing_key": "analyze.web" + }, { "source": "stackup.core-to-ai", "vhost": "/",