diff --git a/ai/.env.example b/ai/.env.example index 59d079e4..043be9a1 100644 --- a/ai/.env.example +++ b/ai/.env.example @@ -18,3 +18,18 @@ LLM_API_KEY= LLM_BASE_URL=https://factchat-cloud.mindlogic.ai/v1/gateway LLM_PRO_MODEL=gemini-3.1-pro-preview LLM_PRO_TEMPERATURE=0.2 + +# Github 관련 +CORE_INTERNAL_BASE_URL=http://localhost:38010 +CORE_INTERNAL_API_KEY= +CORE_INTERNAL_TIMEOUT_SEC=10 + +GITHUB_FALLBACK_TOKEN= +GITHUB_API_BASE_URL=https://api.github.com +REPO_MAX_SOURCE_FILES=8 +REPO_MAX_SOURCE_FILE_BYTES=50000 +REPO_FETCH_TIMEOUT_SEC=30 + +# 웹 이력서 관련 +WEB_FETCH_TIMEOUT_SEC=20 +WEB_MAX_HTML_BYTES=2000000 diff --git a/ai/CLAUDE.md b/ai/CLAUDE.md index fb5b3fbc..ac7ea660 100644 --- a/ai/CLAUDE.md +++ b/ai/CLAUDE.md @@ -88,11 +88,13 @@ ai/ 본 서버는 RabbitMQ **consumer**로 작동. -| Queue | Bind | -|-------|------| -| `q.ai.resume` | `ai.request.resume.*` | -| `q.ai.repo` | `ai.request.repo.*` | -| `q.ai.session` | `ai.request.session.*` | +| Queue | Routing key | 상태 | +|-------|-------------|------| +| `ai.analyze.resume` | `analyze.resume` | 본 구현 (PDF → MD) | +| `ai.analyze.repository` | `analyze.repository` | 본 구현 (GitHub README + tree + 소스 sampling) | +| `ai.analyze.web` | `analyze.web` | 본 구현 (URL → trafilatura) | +| `ai.generate.questions` | `generate.questions` | 큐만, 코드 미구현 | +| `ai.generate.followup` | `generate.followup` | 큐만, 코드 미구현 | 콜백 발행: `ai.callback.{type}` 익스체인지. 상세 envelope/스키마/재시도: [`/docs/messaging.md`](../docs/messaging.md). diff --git a/ai/pyproject.toml b/ai/pyproject.toml index 380d3396..eea3e136 100644 --- a/ai/pyproject.toml +++ b/ai/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "boto3>=1.42.77", "aiofiles>=24.1.0", "pypdf>=5.1.0", + "trafilatura>=2.0.0", "langchain>=1.2.13", "langchain-core>=1.2.22", "langchain-community>=0.4.1", diff --git a/ai/src/ai_server/analyzer/repository_analyzer.py b/ai/src/ai_server/analyzer/repository_analyzer.py new file mode 100644 index 00000000..d08ca797 --- /dev/null +++ b/ai/src/ai_server/analyzer/repository_analyzer.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import structlog + +from ai_server.analyzer.sources.github_repo import ( + GitHubRepoSourceExtractor, + RepositoryFetchError, +) +from ai_server.chain.document_analysis_chain import DocumentAnalyzer +from ai_server.core.client import CoreClient, CoreTokenError +from ai_server.storage.base import ObjectStorage + +log = structlog.get_logger(__name__) + + +class RepositoryAnalyzeError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class RepositoryAnalysisResult: + summary: str + tech_stack: list[str] + document_path: str + + +# 코어서버에서 토큰 받아오고 레포 가져온다 +# 이후 LLM 분석하고 마크다운 저장 +class RepositoryAnalyzer: + def __init__( + self, + *, + extractor: GitHubRepoSourceExtractor, + core_client: CoreClient, + chain: DocumentAnalyzer, + storage: ObjectStorage, + analyzed_key_template: str, + ) -> None: + self._extractor = extractor + self._core_client = core_client + self._chain = chain + self._storage = storage + self._analyzed_key_template = analyzed_key_template + + async def analyze( + self, + *, + repository_id: int, + repo_full_name: str, + default_branch: str = "main", + user_id: int | None, + ) -> RepositoryAnalysisResult: + if user_id is None: + raise RepositoryAnalyzeError( + code="MISSING_USER_ID", + message="envelope.context.userId 없이는 GitHub token을 가져올 수 없음", + retriable=False, + ) + + log.info( + "repository.token.fetch", + user_id=user_id, + repository_id=repository_id, + ) + try: + access_token = await self._core_client.fetch_github_token(user_id) + except CoreTokenError as err: + raise RepositoryAnalyzeError( + code=err.code, + message=err.message, + retriable=err.retriable, + ) from err + + log.info( + "repository.extract.start", + repository_id=repository_id, + repo_full_name=repo_full_name, + default_branch=default_branch, + ) + try: + source = await self._extractor.extract( + repo_full_name, + access_token=access_token, + ) + except RepositoryFetchError as err: + raise RepositoryAnalyzeError( + code=err.code, + message=err.message, + retriable=err.retriable, + ) from err + + if not source.text.strip(): + raise RepositoryAnalyzeError( + code="EMPTY_REPO_CONTENT", + message="레포에서 추출된 텍스트가 비어 있음", + retriable=False, + ) + + log.info( + "repository.llm.start", + repository_id=repository_id, + text_chars=len(source.text), + ) + analysis = await self._chain.analyze( + text=source.text, + source_type=source.source_type, + ) + + out_key = self._analyzed_key_template.format(repository_id=repository_id) + await self._storage.put_text(out_key, analysis.markdown) + log.info( + "repository.markdown.saved", + repository_id=repository_id, + key=out_key, + md_chars=len(analysis.markdown), + ) + + return RepositoryAnalysisResult( + summary=analysis.summary, + tech_stack=list(analysis.tech_stack), + document_path=out_key, + ) diff --git a/ai/src/ai_server/analyzer/sources/__init__.py b/ai/src/ai_server/analyzer/sources/__init__.py index d8929a8d..53344b6e 100644 --- a/ai/src/ai_server/analyzer/sources/__init__.py +++ b/ai/src/ai_server/analyzer/sources/__init__.py @@ -3,11 +3,20 @@ SourceExtractor, SourceType, ) +from ai_server.analyzer.sources.github_repo import ( + GitHubRepoSourceExtractor, + RepositoryFetchError, +) from ai_server.analyzer.sources.pdf import PdfSourceExtractor +from ai_server.analyzer.sources.web import WebFetchError, WebSourceExtractor __all__ = [ "ExtractedSource", "SourceExtractor", "SourceType", "PdfSourceExtractor", + "GitHubRepoSourceExtractor", + "RepositoryFetchError", + "WebSourceExtractor", + "WebFetchError", ] diff --git a/ai/src/ai_server/analyzer/sources/github_repo.py b/ai/src/ai_server/analyzer/sources/github_repo.py new file mode 100644 index 00000000..2692b895 --- /dev/null +++ b/ai/src/ai_server/analyzer/sources/github_repo.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import base64 +from dataclasses import dataclass + +import httpx +import structlog + +from ai_server.analyzer.sources.base import ExtractedSource, SourceExtractor + +log = structlog.get_logger(__name__) + + +# 프로젝트 설정 파일을 먼저 읽는다 +_PRIORITY_FILES: tuple[str, ...] = ( + "package.json", + "pyproject.toml", + "requirements.txt", + "go.mod", + "build.gradle", + "build.gradle.kts", + "pom.xml", + "Cargo.toml", + "composer.json", + "Gemfile", + "Dockerfile", +) + +_TEXT_EXTENSIONS: frozenset[str] = frozenset( + { + ".py", + ".ts", + ".tsx", + ".js", + ".jsx", + ".go", + ".java", + ".kt", + ".rs", + ".rb", + ".php", + ".cs", + ".swift", + ".md", + ".yaml", + ".yml", + ".toml", + ".json", + ".sh", + } +) + + +class RepositoryFetchError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class _RepoConfig: + api_base_url: str + fallback_token: str + max_files: int + max_file_bytes: int + timeout_sec: float + + +# 리드미, 주요 소스를 읽는다 +class GitHubRepoSourceExtractor(SourceExtractor): + def __init__( + self, + *, + api_base_url: str, + fallback_token: str = "", + max_files: int = 8, + max_file_bytes: int = 50_000, + timeout_sec: float = 30.0, + client: httpx.AsyncClient | None = None, + ) -> None: + self._cfg = _RepoConfig( + api_base_url=api_base_url.rstrip("/"), + fallback_token=fallback_token, + max_files=max_files, + max_file_bytes=max_file_bytes, + timeout_sec=timeout_sec, + ) + self._client = client + + async def extract( + self, + locator: str, + *, + access_token: str | None = None, + ) -> ExtractedSource: + """`locator`는 "owner/repo" 형태의 full name. + + `access_token`이 주어지면 그 token으로 호출(권장 — Core에서 받은 값). + None이면 fallback_token(public repo용)으로 호출. + """ + owner_repo = locator.strip().strip("/") + if "/" not in owner_repo: + raise RepositoryFetchError( + code="INVALID_REPO_LOCATOR", + message=f"locator must be 'owner/repo', got: {locator!r}", + retriable=False, + ) + + effective_token = access_token or self._cfg.fallback_token + + if self._client is not None: + return await self._extract_with_client(self._client, owner_repo) + + async with self._build_client(effective_token) as client: + return await self._extract_with_client(client, owner_repo) + + def _build_client(self, token: str) -> httpx.AsyncClient: + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + return httpx.AsyncClient( + base_url=self._cfg.api_base_url, + headers=headers, + timeout=self._cfg.timeout_sec, + ) + + async def _extract_with_client( + self, + client: httpx.AsyncClient, + owner_repo: str, + ) -> ExtractedSource: + repo_info = await self._fetch_json(client, f"/repos/{owner_repo}") + default_branch = repo_info.get("default_branch") or "main" + + readme_text = await self._fetch_readme(client, owner_repo) + tree_paths, truncated = await self._fetch_tree( + client, owner_repo, default_branch + ) + picked = _select_files(tree_paths, self._cfg.max_files) + snippets = await self._fetch_snippets( + client, owner_repo, default_branch, picked + ) + + parts: list[str] = [] + parts.append(f"# {owner_repo} (branch: {default_branch})") + if desc := repo_info.get("description"): + parts.append(f"\n> {desc}") + + if readme_text: + parts.append("\n## README\n") + parts.append(readme_text) + + parts.append("\n## File tree (truncated)\n") + parts.append("\n".join(f"- {p}" for p in tree_paths[:200])) + if truncated: + parts.append("\n(tree truncated by GitHub API)") + + if snippets: + parts.append("\n## Sampled sources\n") + for path, body in snippets: + parts.append(f"\n### `{path}`\n```\n{body}\n```") + + text = "\n".join(parts).strip() + return ExtractedSource( + text=text, + source_type="REPOSITORY", + metadata={ + "locator": owner_repo, + "default_branch": default_branch, + "tree_size": len(tree_paths), + "tree_truncated": truncated, + "sampled_files": [p for p, _ in snippets], + }, + ) + + async def _fetch_json(self, client: httpx.AsyncClient, path: str) -> dict: + resp = await client.get(path) + if resp.status_code == 404: + raise RepositoryFetchError( + code="REPO_NOT_FOUND", + message=f"GitHub returned 404 for {path}", + retriable=False, + ) + if resp.status_code in (401, 403): + raise RepositoryFetchError( + code="REPO_AUTH_OR_RATE", + message=f"GitHub auth/rate-limit: {resp.status_code} {resp.text[:200]}", + retriable=True, + ) + resp.raise_for_status() + return resp.json() + + async def _fetch_readme( + self, + client: httpx.AsyncClient, + owner_repo: str, + ) -> str: + try: + data = await self._fetch_json(client, f"/repos/{owner_repo}/readme") + except RepositoryFetchError as err: + if err.code == "REPO_NOT_FOUND": + return "" # README 없는 레포는 정상 + raise + if data.get("encoding") != "base64": + return "" + try: + return base64.b64decode(data.get("content", "")).decode( + "utf-8", errors="replace" + ) + except Exception: + return "" + + async def _fetch_tree( + self, + client: httpx.AsyncClient, + owner_repo: str, + branch: str, + ) -> tuple[list[str], bool]: + data = await self._fetch_json( + client, + f"/repos/{owner_repo}/git/trees/{branch}?recursive=1", + ) + tree = data.get("tree", []) + paths = [ + item["path"] + for item in tree + if item.get("type") == "blob" and item.get("path") + ] + return paths, bool(data.get("truncated")) + + async def _fetch_snippets( + self, + client: httpx.AsyncClient, + owner_repo: str, + branch: str, + picked: list[str], + ) -> list[tuple[str, str]]: + out: list[tuple[str, str]] = [] + for path in picked: + try: + data = await self._fetch_json( + client, f"/repos/{owner_repo}/contents/{path}?ref={branch}" + ) + except RepositoryFetchError as err: + log.warning( + "repo.snippet.skip", + path=path, + code=err.code, + ) + continue + if isinstance(data, list) or data.get("encoding") != "base64": + continue + try: + body = base64.b64decode(data.get("content", "")).decode( + "utf-8", errors="replace" + ) + except Exception: # noqa: BLE001 + continue + if len(body.encode("utf-8")) > self._cfg.max_file_bytes: + body = body[: self._cfg.max_file_bytes] + "\n... (truncated)" + out.append((path, body)) + return out + + +def _select_files(paths: list[str], cap: int) -> list[str]: + """우선순위 → 디렉토리당 대표 텍스트 파일 1개씩, 합쳐서 cap까지.""" + chosen: list[str] = [] + seen: set[str] = set() + + for needle in _PRIORITY_FILES: + for p in paths: + if p == needle or p.endswith("/" + needle): + if p not in seen: + chosen.append(p) + seen.add(p) + break + if len(chosen) >= cap: + return chosen + + by_dir: dict[str, str] = {} + for p in paths: + if p in seen: + continue + if "." not in p.rsplit("/", 1)[-1]: + continue + ext = "." + p.rsplit(".", 1)[-1].lower() + if ext not in _TEXT_EXTENSIONS: + continue + d = p.rsplit("/", 1)[0] if "/" in p else "" + if d not in by_dir: + by_dir[d] = p + for p in by_dir.values(): + if p not in seen: + chosen.append(p) + seen.add(p) + if len(chosen) >= cap: + break + return chosen diff --git a/ai/src/ai_server/analyzer/sources/web.py b/ai/src/ai_server/analyzer/sources/web.py new file mode 100644 index 00000000..8ed4c72f --- /dev/null +++ b/ai/src/ai_server/analyzer/sources/web.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import asyncio + +import httpx +import structlog +import trafilatura + +from ai_server.analyzer.sources.base import ExtractedSource, SourceExtractor + +log = structlog.get_logger(__name__) + + +class WebFetchError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +# 라이브러리로 본문 추출 +class WebSourceExtractor(SourceExtractor): + def __init__( + self, + *, + timeout_sec: float = 20.0, + max_html_bytes: int = 2_000_000, + client: httpx.AsyncClient | None = None, + ) -> None: + self._timeout_sec = timeout_sec + self._max_html_bytes = max_html_bytes + self._client = client + + async def extract(self, locator: str) -> ExtractedSource: + url = locator.strip() + if not (url.startswith("http://") or url.startswith("https://")): + raise WebFetchError( + code="INVALID_WEB_URL", + message=f"locator must be http(s) URL, got: {locator!r}", + retriable=False, + ) + + html, final_url, content_type = await self._fetch_html(url) + text = await asyncio.to_thread(_extract_main_text, html, final_url) + + if not text.strip(): + raise WebFetchError( + code="EMPTY_WEB_BODY", + message="추출된 본문이 비어 있음 — JS 렌더링 페이지이거나 비어있는 문서일 가능성", + retriable=False, + ) + + return ExtractedSource( + text=text, + source_type="WEB", + metadata={ + "locator": url, + "final_url": final_url, + "content_type": content_type, + "html_bytes": len(html.encode("utf-8")), + "text_chars": len(text), + }, + ) + + async def _fetch_html(self, url: str) -> tuple[str, str, str]: + if self._client is not None: + return await self._do_fetch(self._client, url) + async with httpx.AsyncClient( + timeout=self._timeout_sec, + follow_redirects=True, + headers={ + "User-Agent": "StackUp-AI/1.0 (+resume web extractor)", + "Accept": "text/html,application/xhtml+xml", + }, + ) as client: + return await self._do_fetch(client, url) + + async def _do_fetch( + self, + client: httpx.AsyncClient, + url: str, + ) -> tuple[str, str, str]: + try: + resp = await client.get(url) + except httpx.HTTPError as exc: + raise WebFetchError( + code="WEB_FETCH_FAILED", + message=f"HTTP 요청 실패: {exc}", + retriable=True, + ) from exc + + if resp.status_code >= 400: + raise WebFetchError( + code="WEB_HTTP_STATUS", + message=f"HTTP {resp.status_code}", + retriable=resp.status_code >= 500, + ) + + content_type = resp.headers.get("content-type", "") + if "html" not in content_type.lower(): + raise WebFetchError( + code="WEB_NOT_HTML", + message=f"HTML이 아님 (content-type={content_type})", + retriable=False, + ) + + raw = resp.content + if len(raw) > self._max_html_bytes: + raise WebFetchError( + code="WEB_HTML_TOO_LARGE", + message=f"HTML이 한도({self._max_html_bytes}B)를 초과: {len(raw)}B", + retriable=False, + ) + + html = resp.text + return html, str(resp.url), content_type + + +def _extract_main_text(html: str, url: str) -> str: + result = trafilatura.extract( + html, + url=url, + include_comments=False, + include_tables=True, + favor_recall=True, + ) + return (result or "").strip() diff --git a/ai/src/ai_server/analyzer/web_resume_analyzer.py b/ai/src/ai_server/analyzer/web_resume_analyzer.py new file mode 100644 index 00000000..d8735b77 --- /dev/null +++ b/ai/src/ai_server/analyzer/web_resume_analyzer.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import structlog + +from ai_server.analyzer.sources.base import SourceExtractor +from ai_server.analyzer.sources.web import WebFetchError +from ai_server.chain.document_analysis_chain import DocumentAnalyzer +from ai_server.storage.base import ObjectStorage + +log = structlog.get_logger(__name__) + + +class WebResumeAnalyzeError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +@dataclass(frozen=True) +class WebResumeAnalysisResult: + summary: str + tech_stack: list[str] + document_path: str + + +# 웹포폴 -> 라이브러리로 텍스트만 - > LLM -> 마크다운 +class WebResumeAnalyzer: + def __init__( + self, + *, + extractor: SourceExtractor, + chain: DocumentAnalyzer, + storage: ObjectStorage, + analyzed_key_template: str, + ) -> None: + self._extractor = extractor + self._chain = chain + self._storage = storage + self._analyzed_key_template = analyzed_key_template + + async def analyze( + self, + *, + resume_id: int, + url: str, + ) -> WebResumeAnalysisResult: + log.info("web_resume.extract.start", resume_id=resume_id, url=url) + try: + source = await self._extractor.extract(url) + except WebFetchError as err: + raise WebResumeAnalyzeError( + code=err.code, + message=err.message, + retriable=err.retriable, + ) from err + + log.info( + "web_resume.llm.start", + resume_id=resume_id, + text_chars=len(source.text), + ) + analysis = await self._chain.analyze( + text=source.text, + source_type=source.source_type, + ) + + out_key = self._analyzed_key_template.format(resume_id=resume_id) + await self._storage.put_text(out_key, analysis.markdown) + log.info( + "web_resume.markdown.saved", + resume_id=resume_id, + key=out_key, + md_chars=len(analysis.markdown), + ) + + return WebResumeAnalysisResult( + summary=analysis.summary, + tech_stack=list(analysis.tech_stack), + document_path=out_key, + ) diff --git a/ai/src/ai_server/config/settings.py b/ai/src/ai_server/config/settings.py index 9b719897..724a0f6a 100644 --- a/ai/src/ai_server/config/settings.py +++ b/ai/src/ai_server/config/settings.py @@ -20,6 +20,8 @@ class Settings(BaseSettings): # AI consumer (큐 이름, prefetch, 콜백 라우팅 등) ai_queue_resume: str = "ai.analyze.resume" + ai_queue_repository: str = "ai.analyze.repository" + ai_queue_web: str = "ai.analyze.web" ai_queue_prefetch: int = 10 ai_callback_exchange: str = "stackup.ai-to-core" ai_callback_routing_analysis: str = "callback.analysis" @@ -42,6 +44,29 @@ class Settings(BaseSettings): llm_pro_temperature: float = 0.2 analyzed_resume_md_key_template: str = "analyzed/resume/{resume_id}/summary.md" + analyzed_repository_md_key_template: str = ( + "analyzed/repository/{repository_id}/summary.md" + ) + analyzed_web_resume_md_key_template: str = ( + "analyzed/web-resume/{resume_id}/summary.md" + ) + + # Core 서버 internal API (사용자별 GitHub access_token 조회 등) + core_internal_base_url: str = "http://localhost:38010" + core_internal_api_key: str = "" + core_internal_timeout_sec: float = 10.0 + + # GitHub repo 분석용 + github_api_base_url: str = "https://api.github.com" + # 사용자별 token은 Core에서 받음. 아래 token은 fallback (public repo 한정). + github_fallback_token: str = "" + repo_max_source_files: int = 8 + repo_max_source_file_bytes: int = 50_000 + repo_fetch_timeout_sec: float = 30.0 + + # 웹 이력서 fetch + web_fetch_timeout_sec: float = 20.0 + web_max_html_bytes: int = 2_000_000 # 2MB 상한 def get_settings() -> Settings: diff --git a/ai/src/ai_server/core/__init__.py b/ai/src/ai_server/core/__init__.py new file mode 100644 index 00000000..02053ab9 --- /dev/null +++ b/ai/src/ai_server/core/__init__.py @@ -0,0 +1,11 @@ +from ai_server.core.client import ( + CoreClient, + CoreTokenError, + HttpCoreClient, +) + +__all__ = [ + "CoreClient", + "CoreTokenError", + "HttpCoreClient", +] diff --git a/ai/src/ai_server/core/client.py b/ai/src/ai_server/core/client.py new file mode 100644 index 00000000..73cefb6f --- /dev/null +++ b/ai/src/ai_server/core/client.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from typing import Protocol + +import httpx +import structlog + +log = structlog.get_logger(__name__) + + +class CoreTokenError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable + + +# 코어 서버 API 호출용 +class CoreClient(Protocol): + async def fetch_github_token(self, user_id: int) -> str: ... + + +class HttpCoreClient: + def __init__( + self, + *, + base_url: str, + api_key: str, + timeout_sec: float = 10.0, + client: httpx.AsyncClient | None = None, + ) -> None: + if not base_url: + raise ValueError("Core internal base_url 누락") + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._timeout_sec = timeout_sec + self._client = client + + async def fetch_github_token(self, user_id: int) -> str: + path = f"/api/internal/users/{user_id}/github-token" + if self._client is not None: + return await self._do_fetch(self._client, path) + async with self._build_client() as client: + return await self._do_fetch(client, path) + + def _build_client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + base_url=self._base_url, + headers={"X-Internal-API-Key": self._api_key}, + timeout=self._timeout_sec, + ) + + async def _do_fetch(self, client: httpx.AsyncClient, path: str) -> str: + try: + resp = await client.get(path) + except httpx.HTTPError as exc: + raise CoreTokenError( + code="CORE_UNAVAILABLE", + message=f"Core API 호출 실패: {exc}", + retriable=True, + ) from exc + + status = resp.status_code + if status == 404: + raise CoreTokenError( + code="USER_NOT_FOUND", + message=f"Core에 사용자 또는 token 없음: {path}", + retriable=False, + ) + if status in (401, 403): + raise CoreTokenError( + code="CORE_AUTH_FAILED", + message=f"Core internal 인증 실패: {status}", + retriable=False, + ) + if status >= 500: + raise CoreTokenError( + code="CORE_UNAVAILABLE", + message=f"Core 5xx: {status}", + retriable=True, + ) + if status >= 400: + raise CoreTokenError( + code="CORE_BAD_REQUEST", + message=f"Core {status}: {resp.text[:200]}", + retriable=False, + ) + + try: + data = resp.json() + except ValueError as exc: + raise CoreTokenError( + code="CORE_BAD_RESPONSE", + message=f"Core 응답 JSON 파싱 실패: {exc}", + retriable=True, + ) from exc + + token = data.get("accessToken") or data.get("access_token") + if not token or not isinstance(token, str): + raise CoreTokenError( + code="CORE_TOKEN_EMPTY", + message="Core 응답에 accessToken 없음", + retriable=True, + ) + return token diff --git a/ai/src/ai_server/messaging/consumers/repository_consumer.py b/ai/src/ai_server/messaging/consumers/repository_consumer.py new file mode 100644 index 00000000..553ddbb4 --- /dev/null +++ b/ai/src/ai_server/messaging/consumers/repository_consumer.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import structlog +from aio_pika.abc import AbstractIncomingMessage + +from ai_server.analyzer.repository_analyzer import ( + RepositoryAnalyzeError, + RepositoryAnalyzer, +) +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.messaging.publisher import CallbackPublisher +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + RepositoryAnalyzeRequest, +) + +log = structlog.get_logger(__name__) + + +class RepositoryConsumer: + def __init__( + self, + *, + analyzer: RepositoryAnalyzer, + publisher: CallbackPublisher, + idempotency: LruIdempotencyStore, + callback_routing_key: str, + ) -> None: + self._analyzer = analyzer + self._publisher = publisher + self._idempotency = idempotency + self._callback_routing_key = callback_routing_key + + async def handle(self, message: AbstractIncomingMessage) -> None: + async with message.process(requeue=False): + try: + envelope = Envelope[RepositoryAnalyzeRequest].model_validate_json( + message.body + ) + except Exception as exc: + log.error( + "repository.parse.failed", + error=str(exc), + delivery_tag=message.delivery_tag, + ) + raise + + if self._idempotency.is_seen_then_mark(envelope.message_id): + log.info( + "repository.idempotent.skip", + message_id=envelope.message_id, + trace_id=envelope.trace_id, + ) + return + + req = envelope.payload + user_id = envelope.context.user_id + log.info( + "repository.analyze.start", + message_id=envelope.message_id, + repository_id=req.repository_id, + repo_full_name=req.repo_full_name, + user_id=user_id, + trace_id=envelope.trace_id, + ) + + payload = await self._run_and_build_payload( + req, user_id=user_id, trace_id=envelope.trace_id + ) + + await self._publisher.publish( + routing_key=self._callback_routing_key, + message_type="callback.analysis", + payload=payload, + trace_id=envelope.trace_id, + correlation_id=envelope.message_id, + context=envelope.context, + ) + log.info( + "repository.analyze.done", + message_id=envelope.message_id, + repository_id=req.repository_id, + status=payload.status, + trace_id=envelope.trace_id, + ) + + async def _run_and_build_payload( + self, + req: RepositoryAnalyzeRequest, + *, + user_id: int | None, + trace_id: str, + ) -> AnalysisCallbackPayload: + try: + result = await self._analyzer.analyze( + repository_id=req.repository_id, + repo_full_name=req.repo_full_name, + default_branch=req.default_branch, + user_id=user_id, + ) + except RepositoryAnalyzeError as err: + log.warning( + "repository.analyze.domain_failed", + repository_id=req.repository_id, + code=err.code, + retriable=err.retriable, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="REPOSITORY", + target_id=req.repository_id, + status="FAILED", + error_code=err.code, + error_message=err.message, + retriable=err.retriable, + ) + except Exception as exc: + log.exception( + "repository.analyze.unexpected_failed", + repository_id=req.repository_id, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="REPOSITORY", + target_id=req.repository_id, + status="FAILED", + error_code="UNEXPECTED", + error_message=str(exc), + retriable=True, + ) + + return AnalysisCallbackPayload( + target_type="REPOSITORY", + target_id=req.repository_id, + status="ANALYZED", + summary=result.summary, + tech_stack=result.tech_stack, + document_path=result.document_path, + embedding_chunk_count=0, + ) diff --git a/ai/src/ai_server/messaging/consumers/web_consumer.py b/ai/src/ai_server/messaging/consumers/web_consumer.py new file mode 100644 index 00000000..51a6f74f --- /dev/null +++ b/ai/src/ai_server/messaging/consumers/web_consumer.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import structlog +from aio_pika.abc import AbstractIncomingMessage + +from ai_server.analyzer.web_resume_analyzer import ( + WebResumeAnalyzeError, + WebResumeAnalyzer, +) +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.messaging.publisher import CallbackPublisher +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + WebResumeAnalyzeRequest, +) + +log = structlog.get_logger(__name__) + + +class WebResumeConsumer: + def __init__( + self, + *, + analyzer: WebResumeAnalyzer, + publisher: CallbackPublisher, + idempotency: LruIdempotencyStore, + callback_routing_key: str, + ) -> None: + self._analyzer = analyzer + self._publisher = publisher + self._idempotency = idempotency + self._callback_routing_key = callback_routing_key + + async def handle(self, message: AbstractIncomingMessage) -> None: + async with message.process(requeue=False): + try: + envelope = Envelope[WebResumeAnalyzeRequest].model_validate_json( + message.body + ) + except Exception as exc: + log.error( + "web_resume.parse.failed", + error=str(exc), + delivery_tag=message.delivery_tag, + ) + raise + + if self._idempotency.is_seen_then_mark(envelope.message_id): + log.info( + "web_resume.idempotent.skip", + message_id=envelope.message_id, + trace_id=envelope.trace_id, + ) + return + + req = envelope.payload + log.info( + "web_resume.analyze.start", + message_id=envelope.message_id, + resume_id=req.resume_id, + url=req.url, + trace_id=envelope.trace_id, + ) + + payload = await self._run_and_build_payload(req, envelope.trace_id) + + await self._publisher.publish( + routing_key=self._callback_routing_key, + message_type="callback.analysis", + payload=payload, + trace_id=envelope.trace_id, + correlation_id=envelope.message_id, + context=envelope.context, + ) + log.info( + "web_resume.analyze.done", + message_id=envelope.message_id, + resume_id=req.resume_id, + status=payload.status, + trace_id=envelope.trace_id, + ) + + async def _run_and_build_payload( + self, + req: WebResumeAnalyzeRequest, + trace_id: str, + ) -> AnalysisCallbackPayload: + try: + result = await self._analyzer.analyze( + resume_id=req.resume_id, + url=req.url, + ) + except WebResumeAnalyzeError as err: + log.warning( + "web_resume.analyze.domain_failed", + resume_id=req.resume_id, + code=err.code, + retriable=err.retriable, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="WEB", + target_id=req.resume_id, + status="FAILED", + error_code=err.code, + error_message=err.message, + retriable=err.retriable, + ) + except Exception as exc: + log.exception( + "web_resume.analyze.unexpected_failed", + resume_id=req.resume_id, + trace_id=trace_id, + ) + return AnalysisCallbackPayload( + target_type="WEB", + target_id=req.resume_id, + status="FAILED", + error_code="UNEXPECTED", + error_message=str(exc), + retriable=True, + ) + + return AnalysisCallbackPayload( + target_type="WEB", + target_id=req.resume_id, + status="ANALYZED", + summary=result.summary, + tech_stack=result.tech_stack, + document_path=result.document_path, + embedding_chunk_count=0, + ) diff --git a/ai/src/ai_server/messaging/runner.py b/ai/src/ai_server/messaging/runner.py index 92ac3738..98710c9b 100644 --- a/ai/src/ai_server/messaging/runner.py +++ b/ai/src/ai_server/messaging/runner.py @@ -3,15 +3,22 @@ import structlog from aio_pika.abc import AbstractRobustQueue +from ai_server.analyzer.repository_analyzer import RepositoryAnalyzer from ai_server.analyzer.resume_analyzer import ResumeAnalyzer +from ai_server.analyzer.sources.github_repo import GitHubRepoSourceExtractor from ai_server.analyzer.sources.pdf import PdfSourceExtractor +from ai_server.analyzer.sources.web import WebSourceExtractor +from ai_server.analyzer.web_resume_analyzer import WebResumeAnalyzer from ai_server.chain.document_analysis_chain import ( LlmDocumentAnalyzer, build_document_analysis_chain, ) from ai_server.config.settings import Settings +from ai_server.core.client import HttpCoreClient from ai_server.messaging.connection import RabbitConnection +from ai_server.messaging.consumers.repository_consumer import RepositoryConsumer from ai_server.messaging.consumers.resume_consumer import ResumeConsumer +from ai_server.messaging.consumers.web_consumer import WebResumeConsumer from ai_server.messaging.idempotency import LruIdempotencyStore from ai_server.messaging.publisher import CallbackPublisher from ai_server.storage.factory import build_storage @@ -39,42 +46,105 @@ def __init__(self, settings: Settings) -> None: storage = build_storage(settings) chain = build_document_analysis_chain(settings) - analyzer = ResumeAnalyzer( + chain_analyzer = LlmDocumentAnalyzer(chain) + + # 이력서 PDF + resume_analyzer = ResumeAnalyzer( extractor=PdfSourceExtractor(storage=storage), - chain=LlmDocumentAnalyzer(chain), + chain=chain_analyzer, storage=storage, analyzed_key_template=settings.analyzed_resume_md_key_template, ) + # 리포지토리 + core_client = HttpCoreClient( + base_url=settings.core_internal_base_url, + api_key=settings.core_internal_api_key, + timeout_sec=settings.core_internal_timeout_sec, + ) + repo_analyzer = RepositoryAnalyzer( + extractor=GitHubRepoSourceExtractor( + api_base_url=settings.github_api_base_url, + fallback_token=settings.github_fallback_token, + max_files=settings.repo_max_source_files, + max_file_bytes=settings.repo_max_source_file_bytes, + timeout_sec=settings.repo_fetch_timeout_sec, + ), + core_client=core_client, + chain=chain_analyzer, + storage=storage, + analyzed_key_template=settings.analyzed_repository_md_key_template, + ) + + # 웹 이력서 + web_analyzer = WebResumeAnalyzer( + extractor=WebSourceExtractor( + timeout_sec=settings.web_fetch_timeout_sec, + max_html_bytes=settings.web_max_html_bytes, + ), + chain=chain_analyzer, + storage=storage, + analyzed_key_template=settings.analyzed_web_resume_md_key_template, + ) + self._resume_consumer = ResumeConsumer( - analyzer=analyzer, + analyzer=resume_analyzer, + publisher=self._publisher, + idempotency=self._idempotency, + callback_routing_key=settings.ai_callback_routing_analysis, + ) + self._repository_consumer = RepositoryConsumer( + analyzer=repo_analyzer, publisher=self._publisher, idempotency=self._idempotency, callback_routing_key=settings.ai_callback_routing_analysis, ) - self._resume_queue: AbstractRobustQueue | None = None - self._consumer_tag: str | None = None + self._web_consumer = WebResumeConsumer( + analyzer=web_analyzer, + publisher=self._publisher, + idempotency=self._idempotency, + callback_routing_key=settings.ai_callback_routing_analysis, + ) + + self._consumers: list[tuple[AbstractRobustQueue, str]] = [] async def start(self) -> None: await self._connection.open() await self._publisher.open() channel = self._connection.channel + + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_resume, + handler=self._resume_consumer.handle, + ) + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_repository, + handler=self._repository_consumer.handle, + ) + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_web, + handler=self._web_consumer.handle, + ) + + async def _start_consumer(self, channel, *, queue_name, handler) -> None: queue = await channel.declare_queue( - self._settings.ai_queue_resume, + queue_name, durable=True, - passive=True, # 정의 파일이 이미 선언함 - ) - self._resume_queue = queue - self._consumer_tag = await queue.consume(self._resume_consumer.handle) - log.info( - "ai.consumer.started", - queue=self._settings.ai_queue_resume, - consumer_tag=self._consumer_tag, + passive=True, ) + tag = await queue.consume(handler) + self._consumers.append((queue, tag)) + log.info("ai.consumer.started", queue=queue_name, consumer_tag=tag) async def stop(self) -> None: - if self._resume_queue is not None and self._consumer_tag is not None: - await self._resume_queue.cancel(self._consumer_tag) - log.info("ai.consumer.stopped", consumer_tag=self._consumer_tag) + for queue, tag in self._consumers: + try: + await queue.cancel(tag) + log.info("ai.consumer.stopped", consumer_tag=tag) + except Exception: + log.exception("ai.consumer.cancel_failed", consumer_tag=tag) await self._connection.close() diff --git a/ai/src/ai_server/model/messages/analyze.py b/ai/src/ai_server/model/messages/analyze.py index 6970a0b1..299e976a 100644 --- a/ai/src/ai_server/model/messages/analyze.py +++ b/ai/src/ai_server/model/messages/analyze.py @@ -4,7 +4,7 @@ from ai_server.model._config import camel_config -TargetType = Literal["RESUME", "REPOSITORY"] +TargetType = Literal["RESUME", "REPOSITORY", "WEB"] AnalysisStatus = Literal["ANALYZED", "FAILED"] @@ -15,6 +15,21 @@ class ResumeAnalyzeRequest(BaseModel): file_path: str +class RepositoryAnalyzeRequest(BaseModel): + model_config = camel_config() + + repository_id: int + repo_full_name: str + default_branch: str = "main" + + +class WebResumeAnalyzeRequest(BaseModel): + model_config = camel_config() + + resume_id: int + url: str + + class AnalysisCallbackPayload(BaseModel): model_config = camel_config() diff --git a/ai/tests/test_core_client.py b/ai/tests/test_core_client.py new file mode 100644 index 00000000..7b7e492e --- /dev/null +++ b/ai/tests/test_core_client.py @@ -0,0 +1,122 @@ +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from ai_server.core.client import CoreTokenError, HttpCoreClient + + +def _make_client( + *, + status: int = 200, + json_body: dict | None = None, + text: str = "", + raise_exc: Exception | None = None, +) -> MagicMock: + client = MagicMock() + resp = MagicMock(spec=httpx.Response) + resp.status_code = status + resp.text = text + if json_body is None: + resp.json = MagicMock(side_effect=ValueError("no json")) + else: + resp.json = MagicMock(return_value=json_body) + if raise_exc is not None: + client.get = AsyncMock(side_effect=raise_exc) + else: + client.get = AsyncMock(return_value=resp) + return client + + +def _build(client: MagicMock) -> HttpCoreClient: + return HttpCoreClient(base_url="http://core:38010", api_key="k", client=client) + + +@pytest.mark.asyncio +async def test_returns_access_token_on_200_camel() -> None: + client = _make_client(json_body={"accessToken": "tok-1"}) + core = _build(client) + token = await core.fetch_github_token(user_id=42) + assert token == "tok-1" + client.get.assert_awaited_once_with("/api/internal/users/42/github-token") + + +@pytest.mark.asyncio +async def test_returns_access_token_on_snake_case_response() -> None: + client = _make_client(json_body={"access_token": "tok-s"}) + core = _build(client) + assert await core.fetch_github_token(user_id=1) == "tok-s" + + +@pytest.mark.asyncio +async def test_404_translates_to_user_not_found() -> None: + client = _make_client(status=404) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "USER_NOT_FOUND" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_401_translates_to_core_auth_failed() -> None: + client = _make_client(status=401) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_AUTH_FAILED" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_5xx_translates_to_retriable_unavailable() -> None: + client = _make_client(status=503) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_UNAVAILABLE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_other_4xx_non_retriable() -> None: + client = _make_client(status=400, text="bad") + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_BAD_REQUEST" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_httpx_error_translates_to_retriable() -> None: + client = _make_client(raise_exc=httpx.ConnectError("dns fail")) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_UNAVAILABLE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_empty_token_field_treated_as_retriable() -> None: + client = _make_client(json_body={"accessToken": ""}) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_TOKEN_EMPTY" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_invalid_json_translates_to_bad_response() -> None: + client = _make_client(status=200, json_body=None) + core = _build(client) + with pytest.raises(CoreTokenError) as exc_info: + await core.fetch_github_token(user_id=42) + assert exc_info.value.code == "CORE_BAD_RESPONSE" + + +def test_constructor_requires_base_url() -> None: + with pytest.raises(ValueError): + HttpCoreClient(base_url="", api_key="x") diff --git a/ai/tests/test_github_repo_extractor.py b/ai/tests/test_github_repo_extractor.py new file mode 100644 index 00000000..9fa165ab --- /dev/null +++ b/ai/tests/test_github_repo_extractor.py @@ -0,0 +1,191 @@ +import base64 +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from ai_server.analyzer.sources.github_repo import ( + GitHubRepoSourceExtractor, + RepositoryFetchError, + _select_files, +) + + +def _make_client(routes: dict[str, dict | int]) -> tuple[MagicMock, AsyncMock]: + """`routes`: path → response dict (JSON) or int (HTTP status).""" + client = MagicMock() + + async def _get(path: str) -> MagicMock: + body = routes.get(path) + resp = MagicMock(spec=httpx.Response) + if isinstance(body, int): + resp.status_code = body + resp.text = "" + resp.raise_for_status = MagicMock() + return resp + resp.status_code = 200 + resp.json = MagicMock(return_value=body) + resp.raise_for_status = MagicMock() + return resp + + client.get = AsyncMock(side_effect=_get) + return client, client.get + + +def _b64(text: str) -> str: + return base64.b64encode(text.encode("utf-8")).decode("ascii") + + +@pytest.mark.asyncio +async def test_extract_assembles_readme_tree_and_snippets() -> None: + routes = { + "/repos/user/repo": { + "default_branch": "dev", + "description": "demo project", + }, + "/repos/user/repo/readme": { + "encoding": "base64", + "content": _b64("# Hello\nReadme body"), + }, + "/repos/user/repo/git/trees/dev?recursive=1": { + "tree": [ + {"path": "pyproject.toml", "type": "blob"}, + {"path": "src/app.py", "type": "blob"}, + {"path": "tests/test_app.py", "type": "blob"}, + {"path": "docs", "type": "tree"}, + ], + "truncated": False, + }, + "/repos/user/repo/contents/pyproject.toml?ref=dev": { + "encoding": "base64", + "content": _b64('[project]\nname = "app"'), + }, + "/repos/user/repo/contents/src/app.py?ref=dev": { + "encoding": "base64", + "content": _b64("def main():\n return 1"), + }, + "/repos/user/repo/contents/tests/test_app.py?ref=dev": { + "encoding": "base64", + "content": _b64("def test(): pass"), + }, + } + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + result = await extractor.extract("user/repo") + + assert result.source_type == "REPOSITORY" + assert "user/repo" in result.text + assert "Readme body" in result.text + assert "pyproject.toml" in result.text + assert "def main()" in result.text + assert result.metadata["default_branch"] == "dev" + assert result.metadata["tree_size"] == 3 + assert "pyproject.toml" in result.metadata["sampled_files"] + + +@pytest.mark.asyncio +async def test_extract_handles_missing_readme_gracefully() -> None: + routes = { + "/repos/user/repo": {"default_branch": "main"}, + "/repos/user/repo/readme": 404, + "/repos/user/repo/git/trees/main?recursive=1": {"tree": [], "truncated": False}, + } + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + result = await extractor.extract("user/repo") + assert "user/repo" in result.text + assert result.metadata["tree_size"] == 0 + + +@pytest.mark.asyncio +async def test_extract_raises_on_repo_not_found() -> None: + routes = {"/repos/user/repo": 404} + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + with pytest.raises(RepositoryFetchError) as exc_info: + await extractor.extract("user/repo") + assert exc_info.value.code == "REPO_NOT_FOUND" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_extract_raises_on_rate_limit_as_retriable() -> None: + routes = {"/repos/user/repo": 403} + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + with pytest.raises(RepositoryFetchError) as exc_info: + await extractor.extract("user/repo") + assert exc_info.value.code == "REPO_AUTH_OR_RATE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_extract_rejects_malformed_locator() -> None: + client, _ = _make_client({}) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + with pytest.raises(RepositoryFetchError) as exc_info: + await extractor.extract("not-a-full-name") + assert exc_info.value.code == "INVALID_REPO_LOCATOR" + + +def test_select_files_prioritizes_manifests_then_one_per_dir() -> None: + paths = [ + "package.json", + "src/index.ts", + "src/util/format.ts", + "tests/index.test.ts", + "README.md", + ] + picked = _select_files(paths, cap=4) + # manifest 우선 + assert "package.json" in picked + # 디렉토리당 1개씩 sampling (src/, tests/, root) + assert any(p.startswith("src/") for p in picked) + assert any(p.startswith("tests/") for p in picked) + assert len(picked) <= 4 + + +def test_select_files_respects_cap() -> None: + paths = [f"src/a{i}.py" for i in range(20)] + picked = _select_files(paths, cap=3) + assert len(picked) <= 3 + + +def test_build_client_includes_authorization_with_user_token() -> None: + extractor = GitHubRepoSourceExtractor(api_base_url="https://api.github.com") + client = extractor._build_client("user-token-xyz") + try: + assert client.headers.get("Authorization") == "Bearer user-token-xyz" + finally: + # 비동기 cleanup 없이 닫기 OK — 실제 연결은 안 함 + pass + + +def test_build_client_omits_authorization_when_token_empty() -> None: + extractor = GitHubRepoSourceExtractor(api_base_url="https://api.github.com") + client = extractor._build_client("") + assert "Authorization" not in client.headers + + +def test_fallback_token_used_when_no_per_call_token() -> None: + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", + fallback_token="fallback-tok", + ) + # access_token=None → fallback 사용 + client = extractor._build_client(extractor._cfg.fallback_token) + assert client.headers.get("Authorization") == "Bearer fallback-tok" diff --git a/ai/tests/test_messages_analyze.py b/ai/tests/test_messages_analyze.py index a699dd2d..c8e211a2 100644 --- a/ai/tests/test_messages_analyze.py +++ b/ai/tests/test_messages_analyze.py @@ -3,7 +3,9 @@ from ai_server.model.messages.analyze import ( AnalysisCallbackPayload, + RepositoryAnalyzeRequest, ResumeAnalyzeRequest, + WebResumeAnalyzeRequest, ) @@ -68,3 +70,45 @@ def test_callback_target_type_must_be_known() -> None: target_id=1, status="ANALYZED", ) + + +def test_repository_request_parses_with_default_branch() -> None: + req = RepositoryAnalyzeRequest.model_validate( + {"repositoryId": 5, "repoFullName": "user/repo"} + ) + assert req.repository_id == 5 + assert req.repo_full_name == "user/repo" + assert req.default_branch == "main" + + +def test_repository_request_serializes_camel_case() -> None: + req = RepositoryAnalyzeRequest( + repository_id=5, repo_full_name="user/repo", default_branch="dev" + ) + dumped = req.model_dump(by_alias=True) + assert dumped == { + "repositoryId": 5, + "repoFullName": "user/repo", + "defaultBranch": "dev", + } + + +def test_web_resume_request_parses() -> None: + req = WebResumeAnalyzeRequest.model_validate( + {"resumeId": 9, "url": "https://example.com/me"} + ) + assert req.resume_id == 9 + assert req.url == "https://example.com/me" + + +def test_callback_accepts_web_target_type() -> None: + cb = AnalysisCallbackPayload( + target_type="WEB", + target_id=9, + status="ANALYZED", + summary="ok", + document_path="analyzed/web-resume/9/summary.md", + ) + dumped = cb.model_dump(by_alias=True) + assert dumped["targetType"] == "WEB" + assert dumped["documentPath"] == "analyzed/web-resume/9/summary.md" diff --git a/ai/tests/test_repository_analyzer.py b/ai/tests/test_repository_analyzer.py new file mode 100644 index 00000000..d0a017ab --- /dev/null +++ b/ai/tests/test_repository_analyzer.py @@ -0,0 +1,144 @@ +from unittest.mock import AsyncMock + +import pytest + +from ai_server.analyzer.repository_analyzer import ( + RepositoryAnalyzeError, + RepositoryAnalyzer, +) +from ai_server.analyzer.sources.base import ExtractedSource +from ai_server.analyzer.sources.github_repo import RepositoryFetchError +from ai_server.chain.document_analysis_chain import DocumentAnalysisResult +from ai_server.core.client import CoreTokenError + + +def _make_analyzer( + *, + extract_result: ExtractedSource | Exception | None = None, + analysis: DocumentAnalysisResult | None = None, + token_result: str | Exception = "user-token", +) -> tuple[RepositoryAnalyzer, AsyncMock, AsyncMock, AsyncMock, AsyncMock]: + extractor = AsyncMock() + if isinstance(extract_result, Exception): + extractor.extract = AsyncMock(side_effect=extract_result) + else: + extractor.extract = AsyncMock(return_value=extract_result) + + chain = AsyncMock() + if analysis is not None: + chain.analyze = AsyncMock(return_value=analysis) + + storage = AsyncMock() + + core_client = AsyncMock() + if isinstance(token_result, Exception): + core_client.fetch_github_token = AsyncMock(side_effect=token_result) + else: + core_client.fetch_github_token = AsyncMock(return_value=token_result) + + analyzer = RepositoryAnalyzer( + extractor=extractor, + core_client=core_client, + chain=chain, + storage=storage, + analyzed_key_template="analyzed/repository/{repository_id}/summary.md", + ) + return analyzer, extractor, chain, storage, core_client + + +@pytest.mark.asyncio +async def test_happy_path_fetches_token_then_extracts_with_it() -> None: + analyzer, extractor, chain, storage, core_client = _make_analyzer( + extract_result=ExtractedSource(text="README + tree", source_type="REPOSITORY"), + analysis=DocumentAnalysisResult( + summary="요약", tech_stack=["Go"], markdown="## 개요\nx" + ), + token_result="user-abc", + ) + result = await analyzer.analyze( + repository_id=7, + repo_full_name="user/repo", + default_branch="main", + user_id=42, + ) + + core_client.fetch_github_token.assert_awaited_once_with(42) + extractor.extract.assert_awaited_once_with("user/repo", access_token="user-abc") + chain.analyze.assert_awaited_once_with( + text="README + tree", source_type="REPOSITORY" + ) + storage.put_text.assert_awaited_once_with( + "analyzed/repository/7/summary.md", "## 개요\nx" + ) + assert result.summary == "요약" + assert result.document_path == "analyzed/repository/7/summary.md" + + +@pytest.mark.asyncio +async def test_missing_user_id_raises_before_token_fetch() -> None: + analyzer, extractor, _, storage, core_client = _make_analyzer() + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze( + repository_id=1, + repo_full_name="user/repo", + user_id=None, + ) + assert exc_info.value.code == "MISSING_USER_ID" + assert exc_info.value.retriable is False + core_client.fetch_github_token.assert_not_called() + extractor.extract.assert_not_called() + storage.put_text.assert_not_called() + + +@pytest.mark.asyncio +async def test_token_fetch_error_translates_to_domain_error() -> None: + analyzer, extractor, _, storage, _ = _make_analyzer( + token_result=CoreTokenError( + code="USER_NOT_FOUND", message="404", retriable=False + ), + ) + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze(repository_id=1, repo_full_name="user/repo", user_id=42) + assert exc_info.value.code == "USER_NOT_FOUND" + assert exc_info.value.retriable is False + extractor.extract.assert_not_called() + storage.put_text.assert_not_called() + + +@pytest.mark.asyncio +async def test_token_fetch_retriable_propagates_retriable() -> None: + analyzer, _, _, _, _ = _make_analyzer( + token_result=CoreTokenError( + code="CORE_UNAVAILABLE", message="5xx", retriable=True + ), + ) + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze(repository_id=1, repo_full_name="user/repo", user_id=42) + assert exc_info.value.code == "CORE_UNAVAILABLE" + assert exc_info.value.retriable is True + + +@pytest.mark.asyncio +async def test_fetch_error_translates_to_domain_error() -> None: + analyzer, _, chain, storage, _ = _make_analyzer( + extract_result=RepositoryFetchError( + code="REPO_NOT_FOUND", message="not found", retriable=False + ), + ) + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze(repository_id=1, repo_full_name="user/repo", user_id=42) + assert exc_info.value.code == "REPO_NOT_FOUND" + chain.analyze.assert_not_called() + storage.put_text.assert_not_called() + + +@pytest.mark.asyncio +async def test_empty_text_raises_empty_repo_content() -> None: + analyzer, _, chain, storage, _ = _make_analyzer( + extract_result=ExtractedSource(text=" ", source_type="REPOSITORY"), + ) + with pytest.raises(RepositoryAnalyzeError) as exc_info: + await analyzer.analyze(repository_id=1, repo_full_name="user/repo", user_id=42) + assert exc_info.value.code == "EMPTY_REPO_CONTENT" + chain.analyze.assert_not_called() + storage.put_text.assert_not_called() diff --git a/ai/tests/test_repository_consumer.py b/ai/tests/test_repository_consumer.py new file mode 100644 index 00000000..278ab284 --- /dev/null +++ b/ai/tests/test_repository_consumer.py @@ -0,0 +1,148 @@ +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ai_server.analyzer.repository_analyzer import ( + RepositoryAnalysisResult, + RepositoryAnalyzeError, +) +from ai_server.messaging.consumers.repository_consumer import RepositoryConsumer +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + RepositoryAnalyzeRequest, +) + + +def _request_envelope(message_id: str = "req-1", repository_id: int = 7) -> bytes: + env = Envelope[RepositoryAnalyzeRequest].model_validate( + { + "messageId": message_id, + "messageType": "analyze.repository", + "version": "v1", + "traceId": "trace-r", + "publishedAt": datetime(2026, 5, 14, tzinfo=timezone.utc), + "publisher": "core-server", + "payload": { + "repositoryId": repository_id, + "repoFullName": "user/repo", + "defaultBranch": "main", + }, + "context": {"userId": 1}, + } + ) + return env.model_dump_json(by_alias=True).encode("utf-8") + + +def _incoming_message(body: bytes) -> MagicMock: + msg = MagicMock() + msg.body = body + msg.delivery_tag = 1 + process_ctx = AsyncMock() + process_ctx.__aenter__.return_value = None + process_ctx.__aexit__.return_value = False + msg.process = MagicMock(return_value=process_ctx) + return msg + + +def _make_consumer( + analyzer: AsyncMock, +) -> tuple[RepositoryConsumer, AsyncMock]: + publisher = AsyncMock() + consumer = RepositoryConsumer( + analyzer=analyzer, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=16), + callback_routing_key="callback.analysis", + ) + return consumer, publisher + + +def _captured_payload(publisher: AsyncMock) -> AnalysisCallbackPayload: + publisher.publish.assert_awaited_once() + kwargs = publisher.publish.await_args.kwargs + assert kwargs["message_type"] == "callback.analysis" + payload = kwargs["payload"] + assert isinstance(payload, AnalysisCallbackPayload) + return payload + + +@pytest.mark.asyncio +async def test_happy_path_publishes_analyzed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=RepositoryAnalysisResult( + summary="요약", + tech_stack=["Go"], + document_path="analyzed/repository/7/summary.md", + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + analyzer.analyze.assert_awaited_once_with( + repository_id=7, + repo_full_name="user/repo", + default_branch="main", + user_id=1, + ) + payload = _captured_payload(publisher) + assert payload.status == "ANALYZED" + assert payload.target_type == "REPOSITORY" + assert payload.target_id == 7 + assert payload.summary == "요약" + assert payload.tech_stack == ["Go"] + assert payload.document_path == "analyzed/repository/7/summary.md" + assert payload.embedding_chunk_count == 0 + + +@pytest.mark.asyncio +async def test_domain_error_publishes_failed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + side_effect=RepositoryAnalyzeError( + code="REPO_NOT_FOUND", message="404", retriable=False + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "REPO_NOT_FOUND" + assert payload.retriable is False + assert payload.target_type == "REPOSITORY" + + +@pytest.mark.asyncio +async def test_unexpected_error_publishes_retriable_failure() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock(side_effect=RuntimeError("boom")) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "UNEXPECTED" + assert payload.retriable is True + + +@pytest.mark.asyncio +async def test_idempotent_duplicate_skipped() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=RepositoryAnalysisResult( + summary="x", + tech_stack=[], + document_path="analyzed/repository/7/summary.md", + ) + ) + consumer, publisher = _make_consumer(analyzer) + body = _request_envelope(message_id="dup-r") + await consumer.handle(_incoming_message(body)) + await consumer.handle(_incoming_message(body)) + + assert analyzer.analyze.await_count == 1 + assert publisher.publish.await_count == 1 diff --git a/ai/tests/test_web_consumer.py b/ai/tests/test_web_consumer.py new file mode 100644 index 00000000..9782f2ba --- /dev/null +++ b/ai/tests/test_web_consumer.py @@ -0,0 +1,103 @@ +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ai_server.analyzer.web_resume_analyzer import ( + WebResumeAnalysisResult, + WebResumeAnalyzeError, +) +from ai_server.messaging.consumers.web_consumer import WebResumeConsumer +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.model.envelope import Envelope +from ai_server.model.messages.analyze import ( + AnalysisCallbackPayload, + WebResumeAnalyzeRequest, +) + + +def _request_envelope(message_id: str = "req-1", resume_id: int = 11) -> bytes: + env = Envelope[WebResumeAnalyzeRequest].model_validate( + { + "messageId": message_id, + "messageType": "analyze.web", + "version": "v1", + "traceId": "trace-w", + "publishedAt": datetime(2026, 5, 14, tzinfo=timezone.utc), + "publisher": "core-server", + "payload": {"resumeId": resume_id, "url": "https://example.com/me"}, + "context": {"userId": 1}, + } + ) + return env.model_dump_json(by_alias=True).encode("utf-8") + + +def _incoming_message(body: bytes) -> MagicMock: + msg = MagicMock() + msg.body = body + msg.delivery_tag = 1 + process_ctx = AsyncMock() + process_ctx.__aenter__.return_value = None + process_ctx.__aexit__.return_value = False + msg.process = MagicMock(return_value=process_ctx) + return msg + + +def _make_consumer(analyzer: AsyncMock) -> tuple[WebResumeConsumer, AsyncMock]: + publisher = AsyncMock() + consumer = WebResumeConsumer( + analyzer=analyzer, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=16), + callback_routing_key="callback.analysis", + ) + return consumer, publisher + + +def _captured_payload(publisher: AsyncMock) -> AnalysisCallbackPayload: + publisher.publish.assert_awaited_once() + payload = publisher.publish.await_args.kwargs["payload"] + assert isinstance(payload, AnalysisCallbackPayload) + return payload + + +@pytest.mark.asyncio +async def test_happy_path_publishes_analyzed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + return_value=WebResumeAnalysisResult( + summary="요약", + tech_stack=["React"], + document_path="analyzed/web-resume/11/summary.md", + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + analyzer.analyze.assert_awaited_once_with( + resume_id=11, url="https://example.com/me" + ) + payload = _captured_payload(publisher) + assert payload.status == "ANALYZED" + assert payload.target_type == "WEB" + assert payload.target_id == 11 + assert payload.summary == "요약" + assert payload.document_path == "analyzed/web-resume/11/summary.md" + + +@pytest.mark.asyncio +async def test_domain_error_publishes_failed_callback() -> None: + analyzer = AsyncMock() + analyzer.analyze = AsyncMock( + side_effect=WebResumeAnalyzeError( + code="WEB_HTTP_STATUS", message="404", retriable=False + ) + ) + consumer, publisher = _make_consumer(analyzer) + await consumer.handle(_incoming_message(_request_envelope())) + + payload = _captured_payload(publisher) + assert payload.status == "FAILED" + assert payload.error_code == "WEB_HTTP_STATUS" + assert payload.retriable is False + assert payload.target_type == "WEB" diff --git a/ai/tests/test_web_extractor.py b/ai/tests/test_web_extractor.py new file mode 100644 index 00000000..3bd75c66 --- /dev/null +++ b/ai/tests/test_web_extractor.py @@ -0,0 +1,117 @@ +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from ai_server.analyzer.sources.web import WebFetchError, WebSourceExtractor + + +def _make_client( + *, + status: int = 200, + content_type: str = "text/html; charset=utf-8", + body: ( + str | bytes + ) = "

주요 내용

본문 텍스트입니다.

", + final_url: str = "https://example.com/r", + raise_exc: Exception | None = None, +) -> MagicMock: + client = MagicMock() + resp = MagicMock(spec=httpx.Response) + resp.status_code = status + resp.headers = {"content-type": content_type} + resp.text = body if isinstance(body, str) else body.decode("utf-8", errors="ignore") + resp.content = body.encode("utf-8") if isinstance(body, str) else body + resp.url = final_url + if raise_exc is not None: + client.get = AsyncMock(side_effect=raise_exc) + else: + client.get = AsyncMock(return_value=resp) + return client + + +@pytest.mark.asyncio +async def test_extract_returns_main_body_text() -> None: + html = ( + "이력서" + "

김OO

" + "

백엔드 개발자. Spring Boot 3년차.

" + "" + ) + client = _make_client(body=html) + extractor = WebSourceExtractor(client=client) + result = await extractor.extract("https://example.com/r") + + assert result.source_type == "WEB" + assert "백엔드 개발자" in result.text + assert "김OO" in result.text + assert result.metadata["final_url"] == "https://example.com/r" + assert result.metadata["text_chars"] > 0 + + +@pytest.mark.asyncio +async def test_rejects_non_http_locator() -> None: + extractor = WebSourceExtractor(client=_make_client()) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("ftp://example.com/x") + assert exc_info.value.code == "INVALID_WEB_URL" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_raises_on_http_error_status() -> None: + client = _make_client(status=503) + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_HTTP_STATUS" + assert exc_info.value.retriable is True # 5xx → retriable + + +@pytest.mark.asyncio +async def test_raises_on_4xx_as_non_retriable() -> None: + client = _make_client(status=404) + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_HTTP_STATUS" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_rejects_non_html_content_type() -> None: + client = _make_client(content_type="application/pdf") + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_NOT_HTML" + + +@pytest.mark.asyncio +async def test_rejects_oversized_html() -> None: + big = b"" + b"a" * 1024 + b"" + client = _make_client(body=big) + extractor = WebSourceExtractor(client=client, max_html_bytes=512) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_HTML_TOO_LARGE" + + +@pytest.mark.asyncio +async def test_raises_on_empty_body() -> None: + client = _make_client(body="") + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "EMPTY_WEB_BODY" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_raises_on_httpx_error_as_retriable() -> None: + client = _make_client(raise_exc=httpx.ConnectError("dns fail")) + extractor = WebSourceExtractor(client=client) + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/r") + assert exc_info.value.code == "WEB_FETCH_FAILED" + assert exc_info.value.retriable is True diff --git a/ai/tests/test_web_resume_analyzer.py b/ai/tests/test_web_resume_analyzer.py new file mode 100644 index 00000000..6ccaf4f9 --- /dev/null +++ b/ai/tests/test_web_resume_analyzer.py @@ -0,0 +1,69 @@ +from unittest.mock import AsyncMock + +import pytest + +from ai_server.analyzer.sources.base import ExtractedSource +from ai_server.analyzer.sources.web import WebFetchError +from ai_server.analyzer.web_resume_analyzer import ( + WebResumeAnalyzeError, + WebResumeAnalyzer, +) +from ai_server.chain.document_analysis_chain import DocumentAnalysisResult + + +def _make_analyzer( + *, + extract_result: ExtractedSource | Exception | None = None, + analysis: DocumentAnalysisResult | None = None, +) -> tuple[WebResumeAnalyzer, AsyncMock, AsyncMock, AsyncMock]: + extractor = AsyncMock() + if isinstance(extract_result, Exception): + extractor.extract = AsyncMock(side_effect=extract_result) + else: + extractor.extract = AsyncMock(return_value=extract_result) + chain = AsyncMock() + if analysis is not None: + chain.analyze = AsyncMock(return_value=analysis) + storage = AsyncMock() + analyzer = WebResumeAnalyzer( + extractor=extractor, + chain=chain, + storage=storage, + analyzed_key_template="analyzed/web-resume/{resume_id}/summary.md", + ) + return analyzer, extractor, chain, storage + + +@pytest.mark.asyncio +async def test_happy_path() -> None: + analyzer, extractor, chain, storage = _make_analyzer( + extract_result=ExtractedSource(text="본문 텍스트", source_type="WEB"), + analysis=DocumentAnalysisResult( + summary="요약", tech_stack=["TypeScript"], markdown="## 개요\nweb" + ), + ) + result = await analyzer.analyze(resume_id=11, url="https://example.com/me") + + extractor.extract.assert_awaited_once_with("https://example.com/me") + chain.analyze.assert_awaited_once_with(text="본문 텍스트", source_type="WEB") + storage.put_text.assert_awaited_once_with( + "analyzed/web-resume/11/summary.md", "## 개요\nweb" + ) + assert result.summary == "요약" + assert result.tech_stack == ["TypeScript"] + assert result.document_path == "analyzed/web-resume/11/summary.md" + + +@pytest.mark.asyncio +async def test_fetch_error_translates_to_domain_error() -> None: + analyzer, _, chain, storage = _make_analyzer( + extract_result=WebFetchError( + code="WEB_HTTP_STATUS", message="404", retriable=False + ), + ) + with pytest.raises(WebResumeAnalyzeError) as exc_info: + await analyzer.analyze(resume_id=1, url="https://example.com/x") + assert exc_info.value.code == "WEB_HTTP_STATUS" + assert exc_info.value.retriable is False + chain.analyze.assert_not_called() + storage.put_text.assert_not_called() diff --git a/ai/uv.lock b/ai/uv.lock index f2fca866..78798338 100644 --- a/ai/uv.lock +++ b/ai/uv.lock @@ -174,6 +174,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + [[package]] name = "black" version = "26.3.1" @@ -316,6 +325,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "courlan" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "tld" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/54/6d6ceeff4bed42e7a10d6064d35ee43a810e7b3e8beb4abeae8cff4713ae/courlan-1.3.2.tar.gz", hash = "sha256:0b66f4db3a9c39a6e22dd247c72cfaa57d68ea660e94bb2c84ec7db8712af190", size = 206382, upload-time = "2024-10-29T16:40:20.994Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/ca/6a667ccbe649856dcd3458bab80b016681b274399d6211187c6ab969fc50/courlan-1.3.2-py3-none-any.whl", hash = "sha256:d0dab52cf5b5b1000ee2839fbc2837e93b2514d3cb5bb61ae158a55b7a04c6be", size = 33848, upload-time = "2024-10-29T16:40:18.325Z" }, +] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -329,6 +352,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, ] +[[package]] +name = "dateparser" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "regex" }, + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/2d/a0ccdb78788064fa0dc901b8524e50615c42be1d78b78d646d0b28d09180/dateparser-1.4.0.tar.gz", hash = "sha256:97a21840d5ecdf7630c584f673338a5afac5dfe84f647baf4d7e8df98f9354a4", size = 321512, upload-time = "2026-03-26T09:56:10.292Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/0b/3c3bb7cbe757279e693a0be6049048012f794d01f81099609ecd53b899f0/dateparser-1.4.0-py3-none-any.whl", hash = "sha256:7902b8e85d603494bf70a5a0b1decdddb2270b9c6e6b2bc8a57b93476c0df378", size = 300379, upload-time = "2026-03-26T09:56:08.409Z" }, +] + [[package]] name = "dill" version = "0.4.1" @@ -490,6 +528,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "htmldate" +version = "1.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "dateparser" }, + { name = "lxml" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/10/ead9dabc999f353c3aa5d0dc0835b1e355215a5ecb489a7f4ef2ddad5e33/htmldate-1.9.4.tar.gz", hash = "sha256:1129063e02dd0354b74264de71e950c0c3fcee191178321418ccad2074cc8ed0", size = 44690, upload-time = "2025-11-04T17:46:44.983Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/bd/adfcdaaad5805c0c5156aeefd64c1e868c05e9c1cd6fd21751f168cd88c7/htmldate-1.9.4-py3-none-any.whl", hash = "sha256:1b94bcc4e08232a5b692159903acf95548b6a7492dddca5bb123d89d6325921c", size = 31558, upload-time = "2025-11-04T17:46:43.258Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -660,6 +714,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" }, ] +[[package]] +name = "justext" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml", extra = ["html-clean"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/f3/45890c1b314f0d04e19c1c83d534e611513150939a7cf039664d9ab1e649/justext-3.0.2.tar.gz", hash = "sha256:13496a450c44c4cd5b5a75a5efcd9996066d2a189794ea99a49949685a0beb05", size = 828521, upload-time = "2025-02-25T20:21:49.934Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/ac/52f4e86d1924a7fc05af3aeb34488570eccc39b4af90530dd6acecdf16b5/justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7", size = 837940, upload-time = "2025-02-25T20:21:44.179Z" }, +] + [[package]] name = "langchain" version = "1.2.13" @@ -849,6 +915,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/94/1f5d72655ab6534129540843776c40eff757387b88e798d8b3bf7e313fd4/langsmith-0.7.22-py3-none-any.whl", hash = "sha256:6e9d5148314d74e86748cb9d3898632cad0320c9323d95f70f969e5bc078eee4", size = 359927, upload-time = "2026-03-19T22:45:21.603Z" }, ] +[[package]] +name = "lxml" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/28/30/9abc9e34c657c33834eaf6cd02124c61bdf5944d802aa48e69be8da3585d/lxml-6.1.0.tar.gz", hash = "sha256:bfd57d8008c4965709a919c3e9a98f76c2c7cb319086b3d26858250620023b13", size = 4197006, upload-time = "2026-04-18T04:32:51.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/03/69347590f1cf4a6d5a4944bb6099e6d37f334784f16062234e1f892fdb1d/lxml-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a0092f2b107b69601adf562a57c956fbb596e05e3e6651cabd3054113b007e45", size = 8559689, upload-time = "2026-04-18T04:31:57.785Z" }, + { url = "https://files.pythonhosted.org/packages/3f/58/25e00bb40b185c974cfe156c110474d9a8a8390d5f7c92a4e328189bb60e/lxml-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fc7140d7a7386e6b545d41b7358f4d02b656d4053f5fa6859f92f4b9c2572c4d", size = 4617892, upload-time = "2026-04-18T04:32:01.78Z" }, + { url = "https://files.pythonhosted.org/packages/f5/54/92ad98a94ac318dc4f97aaac22ff8d1b94212b2ae8af5b6e9b354bf825f7/lxml-6.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:419c58fc92cc3a2c3fa5f78c63dbf5da70c1fa9c1b25f25727ecee89a96c7de2", size = 4923489, upload-time = "2026-04-18T04:33:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/15/3b/a20aecfab42bdf4f9b390590d345857ad3ffd7c51988d1c89c53a0c73faf/lxml-6.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:37fabd1452852636cf38ecdcc9dd5ca4bba7a35d6c53fa09725deeb894a87491", size = 5082162, upload-time = "2026-04-18T04:33:34.262Z" }, + { url = "https://files.pythonhosted.org/packages/45/26/2cdb3d281ac1bd175603e290cbe4bad6eff127c0f8de90bafd6f8548f0fd/lxml-6.1.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2853c8b2170cc6cd54a6b4d50d2c1a8a7aeca201f23804b4898525c7a152cfc", size = 4993247, upload-time = "2026-04-18T04:33:36.674Z" }, + { url = "https://files.pythonhosted.org/packages/f6/05/d735aef963740022a08185c84821f689fc903acb3d50326e6b1e9886cc22/lxml-6.1.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e369cbd690e788c8d15e56222d91a09c6a417f49cbc543040cba0fe2e25a79e", size = 5613042, upload-time = "2026-04-18T04:33:39.205Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b8/ead7c10efff731738c72e59ed6eb5791854879fbed7ae98781a12006263a/lxml-6.1.0-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e69aa6805905807186eb00e66c6d97a935c928275182eb02ee40ba00da9623b2", size = 5228304, upload-time = "2026-04-18T04:33:41.647Z" }, + { url = "https://files.pythonhosted.org/packages/6b/10/e9842d2ec322ea65f0a7270aa0315a53abed06058b88ef1b027f620e7a5f/lxml-6.1.0-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:4bd1bdb8a9e0e2dd229de19b5f8aebac80e916921b4b2c6ef8a52bc131d0c1f9", size = 5341578, upload-time = "2026-04-18T04:33:44.596Z" }, + { url = "https://files.pythonhosted.org/packages/89/54/40d9403d7c2775fa7301d3ddd3464689bfe9ba71acc17dfff777071b4fdc/lxml-6.1.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:cbd7b79cdcb4986ad78a2662625882747f09db5e4cd7b2ae178a88c9c51b3dfe", size = 4700209, upload-time = "2026-04-18T04:33:47.552Z" }, + { url = "https://files.pythonhosted.org/packages/85/b2/bbdcc2cf45dfc7dfffef4fd97e5c47b15919b6a365247d95d6f684ef5e82/lxml-6.1.0-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:43e4d297f11080ec9d64a4b1ad7ac02b4484c9f0e2179d9c4ef78e886e747b88", size = 5232365, upload-time = "2026-04-18T04:33:50.249Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/b06875665e53aaba7127611a7bed3b7b9658e20b22bc2dd217a0b7ab0091/lxml-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cc16682cc987a3da00aa56a3aa3075b08edb10d9b1e476938cfdbee8f3b67181", size = 5043654, upload-time = "2026-04-18T04:33:52.71Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9c/e71a069d09641c1a7abeb30e693f828c7c90a41cbe3d650b2d734d876f85/lxml-6.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:d6d8efe71429635f0559579092bb5e60560d7b9115ee38c4adbea35632e7fa24", size = 4769326, upload-time = "2026-04-18T04:33:55.244Z" }, + { url = "https://files.pythonhosted.org/packages/cc/06/7a9cd84b3d4ed79adf35f874750abb697dec0b4a81a836037b36e47c091a/lxml-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e39ab3a28af7784e206d8606ec0e4bcad0190f63a492bca95e94e5a4aef7f6e", size = 5635879, upload-time = "2026-04-18T04:33:58.509Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f0/9d57916befc1e54c451712c7ee48e9e74e80ae4d03bdce49914e0aee42cd/lxml-6.1.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9eb667bf50856c4a58145f8ca2d5e5be160191e79eb9e30855a476191b3c3495", size = 5224048, upload-time = "2026-04-18T04:34:00.943Z" }, + { url = "https://files.pythonhosted.org/packages/99/75/90c4eefda0c08c92221fe0753db2d6699a4c628f76ff4465ec20dea84cc1/lxml-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7f4a77d6f7edf9230cee3e1f7f6764722a41604ee5681844f18db9a81ea0ec33", size = 5250241, upload-time = "2026-04-18T04:34:03.365Z" }, + { url = "https://files.pythonhosted.org/packages/5e/73/16596f7e4e38fa33084b9ccbccc22a15f82a290a055126f2c1541236d2ff/lxml-6.1.0-cp313-cp313-win32.whl", hash = "sha256:28902146ffbe5222df411c5d19e5352490122e14447e98cd118907ee3fd6ee62", size = 3596938, upload-time = "2026-04-18T04:31:56.206Z" }, + { url = "https://files.pythonhosted.org/packages/8e/63/981401c5680c1eb30893f00a19641ac80db5d1e7086c62cb4b13ed813038/lxml-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a1503c56e4e2b38dc76f2f2da7bae69670c0f1933e27cfa34b2fa5876410b16", size = 3995728, upload-time = "2026-04-18T04:31:58.763Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e8/c358a38ac3e541d16a1b527e4e9cb78c0419b0506a070ace11777e5e8404/lxml-6.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:e0af85773850417d994d019741239b901b22c6680206f46a34766926e466141d", size = 3658372, upload-time = "2026-04-18T04:32:03.629Z" }, + { url = "https://files.pythonhosted.org/packages/eb/45/cee4cf203ef0bab5c52afc118da61d6b460c928f2893d40023cfa27e0b80/lxml-6.1.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ab863fd37458fed6456525f297d21239d987800c46e67da5ef04fc6b3dd93ac8", size = 8576713, upload-time = "2026-04-18T04:32:06.831Z" }, + { url = "https://files.pythonhosted.org/packages/8a/a7/eda05babeb7e046839204eaf254cd4d7c9130ce2bbf0d9e90ea41af5654d/lxml-6.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6fd8b1df8254ff4fd93fd31da1fc15770bde23ac045be9bb1f87425702f61cc9", size = 4623874, upload-time = "2026-04-18T04:32:10.755Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e9/db5846de9b436b91890a62f29d80cd849ea17948a49bf532d5278ee69a9e/lxml-6.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:47024feaae386a92a146af0d2aeed65229bf6fff738e6a11dda6b0015fb8fd03", size = 4949535, upload-time = "2026-04-18T04:34:06.657Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ba/0d3593373dcae1d68f40dc3c41a5a92f2544e68115eb2f62319a4c2a6500/lxml-6.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f00972f84450204cd5d93a5395965e348956aaceaadec693a22ec743f8ae3eb", size = 5086881, upload-time = "2026-04-18T04:34:09.556Z" }, + { url = "https://files.pythonhosted.org/packages/43/76/759a7484539ad1af0d125a9afe9c3fb5f82a8779fd1f5f56319d9e4ea2fd/lxml-6.1.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97faa0860e13b05b15a51fb4986421ef7a30f0b3334061c416e0981e9450ca4c", size = 5031305, upload-time = "2026-04-18T04:34:12.336Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b9/c1f0daf981a11e47636126901fd4ab82429e18c57aeb0fc3ad2940b42d8b/lxml-6.1.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:972a6451204798675407beaad97b868d0c733d9a74dafefc63120b81b8c2de28", size = 5647522, upload-time = "2026-04-18T04:34:14.89Z" }, + { url = "https://files.pythonhosted.org/packages/31/e6/1f533dcd205275363d9ba3511bcec52fa2df86abf8abe6a5f2c599f0dc31/lxml-6.1.0-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe022f20bc4569ec66b63b3fb275a3d628d9d32da6326b2982584104db6d3086", size = 5239310, upload-time = "2026-04-18T04:34:17.652Z" }, + { url = "https://files.pythonhosted.org/packages/c3/8c/4175fb709c78a6e315ed814ed33be3defd8b8721067e70419a6cf6f971da/lxml-6.1.0-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:75c4c7c619a744f972f4451bf5adf6d0fb00992a1ffc9fd78e13b0bc817cc99f", size = 5350799, upload-time = "2026-04-18T04:34:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/6ffdebc5994975f0dde4acb59761902bd9d9bb84422b9a0bd239a7da9ca8/lxml-6.1.0-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3648f20d25102a22b6061c688beb3a805099ea4beb0a01ce62975d926944d292", size = 4697693, upload-time = "2026-04-18T04:34:23.541Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f1/565f36bd5c73294602d48e04d23f81ff4c8736be6ba5e1d1ec670ac9be80/lxml-6.1.0-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:77b9f99b17cbf14026d1e618035077060fc7195dd940d025149f3e2e830fbfcb", size = 5250708, upload-time = "2026-04-18T04:34:26.001Z" }, + { url = "https://files.pythonhosted.org/packages/5a/11/a68ab9dd18c5c499404deb4005f4bc4e0e88e5b72cd755ad96efec81d18d/lxml-6.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32662519149fd7a9db354175aa5e417d83485a8039b8aaa62f873ceee7ea4cad", size = 5084737, upload-time = "2026-04-18T04:34:28.32Z" }, + { url = "https://files.pythonhosted.org/packages/ab/78/e8f41e2c74f4af564e6a0348aea69fb6daaefa64bc071ef469823d22cc18/lxml-6.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:73d658216fc173cf2c939e90e07b941c5e12736b0bf6a99e7af95459cfe8eabb", size = 4737817, upload-time = "2026-04-18T04:34:30.784Z" }, + { url = "https://files.pythonhosted.org/packages/06/2d/aa4e117aa2ce2f3b35d9ff246be74a2f8e853baba5d2a92c64744474603a/lxml-6.1.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ac4db068889f8772a4a698c5980ec302771bb545e10c4b095d4c8be26749616f", size = 5670753, upload-time = "2026-04-18T04:34:33.675Z" }, + { url = "https://files.pythonhosted.org/packages/08/f5/dd745d50c0409031dbfcc4881740542a01e54d6f0110bd420fa7782110b8/lxml-6.1.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:45e9dfbd1b661eb64ba0d4dbe762bd210c42d86dd1e5bd2bdf89d634231beb43", size = 5238071, upload-time = "2026-04-18T04:34:36.12Z" }, + { url = "https://files.pythonhosted.org/packages/3e/74/ad424f36d0340a904665867dab310a3f1f4c96ff4039698de83b77f44c1f/lxml-6.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:89e8d73d09ac696a5ba42ec69787913d53284f12092f651506779314f10ba585", size = 5264319, upload-time = "2026-04-18T04:34:39.035Z" }, + { url = "https://files.pythonhosted.org/packages/53/36/a15d8b3514ec889bfd6aa3609107fcb6c9189f8dc347f1c0b81eded8d87c/lxml-6.1.0-cp314-cp314-win32.whl", hash = "sha256:ebe33f4ec1b2de38ceb225a1749a2965855bffeef435ba93cd2d5d540783bf2f", size = 3657139, upload-time = "2026-04-18T04:32:20.006Z" }, + { url = "https://files.pythonhosted.org/packages/1a/a4/263ebb0710851a3c6c937180a9a86df1206fdfe53cc43005aa2237fd7736/lxml-6.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:398443df51c538bd578529aa7e5f7afc6c292644174b47961f3bf87fe5741120", size = 4064195, upload-time = "2026-04-18T04:32:23.876Z" }, + { url = "https://files.pythonhosted.org/packages/80/68/2000f29d323b6c286de077ad20b429fc52272e44eae6d295467043e56012/lxml-6.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:8c8984e1d8c4b3949e419158fda14d921ff703a9ed8a47236c6eb7a2b6cb4946", size = 3741870, upload-time = "2026-04-18T04:32:27.922Z" }, + { url = "https://files.pythonhosted.org/packages/30/e9/21383c7c8d43799f0da90224c0d7c921870d476ec9b3e01e1b2c0b8237c5/lxml-6.1.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:1081dd10bc6fa437db2500e13993abf7cc30716d0a2f40e65abb935f02ec559c", size = 8827548, upload-time = "2026-04-18T04:32:15.094Z" }, + { url = "https://files.pythonhosted.org/packages/a5/01/c6bc11cd587030dd4f719f65c5657960649fe3e19196c844c75bf32cd0d6/lxml-6.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dabecc48db5f42ba348d1f5d5afdc54c6c4cc758e676926c7cd327045749517d", size = 4735866, upload-time = "2026-04-18T04:32:18.924Z" }, + { url = "https://files.pythonhosted.org/packages/f3/01/757132fff5f4acf25463b5298f1a46099f3a94480b806547b29ce5e385de/lxml-6.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e3dd5fe19c9e0ac818a9c7f132a5e43c1339ec1cbbfecb1a938bd3a47875b7c9", size = 4969476, upload-time = "2026-04-18T04:34:41.889Z" }, + { url = "https://files.pythonhosted.org/packages/fd/fb/1bc8b9d27ed64be7c8903db6c89e74dc8c2cd9ec630a7462e4654316dc5b/lxml-6.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7b0a4ca6dcc007a4cef00a761bba2dea959de4bd2df98f926b33c92ca5dfb9", size = 5103719, upload-time = "2026-04-18T04:34:44.797Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e7/5bf82fa28133536a54601aae633b14988e89ed61d4c1eb6b899b023233aa/lxml-6.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d27bbe326c6b539c64b42638b18bc6003a8d88f76213a97ac9ed4f885efeab7", size = 5027890, upload-time = "2026-04-18T04:34:47.634Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/e048db5d4b4ea0366648aa595f26bb764b2670903fc585b87436d0a5032c/lxml-6.1.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4e425db0c5445ef0ad56b0eec54f89b88b2d884656e536a90b2f52aecb4ca86", size = 5596008, upload-time = "2026-04-18T04:34:51.503Z" }, + { url = "https://files.pythonhosted.org/packages/9a/c2/d10807bc8da4824b39e5bd01b5d05c077b6fd01bd91584167edf6b269d22/lxml-6.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b89b098105b8599dc57adac95d1813409ac476d3c948a498775d3d0c6124bfb", size = 5224451, upload-time = "2026-04-18T04:34:54.263Z" }, + { url = "https://files.pythonhosted.org/packages/3c/15/2ebea45bea427e7f0057e9ce7b2d62c5aba20c6b001cca89ed0aadb3ad41/lxml-6.1.0-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:c4a699432846df86cc3de502ee85f445ebad748a1c6021d445f3e514d2cd4b1c", size = 5312135, upload-time = "2026-04-18T04:34:56.818Z" }, + { url = "https://files.pythonhosted.org/packages/31/e2/87eeae151b0be2a308d49a7ec444ff3eb192b14251e62addb29d0bf3778f/lxml-6.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:30e7b2ed63b6c8e97cca8af048589a788ab5c9c905f36d9cf1c2bb549f450d2f", size = 4639126, upload-time = "2026-04-18T04:34:59.704Z" }, + { url = "https://files.pythonhosted.org/packages/a3/51/8a3f6a20902ad604dd746ec7b4000311b240d389dac5e9d95adefd349e0c/lxml-6.1.0-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:022981127642fe19866d2907d76241bb07ed21749601f727d5d5dd1ce5d1b773", size = 5232579, upload-time = "2026-04-18T04:35:02.658Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d2/650d619bdbe048d2c3f2c31edb00e35670a5e2d65b4fe3b61bce37b19121/lxml-6.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:23cad0cc86046d4222f7f418910e46b89971c5a45d3c8abfad0f64b7b05e4a9b", size = 5084206, upload-time = "2026-04-18T04:35:05.175Z" }, + { url = "https://files.pythonhosted.org/packages/dd/8a/672ca1a3cbeabd1f511ca275a916c0514b747f4b85bdaae103b8fa92f307/lxml-6.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:21c3302068f50d1e8728c67c87ba92aa87043abee517aa2576cca1855326b405", size = 4758906, upload-time = "2026-04-18T04:35:08.098Z" }, + { url = "https://files.pythonhosted.org/packages/be/f1/ef4b691da85c916cb2feb1eec7414f678162798ac85e042fa164419ac05c/lxml-6.1.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:be10838781cb3be19251e276910cd508fe127e27c3242e50521521a0f3781690", size = 5620553, upload-time = "2026-04-18T04:35:11.23Z" }, + { url = "https://files.pythonhosted.org/packages/59/17/94e81def74107809755ac2782fdad4404420f1c92ca83433d117a6d5acf0/lxml-6.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2173a7bffe97667bbf0767f8a99e587740a8c56fdf3befac4b09cb29a80276fd", size = 5229458, upload-time = "2026-04-18T04:35:14.254Z" }, + { url = "https://files.pythonhosted.org/packages/21/55/c4be91b0f830a871fc1b0d730943d56013b683d4671d5198260e2eae722b/lxml-6.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c6854e9cf99c84beb004eecd7d3a3868ef1109bf2b1df92d7bc11e96a36c2180", size = 5247861, upload-time = "2026-04-18T04:35:17.006Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ca/77123e4d77df3cb1e968ade7b1f808f5d3a5c1c96b18a33895397de292c1/lxml-6.1.0-cp314-cp314t-win32.whl", hash = "sha256:00750d63ef0031a05331b9223463b1c7c02b9004cef2346a5b2877f0f9494dd2", size = 3897377, upload-time = "2026-04-18T04:32:07.656Z" }, + { url = "https://files.pythonhosted.org/packages/64/ce/3554833989d074267c063209bae8b09815e5656456a2d332b947806b05ff/lxml-6.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:80410c3a7e3c617af04de17caa9f9f20adaa817093293d69eae7d7d0522836f5", size = 4392701, upload-time = "2026-04-18T04:32:12.113Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a0/9b916c68c0e57752c07f8f64b30138d9d4059dbeb27b90274dedbea128ff/lxml-6.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:26dd9f57ee3bd41e7d35b4c98a2ffd89ed11591649f421f0ec19f67d50ec67ac", size = 3817120, upload-time = "2026-04-18T04:32:15.803Z" }, +] + +[package.optional-dependencies] +html-clean = [ + { name = "lxml-html-clean" }, +] + +[[package]] +name = "lxml-html-clean" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/a4/5c62acfacd69ff4f5db395100f5cfb9b54e7ac8c69a235e4e939fd13f021/lxml_html_clean-0.4.4.tar.gz", hash = "sha256:58f39a9d632711202ed1d6d0b9b47a904e306c85de5761543b90e3e3f736acfb", size = 23899, upload-time = "2026-02-27T09:35:52.911Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/76/7ffc1d3005cf7749123bc47cb3ea343cd97b0ac2211bab40f57283577d0e/lxml_html_clean-0.4.4-py3-none-any.whl", hash = "sha256:ce2ef506614ecb85ee1c5fe0a2aa45b06a19514ec7949e9c8f34f06925cfabcb", size = 14565, upload-time = "2026-02-27T09:35:51.86Z" }, +] + [[package]] name = "marshmallow" version = "3.26.2" @@ -1420,6 +1565,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] +[[package]] +name = "pytz" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1642,6 +1796,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pypdf" }, { name = "structlog" }, + { name = "trafilatura" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -1674,6 +1829,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" }, { name = "structlog", specifier = ">=25.5.0" }, + { name = "trafilatura", specifier = ">=2.0.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.42.0" }, ] provides-extras = ["dev"] @@ -1748,6 +1904,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, ] +[[package]] +name = "tld" +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5d/76b4383ac4e5b5e254e50c09807b3e13820bed6d6c11cd540264988d6802/tld-0.13.2.tar.gz", hash = "sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345", size = 467175, upload-time = "2026-03-06T23:50:34.498Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/90/39a85a4b63c84213e78b3c17d22e1bf45328acf8ebb33ef93be30d0a3911/tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c", size = 296743, upload-time = "2026-03-06T23:50:32.465Z" }, +] + [[package]] name = "tomlkit" version = "0.14.0" @@ -1769,6 +1934,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "trafilatura" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "courlan" }, + { name = "htmldate" }, + { name = "justext" }, + { name = "lxml" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/25/e3ebeefdebfdfae8c4a4396f5a6ea51fc6fa0831d63ce338e5090a8003dc/trafilatura-2.0.0.tar.gz", hash = "sha256:ceb7094a6ecc97e72fea73c7dba36714c5c5b577b6470e4520dca893706d6247", size = 253404, upload-time = "2024-12-03T15:23:24.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/b6/097367f180b6383a3581ca1b86fcae284e52075fa941d1232df35293363c/trafilatura-2.0.0-py3-none-any.whl", hash = "sha256:77eb5d1e993747f6f20938e1de2d840020719735690c840b9a1024803a4cd51d", size = 132557, upload-time = "2024-12-03T15:23:21.41Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -1803,6 +1986,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] +[[package]] +name = "tzdata" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "urllib3" version = "2.6.3" diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json index 88e92eb7..31972b86 100644 --- a/infra/rabbitmq/definitions.json +++ b/infra/rabbitmq/definitions.json @@ -57,6 +57,13 @@ "auto_delete": false, "arguments": {} }, + { + "name": "ai.analyze.web", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, { "name": "ai.generate.questions", "vhost": "/", @@ -108,6 +115,13 @@ "destination_type": "queue", "routing_key": "analyze.resume" }, + { + "source": "stackup.core-to-ai", + "vhost": "/", + "destination": "ai.analyze.web", + "destination_type": "queue", + "routing_key": "analyze.web" + }, { "source": "stackup.core-to-ai", "vhost": "/",