From f1107995771db9cd9bb5b7144b324afb4b401967 Mon Sep 17 00:00:00 2001 From: jmj Date: Tue, 2 Jun 2026 23:45:50 +0900 Subject: [PATCH] =?UTF-8?q?feat(analysis):=20=EB=B6=84=EC=84=9D=20?= =?UTF-8?q?=ED=8C=8C=EC=9D=B4=ED=94=84=EB=9D=BC=EC=9D=B8=20=EA=B3=A0?= =?UTF-8?q?=EB=8F=84=ED=99=94=20=E2=80=94=20=EA=B5=AC=EC=A1=B0=ED=99=94?= =?UTF-8?q?=EC=B6=94=EC=B6=9C=C2=B7=EA=B8=B0=EC=97=AC=EB=8F=84=C2=B7Playwr?= =?UTF-8?q?ight=C2=B7Vision=20PDF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 섹션1 BP 중 입력처리/분석체인 품질 항목 구현. 콜백 계약·Core·DB·프론트 무변경(AI 내부 한정). - 분석 체인(A): DocumentAnalysisResult 에 projects/experiences/skills(+source_quote) 추가, 추출→요약 2단계 프롬프트(근거 인용 강제, 환각 차단), with_retry 안전장치. 콜백은 기존 summary/tech_stack/markdown 그대로. - GitHub 기여도(B): GET /user 로 지원자 login → 커밋 author 필터 → 지원자 작성 파일 우선 샘플링 + "## 지원자 기여" 섹션. 포크/남의코드 구분. graceful fallback. - Web Playwright(C): trafilatura 빈 본문 시 헤드리스 chromium 렌더 후 재추출 → JS 렌더링 SPA 포폴 처리. playwright dep + Dockerfile chromium 설치. - PDF Vision(D): pypdf 텍스트 빈약 시 pymupdf 렌더 → 게이트웨이 멀티모달로 추출(새 키 0). pymupdf dep. 텍스트 충분 시 기존 경로, 실패 시 graceful. 테스트: AI 203 passed. Docker 빌드(playwright/chromium) 검증은 Docker 환경에서 후속. Co-Authored-By: Claude Opus 4.8 --- ai/Dockerfile | 3 + ai/pyproject.toml | 2 + .../ai_server/analyzer/sources/github_repo.py | 116 +++++++++++++++++- ai/src/ai_server/analyzer/sources/pdf.py | 35 +++++- ai/src/ai_server/analyzer/sources/web.py | 37 ++++++ .../chain/document_analysis_chain.py | 44 +++++-- ai/src/ai_server/chain/pdf_vision.py | 84 +++++++++++++ .../chain/prompts/document_analysis.py | 26 ++-- ai/src/ai_server/chain/prompts/pdf_vision.py | 10 ++ ai/src/ai_server/config/settings.py | 4 + ai/src/ai_server/messaging/runner.py | 6 +- ai/tests/test_document_chain.py | 26 ++++ ai/tests/test_github_repo_extractor.py | 78 ++++++++++++ ai/tests/test_pdf_extractor.py | 50 ++++++++ ai/tests/test_pdf_vision.py | 58 +++++++++ ai/tests/test_web_extractor.py | 29 ++++- ai/uv.lock | 54 ++++++++ 17 files changed, 638 insertions(+), 24 deletions(-) create mode 100644 ai/src/ai_server/chain/pdf_vision.py create mode 100644 ai/src/ai_server/chain/prompts/pdf_vision.py create mode 100644 ai/tests/test_pdf_vision.py diff --git a/ai/Dockerfile b/ai/Dockerfile index ce3a4b3b..433b73b8 100644 --- a/ai/Dockerfile +++ b/ai/Dockerfile @@ -7,6 +7,9 @@ WORKDIR /app COPY pyproject.toml uv.lock ./ RUN uv sync --frozen --no-dev +# 웹 이력서(JS 렌더링 SPA) 폴백용 헤드리스 chromium + 시스템 의존성. +RUN uv run playwright install --with-deps chromium + COPY src/ src/ EXPOSE 38030 diff --git a/ai/pyproject.toml b/ai/pyproject.toml index d6ad403a..8d7c78cb 100644 --- a/ai/pyproject.toml +++ b/ai/pyproject.toml @@ -16,7 +16,9 @@ dependencies = [ "boto3>=1.42.77", "aiofiles>=24.1.0", "pypdf>=5.1.0", + "pymupdf>=1.24.0", "trafilatura>=2.0.0", + "playwright>=1.49.0", "langchain-text-splitters>=0.3.0", "google-genai>=1.0.0", "langchain>=1.2.13", diff --git a/ai/src/ai_server/analyzer/sources/github_repo.py b/ai/src/ai_server/analyzer/sources/github_repo.py index 70ea5e47..123b9b42 100644 --- a/ai/src/ai_server/analyzer/sources/github_repo.py +++ b/ai/src/ai_server/analyzer/sources/github_repo.py @@ -59,6 +59,13 @@ def __init__(self, *, code: str, message: str, retriable: bool) -> None: self.retriable = retriable +@dataclass(frozen=True) +class ContributionInfo: + login: str + commit_count: int + authored_files: list[str] + + @dataclass(frozen=True) class _RepoConfig: api_base_url: str @@ -66,6 +73,7 @@ class _RepoConfig: max_files: int max_file_bytes: int timeout_sec: float + max_contrib_commits: int # 리드미, 주요 소스를 읽는다 @@ -78,6 +86,7 @@ def __init__( max_files: int = 8, max_file_bytes: int = 50_000, timeout_sec: float = 30.0, + max_contrib_commits: int = 15, client: httpx.AsyncClient | None = None, ) -> None: self._cfg = _RepoConfig( @@ -86,6 +95,7 @@ def __init__( max_files=max_files, max_file_bytes=max_file_bytes, timeout_sec=timeout_sec, + max_contrib_commits=max_contrib_commits, ) self._client = client @@ -109,12 +119,18 @@ async def extract( ) effective_token = access_token or self._cfg.fallback_token + # 기여도 분석은 지원자 본인 token 이 있을 때만 의미 있음(GET /user = 지원자). + has_user_token = bool(access_token) if self._client is not None: - return await self._extract_with_client(self._client, owner_repo) + return await self._extract_with_client( + self._client, owner_repo, has_user_token=has_user_token + ) async with self._build_client(effective_token) as client: - return await self._extract_with_client(client, owner_repo) + return await self._extract_with_client( + client, owner_repo, has_user_token=has_user_token + ) def _build_client(self, token: str) -> httpx.AsyncClient: headers = { @@ -133,15 +149,24 @@ async def _extract_with_client( self, client: httpx.AsyncClient, owner_repo: str, + *, + has_user_token: bool = False, ) -> ExtractedSource: repo_info = await self._fetch_json(client, f"/repos/{owner_repo}") default_branch = repo_info.get("default_branch") or "main" + contribution: ContributionInfo | None = None + if has_user_token: + contribution = await self._fetch_contribution( + client, owner_repo, default_branch + ) + readme_text = await self._fetch_readme(client, owner_repo) tree_paths, truncated = await self._fetch_tree( client, owner_repo, default_branch ) - picked = _select_files(tree_paths, self._cfg.max_files) + authored = contribution.authored_files if contribution else [] + picked = _select_files(tree_paths, self._cfg.max_files, prioritized=authored) snippets = await self._fetch_snippets( client, owner_repo, default_branch, picked ) @@ -151,6 +176,16 @@ async def _extract_with_client( if desc := repo_info.get("description"): parts.append(f"\n> {desc}") + # 다중 기여자 레포에서 "지원자가 실제 쓴 코드" 를 구분하기 위한 기여 요약. + if contribution and contribution.commit_count: + parts.append("\n## 지원자 기여\n") + parts.append( + f"- 지원자(@{contribution.login}) 커밋 {contribution.commit_count}개" + ) + if contribution.authored_files: + top = contribution.authored_files[:10] + parts.append("- 주요 작성 파일:\n" + "\n".join(f" - {p}" for p in top)) + if readme_text: parts.append("\n## README\n") parts.append(readme_text) @@ -175,9 +210,64 @@ async def _extract_with_client( "tree_size": len(tree_paths), "tree_truncated": truncated, "sampled_files": [p for p, _ in snippets], + "contributor_login": contribution.login if contribution else None, + "contrib_commit_count": ( + contribution.commit_count if contribution else 0 + ), }, ) + # 지원자 token 으로 GET /user → login 확보 후, 그 login 이 author 인 커밋을 + # 집계해 기여 파일을 추린다. 실패(권한/rate limit/공개레포)는 None 으로 graceful. + async def _fetch_contribution( + self, + client: httpx.AsyncClient, + owner_repo: str, + branch: str, + ) -> ContributionInfo | None: + try: + me = await self._fetch_json(client, "/user") + except RepositoryFetchError as err: + log.warning("repo.contrib.user_skip", code=err.code) + return None + login = me.get("login") if isinstance(me, dict) else None + if not login: + return None + + try: + commits = await self._fetch_json( + client, + f"/repos/{owner_repo}/commits" + f"?author={login}&sha={branch}&per_page=100", + ) + except RepositoryFetchError as err: + log.warning("repo.contrib.commits_skip", code=err.code) + return None + if not isinstance(commits, list) or not commits: + return ContributionInfo(login=login, commit_count=0, authored_files=[]) + + file_counter: dict[str, int] = {} + for commit in commits[: self._cfg.max_contrib_commits]: + sha = commit.get("sha") if isinstance(commit, dict) else None + if not sha: + continue + try: + detail = await self._fetch_json( + client, f"/repos/{owner_repo}/commits/{sha}" + ) + except RepositoryFetchError: + continue + files = detail.get("files") if isinstance(detail, dict) else None + for f in files or []: + path = f.get("filename") if isinstance(f, dict) else None + if path: + file_counter[path] = file_counter.get(path, 0) + 1 + + authored = sorted(file_counter, key=lambda p: file_counter[p], reverse=True) + return ContributionInfo( + login=login, commit_count=len(commits), authored_files=authored + ) + async def _fetch_json(self, client: httpx.AsyncClient, path: str) -> dict: resp = await client.get(path) if resp.status_code == 404: @@ -267,10 +357,26 @@ async def _fetch_snippets( return out -def _select_files(paths: list[str], cap: int) -> list[str]: - """우선순위 → 디렉토리당 대표 텍스트 파일 1개씩, 합쳐서 cap까지.""" +def _select_files( + paths: list[str], cap: int, *, prioritized: list[str] | None = None +) -> list[str]: + """지원자 기여 파일 → 설정파일 우선순위 → 디렉토리당 대표 텍스트 파일, cap까지.""" chosen: list[str] = [] seen: set[str] = set() + tree = set(paths) + + # 지원자가 실제 작성한 텍스트 파일을 가장 먼저 (트리에 존재 + 텍스트 확장자). + for p in prioritized or []: + if p in seen or p not in tree: + continue + if "." not in p.rsplit("/", 1)[-1]: + continue + if ("." + p.rsplit(".", 1)[-1].lower()) not in _TEXT_EXTENSIONS: + continue + chosen.append(p) + seen.add(p) + if len(chosen) >= cap: + return chosen for needle in _PRIORITY_FILES: for p in paths: diff --git a/ai/src/ai_server/analyzer/sources/pdf.py b/ai/src/ai_server/analyzer/sources/pdf.py index 8dcf38b3..92af3c9f 100644 --- a/ai/src/ai_server/analyzer/sources/pdf.py +++ b/ai/src/ai_server/analyzer/sources/pdf.py @@ -2,12 +2,16 @@ import asyncio import io +from typing import Protocol +import structlog from pypdf import PdfReader from ai_server.analyzer.sources.base import ExtractedSource, SourceExtractor from ai_server.storage.base import ObjectStorage +log = structlog.get_logger(__name__) + def _extract_pdf_text(data: bytes) -> str: reader = PdfReader(io.BytesIO(data)) @@ -17,17 +21,44 @@ def _extract_pdf_text(data: bytes) -> str: return "\n\n".join(p for p in parts if p).strip() +# 이미지/스캔 PDF 의 텍스트를 비전 모델로 추출하는 어댑터 (구현체는 chain/pdf_vision). +class VisionPdfReader(Protocol): + async def extract_text(self, pdf_bytes: bytes) -> str: ... + + # PDF 를 읽어 페이지 텍스트를 이어붙인다. +# 텍스트 레이어가 비거나 너무 짧으면(스캔/이미지 PDF) vision_reader 로 폴백. class PdfSourceExtractor(SourceExtractor): - def __init__(self, storage: ObjectStorage) -> None: + def __init__( + self, + storage: ObjectStorage, + *, + vision_reader: VisionPdfReader | None = None, + min_text_chars: int = 50, + ) -> None: self._storage = storage + self._vision_reader = vision_reader + self._min_text_chars = min_text_chars async def extract(self, locator: str) -> ExtractedSource: data = await self._storage.get_bytes(locator) text = await asyncio.to_thread(_extract_pdf_text, data) + used_vision = False + + # 텍스트가 빈약하면(스캔/이미지 PDF) 비전 모델로 추출 시도. + if len(text.strip()) < self._min_text_chars and self._vision_reader is not None: + try: + vision_text = await self._vision_reader.extract_text(data) + except Exception as exc: # noqa: BLE001 + log.warning("pdf.vision.failed", locator=locator, error=str(exc)) + vision_text = "" + if vision_text.strip(): + text = vision_text.strip() + used_vision = True + return ExtractedSource( text=text, source_type="PDF", - metadata={"locator": locator, "bytes": len(data)}, + metadata={"locator": locator, "bytes": len(data), "vision": used_vision}, ) diff --git a/ai/src/ai_server/analyzer/sources/web.py b/ai/src/ai_server/analyzer/sources/web.py index ca44f9c5..130d4590 100644 --- a/ai/src/ai_server/analyzer/sources/web.py +++ b/ai/src/ai_server/analyzer/sources/web.py @@ -26,10 +26,12 @@ def __init__( *, timeout_sec: float = 20.0, max_html_bytes: int = 2_000_000, + enable_render_fallback: bool = True, client: httpx.AsyncClient | None = None, ) -> None: self._timeout_sec = timeout_sec self._max_html_bytes = max_html_bytes + self._enable_render_fallback = enable_render_fallback self._client = client async def extract(self, locator: str) -> ExtractedSource: @@ -43,6 +45,15 @@ async def extract(self, locator: str) -> ExtractedSource: html, final_url, content_type = await self._fetch_html(url) text = await asyncio.to_thread(_extract_main_text, html, final_url) + rendered = False + + # 본문이 비면 JS 렌더링 SPA(React 포폴 등)일 가능성 → Playwright 로 렌더 후 재추출. + if not text.strip() and self._enable_render_fallback: + rendered_html = await self._render(url) + if rendered_html: + html = rendered_html + text = await asyncio.to_thread(_extract_main_text, html, final_url) + rendered = True if not text.strip(): raise WebFetchError( @@ -60,9 +71,35 @@ async def extract(self, locator: str) -> ExtractedSource: "content_type": content_type, "html_bytes": len(html.encode("utf-8")), "text_chars": len(text), + "rendered": rendered, }, ) + async def _render(self, url: str) -> str | None: + """헤드리스 chromium 으로 페이지를 렌더해 최종 HTML 을 반환. + playwright 미설치/브라우저 미설치/렌더 실패는 None 으로 graceful.""" + try: + from playwright.async_api import async_playwright + except ImportError: + log.warning("web.render.playwright_unavailable", url=url) + return None + try: + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + try: + page = await browser.new_page() + await page.goto( + url, + wait_until="networkidle", + timeout=int(self._timeout_sec * 1000), + ) + return await page.content() + finally: + await browser.close() + except Exception as exc: # noqa: BLE001 + log.warning("web.render.failed", url=url, error=str(exc)) + return None + async def _fetch_html(self, url: str) -> tuple[str, str, str]: if self._client is not None: return await self._do_fetch(self._client, url) diff --git a/ai/src/ai_server/chain/document_analysis_chain.py b/ai/src/ai_server/chain/document_analysis_chain.py index ac949df2..f3c5dc2f 100644 --- a/ai/src/ai_server/chain/document_analysis_chain.py +++ b/ai/src/ai_server/chain/document_analysis_chain.py @@ -14,9 +14,32 @@ from ai_server.observability.llm_logging_callback import CoreAiLogCallback +class Project(BaseModel): + name: str + role: str = "" + contribution: str = "" + stack: list[str] = Field(default_factory=list) + source_quote: str = Field("", description="원문에서 그대로 따온 근거 인용") + + +class Experience(BaseModel): + title: str + detail: str = "" + source_quote: str = Field("", description="원문 근거 인용") + + +class Skill(BaseModel): + name: str + evidence: str = Field("", description="원문 근거 인용") + + class DocumentAnalysisResult(BaseModel): summary: str = Field(..., description="2~4 sentence Korean summary") tech_stack: list[str] = Field(default_factory=list) + # 구조화 추출 (AI 내부용 — 콜백 계약에는 포함되지 않음, markdown 에 녹여 사용). + projects: list[Project] = Field(default_factory=list) + experiences: list[Experience] = Field(default_factory=list) + skills: list[Skill] = Field(default_factory=list) markdown: str = Field(..., description="Interviewer-facing markdown") @@ -43,8 +66,11 @@ async def analyze( return result -# 프롬프트 -> LLM -> 파서 하나로 묶어서 처리함 -def build_document_analysis_chain(settings: Settings, core_client: CoreClient | None = None) -> Runnable: +# 프롬프트 -> LLM -> 파서 하나로 묶어서 처리함. +# 스키마가 커져 파싱 실패 가능성이 있으므로 with_retry 로 1회 재시도(저비용 안전장치). +def build_document_analysis_chain( + settings: Settings, core_client: CoreClient | None = None +) -> Runnable: from langchain_openai import ChatOpenAI parser = PydanticOutputParser(pydantic_object=DocumentAnalysisResult) @@ -57,11 +83,13 @@ def build_document_analysis_chain(settings: Settings, core_client: CoreClient | callbacks = [] if core_client is not None: - callbacks.append(CoreAiLogCallback( - core_client=core_client, - request_type="analyze.document", - default_model=settings.llm_pro_model, - )) + callbacks.append( + CoreAiLogCallback( + core_client=core_client, + request_type="analyze.document", + default_model=settings.llm_pro_model, + ) + ) llm = ChatOpenAI( model=settings.llm_pro_model, @@ -70,4 +98,4 @@ def build_document_analysis_chain(settings: Settings, core_client: CoreClient | base_url=settings.llm_base_url, callbacks=callbacks, ) - return prompt | llm | parser + return (prompt | llm | parser).with_retry(stop_after_attempt=2) diff --git a/ai/src/ai_server/chain/pdf_vision.py b/ai/src/ai_server/chain/pdf_vision.py new file mode 100644 index 00000000..f66cf244 --- /dev/null +++ b/ai/src/ai_server/chain/pdf_vision.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import asyncio +import base64 + +import structlog +from langchain_core.messages import HumanMessage + +from ai_server.chain.prompts.pdf_vision import VISION_PROMPT +from ai_server.config.settings import Settings +from ai_server.core.client import CoreClient +from ai_server.observability.llm_logging_callback import CoreAiLogCallback + +log = structlog.get_logger(__name__) + + +def _render_pdf_to_pngs(data: bytes, max_pages: int, dpi: int) -> list[bytes]: + import fitz # pymupdf + + images: list[bytes] = [] + with fitz.open(stream=data, filetype="pdf") as doc: + for i, page in enumerate(doc): + if i >= max_pages: + break + pix = page.get_pixmap(dpi=dpi) + images.append(pix.tobytes("png")) + return images + + +# 게이트웨이 멀티모달 모델로 PDF 페이지 이미지에서 텍스트를 추출한다 (호출 1회). +class LlmVisionPdfReader: + def __init__(self, llm, *, max_pages: int = 5, dpi: int = 150) -> None: + self._llm = llm + self._max_pages = max_pages + self._dpi = dpi + + async def extract_text(self, pdf_bytes: bytes) -> str: + images = await asyncio.to_thread( + _render_pdf_to_pngs, pdf_bytes, self._max_pages, self._dpi + ) + if not images: + return "" + content: list[dict] = [{"type": "text", "text": VISION_PROMPT}] + for png in images: + b64 = base64.b64encode(png).decode("ascii") + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + } + ) + resp = await self._llm.ainvoke([HumanMessage(content=content)]) + text = resp.content + if isinstance(text, list): # 일부 provider 는 content 를 블록 리스트로 반환 + text = "".join(b.get("text", "") for b in text if isinstance(b, dict)) + return text if isinstance(text, str) else "" + + +def build_vision_pdf_reader( + settings: Settings, core_client: CoreClient | None = None +) -> LlmVisionPdfReader: + from langchain_openai import ChatOpenAI + + callbacks = [] + if core_client is not None: + callbacks.append( + CoreAiLogCallback( + core_client=core_client, + request_type="analyze.pdf_vision", + default_model=settings.llm_pro_model, + ) + ) + llm = ChatOpenAI( + model=settings.llm_pro_model, # 멀티모달(gemini-3.1-pro) — 게이트웨이 경유 + temperature=0.0, + api_key=settings.llm_api_key or None, + base_url=settings.llm_base_url, + callbacks=callbacks, + ) + return LlmVisionPdfReader( + llm, + max_pages=settings.pdf_vision_max_pages, + dpi=settings.pdf_vision_dpi, + ) diff --git a/ai/src/ai_server/chain/prompts/document_analysis.py b/ai/src/ai_server/chain/prompts/document_analysis.py index 6cc9784c..c6ce8492 100644 --- a/ai/src/ai_server/chain/prompts/document_analysis.py +++ b/ai/src/ai_server/chain/prompts/document_analysis.py @@ -1,10 +1,17 @@ # 문서 분석 프롬프트 템플릿 -# source_type 으로 PDF, 웹, 리포지토리 구분함 +# source_type 으로 PDF, 웹, 리포지토리 구분함. +# 2단계 지향: ① 원문 근거(source_quote)와 함께 사실을 구조화 추출 → +# ② 추출 결과만 근거로 summary·markdown 생성 (환각 차단). SYSTEM_PROMPT = ( "당신은 IT 직군 채용을 진행하는 시니어 면접관입니다. " - "지원자 자료(이력서·레포 README·기술 블로그 등)를 분석하여 면접 준비에 사용할 " + "지원자 자료(이력서·레포 README·기술 블로그 등)를 분석하여 면접 준비에 쓸 " "구조화된 결과를 산출하세요.\n" - "- 사실에 근거해서 작성하고, 자료에 없는 내용을 추측해 채우지 마세요.\n" + "작업 순서:\n" + "1) 먼저 자료에서 사실을 구조화 추출합니다(projects·experiences·skills). " + "각 항목에는 반드시 원문에서 그대로 따온 짧은 근거 인용(source_quote/evidence)을 답니다. " + "근거를 찾을 수 없는 항목은 추출하지 마세요.\n" + "2) 그 다음, **추출한 항목만을 근거로** summary 와 markdown 을 작성합니다. " + "추출에 없는 내용을 summary/markdown 에 새로 지어내지 마세요.\n" "- 한국어로 작성하되 기술 용어는 영문 원어를 그대로 둡니다.\n" "- 응답은 반드시 지정된 JSON 스키마를 따릅니다." ) @@ -15,9 +22,14 @@ "{text}\n" "---\n\n" "요구 사항:\n" - "1. summary: 2~4문장 한국어 요약.\n" - "2. tech_stack: 자료에 명시된 핵심 기술 5~15개. 영문 표기.\n" - "3. markdown: 면접관이 훑어볼 한국어 마크다운. 섹션 구조는 " - "'## 개요', '## 주요 경험', '## 기술', '## 추가 메모' 사용.\n\n" + "1. projects: 자료에 나타난 프로젝트. 각 항목에 name, role, contribution, " + "stack, source_quote(원문 인용).\n" + "2. experiences: 경력/활동. 각 항목에 title, detail, source_quote.\n" + "3. skills: 핵심 기술. 각 항목에 name, evidence(원문 인용).\n" + "4. tech_stack: skills/projects 에서 도출한 핵심 기술 5~15개. 영문 표기.\n" + "5. summary: 위 추출 내용만 근거로 한 2~4문장 한국어 요약.\n" + "6. markdown: 면접관이 훑어볼 한국어 마크다운. 섹션 구조는 " + "'## 개요', '## 주요 경험', '## 기술', '## 추가 메모' 사용. " + "추출된 projects/experiences/skills 를 반영하되 추측은 넣지 마세요.\n\n" "{format_instructions}" ) diff --git a/ai/src/ai_server/chain/prompts/pdf_vision.py b/ai/src/ai_server/chain/prompts/pdf_vision.py new file mode 100644 index 00000000..44a5438a --- /dev/null +++ b/ai/src/ai_server/chain/prompts/pdf_vision.py @@ -0,0 +1,10 @@ +# 이미지/스캔 PDF 비전 추출 프롬프트. +# 렌더된 페이지 이미지를 받아 원문 텍스트를 그대로 옮긴다 (요약/해석 금지). +VISION_PROMPT = ( + "다음은 지원자 이력서 PDF 를 페이지별로 렌더한 이미지입니다. " + "각 이미지에 보이는 텍스트를 **있는 그대로** 추출해 한국어/영문 원문 그대로 옮기세요.\n" + "- 요약하거나 해석하지 말고, 보이는 내용을 그대로 전사합니다.\n" + "- 표/목록 구조는 가능한 한 유지합니다.\n" + "- 이미지에 없는 내용을 지어내지 마세요.\n" + "- 텍스트만 출력하세요(설명 문구 없이)." +) diff --git a/ai/src/ai_server/config/settings.py b/ai/src/ai_server/config/settings.py index 42fa0fa5..01d8eaa2 100644 --- a/ai/src/ai_server/config/settings.py +++ b/ai/src/ai_server/config/settings.py @@ -130,6 +130,10 @@ class Settings(BaseSettings): embedding_chunk_overlap: int = 200 embedding_batch_size: int = 32 + # PDF Vision (이미지/스캔 PDF 폴백 — 게이트웨이 멀티모달) + pdf_vision_max_pages: int = 5 + pdf_vision_dpi: int = 150 + gemini_api_key: str = "" diff --git a/ai/src/ai_server/messaging/runner.py b/ai/src/ai_server/messaging/runner.py index a270b04d..ff1489fe 100644 --- a/ai/src/ai_server/messaging/runner.py +++ b/ai/src/ai_server/messaging/runner.py @@ -13,6 +13,7 @@ LlmDocumentAnalyzer, build_document_analysis_chain, ) +from ai_server.chain.pdf_vision import build_vision_pdf_reader from ai_server.chain.followup_generation_chain import ( LlmFollowupGenerator, build_followup_generation_chain, @@ -97,10 +98,13 @@ def __init__(self, settings: Settings) -> None: gemini_api_key=settings.gemini_api_key, ) reranker = build_reranker(settings, core_client=core_client) + vision_pdf_reader = build_vision_pdf_reader(settings, core_client=core_client) # 이력서 PDF resume_analyzer = ResumeAnalyzer( - extractor=PdfSourceExtractor(storage=storage), + extractor=PdfSourceExtractor( + storage=storage, vision_reader=vision_pdf_reader + ), chain=chain_analyzer, storage=storage, chunker=chunker, diff --git a/ai/tests/test_document_chain.py b/ai/tests/test_document_chain.py index 32317641..2c2f4b46 100644 --- a/ai/tests/test_document_chain.py +++ b/ai/tests/test_document_chain.py @@ -33,3 +33,29 @@ async def fake_invoke(_: dict) -> dict: analyzer = LlmDocumentAnalyzer(RunnableLambda(fake_invoke)) with pytest.raises(TypeError): await analyzer.analyze(text="x", source_type="PDF") + + +def test_parser_parses_structured_extraction_with_source_quotes() -> None: + from langchain_core.output_parsers import PydanticOutputParser + + parser = PydanticOutputParser(pydantic_object=DocumentAnalysisResult) + obj = parser.parse( + '{"summary":"백엔드","tech_stack":["Go"],' + '"projects":[{"name":"결제","role":"BE","contribution":"분산락","stack":["Go"],' + '"source_quote":"결제 시스템에서 분산락을 도입"}],' + '"experiences":[{"title":"스타트업","detail":"3년","source_quote":"3년 차"}],' + '"skills":[{"name":"Kafka","evidence":"Kafka 로 처리량 3배"}],' + '"markdown":"## 개요\\n..."}' + ) + assert obj.projects[0].name == "결제" + assert obj.projects[0].source_quote == "결제 시스템에서 분산락을 도입" + assert obj.experiences[0].title == "스타트업" + assert obj.skills[0].evidence == "Kafka 로 처리량 3배" + + +def test_result_backward_compatible_defaults() -> None: + # 구조화 필드 없이도(기존 호출부) 생성 가능 — 콜백 계약 불변 보장 + r = DocumentAnalysisResult(summary="s", tech_stack=["x"], markdown="## 개요") + assert r.projects == [] + assert r.experiences == [] + assert r.skills == [] diff --git a/ai/tests/test_github_repo_extractor.py b/ai/tests/test_github_repo_extractor.py index 9fa165ab..9ecb6b7c 100644 --- a/ai/tests/test_github_repo_extractor.py +++ b/ai/tests/test_github_repo_extractor.py @@ -142,6 +142,84 @@ async def test_extract_rejects_malformed_locator() -> None: assert exc_info.value.code == "INVALID_REPO_LOCATOR" +@pytest.mark.asyncio +async def test_extract_includes_contributor_analysis_with_user_token() -> None: + routes = { + "/repos/user/repo": {"default_branch": "dev", "description": "demo"}, + "/user": {"login": "alice"}, + "/repos/user/repo/commits?author=alice&sha=dev&per_page=100": [ + {"sha": "c1"}, + {"sha": "c2"}, + ], + "/repos/user/repo/commits/c1": {"files": [{"filename": "src/pay.py"}]}, + "/repos/user/repo/commits/c2": { + "files": [{"filename": "src/pay.py"}, {"filename": "src/util.py"}] + }, + "/repos/user/repo/readme": 404, + "/repos/user/repo/git/trees/dev?recursive=1": { + "tree": [ + {"path": "src/pay.py", "type": "blob"}, + {"path": "src/util.py", "type": "blob"}, + {"path": "src/other.py", "type": "blob"}, + ], + "truncated": False, + }, + "/repos/user/repo/contents/src/pay.py?ref=dev": { + "encoding": "base64", + "content": _b64("def pay(): ..."), + }, + "/repos/user/repo/contents/src/util.py?ref=dev": { + "encoding": "base64", + "content": _b64("def util(): ..."), + }, + "/repos/user/repo/contents/src/other.py?ref=dev": { + "encoding": "base64", + "content": _b64("def other(): ..."), + }, + } + client, _ = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + + result = await extractor.extract("user/repo", access_token="alice-token") + + assert "## 지원자 기여" in result.text + assert "@alice" in result.text + assert "커밋 2개" in result.text + assert result.metadata["contributor_login"] == "alice" + assert result.metadata["contrib_commit_count"] == 2 + # 기여 파일(src/pay.py)이 샘플링 우선순위에 들어감 + assert "src/pay.py" in result.metadata["sampled_files"] + + +@pytest.mark.asyncio +async def test_extract_skips_contribution_without_user_token() -> None: + routes = { + "/repos/user/repo": {"default_branch": "main"}, + "/repos/user/repo/readme": 404, + "/repos/user/repo/git/trees/main?recursive=1": { + "tree": [], + "truncated": False, + }, + } + client, getter = _make_client(routes) + extractor = GitHubRepoSourceExtractor( + api_base_url="https://api.github.com", client=client + ) + result = await extractor.extract("user/repo") # access_token 없음 + # /user 를 호출하지 않음 (기여도 분석 스킵) + called_paths = [c.args[0] for c in getter.call_args_list] + assert "/user" not in called_paths + assert result.metadata["contrib_commit_count"] == 0 + + +def test_select_files_prioritizes_contributed_files_first() -> None: + paths = ["package.json", "src/index.ts", "src/feature/pay.ts"] + picked = _select_files(paths, cap=2, prioritized=["src/feature/pay.ts"]) + assert picked[0] == "src/feature/pay.ts" # 기여 파일이 최우선 + + def test_select_files_prioritizes_manifests_then_one_per_dir() -> None: paths = [ "package.json", diff --git a/ai/tests/test_pdf_extractor.py b/ai/tests/test_pdf_extractor.py index 1d4bb988..0ed23dfa 100644 --- a/ai/tests/test_pdf_extractor.py +++ b/ai/tests/test_pdf_extractor.py @@ -64,3 +64,53 @@ async def test_extract_returns_empty_on_empty_pdf() -> None: assert result.text == "" assert result.source_type == "PDF" + + +@pytest.mark.asyncio +async def test_extract_falls_back_to_vision_when_text_sparse() -> None: + storage = AsyncMock() + storage.get_bytes.return_value = b"PDF-BYTES" + vision = AsyncMock() + vision.extract_text = AsyncMock(return_value="비전으로 추출한 이력서 본문") + + with patch("ai_server.analyzer.sources.pdf._extract_pdf_text", return_value=""): + extractor = PdfSourceExtractor(storage=storage, vision_reader=vision) + result = await extractor.extract("scanned.pdf") + + vision.extract_text.assert_awaited_once_with(b"PDF-BYTES") + assert result.text == "비전으로 추출한 이력서 본문" + assert result.metadata["vision"] is True + + +@pytest.mark.asyncio +async def test_extract_skips_vision_when_text_sufficient() -> None: + storage = AsyncMock() + storage.get_bytes.return_value = b"PDF-BYTES" + vision = AsyncMock() + vision.extract_text = AsyncMock(return_value="should not be used") + long_text = "충분히 긴 텍스트 레이어가 있는 일반 PDF 입니다. " * 3 + + with patch( + "ai_server.analyzer.sources.pdf._extract_pdf_text", return_value=long_text + ): + extractor = PdfSourceExtractor(storage=storage, vision_reader=vision) + result = await extractor.extract("normal.pdf") + + vision.extract_text.assert_not_awaited() + assert "충분히 긴 텍스트 레이어" in result.text + assert result.metadata["vision"] is False + + +@pytest.mark.asyncio +async def test_extract_vision_failure_falls_back_gracefully() -> None: + storage = AsyncMock() + storage.get_bytes.return_value = b"PDF-BYTES" + vision = AsyncMock() + vision.extract_text = AsyncMock(side_effect=RuntimeError("vision down")) + + with patch("ai_server.analyzer.sources.pdf._extract_pdf_text", return_value=""): + extractor = PdfSourceExtractor(storage=storage, vision_reader=vision) + result = await extractor.extract("scanned.pdf") + + assert result.text == "" # 비전 실패 → 원래 텍스트(빈값)로 graceful + assert result.metadata["vision"] is False diff --git a/ai/tests/test_pdf_vision.py b/ai/tests/test_pdf_vision.py new file mode 100644 index 00000000..9fbb295b --- /dev/null +++ b/ai/tests/test_pdf_vision.py @@ -0,0 +1,58 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from ai_server.chain.pdf_vision import LlmVisionPdfReader + + +@pytest.mark.asyncio +async def test_vision_reader_sends_images_and_returns_text() -> None: + fake_llm = SimpleNamespace( + ainvoke=AsyncMock(return_value=SimpleNamespace(content="추출된 텍스트")) + ) + reader = LlmVisionPdfReader(fake_llm, max_pages=3, dpi=100) + + with patch( + "ai_server.chain.pdf_vision._render_pdf_to_pngs", + return_value=[b"png-a", b"png-b"], + ): + out = await reader.extract_text(b"PDF") + + assert out == "추출된 텍스트" + # LLM 에 텍스트 프롬프트 + 이미지 2장이 한 메시지로 전달됨 + msgs = fake_llm.ainvoke.await_args.args[0] + content = msgs[0].content + assert content[0]["type"] == "text" + image_blocks = [c for c in content if c["type"] == "image_url"] + assert len(image_blocks) == 2 + assert image_blocks[0]["image_url"]["url"].startswith("data:image/png;base64,") + + +@pytest.mark.asyncio +async def test_vision_reader_empty_when_no_pages() -> None: + fake_llm = SimpleNamespace(ainvoke=AsyncMock()) + reader = LlmVisionPdfReader(fake_llm) + with patch("ai_server.chain.pdf_vision._render_pdf_to_pngs", return_value=[]): + out = await reader.extract_text(b"PDF") + assert out == "" + fake_llm.ainvoke.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_vision_reader_handles_block_list_content() -> None: + # 일부 provider 는 content 를 [{type,text}] 블록 리스트로 반환 + fake_llm = SimpleNamespace( + ainvoke=AsyncMock( + return_value=SimpleNamespace( + content=[ + {"type": "text", "text": "조각1 "}, + {"type": "text", "text": "조각2"}, + ] + ) + ) + ) + reader = LlmVisionPdfReader(fake_llm) + with patch("ai_server.chain.pdf_vision._render_pdf_to_pngs", return_value=[b"png"]): + out = await reader.extract_text(b"PDF") + assert out == "조각1 조각2" diff --git a/ai/tests/test_web_extractor.py b/ai/tests/test_web_extractor.py index 3bd75c66..93619f02 100644 --- a/ai/tests/test_web_extractor.py +++ b/ai/tests/test_web_extractor.py @@ -100,13 +100,40 @@ async def test_rejects_oversized_html() -> None: @pytest.mark.asyncio async def test_raises_on_empty_body() -> None: client = _make_client(body="") - extractor = WebSourceExtractor(client=client) + extractor = WebSourceExtractor(client=client, enable_render_fallback=False) with pytest.raises(WebFetchError) as exc_info: await extractor.extract("https://example.com/r") assert exc_info.value.code == "EMPTY_WEB_BODY" assert exc_info.value.retriable is False +@pytest.mark.asyncio +async def test_empty_body_falls_back_to_render() -> None: + # 1차 fetch = JS 셸(본문 없음) → 렌더 폴백으로 본문 확보 + client = _make_client(body='
') + extractor = WebSourceExtractor(client=client) + rendered_html = ( + "

김OO

" + "

프론트엔드 개발자. React 포트폴리오.

" + ) + extractor._render = AsyncMock(return_value=rendered_html) + + result = await extractor.extract("https://example.com/spa") + assert "프론트엔드 개발자" in result.text + assert result.metadata["rendered"] is True + extractor._render.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_render_fallback_returning_none_raises_empty() -> None: + client = _make_client(body='
') + extractor = WebSourceExtractor(client=client) + extractor._render = AsyncMock(return_value=None) # 렌더 실패/불가 + with pytest.raises(WebFetchError) as exc_info: + await extractor.extract("https://example.com/spa") + assert exc_info.value.code == "EMPTY_WEB_BODY" + + @pytest.mark.asyncio async def test_raises_on_httpx_error_as_retriable() -> None: client = _make_client(raise_exc=httpx.ConnectError("dns fail")) diff --git a/ai/uv.lock b/ai/uv.lock index 6eba5261..91f9862c 100644 --- a/ai/uv.lock +++ b/ai/uv.lock @@ -634,6 +634,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/48/f8b875fa7dea7dd9b33245e37f065af59df6a25af2f9561efa8d822fde51/greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4", size = 279120, upload-time = "2026-02-20T20:19:01.9Z" }, { url = "https://files.pythonhosted.org/packages/49/8d/9771d03e7a8b1ee456511961e1b97a6d77ae1dea4a34a5b98eee706689d3/greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986", size = 603238, upload-time = "2026-02-20T20:47:32.873Z" }, { url = "https://files.pythonhosted.org/packages/59/0e/4223c2bbb63cd5c97f28ffb2a8aee71bdfb30b323c35d409450f51b91e3e/greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92", size = 614219, upload-time = "2026-02-20T20:55:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/94/2b/4d012a69759ac9d77210b8bfb128bc621125f5b20fc398bce3940d036b1c/greenlet-3.3.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccd21bb86944ca9be6d967cf7691e658e43417782bce90b5d2faeda0ff78a7dd", size = 628268, upload-time = "2026-02-20T21:02:48.024Z" }, { url = "https://files.pythonhosted.org/packages/7a/34/259b28ea7a2a0c904b11cd36c79b8cef8019b26ee5dbe24e73b469dea347/greenlet-3.3.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6997d360a4e6a4e936c0f9625b1c20416b8a0ea18a8e19cabbefc712e7397ab", size = 616774, upload-time = "2026-02-20T20:21:02.454Z" }, { url = "https://files.pythonhosted.org/packages/0a/03/996c2d1689d486a6e199cb0f1cf9e4aa940c500e01bdf201299d7d61fa69/greenlet-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64970c33a50551c7c50491671265d8954046cb6e8e2999aacdd60e439b70418a", size = 1571277, upload-time = "2026-02-20T20:49:34.795Z" }, { url = "https://files.pythonhosted.org/packages/d9/c4/2570fc07f34a39f2caf0bf9f24b0a1a0a47bc2e8e465b2c2424821389dfc/greenlet-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a9172f5bf6bd88e6ba5a84e0a68afeac9dc7b6b412b245dd64f52d83c81e55b", size = 1640455, upload-time = "2026-02-20T20:21:10.261Z" }, @@ -642,6 +643,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/ae/8bffcbd373b57a5992cd077cbe8858fff39110480a9d50697091faea6f39/greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab", size = 279650, upload-time = "2026-02-20T20:18:00.783Z" }, { url = "https://files.pythonhosted.org/packages/d1/c0/45f93f348fa49abf32ac8439938726c480bd96b2a3c6f4d949ec0124b69f/greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082", size = 650295, upload-time = "2026-02-20T20:47:34.036Z" }, { url = "https://files.pythonhosted.org/packages/b3/de/dd7589b3f2b8372069ab3e4763ea5329940fc7ad9dcd3e272a37516d7c9b/greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9", size = 662163, upload-time = "2026-02-20T20:56:01.295Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ac/85804f74f1ccea31ba518dcc8ee6f14c79f73fe36fa1beba38930806df09/greenlet-3.3.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9", size = 675371, upload-time = "2026-02-20T21:02:49.664Z" }, { url = "https://files.pythonhosted.org/packages/d2/d8/09bfa816572a4d83bccd6750df1926f79158b1c36c5f73786e26dbe4ee38/greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506", size = 664160, upload-time = "2026-02-20T20:21:04.015Z" }, { url = "https://files.pythonhosted.org/packages/48/cf/56832f0c8255d27f6c35d41b5ec91168d74ec721d85f01a12131eec6b93c/greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce", size = 1619181, upload-time = "2026-02-20T20:49:36.052Z" }, { url = "https://files.pythonhosted.org/packages/0a/23/b90b60a4aabb4cec0796e55f25ffbfb579a907c3898cd2905c8918acaa16/greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5", size = 1687713, upload-time = "2026-02-20T20:21:11.684Z" }, @@ -650,6 +652,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/6d/8f2ef704e614bcf58ed43cfb8d87afa1c285e98194ab2cfad351bf04f81e/greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54", size = 286617, upload-time = "2026-02-20T20:19:29.856Z" }, { url = "https://files.pythonhosted.org/packages/5e/0d/93894161d307c6ea237a43988f27eba0947b360b99ac5239ad3fe09f0b47/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4", size = 655189, upload-time = "2026-02-20T20:47:35.742Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/d2d506ebd8abcb57386ec4f7ba20f4030cbe56eae541bc6fd6ef399c0b41/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff", size = 658225, upload-time = "2026-02-20T20:56:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/d1/67/8197b7e7e602150938049d8e7f30de1660cfb87e4c8ee349b42b67bdb2e1/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:59b3e2c40f6706b05a9cd299c836c6aa2378cabe25d021acd80f13abf81181cf", size = 666581, upload-time = "2026-02-20T21:02:51.526Z" }, { url = "https://files.pythonhosted.org/packages/8e/30/3a09155fbf728673a1dea713572d2d31159f824a37c22da82127056c44e4/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4", size = 657907, upload-time = "2026-02-20T20:21:05.259Z" }, { url = "https://files.pythonhosted.org/packages/f3/fd/d05a4b7acd0154ed758797f0a43b4c0962a843bedfe980115e842c5b2d08/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727", size = 1618857, upload-time = "2026-02-20T20:49:37.309Z" }, { url = "https://files.pythonhosted.org/packages/6f/e1/50ee92a5db521de8f35075b5eff060dd43d39ebd46c2181a2042f7070385/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e", size = 1680010, upload-time = "2026-02-20T20:21:13.427Z" }, @@ -1415,6 +1418,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/d7/97f7e3a6abb67d8080dd406fd4df842c2be0efaf712d1c899c32a075027c/platformdirs-4.9.4-py3-none-any.whl", hash = "sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868", size = 21216, upload-time = "2026-03-05T18:34:12.172Z" }, ] +[[package]] +name = "playwright" +version = "1.60.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/f0/832bd9677194908da118064eef20082f2791e3d18215cc6d9391ee2c5a67/playwright-1.60.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:6a8cd0fec171fb3089e95e898c8bc8a6f35dea0b78b399e12fcc19427e91b1d7", size = 43474635, upload-time = "2026-05-18T12:00:31.969Z" }, + { url = "https://files.pythonhosted.org/packages/59/7b/e1d32ae8a3ed937ec2be3721c5f728b13d731a0b7c6442e0b3bec5094ac0/playwright-1.60.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:39b5420ba6145045b69ced4c5c47d4d9fe5bddfc8ff816c518913afcb25ec7a5", size = 42261327, upload-time = "2026-05-18T12:00:35.638Z" }, + { url = "https://files.pythonhosted.org/packages/d7/bc/23de499ded6411c188a20c5a0dea6f0cd4ed5d2b3cc6042a5dbd3ed609aa/playwright-1.60.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:2581d0e6a3392c71f91b27460c7fd093356818dc430f48153896c8aeeaef7705", size = 43474636, upload-time = "2026-05-18T12:00:39.294Z" }, + { url = "https://files.pythonhosted.org/packages/22/7b/1d679f4fced4ea94efadd17103856d8c565384f68382a1681264e46f5925/playwright-1.60.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:1c2bfae7884fb3fb05b853290eab8f343d524e5016f2f1def702acbbdf14c93e", size = 47467220, upload-time = "2026-05-18T12:00:43.179Z" }, + { url = "https://files.pythonhosted.org/packages/84/c2/1528d267d4442bd2c6b8eaeab819dd52c2030bf80e89293f0ba1f687473b/playwright-1.60.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43e66564125ee31b07a58cefb21e256d62d67d8d1713e6858df7a3019d8ed353", size = 47154856, upload-time = "2026-05-18T12:00:46.715Z" }, + { url = "https://files.pythonhosted.org/packages/bb/4e/b008b6440a7a1624378041da94829956d4b8f7ab9ef5aad22d0dc3f2e26d/playwright-1.60.0-py3-none-win32.whl", hash = "sha256:ec94e416ea320711e0ad4bf185dcbf41833672961e90773e1885255d7db7b7e7", size = 37902157, upload-time = "2026-05-18T12:00:50.374Z" }, + { url = "https://files.pythonhosted.org/packages/55/f0/0541524133104f9cc20bf900870ff4a736b76a23483f3a55295ddfa58409/playwright-1.60.0-py3-none-win_amd64.whl", hash = "sha256:9566821ce6030a1f9e7146a24e19355ab0d98805fd0f9be50bb3d8fef1750c02", size = 37902159, upload-time = "2026-05-18T12:00:53.728Z" }, + { url = "https://files.pythonhosted.org/packages/80/c8/210f282d278e4709cdd71b12a31af45a30a22ab3207b387e29b37e478713/playwright-1.60.0-py3-none-win_arm64.whl", hash = "sha256:6e4f6700a4c2250efff8e690a81d66e3855754fb587b6b87cf5c784014f91537", size = 34037981, upload-time = "2026-05-18T12:00:57.584Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1614,6 +1636,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/4b/ccc026168948fec4f7555b9164c724cf4125eac006e176541483d2c959be/pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237", size = 58929, upload-time = "2026-02-19T13:45:06.034Z" }, ] +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + [[package]] name = "pyflakes" version = "3.4.0" @@ -1650,6 +1684,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/6f/9ac2548e290764781f9e7e2aaf0685b086379dabfb29ca38536985471eaf/pylint-4.0.5-py3-none-any.whl", hash = "sha256:00f51c9b14a3b3ae08cff6b2cdd43f28165c78b165b628692e428fb1f8dc2cf2", size = 536694, upload-time = "2026-02-20T09:07:31.028Z" }, ] +[[package]] +name = "pymupdf" +version = "1.27.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/32/708bedc9dde7b328d45abbc076091769d44f2f24ad151ad92d56a6ec142b/pymupdf-1.27.2.3.tar.gz", hash = "sha256:7a92faa25129e8bbec5e50eeb9214f187665428c31b05c4ef6e36c58c0b1c6d2", size = 85759618, upload-time = "2026-04-24T14:13:14.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/09/ddbdfa7ee91fbabd6f63d7d744884cbdfe3e7ff9b8604749fb38bddf5c5d/pymupdf-1.27.2.3-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc1bc3cae6e9e150b0dbb0a9221bdfd411d65f0db2fe359eaa22467d7cc2a05f", size = 24002636, upload-time = "2026-04-24T14:09:17.459Z" }, + { url = "https://files.pythonhosted.org/packages/01/89/3f8edd6c4f50ca370e2a2f2a3011face36f3760728ffe76dffec91c0fca0/pymupdf-1.27.2.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:660d93cb6da5bbddf11d3982ae27745dd3a9902d9f24cdb69adab83962294b5a", size = 23278238, upload-time = "2026-04-24T14:09:32.882Z" }, + { url = "https://files.pythonhosted.org/packages/c3/26/b7e5a70eb83bd189f8b5df87ec442746b992f2f632662839b288170d357d/pymupdf-1.27.2.3-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1dd460a3ae4597a755f00a3bd9771f5ebf1531dc111f6a36bf05dd00a6b84425", size = 24333923, upload-time = "2026-04-24T14:09:47.341Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a0/aa1ee2240f29481a04a827c313333b4ecd8a14d6ac3e15d3f41a30574781/pymupdf-1.27.2.3-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:857842b4888827bd6155a1131341b2822a7ebe9a8c15a975fd7d490d7a64a30c", size = 24963198, upload-time = "2026-04-24T14:10:07.408Z" }, + { url = "https://files.pythonhosted.org/packages/69/49/4f742451f980840829fc00ba158bebb25d389c846d8f4f8c65936ee55de8/pymupdf-1.27.2.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:580983849c64a08d08344ca3d1580e87c01f046a8392421797bc850efd72a5b6", size = 25184609, upload-time = "2026-04-24T14:10:22.911Z" }, + { url = "https://files.pythonhosted.org/packages/f6/3f/3853d6608f394faf6eec2bd4e8ea9f6a00beea329b071abdb29f4164cc3d/pymupdf-1.27.2.3-cp310-abi3-win32.whl", hash = "sha256:a5c1088a87189891a4946ab314a14b7934ac4c5b6077f7e74ebee956f8906d0e", size = 18019286, upload-time = "2026-04-24T14:10:34.239Z" }, + { url = "https://files.pythonhosted.org/packages/44/47/5fb10fe73f96b31253a41647c362ea9e0380920bddf16028414a051247fc/pymupdf-1.27.2.3-cp310-abi3-win_amd64.whl", hash = "sha256:d20f68ef15195e073071dbc4ae7455257c7889af7584e39df490c0a92728526e", size = 19249102, upload-time = "2026-04-24T14:10:46.72Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/b9e91aac82293f9c954654c85581ee8212b5b05efadc534b581141241e6f/pymupdf-1.27.2.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:77691604c5d1d0233827139bbcdea61fd57879c84712b8e49b1f45520f7ab9c2", size = 25000393, upload-time = "2026-04-24T14:11:01.669Z" }, +] + [[package]] name = "pypdf" version = "6.11.0" @@ -1961,8 +2011,10 @@ dependencies = [ { name = "langchain-core" }, { name = "langchain-openai" }, { name = "langchain-text-splitters" }, + { name = "playwright" }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "pymupdf" }, { name = "pypdf" }, { name = "structlog" }, { name = "trafilatura" }, @@ -1994,9 +2046,11 @@ requires-dist = [ { name = "langchain-core", specifier = ">=1.2.22" }, { name = "langchain-openai", specifier = ">=0.3.0" }, { name = "langchain-text-splitters", specifier = ">=0.3.0" }, + { name = "playwright", specifier = ">=1.49.0" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "pydantic-settings", specifier = ">=2.13.1" }, { name = "pylint", marker = "extra == 'dev'", specifier = ">=4.0.5" }, + { name = "pymupdf", specifier = ">=1.24.0" }, { name = "pypdf", specifier = ">=5.1.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.3.0" },