Merge pull request #62 from Team-StackUp/feature/analysis-quality

feat(analysis): 분석 파이프라인 고도화 — 구조화추출·기여도·Playwright·Vision PDF

Author: i3months

ai/Dockerfile

@@ -7,6 +7,9 @@ WORKDIR /app
 COPY pyproject.toml uv.lock ./
 RUN uv sync --frozen --no-dev
 
+# 웹 이력서(JS 렌더링 SPA) 폴백용 헤드리스 chromium + 시스템 의존성.
+RUN uv run playwright install --with-deps chromium
+
 COPY src/ src/
 
 EXPOSE 38030

ai/pyproject.toml

@@ -16,7 +16,9 @@ dependencies = [
     "boto3>=1.42.77",
     "aiofiles>=24.1.0",
     "pypdf>=5.1.0",
+    "pymupdf>=1.24.0",
     "trafilatura>=2.0.0",
+    "playwright>=1.49.0",
     "langchain-text-splitters>=0.3.0",
     "google-genai>=1.0.0",
     "langchain>=1.2.13",

ai/src/ai_server/analyzer/sources/github_repo.py

@@ -59,13 +59,21 @@ def __init__(self, *, code: str, message: str, retriable: bool) -> None:
         self.retriable = retriable
 
 
+@dataclass(frozen=True)
+class ContributionInfo:
+    login: str
+    commit_count: int
+    authored_files: list[str]
+
+
 @dataclass(frozen=True)
 class _RepoConfig:
     api_base_url: str
     fallback_token: str
     max_files: int
     max_file_bytes: int
     timeout_sec: float
+    max_contrib_commits: int
 
 
 # 리드미, 주요 소스를 읽는다
@@ -78,6 +86,7 @@ def __init__(
         max_files: int = 8,
         max_file_bytes: int = 50_000,
         timeout_sec: float = 30.0,
+        max_contrib_commits: int = 15,
         client: httpx.AsyncClient | None = None,
     ) -> None:
         self._cfg = _RepoConfig(
@@ -86,6 +95,7 @@ def __init__(
             max_files=max_files,
             max_file_bytes=max_file_bytes,
             timeout_sec=timeout_sec,
+            max_contrib_commits=max_contrib_commits,
         )
         self._client = client
 
@@ -109,12 +119,18 @@ async def extract(
             )
 
         effective_token = access_token or self._cfg.fallback_token
+        # 기여도 분석은 지원자 본인 token 이 있을 때만 의미 있음(GET /user = 지원자).
+        has_user_token = bool(access_token)
 
         if self._client is not None:
-            return await self._extract_with_client(self._client, owner_repo)
+            return await self._extract_with_client(
+                self._client, owner_repo, has_user_token=has_user_token
+            )
 
         async with self._build_client(effective_token) as client:
-            return await self._extract_with_client(client, owner_repo)
+            return await self._extract_with_client(
+                client, owner_repo, has_user_token=has_user_token
+            )
 
     def _build_client(self, token: str) -> httpx.AsyncClient:
         headers = {
@@ -133,15 +149,24 @@ async def _extract_with_client(
         self,
         client: httpx.AsyncClient,
         owner_repo: str,
+        *,
+        has_user_token: bool = False,
     ) -> ExtractedSource:
         repo_info = await self._fetch_json(client, f"/repos/{owner_repo}")
         default_branch = repo_info.get("default_branch") or "main"
 
+        contribution: ContributionInfo | None = None
+        if has_user_token:
+            contribution = await self._fetch_contribution(
+                client, owner_repo, default_branch
+            )
+
         readme_text = await self._fetch_readme(client, owner_repo)
         tree_paths, truncated = await self._fetch_tree(
             client, owner_repo, default_branch
         )
-        picked = _select_files(tree_paths, self._cfg.max_files)
+        authored = contribution.authored_files if contribution else []
+        picked = _select_files(tree_paths, self._cfg.max_files, prioritized=authored)
         snippets = await self._fetch_snippets(
             client, owner_repo, default_branch, picked
         )
@@ -151,6 +176,16 @@ async def _extract_with_client(
         if desc := repo_info.get("description"):
             parts.append(f"\n> {desc}")
 
+        # 다중 기여자 레포에서 "지원자가 실제 쓴 코드" 를 구분하기 위한 기여 요약.
+        if contribution and contribution.commit_count:
+            parts.append("\n## 지원자 기여\n")
+            parts.append(
+                f"- 지원자(@{contribution.login}) 커밋 {contribution.commit_count}개"
+            )
+            if contribution.authored_files:
+                top = contribution.authored_files[:10]
+                parts.append("- 주요 작성 파일:\n" + "\n".join(f"  - {p}" for p in top))
+
         if readme_text:
             parts.append("\n## README\n")
             parts.append(readme_text)
@@ -175,9 +210,64 @@ async def _extract_with_client(
                 "tree_size": len(tree_paths),
                 "tree_truncated": truncated,
                 "sampled_files": [p for p, _ in snippets],
+                "contributor_login": contribution.login if contribution else None,
+                "contrib_commit_count": (
+                    contribution.commit_count if contribution else 0
+                ),
             },
         )
 
+    # 지원자 token 으로 GET /user → login 확보 후, 그 login 이 author 인 커밋을
+    # 집계해 기여 파일을 추린다. 실패(권한/rate limit/공개레포)는 None 으로 graceful.
+    async def _fetch_contribution(
+        self,
+        client: httpx.AsyncClient,
+        owner_repo: str,
+        branch: str,
+    ) -> ContributionInfo | None:
+        try:
+            me = await self._fetch_json(client, "/user")
+        except RepositoryFetchError as err:
+            log.warning("repo.contrib.user_skip", code=err.code)
+            return None
+        login = me.get("login") if isinstance(me, dict) else None
+        if not login:
+            return None
+
+        try:
+            commits = await self._fetch_json(
+                client,
+                f"/repos/{owner_repo}/commits"
+                f"?author={login}&sha={branch}&per_page=100",
+            )
+        except RepositoryFetchError as err:
+            log.warning("repo.contrib.commits_skip", code=err.code)
+            return None
+        if not isinstance(commits, list) or not commits:
+            return ContributionInfo(login=login, commit_count=0, authored_files=[])
+
+        file_counter: dict[str, int] = {}
+        for commit in commits[: self._cfg.max_contrib_commits]:
+            sha = commit.get("sha") if isinstance(commit, dict) else None
+            if not sha:
+                continue
+            try:
+                detail = await self._fetch_json(
+                    client, f"/repos/{owner_repo}/commits/{sha}"
+                )
+            except RepositoryFetchError:
+                continue
+            files = detail.get("files") if isinstance(detail, dict) else None
+            for f in files or []:
+                path = f.get("filename") if isinstance(f, dict) else None
+                if path:
+                    file_counter[path] = file_counter.get(path, 0) + 1
+
+        authored = sorted(file_counter, key=lambda p: file_counter[p], reverse=True)
+        return ContributionInfo(
+            login=login, commit_count=len(commits), authored_files=authored
+        )
+
     async def _fetch_json(self, client: httpx.AsyncClient, path: str) -> dict:
         resp = await client.get(path)
         if resp.status_code == 404:
@@ -267,10 +357,26 @@ async def _fetch_snippets(
         return out
 
 
-def _select_files(paths: list[str], cap: int) -> list[str]:
-    """우선순위 → 디렉토리당 대표 텍스트 파일 1개씩, 합쳐서 cap까지."""
+def _select_files(
+    paths: list[str], cap: int, *, prioritized: list[str] | None = None
+) -> list[str]:
+    """지원자 기여 파일 → 설정파일 우선순위 → 디렉토리당 대표 텍스트 파일, cap까지."""
     chosen: list[str] = []
     seen: set[str] = set()
+    tree = set(paths)
+
+    # 지원자가 실제 작성한 텍스트 파일을 가장 먼저 (트리에 존재 + 텍스트 확장자).
+    for p in prioritized or []:
+        if p in seen or p not in tree:
+            continue
+        if "." not in p.rsplit("/", 1)[-1]:
+            continue
+        if ("." + p.rsplit(".", 1)[-1].lower()) not in _TEXT_EXTENSIONS:
+            continue
+        chosen.append(p)
+        seen.add(p)
+        if len(chosen) >= cap:
+            return chosen
 
     for needle in _PRIORITY_FILES:
         for p in paths:

ai/src/ai_server/analyzer/sources/pdf.py

@@ -2,12 +2,16 @@
 
 import asyncio
 import io
+from typing import Protocol
 
+import structlog
 from pypdf import PdfReader
 
 from ai_server.analyzer.sources.base import ExtractedSource, SourceExtractor
 from ai_server.storage.base import ObjectStorage
 
+log = structlog.get_logger(__name__)
+
 
 def _extract_pdf_text(data: bytes) -> str:
     reader = PdfReader(io.BytesIO(data))
@@ -17,17 +21,44 @@ def _extract_pdf_text(data: bytes) -> str:
     return "\n\n".join(p for p in parts if p).strip()
 
 
+# 이미지/스캔 PDF 의 텍스트를 비전 모델로 추출하는 어댑터 (구현체는 chain/pdf_vision).
+class VisionPdfReader(Protocol):
+    async def extract_text(self, pdf_bytes: bytes) -> str: ...
+
+
 # PDF 를 읽어 페이지 텍스트를 이어붙인다.
+# 텍스트 레이어가 비거나 너무 짧으면(스캔/이미지 PDF) vision_reader 로 폴백.
 class PdfSourceExtractor(SourceExtractor):
 
-    def __init__(self, storage: ObjectStorage) -> None:
+    def __init__(
+        self,
+        storage: ObjectStorage,
+        *,
+        vision_reader: VisionPdfReader | None = None,
+        min_text_chars: int = 50,
+    ) -> None:
         self._storage = storage
+        self._vision_reader = vision_reader
+        self._min_text_chars = min_text_chars
 
     async def extract(self, locator: str) -> ExtractedSource:
         data = await self._storage.get_bytes(locator)
         text = await asyncio.to_thread(_extract_pdf_text, data)
+        used_vision = False
+
+        # 텍스트가 빈약하면(스캔/이미지 PDF) 비전 모델로 추출 시도.
+        if len(text.strip()) < self._min_text_chars and self._vision_reader is not None:
+            try:
+                vision_text = await self._vision_reader.extract_text(data)
+            except Exception as exc:  # noqa: BLE001
+                log.warning("pdf.vision.failed", locator=locator, error=str(exc))
+                vision_text = ""
+            if vision_text.strip():
+                text = vision_text.strip()
+                used_vision = True
+
         return ExtractedSource(
             text=text,
             source_type="PDF",
-            metadata={"locator": locator, "bytes": len(data)},
+            metadata={"locator": locator, "bytes": len(data), "vision": used_vision},
         )

ai/src/ai_server/analyzer/sources/web.py

@@ -26,10 +26,12 @@ def __init__(
         *,
         timeout_sec: float = 20.0,
         max_html_bytes: int = 2_000_000,
+        enable_render_fallback: bool = True,
         client: httpx.AsyncClient | None = None,
     ) -> None:
         self._timeout_sec = timeout_sec
         self._max_html_bytes = max_html_bytes
+        self._enable_render_fallback = enable_render_fallback
         self._client = client
 
     async def extract(self, locator: str) -> ExtractedSource:
@@ -43,6 +45,15 @@ async def extract(self, locator: str) -> ExtractedSource:
 
         html, final_url, content_type = await self._fetch_html(url)
         text = await asyncio.to_thread(_extract_main_text, html, final_url)
+        rendered = False
+
+        # 본문이 비면 JS 렌더링 SPA(React 포폴 등)일 가능성 → Playwright 로 렌더 후 재추출.
+        if not text.strip() and self._enable_render_fallback:
+            rendered_html = await self._render(url)
+            if rendered_html:
+                html = rendered_html
+                text = await asyncio.to_thread(_extract_main_text, html, final_url)
+                rendered = True
 
         if not text.strip():
             raise WebFetchError(
@@ -60,9 +71,35 @@ async def extract(self, locator: str) -> ExtractedSource:
                 "content_type": content_type,
                 "html_bytes": len(html.encode("utf-8")),
                 "text_chars": len(text),
+                "rendered": rendered,
             },
         )
 
+    async def _render(self, url: str) -> str | None:
+        """헤드리스 chromium 으로 페이지를 렌더해 최종 HTML 을 반환.
+        playwright 미설치/브라우저 미설치/렌더 실패는 None 으로 graceful."""
+        try:
+            from playwright.async_api import async_playwright
+        except ImportError:
+            log.warning("web.render.playwright_unavailable", url=url)
+            return None
+        try:
+            async with async_playwright() as p:
+                browser = await p.chromium.launch(headless=True)
+                try:
+                    page = await browser.new_page()
+                    await page.goto(
+                        url,
+                        wait_until="networkidle",
+                        timeout=int(self._timeout_sec * 1000),
+                    )
+                    return await page.content()
+                finally:
+                    await browser.close()
+        except Exception as exc:  # noqa: BLE001
+            log.warning("web.render.failed", url=url, error=str(exc))
+            return None
+
     async def _fetch_html(self, url: str) -> tuple[str, str, str]:
         if self._client is not None:
             return await self._do_fetch(self._client, url)