Merge pull request #16 from Team-StackUp/feature/us-09-chunking-embedding

Feature/us 09 chunking embedding - Chunking 및 Embedding 작성

Author: i3months

.github/workflows/deploy-app.yml

@@ -43,25 +43,32 @@ jobs:
             --exclude='**/node_modules' \
             "$GITHUB_WORKSPACE/" "$DEPLOY_DIR/"
 
-      - name: Ensure .env exists and contains current LLM_API_KEY
+      - name: Ensure .env exists and contains current secrets
         env:
           LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
         run: |
           cd "$DEPLOY_DIR"
           if [ ! -f .env ]; then
             cp .env.example .env
           fi
-          # Write LLM_API_KEY via a temp file to keep the secret off the
-          # process command line.
-          tmp=$(mktemp)
-          trap 'rm -f "$tmp"' EXIT
-          awk -v key="$LLM_API_KEY" '
-            BEGIN { written = 0 }
-            /^LLM_API_KEY=/ { print "LLM_API_KEY=" key; written = 1; next }
-            { print }
-            END { if (!written) print "LLM_API_KEY=" key }
-          ' .env > "$tmp"
-          mv "$tmp" .env
+          # 비밀 값은 임시 파일을 거쳐서 .env에 주입 — 명령줄 노출 회피.
+          # 키마다 같은 awk 블록을 한 번씩 적용.
+          inject_secret() {
+            local name="$1"
+            local value="$2"
+            local tmp
+            tmp=$(mktemp)
+            awk -v k="$name" -v v="$value" '
+              BEGIN { written = 0 }
+              $0 ~ "^" k "=" { print k "=" v; written = 1; next }
+              { print }
+              END { if (!written) print k "=" v }
+            ' .env > "$tmp"
+            mv "$tmp" .env
+          }
+          inject_secret LLM_API_KEY "$LLM_API_KEY"
+          inject_secret GEMINI_API_KEY "$GEMINI_API_KEY"
           chmod 600 .env
 
       - name: Build and restart app services

ai/.env.example

@@ -33,3 +33,14 @@ REPO_FETCH_TIMEOUT_SEC=30
 # 웹 이력서 관련
 WEB_FETCH_TIMEOUT_SEC=20
 WEB_MAX_HTML_BYTES=2000000
+
+# 개발 및 테스트는 mock 
+# 운영은 gemini
+EMBEDDING_PROVIDER=gemini
+EMBEDDING_MODEL=gemini-embedding-001
+EMBEDDING_DIM=1536
+EMBEDDING_CHUNK_SIZE=1000
+EMBEDDING_CHUNK_OVERLAP=200
+EMBEDDING_BATCH_SIZE=32
+
+GEMINI_API_KEY=

ai/CLAUDE.md

@@ -164,11 +164,14 @@ chain = prompt | llm | PydanticOutputParser(pydantic_object=...)
 
 ## 7. RAG 파이프라인
 
-### 7.1 인제스트
-1. 마크다운 입력
-2. 청킹 (LangChain `RecursiveCharacterTextSplitter`, chunk_size=1000, overlap=200)
-3. 임베딩 생성 (Gemini `text-embedding-004` 또는 OpenAI `text-embedding-3-small`)
-4. Core API 호출 → pgvector INSERT
+### 7.1 인제스트 (본 구현)
+1. 마크다운 입력 (`analyzer/_embedding_step.chunk_embed_and_upsert`)
+2. 청킹 — `rag/chunker.MarkdownChunker` (`RecursiveCharacterTextSplitter`, 기본 1000/200)
+3. 임베딩 생성 — `rag/embedder.EmbeddingProvider`. 현재 구현체:
+   - `MockEmbeddingProvider` (default) — 차원 결정 보류, e2e 흐름 검증용
+   - `openai` / `ollama` 구현체는 후속 PR
+4. Core API 호출 — `CoreClient.upsert_embeddings(document_id, model, dim, chunks)` →
+   `PUT /api/internal/documents/{id}/embeddings` (idempotent upsert)
 
 ### 7.2 검색
 1. 쿼리 텍스트 → 임베딩

ai/pyproject.toml

@@ -17,6 +17,8 @@ dependencies = [
     "aiofiles>=24.1.0",
     "pypdf>=5.1.0",
     "trafilatura>=2.0.0",
+    "langchain-text-splitters>=0.3.0",
+    "google-genai>=1.0.0",
     "langchain>=1.2.13",
     "langchain-core>=1.2.22",
     "langchain-community>=0.4.1",

ai/src/ai_server/analyzer/_embedding_step.py

@@ -0,0 +1,86 @@
+# 공통 임베딩 모듈 
+from __future__ import annotations
+
+import structlog
+
+from ai_server.core.client import (
+    CoreClient,
+    CoreEmbeddingUpsertError,
+    EmbeddingChunkPayload,
+)
+from ai_server.rag.chunker import MarkdownChunker
+from ai_server.rag.embedder import EmbeddingError, EmbeddingProvider
+
+log = structlog.get_logger(__name__)
+
+
+class EmbeddingStepError(Exception):
+    def __init__(self, *, code: str, message: str, retriable: bool) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.retriable = retriable
+
+
+async def chunk_embed_and_upsert(
+    *,
+    document_id: int,
+    markdown: str,
+    chunker: MarkdownChunker,
+    embedder: EmbeddingProvider,
+    core_client: CoreClient,
+    log_prefix: str = "analyze",
+) -> int:
+    chunks = chunker.split(markdown)
+    log.info(
+        f"{log_prefix}.chunk.done",
+        document_id=document_id,
+        chunk_count=len(chunks),
+    )
+
+    if not chunks:
+        return 0
+
+    try:
+        vectors = await embedder.embed([c.text for c in chunks])
+    except EmbeddingError as err:
+        raise EmbeddingStepError(
+            code=err.code, message=err.message, retriable=err.retriable
+        ) from err
+
+    if len(vectors) != len(chunks):
+        raise EmbeddingStepError(
+            code="EMBED_COUNT_MISMATCH",
+            message=(f"embedder가 chunk {len(chunks)}개 중 {len(vectors)}개만 반환"),
+            retriable=True,
+        )
+
+    payloads = [
+        EmbeddingChunkPayload(
+            chunk_index=chunks[i].index,
+            chunk_text=chunks[i].text,
+            embedding=vectors[i],
+        )
+        for i in range(len(chunks))
+    ]
+
+    try:
+        upserted = await core_client.upsert_embeddings(
+            document_id=document_id,
+            model=embedder.model,
+            dim=embedder.dim,
+            chunks=payloads,
+        )
+    except CoreEmbeddingUpsertError as err:
+        raise EmbeddingStepError(
+            code=err.code, message=err.message, retriable=err.retriable
+        ) from err
+
+    log.info(
+        f"{log_prefix}.embed.upserted",
+        document_id=document_id,
+        chunk_count=upserted,
+        model=embedder.model,
+        dim=embedder.dim,
+    )
+    return upserted

ai/src/ai_server/analyzer/repository_analyzer.py

@@ -4,12 +4,18 @@
 
 import structlog
 
+from ai_server.analyzer._embedding_step import (
+    EmbeddingStepError,
+    chunk_embed_and_upsert,
+)
 from ai_server.analyzer.sources.github_repo import (
     GitHubRepoSourceExtractor,
     RepositoryFetchError,
 )
 from ai_server.chain.document_analysis_chain import DocumentAnalyzer
 from ai_server.core.client import CoreClient, CoreTokenError
+from ai_server.rag.chunker import MarkdownChunker
+from ai_server.rag.embedder import EmbeddingProvider
 from ai_server.storage.base import ObjectStorage
 
 log = structlog.get_logger(__name__)
@@ -28,10 +34,10 @@ class RepositoryAnalysisResult:
     summary: str
     tech_stack: list[str]
     document_path: str
+    embedding_chunk_count: int
 
 
-# 코어서버에서 토큰 받아오고 레포 가져온다
-# 이후 LLM 분석하고 마크다운 저장 
+# Core에서 사용자별 GitHub token 수령 → 레포 fetch → LLM 분석 → 마크다운 저장 → 청킹·임베딩
 class RepositoryAnalyzer:
     def __init__(
         self,
@@ -40,12 +46,16 @@ def __init__(
         core_client: CoreClient,
         chain: DocumentAnalyzer,
         storage: ObjectStorage,
+        chunker: MarkdownChunker,
+        embedder: EmbeddingProvider,
         analyzed_key_template: str,
     ) -> None:
         self._extractor = extractor
         self._core_client = core_client
         self._chain = chain
         self._storage = storage
+        self._chunker = chunker
+        self._embedder = embedder
         self._analyzed_key_template = analyzed_key_template
 
     async def analyze(
@@ -55,6 +65,7 @@ async def analyze(
         repo_full_name: str,
         default_branch: str = "main",
         user_id: int | None,
+        analyzed_document_id: int,
     ) -> RepositoryAnalysisResult:
         if user_id is None:
             raise RepositoryAnalyzeError(
@@ -72,9 +83,7 @@ async def analyze(
             access_token = await self._core_client.fetch_github_token(user_id)
         except CoreTokenError as err:
             raise RepositoryAnalyzeError(
-                code=err.code,
-                message=err.message,
-                retriable=err.retriable,
+                code=err.code, message=err.message, retriable=err.retriable
             ) from err
 
         log.info(
@@ -90,9 +99,7 @@ async def analyze(
             )
         except RepositoryFetchError as err:
             raise RepositoryAnalyzeError(
-                code=err.code,
-                message=err.message,
-                retriable=err.retriable,
+                code=err.code, message=err.message, retriable=err.retriable
             ) from err
 
         if not source.text.strip():
@@ -121,8 +128,23 @@ async def analyze(
             md_chars=len(analysis.markdown),
         )
 
+        try:
+            chunk_count = await chunk_embed_and_upsert(
+                document_id=analyzed_document_id,
+                markdown=analysis.markdown,
+                chunker=self._chunker,
+                embedder=self._embedder,
+                core_client=self._core_client,
+                log_prefix="repository",
+            )
+        except EmbeddingStepError as err:
+            raise RepositoryAnalyzeError(
+                code=err.code, message=err.message, retriable=err.retriable
+            ) from err
+
         return RepositoryAnalysisResult(
             summary=analysis.summary,
             tech_stack=list(analysis.tech_stack),
             document_path=out_key,
+            embedding_chunk_count=chunk_count,
         )

ai/src/ai_server/analyzer/resume_analyzer.py

@@ -4,8 +4,15 @@
 
 import structlog
 
+from ai_server.analyzer._embedding_step import (
+    EmbeddingStepError,
+    chunk_embed_and_upsert,
+)
 from ai_server.analyzer.sources.base import SourceExtractor
 from ai_server.chain.document_analysis_chain import DocumentAnalyzer
+from ai_server.core.client import CoreClient
+from ai_server.rag.chunker import MarkdownChunker
+from ai_server.rag.embedder import EmbeddingProvider
 from ai_server.storage.base import ObjectStorage
 
 log = structlog.get_logger(__name__)
@@ -24,28 +31,36 @@ class ResumeAnalysisResult:
     summary: str
     tech_stack: list[str]
     document_path: str
+    embedding_chunk_count: int
 
 
-# 스토리지에서 가져오고 LLM을 통해 분석함
+# 스토리지에서 가져오고 LLM을 통해 분석 후 청킹·임베딩까지 처리
 class ResumeAnalyzer:
     def __init__(
         self,
         *,
         extractor: SourceExtractor,
         chain: DocumentAnalyzer,
         storage: ObjectStorage,
+        chunker: MarkdownChunker,
+        embedder: EmbeddingProvider,
+        core_client: CoreClient,
         analyzed_key_template: str,
     ) -> None:
         self._extractor = extractor
         self._chain = chain
         self._storage = storage
+        self._chunker = chunker
+        self._embedder = embedder
+        self._core_client = core_client
         self._analyzed_key_template = analyzed_key_template
 
     async def analyze(
         self,
         *,
         resume_id: int,
         file_path: str,
+        analyzed_document_id: int,
     ) -> ResumeAnalysisResult:
         log.info(
             "resume.extract.start",
@@ -81,8 +96,23 @@ async def analyze(
             md_chars=len(analysis.markdown),
         )
 
+        try:
+            chunk_count = await chunk_embed_and_upsert(
+                document_id=analyzed_document_id,
+                markdown=analysis.markdown,
+                chunker=self._chunker,
+                embedder=self._embedder,
+                core_client=self._core_client,
+                log_prefix="resume",
+            )
+        except EmbeddingStepError as err:
+            raise ResumeAnalyzeError(
+                code=err.code, message=err.message, retriable=err.retriable
+            ) from err
+
         return ResumeAnalysisResult(
             summary=analysis.summary,
             tech_stack=list(analysis.tech_stack),
             document_path=out_key,
+            embedding_chunk_count=chunk_count,
         )

ai/src/ai_server/analyzer/sources/github_repo.py

@@ -11,7 +11,7 @@
 log = structlog.get_logger(__name__)
 
 
-# 프로젝트 설정 파일을 먼저 읽는다 
+# 프로젝트 설정 파일을 먼저 읽는다
 _PRIORITY_FILES: tuple[str, ...] = (
     "package.json",
     "pyproject.toml",
@@ -68,7 +68,7 @@ class _RepoConfig:
     timeout_sec: float
 
 
-# 리드미, 주요 소스를 읽는다 
+# 리드미, 주요 소스를 읽는다
 class GitHubRepoSourceExtractor(SourceExtractor):
     def __init__(
         self,
@@ -87,7 +87,7 @@ def __init__(
             max_file_bytes=max_file_bytes,
             timeout_sec=timeout_sec,
         )
-        self._client = client  
+        self._client = client
 
     async def extract(
         self,

ai/src/ai_server/analyzer/sources/web.py

@@ -19,7 +19,7 @@ def __init__(self, *, code: str, message: str, retriable: bool) -> None:
         self.retriable = retriable
 
 
-# 라이브러리로 본문 추출 
+# 라이브러리로 본문 추출
 class WebSourceExtractor(SourceExtractor):
     def __init__(
         self,