Merge pull request #112 from Team-StackUp/feature/multi-agent-feedback-panel

feat(ai): 멀티 면접관 패널 피드백 평가 (Phase 1 — AI 단독, 출력 호환)

Author: i3months

ai/src/ai_server/chain/feedback_generation_chain.py

@@ -1,17 +1,23 @@
 from __future__ import annotations
 
+import asyncio
+from dataclasses import dataclass
 from typing import Protocol
 
+import structlog
 from langchain_core.output_parsers import PydanticOutputParser
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.runnables import Runnable
 from pydantic import BaseModel, Field
 
 from ai_server.chain.prompts.feedback_generation import HUMAN_PROMPT, SYSTEM_PROMPT
+from ai_server.chain.prompts import feedback_panel
 from ai_server.config.settings import Settings
 from ai_server.core.client import CoreClient
 from ai_server.observability.llm_logging_callback import CoreAiLogCallback
 
+log = structlog.get_logger(__name__)
+
 
 class FeedbackResult(BaseModel):
     overall_score: float | None = Field(None, description="0~100")
@@ -105,3 +111,221 @@ def build_feedback_generation_chain(
         callbacks=callbacks,
     )
     return prompt | llm | parser
+
+
+# ── 멀티 면접관 패널 ──────────────────────────────────────────────────────────
+# 단일 평가자 대신 직군·논리·커뮤니케이션 평가위원이 각자 한 축을 채점(병렬) →
+# 가중평균으로 종합. A=평가만 / B=직군별(+단일직군도 다관점) / C=가중평균 / D=프롬프트 멀티콜.
+
+
+class EvaluatorResult(BaseModel):
+    score: float | None = Field(None, description="0~100, 산정 불가 시 null")
+    strength: str | None = None
+    weakness: str | None = None
+    keywords: list[str] = Field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class _EvaluatorSpec:
+    key: str  # 'technical' | 'logic' | 'communication'
+    label: str  # 요약 표기용 ('기술'/'인성'/'논리'/'전달')
+    persona: str
+    dimension_name: str
+    dimension_guide: str
+
+
+def _domain_spec(job_category: str, mode: str) -> _EvaluatorSpec:
+    # PERSONALITY 모드는 기술 평가자를 인성·협업 평가자로 교체(사용자 결정).
+    if (mode or "").upper() == "PERSONALITY":
+        return _EvaluatorSpec(
+            key="technical",
+            label="인성",
+            persona="인성·협업 중심 면접관",
+            dimension_name="인성·협업 역량",
+            dimension_guide=(
+                "- 협업/갈등 해결, 성장 경험, 태도, 자기주도성을 봅니다. "
+                "기술 정확도는 평가하지 않습니다."
+            ),
+        )
+    return _EvaluatorSpec(
+        key="technical",
+        label="기술",
+        persona=f"{job_category} 직군 시니어 기술 면접관",
+        dimension_name="기술 정확도·깊이",
+        dimension_guide=(
+            "- 기술 정확성, 깊이, trade-off, 근거를 봅니다. 질문의 '기대 신호'를 "
+            "답변이 얼마나 짚었는지를 핵심 근거로 삼습니다."
+        ),
+    )
+
+
+_LOGIC_SPEC = _EvaluatorSpec(
+    key="logic",
+    label="논리",
+    persona="논리·문제해결 평가위원",
+    dimension_name="논리·인과관계 명확성",
+    dimension_guide=(
+        "- 주장→근거→결론의 인과, trade-off 설명의 일관성, 문제 구조화를 봅니다."
+    ),
+)
+
+_COMM_SPEC = _EvaluatorSpec(
+    key="communication",
+    label="전달",
+    persona="커뮤니케이션·전달력 평가위원",
+    dimension_name="명료성·구조화·전달력",
+    dimension_guide=(
+        "- 답변의 구조(STAR 등)·간결성·명료성을 보고, 음성 분석(WPM·무음·간투어)이 "
+        "있으면 전달력 판단에 적극 활용합니다."
+    ),
+)
+
+
+def build_panel_evaluator_chain(
+    settings: Settings, core_client: CoreClient | None = None
+) -> Runnable:
+    """패널 평가위원 1명용 체인. persona/dimension 을 invoke 변수로 받아 N회 재사용."""
+    from langchain_openai import ChatOpenAI
+
+    parser = PydanticOutputParser(pydantic_object=EvaluatorResult)
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", feedback_panel.SYSTEM_PROMPT),
+            ("human", feedback_panel.HUMAN_PROMPT),
+        ]
+    ).partial(format_instructions=parser.get_format_instructions())
+
+    callbacks = []
+    if core_client is not None:
+        callbacks.append(
+            CoreAiLogCallback(
+                core_client=core_client,
+                request_type="generate.feedback.panel",
+                default_model=settings.llm_pro_model,
+            )
+        )
+
+    llm = ChatOpenAI(
+        model=settings.llm_pro_model,
+        temperature=settings.llm_pro_temperature,
+        api_key=settings.llm_api_key or None,
+        base_url=settings.llm_base_url,
+        callbacks=callbacks,
+    )
+    return prompt | llm | parser
+
+
+def _weighted_overall(pairs: list[tuple[float | None, float]]) -> float | None:
+    """(score, weight) 중 score 가 있는 것만 가중평균. 전부 None 이면 None."""
+    present = [(s, w) for s, w in pairs if s is not None and w > 0]
+    if not present:
+        return None
+    total_w = sum(w for _, w in present)
+    return round(sum(s * w for s, w in present) / total_w)
+
+
+def _merge_notes(items: list[tuple[str, str | None]]) -> str | None:
+    parts = [f"[{label}] {note.strip()}" for label, note in items if note and note.strip()]
+    return " ".join(parts) if parts else None
+
+
+def _dedup_keywords(keywords: list[str], cap: int = 8) -> list[str]:
+    seen: set[str] = set()
+    out: list[str] = []
+    for kw in keywords:
+        k = (kw or "").strip()
+        if k and k not in seen:
+            seen.add(k)
+            out.append(k)
+        if len(out) >= cap:
+            break
+    return out
+
+
+class PanelFeedbackGenerator:
+    """직군·논리·커뮤니케이션 평가위원을 병렬 호출 → 가중평균 종합. FeedbackGenerator 호환."""
+
+    def __init__(self, chain: Runnable, *, weights: tuple[float, float, float] = (0.5, 0.25, 0.25)) -> None:
+        self._chain = chain
+        self._w_tech, self._w_logic, self._w_comm = weights
+
+    async def generate(
+        self,
+        *,
+        job_category: str,
+        mode: str,
+        total_question_count: int | None,
+        end_reason: str | None,
+        transcript: str,
+        rag_context: str,
+        voice_analysis_summary: str = "",
+        score_basis: str = "(없음)",
+    ) -> FeedbackResult:
+        specs = [_domain_spec(job_category, mode), _LOGIC_SPEC, _COMM_SPEC]
+        shared = {
+            "job_category": job_category,
+            "mode": mode,
+            "total_question_count": total_question_count or 0,
+            "end_reason": end_reason or "USER_REQUEST",
+            "transcript": transcript,
+            "score_basis": score_basis or "(없음)",
+            "rag_context": rag_context or "(none)",
+            "voice_analysis_summary": voice_analysis_summary
+            or "No voice analysis summary was provided.",
+        }
+        raw = await asyncio.gather(
+            *(
+                self._chain.ainvoke(
+                    {
+                        **shared,
+                        "persona": s.persona,
+                        "dimension_name": s.dimension_name,
+                        "dimension_guide": s.dimension_guide,
+                    }
+                )
+                for s in specs
+            ),
+            return_exceptions=True,
+        )
+
+        results: dict[str, EvaluatorResult] = {}
+        for spec, r in zip(specs, raw):
+            if isinstance(r, EvaluatorResult):
+                results[spec.key] = r
+            else:
+                log.warning(
+                    "feedback.panel.evaluator_failed",
+                    evaluator=spec.key,
+                    error=str(r),
+                )
+                results[spec.key] = EvaluatorResult()
+
+        tech = results["technical"]
+        logic = results["logic"]
+        comm = results["communication"]
+        domain_label = specs[0].label
+
+        overall = _weighted_overall(
+            [
+                (tech.score, self._w_tech),
+                (logic.score, self._w_logic),
+                (comm.score, self._w_comm),
+            ]
+        )
+        strengths = _merge_notes(
+            [(domain_label, tech.strength), ("논리", logic.strength), ("전달", comm.strength)]
+        )
+        weaknesses = _merge_notes(
+            [(domain_label, tech.weakness), ("논리", logic.weakness), ("전달", comm.weakness)]
+        )
+        keywords = _dedup_keywords(tech.keywords + logic.keywords + comm.keywords)
+
+        return FeedbackResult(
+            overall_score=overall,
+            technical_accuracy=tech.score,
+            logic_score=logic.score,
+            communication_score=comm.score,
+            strengths_summary=strengths,
+            weaknesses_summary=weaknesses,
+            improvement_keywords=keywords,
+        )

ai/src/ai_server/chain/prompts/feedback_panel.py

@@ -0,0 +1,28 @@
+# 멀티 면접관 패널 — 단일 평가위원 프롬프트.
+# persona/평가축(dimension)을 변수로 주입해 같은 체인을 N회(직군·논리·커뮤) 호출한다.
+
+SYSTEM_PROMPT = (
+    "당신은 IT 직군 면접 평가 패널의 한 평가위원입니다.\n"
+    "역할: {persona}\n"
+    "당신은 **오직 '{dimension_name}' 한 축만** 평가합니다. 다른 축은 평가하지 마세요.\n"
+    "{dimension_guide}\n"
+    "- 점수는 0~100 정수, 산정 불가(짧거나 빈 답변 등) 시 null.\n"
+    "- 점수 앵커: 90~100 정확·구체적이며 근거·trade-off까지 깊이 있음 / "
+    "70~89 대체로 정확하나 일부 깊이·근거 부족 / 50~69 방향은 맞으나 추상적 / "
+    "30~49 부분적으로만 타당하고 핵심 누락 多 / 0~29 부정확하거나 거의 무응답.\n"
+    "- '점수 기준값(score_basis)'에 해당 축 기준값이 있으면 그 값에서 ±15점 이내로 산정한다.\n"
+    "- 점수를 매기기 전에 강점/약점 근거를 먼저 정리한 뒤 산정한다(즉흥 점수 금지).\n"
+    "- strength/weakness 는 각각 한 줄(한국어, 구체적으로). keywords 는 이 축에서 보완할 "
+    "개선 키워드 0~4개(짧은 명사구).\n"
+    "- 응답은 반드시 지정된 JSON 스키마를 따른다."
+)
+
+HUMAN_PROMPT = (
+    "직군: {job_category} / 면접 모드: {mode} / 질문 수: {total_question_count} / "
+    "종료 사유: {end_reason}\n\n"
+    "=== 면접 전사 ===\n{transcript}\n\n"
+    "=== 점수 기준값 (per-answer 평가 집계) ===\n{score_basis}\n\n"
+    "=== 참고 문서 컨텍스트(RAG) ===\n{rag_context}\n\n"
+    "=== 음성 분석 ===\n{voice_analysis_summary}\n\n"
+    "{format_instructions}"
+)

ai/src/ai_server/messaging/runner.py

@@ -20,8 +20,8 @@
     build_streaming_followup_generator,
 )
 from ai_server.chain.feedback_generation_chain import (
-    LlmFeedbackGenerator,
-    build_feedback_generation_chain,
+    PanelFeedbackGenerator,
+    build_panel_evaluator_chain,
 )
 from ai_server.chain.question_generation_chain import (
     LlmQuestionGenerator,
@@ -210,9 +210,9 @@ def __init__(self, settings: Settings) -> None:
             rag_timeout_sec=settings.followup_rag_timeout_sec,
         )
 
-        # 종합 피드백 생성 (US-24)
-        feedback_generator = LlmFeedbackGenerator(
-            build_feedback_generation_chain(settings, core_client=core_client)
+        # 종합 피드백 생성 (US-24) — 멀티 면접관 패널(직군·논리·커뮤 평가위원 병렬 → 가중평균)
+        feedback_generator = PanelFeedbackGenerator(
+            build_panel_evaluator_chain(settings, core_client=core_client)
         )
         self._feedback_consumer = FeedbackConsumer(
             generator=feedback_generator,

ai/tests/test_feedback_panel.py

@@ -0,0 +1,96 @@
+import pytest
+
+from ai_server.chain.feedback_generation_chain import (
+    EvaluatorResult,
+    PanelFeedbackGenerator,
+)
+
+# 평가축(dimension_name) 으로 라우팅하는 가짜 체인.
+TECH = "기술 정확도·깊이"
+PERSONALITY = "인성·협업 역량"
+LOGIC = "논리·인과관계 명확성"
+COMM = "명료성·구조화·전달력"
+
+
+class _FakeChain:
+    def __init__(self, by_dim: dict[str, EvaluatorResult]):
+        self._by_dim = by_dim
+        self.calls: list[str] = []
+
+    async def ainvoke(self, variables):
+        dim = variables["dimension_name"]
+        self.calls.append(dim)
+        return self._by_dim[dim]
+
+
+async def _run(by_dim, **kw):
+    gen = PanelFeedbackGenerator(_FakeChain(by_dim))
+    return await gen.generate(
+        job_category=kw.get("job_category", "BACKEND"),
+        mode=kw.get("mode", "TECHNICAL"),
+        total_question_count=5,
+        end_reason="MAX_QUESTIONS_REACHED",
+        transcript="t",
+        rag_context="(none)",
+        voice_analysis_summary="",
+        score_basis="(없음)",
+    )
+
+
+@pytest.mark.asyncio
+async def test_weighted_overall_and_dimension_mapping():
+    r = await _run(
+        {
+            TECH: EvaluatorResult(score=80, strength="설계 깊이", keywords=["JPA"]),
+            LOGIC: EvaluatorResult(score=60, strength="인과 명확", keywords=["trade-off"]),
+            COMM: EvaluatorResult(score=40, strength="간결", keywords=["STAR"]),
+        }
+    )
+    assert r.technical_accuracy == 80
+    assert r.logic_score == 60
+    assert r.communication_score == 40
+    # 0.5*80 + 0.25*60 + 0.25*40 = 65
+    assert r.overall_score == 65
+    assert "[기술]" in r.strengths_summary and "[논리]" in r.strengths_summary
+    assert set(r.improvement_keywords) == {"JPA", "trade-off", "STAR"}
+
+
+@pytest.mark.asyncio
+async def test_overall_reweights_when_a_dimension_is_null():
+    r = await _run(
+        {
+            TECH: EvaluatorResult(score=80),
+            LOGIC: EvaluatorResult(score=None),
+            COMM: EvaluatorResult(score=40),
+        }
+    )
+    # logic None → (80*0.5 + 40*0.25) / 0.75 = 66.67 → 67
+    assert r.logic_score is None
+    assert r.overall_score == 67
+
+
+@pytest.mark.asyncio
+async def test_personality_mode_swaps_domain_to_behavioral():
+    r = await _run(
+        {
+            PERSONALITY: EvaluatorResult(score=70, strength="협업 태도 우수"),
+            LOGIC: EvaluatorResult(score=50),
+            COMM: EvaluatorResult(score=60),
+        },
+        mode="PERSONALITY",
+    )
+    # 기술 평가자 자리가 인성·협업 평가자로 교체됨 → technical_accuracy 슬롯에 인성 점수
+    assert r.technical_accuracy == 70
+    assert "[인성]" in r.strengths_summary
+
+
+@pytest.mark.asyncio
+async def test_keyword_dedup():
+    r = await _run(
+        {
+            TECH: EvaluatorResult(score=70, keywords=["동시성", "트랜잭션"]),
+            LOGIC: EvaluatorResult(score=70, keywords=["트랜잭션"]),
+            COMM: EvaluatorResult(score=70, keywords=["두괄식"]),
+        }
+    )
+    assert r.improvement_keywords == ["동시성", "트랜잭션", "두괄식"]