diff --git a/ai/src/ai_server/chain/feedback_generation_chain.py b/ai/src/ai_server/chain/feedback_generation_chain.py index bc3fea61..4b4cf82c 100644 --- a/ai/src/ai_server/chain/feedback_generation_chain.py +++ b/ai/src/ai_server/chain/feedback_generation_chain.py @@ -34,6 +34,7 @@ async def generate( transcript: str, rag_context: str, voice_analysis_summary: str, + score_basis: str = "(없음)", ) -> FeedbackResult: ... @@ -51,6 +52,7 @@ async def generate( transcript: str, rag_context: str, voice_analysis_summary: str = "", + score_basis: str = "(없음)", ) -> FeedbackResult: result = await self._chain.ainvoke( { @@ -59,6 +61,7 @@ async def generate( "total_question_count": total_question_count or 0, "end_reason": end_reason or "USER_REQUEST", "transcript": transcript, + "score_basis": score_basis or "(없음)", "rag_context": rag_context or "(none)", "voice_analysis_summary": voice_analysis_summary or "No voice analysis summary was provided.", diff --git a/ai/src/ai_server/chain/prompts/feedback_generation.py b/ai/src/ai_server/chain/prompts/feedback_generation.py index 4aab3797..311c713f 100644 --- a/ai/src/ai_server/chain/prompts/feedback_generation.py +++ b/ai/src/ai_server/chain/prompts/feedback_generation.py @@ -32,6 +32,21 @@ " - If voice analysis is absent or sparse, do not invent voice-related findings.\n" ) +# 점수 앵커(캘리브레이션) + per-answer 집계 기준값 제약(하이브리드). +SYSTEM_PROMPT += ( + "\n- 점수 앵커 (0~100, 모든 차원 공통 기준):\n" + " - 90~100: 정확하고 구체적이며 trade-off·근거까지 깊이 있음. 빈틈 거의 없음.\n" + " - 70~89: 대체로 정확·구체적이나 일부 깊이/근거가 부족.\n" + " - 50~69: 방향은 맞으나 추상적이거나 근거·구조가 미흡.\n" + " - 30~49: 부분적으로만 타당하고 핵심 누락이 많음.\n" + " - 0~29: 부정확하거나 거의 무응답.\n" + "- '점수 기준값' 섹션 (per-answer 평가를 집계한 차원별 기준값) 이 주어지면:\n" + " - 각 차원 최종 점수는 그 기준값에서 **±15점 이내**로 산정한다.\n" + " - ±15점을 넘겨야 한다면 그 사유를 strengths/weaknesses 에 반드시 명시한다.\n" + " - 기준값이 '근거 없음'(예: 참고문서 미선택으로 correctness 미산정)인 차원은 " + "전사 내용으로 판단하되, 그 한계를 weaknesses 또는 점수 보수성(과대평가 금지)에 반영한다.\n" +) + HUMAN_PROMPT = ( "직군: {job_category}\n" "면접 모드: {mode}\n" @@ -39,6 +54,8 @@ "종료 사유: {end_reason}\n\n" "=== 메시지 시퀀스 ===\n" "{transcript}\n\n" + "=== 점수 기준값 (per-answer 평가 집계 — 이 값을 기준으로 산정) ===\n" + "{score_basis}\n\n" "=== RAG 컨텍스트 청크 (참고용, 직접 인용 금지) ===\n" "{rag_context}\n\n" "=== Voice Analysis Summary ===\n" diff --git a/ai/src/ai_server/messaging/consumers/feedback_consumer.py b/ai/src/ai_server/messaging/consumers/feedback_consumer.py index 5a468e9b..0378c674 100644 --- a/ai/src/ai_server/messaging/consumers/feedback_consumer.py +++ b/ai/src/ai_server/messaging/consumers/feedback_consumer.py @@ -87,6 +87,7 @@ async def handle(self, message: AbstractIncomingMessage) -> None: ) transcript = _build_transcript(req.messages) + score_basis = _build_score_basis(req.messages) rag_context = await self._build_rag_context(req) voice_analysis_summary = _build_voice_analysis_summary( req.voice_analysis_summary @@ -98,6 +99,7 @@ async def handle(self, message: AbstractIncomingMessage) -> None: total_question_count=req.total_question_count, end_reason=req.end_reason, transcript=transcript, + score_basis=score_basis, rag_context=rag_context, voice_analysis_summary=voice_analysis_summary, ) @@ -191,6 +193,67 @@ def _format_evaluation(e) -> str: return ", ".join(parts) if parts else "(없음)" +_STRUCTURE_SCORE = {"FULL_STAR": 5.0, "PARTIAL_STAR": 2.5, "NONE": 0.0} + + +def _mean(values: list[float]) -> float | None: + vals = [v for v in values if v is not None] + return sum(vals) / len(vals) if vals else None + + +def _to_100(x: float | None) -> int | None: + return None if x is None else round(x * 20) + + +def _build_score_basis(messages: list[FeedbackMessageItem]) -> str: + """per-answer 평가(0~5)를 차원별 0~100 기준값으로 결정론적 집계(하이브리드). + + 이 값을 LLM 에 '기준값'으로 제시하고 ±15점 이내 산정하도록 제약 → 재현성·캘리브레이션. + correctness(RAG 기반)가 전부 null 이면 technical_accuracy 기준값은 '근거 없음'. + """ + evals = [ + m.evaluation + for m in messages + if m.role == "INTERVIEWEE" and m.evaluation is not None + ] + if not evals: + return "(per-answer 평가 없음 — 전사 내용으로만 산정. 과대평가 금지.)" + + spec = _mean([e.specificity for e in evals]) + logic = _mean([e.logic for e in evals]) + corr = _mean([e.correctness for e in evals]) # null 은 자동 제외 + struct = _mean([_STRUCTURE_SCORE.get(e.structure) for e in evals]) + + tech_100 = _to_100(corr) + logic_100 = _to_100(logic) + comm_src = _mean([v for v in (spec, struct) if v is not None]) + comm_100 = _to_100(comm_src) + overall_src = [v for v in (tech_100, logic_100, comm_100) if v is not None] + overall_100 = round(sum(overall_src) / len(overall_src)) if overall_src else None + + def fmt5(x: float | None) -> str: + return f"{x:.1f}/5" if x is not None else "없음" + + corr_count = sum(1 for e in evals if e.correctness is not None) + lines = [ + f"- 채점된 답변 수: {len(evals)} (correctness 산정 {corr_count}건)", + f"- specificity 평균: {fmt5(spec)}, logic 평균: {fmt5(logic)}, " + f"structure 평균: {fmt5(struct)}, correctness 평균: {fmt5(corr)}", + "[차원별 기준값(0~100)]", + ( + f"- technical_accuracy ≈ {tech_100} (correctness=RAG 사실일치 기반)" + if tech_100 is not None + else "- technical_accuracy: 근거 없음(참고문서 미선택→correctness 미산정). " + "전사로 보수적으로 판단." + ), + f"- logic_score ≈ {logic_100 if logic_100 is not None else '근거 없음'}", + f"- communication_score ≈ {comm_100 if comm_100 is not None else '근거 없음'} " + "(specificity 명료성 + structure 구조화)", + f"- overall_score ≈ {overall_100 if overall_100 is not None else '근거 없음'}", + ] + return "\n".join(lines) + + def _build_voice_analysis_summary(summary: VoiceAnalysisSummary | None) -> str: if summary is None: return "No voice analysis summary was provided." diff --git a/ai/tests/test_feedback_consumer.py b/ai/tests/test_feedback_consumer.py index 106b6de9..196e066d 100644 --- a/ai/tests/test_feedback_consumer.py +++ b/ai/tests/test_feedback_consumer.py @@ -9,11 +9,18 @@ FeedbackResult, LlmFeedbackGenerator, ) -from ai_server.chain.prompts.feedback_generation import HUMAN_PROMPT +from ai_server.chain.prompts.feedback_generation import HUMAN_PROMPT, SYSTEM_PROMPT from ai_server.core.client import EmbeddingSearchHit -from ai_server.messaging.consumers.feedback_consumer import FeedbackConsumer +from ai_server.messaging.consumers.feedback_consumer import ( + FeedbackConsumer, + _build_score_basis, +) from ai_server.messaging.idempotency import LruIdempotencyStore -from ai_server.model.messages.feedback import FeedbackCallbackPayload +from ai_server.model.messages.feedback import ( + FeedbackCallbackPayload, + FeedbackMessageItem, + MessageEvaluation, +) VOICE_SUMMARY = { "analyzedMessageCount": 2, @@ -188,6 +195,48 @@ async def test_consumer_accepts_voice_summary_and_passes_it_to_generator(): assert not hasattr(payload, "voice_analysis_summary") +def _answer(seq: int, *, spec=None, logic=None, structure=None, correctness=None): + return FeedbackMessageItem( + id=seq, + sequence_number=seq, + role="INTERVIEWEE", + content="답변", + evaluation=MessageEvaluation( + specificity=spec, logic=logic, structure=structure, correctness=correctness + ), + ) + + +def test_build_score_basis_aggregates_to_0_100(): + msgs = [ + _answer(2, spec=4.0, logic=3.0, structure="FULL_STAR", correctness=3.0), + _answer(4, spec=2.0, logic=4.0, structure="PARTIAL_STAR", correctness=2.0), + ] + basis = _build_score_basis(msgs) + # correctness 평균 2.5 → technical_accuracy ≈ 50 + assert "technical_accuracy ≈ 50" in basis + # logic 평균 3.5 → 70 + assert "logic_score ≈ 70" in basis + assert "채점된 답변 수: 2" in basis + + +def test_build_score_basis_marks_correctness_absent_without_rag(): + # correctness 가 전부 null(참고문서 미선택) → technical_accuracy 근거 없음. + msgs = [_answer(2, spec=3.0, logic=3.0, structure="NONE", correctness=None)] + basis = _build_score_basis(msgs) + assert "technical_accuracy: 근거 없음" in basis + + +def test_build_score_basis_empty_when_no_evaluations(): + assert "per-answer 평가 없음" in _build_score_basis([]) + + +def test_feedback_prompt_has_score_anchor_and_slot(): + assert "점수 앵커" in SYSTEM_PROMPT + assert "±15" in SYSTEM_PROMPT + assert "{score_basis}" in HUMAN_PROMPT + + @pytest.mark.asyncio async def test_llm_feedback_generator_includes_voice_summary_in_chain_input(): class _FakeChain: