diff --git a/ai/src/ai_server/chain/feedback_generation_chain.py b/ai/src/ai_server/chain/feedback_generation_chain.py index 8258701..5f7a7c6 100644 --- a/ai/src/ai_server/chain/feedback_generation_chain.py +++ b/ai/src/ai_server/chain/feedback_generation_chain.py @@ -16,6 +16,7 @@ feedback_panel, feedback_synthesis, job_fit_evaluation, + personality_evaluation, self_intro_evaluation, ) from ai_server.config.settings import Settings @@ -351,6 +352,79 @@ def build_self_intro_evaluation_chain( return prompt | llm | parser +PERSONALITY_EVALUATOR_LABEL = "인성·자소서" +PERSONALITY_DIMENSION = "자소서 소유·인성 답변 구체성/STAR" + + +def build_personality_evaluation_chain( + settings: Settings, core_client: CoreClient | None = None +) -> Runnable: + """인성·자소서 답변(경험형·BEHAVIORAL)을 기술 축과 별개로 평가하는 경량 체인(Flash).""" + from langchain_openai import ChatOpenAI + + parser = PydanticOutputParser(pydantic_object=EvaluatorResult) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", personality_evaluation.SYSTEM_PROMPT), + ("human", personality_evaluation.HUMAN_PROMPT), + ] + ).partial(format_instructions=parser.get_format_instructions()) + + callbacks = [] + if core_client is not None: + callbacks.append( + CoreAiLogCallback( + core_client=core_client, + request_type="generate.feedback.personality", + default_model=settings.llm_flash_model, + ) + ) + + llm = ChatOpenAI( + model=settings.llm_flash_model, + temperature=settings.llm_flash_temperature, + api_key=settings.llm_api_key or None, + base_url=settings.llm_base_url, + callbacks=callbacks, + ) + return prompt | llm | parser + + +class PersonalityEvaluator(Protocol): + async def evaluate( + self, + *, + job_category: str, + mode: str, + transcript: str, + ) -> EvaluatorResult: ... + + +class LlmPersonalityEvaluator: + def __init__(self, chain: Runnable) -> None: + self._chain = chain + + async def evaluate( + self, + *, + job_category: str, + mode: str, + transcript: str, + ) -> EvaluatorResult: + result = await self._chain.ainvoke( + { + "job_category": job_category, + "mode": mode, + "transcript": transcript or "(빈 답변)", + } + ) + if not isinstance(result, EvaluatorResult): + raise TypeError( + f"chain returned {type(result).__name__}, expected EvaluatorResult" + ) + return result + + class SelfIntroEvaluator(Protocol): async def evaluate( self, diff --git a/ai/src/ai_server/chain/prompts/personality_evaluation.py b/ai/src/ai_server/chain/prompts/personality_evaluation.py new file mode 100644 index 0000000..7394041 --- /dev/null +++ b/ai/src/ai_server/chain/prompts/personality_evaluation.py @@ -0,0 +1,28 @@ +# 인성·자소서 평가 프롬프트 (PERSONALITY/INTEGRATED 모드). +# 임원면접식 인성/자소서 답변을 기술 축과 별개로 평가한다. +# 결과는 패널의 '인성·자소서' 평가위원 한 명으로 표시되며, 종합 점수 집계에는 포함되지 않는다. + +SYSTEM_PROMPT = ( + "당신은 IT 직군 임원면접의 **인성·자소서 평가위원**입니다. 지원자의 인성·경험형(자소서 기반) " + "질문과 답변만 보고, 기술 정확성이 아니라 **인성·태도·자소서 소유도**를 평가합니다.\n" + "- 평가 관점(오직 아래만 봅니다. 기술 정확성 평가 금지):\n" + " · 자소서 소유·진정성: 자소서에 쓴 경험을 본인이 진짜 겪고 이해하는가(구체적 상황·본인 역할·" + "결과가 일관되게 드러나는가, 빌린 서사·모호한 총평이 아닌가).\n" + " · 서사 일관성: 지원동기·가치관·성장 스토리가 서로 모순 없이 연결되는가.\n" + " · 표준 인성 주제(리더십·갈등해결·성공/실패·장단점·지원동기·비전 등) 답변의 **구체성**과 " + "**STAR(상황·과제·행동·결과)** 충실도 — 추상적 다짐이 아니라 구체 사례·본인 행동·결과가 있는가.\n" + " · 협업·태도: 갈등·실패·관계 상황에서 드러나는 태도가 성숙한가.\n" + "- 점수는 0~100 정수. 인성/자소서 답변이 없거나 너무 부실해 판단 불가하면 null.\n" + "- 점수 앵커: 90~100 경험 소유·구체성·일관성 모두 뛰어남 / 70~89 무난하나 일부 추상적 / " + "50~69 답은 하나 구체 사례·본인 행동이 약함 / 30~49 상투적이거나 모순 / 0~29 거의 무응답.\n" + "- strength/weakness 는 각각 한 줄(한국어, 구체적으로). keywords 는 보완 키워드 0~3개(짧은 명사구).\n" + "- detail: 답변의 구체적 부분을 인용/지목하며 2~4문장으로 서술(추상적 총평 금지).\n" + "- score_rationale: 그 점수를 준 핵심 근거를 한두 문장으로.\n" + "- 응답은 반드시 지정된 JSON 스키마를 따른다." +) + +HUMAN_PROMPT = ( + "지원 직군: {job_category} / 면접 모드: {mode}\n\n" + "=== 인성·자소서 관련 질문과 답변 ===\n{transcript}\n\n" + "{format_instructions}" +) diff --git a/ai/src/ai_server/messaging/consumers/feedback_consumer.py b/ai/src/ai_server/messaging/consumers/feedback_consumer.py index 4a75dc1..7d9d588 100644 --- a/ai/src/ai_server/messaging/consumers/feedback_consumer.py +++ b/ai/src/ai_server/messaging/consumers/feedback_consumer.py @@ -9,6 +9,8 @@ from ai_server.chain.feedback_generation_chain import ( JOB_FIT_DIMENSION, JOB_FIT_EVALUATOR_LABEL, + PERSONALITY_DIMENSION, + PERSONALITY_EVALUATOR_LABEL, ROLE_UNDERSTANDING_DIMENSION, ROLE_UNDERSTANDING_LABEL, SELF_INTRO_DIMENSION, @@ -17,6 +19,7 @@ EvaluatorResult, FeedbackGenerator, JobFitEvaluator, + PersonalityEvaluator, SelfIntroEvaluator, ) from ai_server.core.client import CoreClient @@ -37,6 +40,9 @@ _SELF_INTRO_CATEGORY = "SELF_INTRODUCTION" _JOB_TAILORED_MODE = "JOB_TAILORED" +_BEHAVIORAL_CATEGORY = "BEHAVIORAL" +# 인성·자소서 평가위원이 도는 모드. 이 모드에서만 인성/경험형 답변을 별도 축으로 평가한다. +_PERSONALITY_MODES = {"PERSONALITY", "INTEGRATED"} # 답변별 코칭 RAG 는 해당 화제에 국한된 소수 청크만. 세션 공용 top_k 보다 작게. _COACHING_RAG_TOP_K = 3 # 세션 RAG 질의 상한(문자). 답변 이어붙임이 임베딩 입력 한도를 넘지 않게. @@ -66,6 +72,7 @@ def __init__( rag_top_k: int = 5, self_intro_evaluator: SelfIntroEvaluator | None = None, job_fit_evaluator: JobFitEvaluator | None = None, + personality_evaluator: PersonalityEvaluator | None = None, answer_coach: AnswerCoach | None = None, coaching_max_answers: int = 30, coaching_concurrency: int = 5, @@ -79,6 +86,7 @@ def __init__( self._rag_top_k = rag_top_k self._self_intro_evaluator = self_intro_evaluator self._job_fit_evaluator = job_fit_evaluator + self._personality_evaluator = personality_evaluator self._answer_coach = answer_coach self._coaching_max_answers = coaching_max_answers self._coaching_concurrency = max(1, coaching_concurrency) @@ -124,26 +132,31 @@ async def handle(self, message: AbstractIncomingMessage) -> None: # 종합 피드백 + 자기소개 첫인상 + 직무 적합도(직무 맞춤 모드)를 병렬 실행. # 첫인상·직무 적합도는 종합 점수(overall)에 미포함 — generator 가 모른 채 계산한 뒤 표시용으로 덧붙인다. - result, self_intro_item, job_fit_items, answer_coaching = ( - await asyncio.gather( - self._generator.generate( - job_category=req.job_category, - mode=req.mode, - total_question_count=req.total_question_count, - end_reason=req.end_reason, - transcript=transcript, - score_basis=score_basis, - rag_context=rag_context, - voice_analysis_summary=voice_analysis_summary, - domain_question_counts=req.domain_question_counts, - ), - self._evaluate_self_intro(req, voice_analysis_summary), - self._evaluate_job_fit(req, transcript, rag_context), - self._coach_answers(req), - ) + ( + result, + self_intro_item, + job_fit_items, + personality_item, + answer_coaching, + ) = await asyncio.gather( + self._generator.generate( + job_category=req.job_category, + mode=req.mode, + total_question_count=req.total_question_count, + end_reason=req.end_reason, + transcript=transcript, + score_basis=score_basis, + rag_context=rag_context, + voice_analysis_summary=voice_analysis_summary, + domain_question_counts=req.domain_question_counts, + ), + self._evaluate_self_intro(req, voice_analysis_summary), + self._evaluate_job_fit(req, transcript, rag_context), + self._evaluate_personality(req), + self._coach_answers(req), ) # 빈 평가위원 항목(점수·내용 모두 없음)은 표시하지 않는다 — LLM 부분 응답이 빈 패널로 새는 것 방지. - extras = [self_intro_item, *job_fit_items] + extras = [self_intro_item, *job_fit_items, personality_item] result.panel_breakdown.extend( e for e in extras if e is not None and _panel_has_content(e) ) @@ -253,6 +266,42 @@ async def _evaluate_job_fit( ), ] + async def _evaluate_personality( + self, req: GenerateFeedbackRequest + ) -> PanelBreakdownItem | None: + """PERSONALITY·INTEGRATED 모드에서 인성·자소서(경험형) 답변을 기술 축과 별개로 평가. + 그 외 모드/대상 없음/실패는 None(패널 미표시). 종합 점수엔 미포함.""" + if self._personality_evaluator is None: + return None + if (req.mode or "") not in _PERSONALITY_MODES: + return None + pairs = _collect_coachable_pairs(req.messages) # 자기소개·빈·짧은확인 제외 + behavioral = [ + (q, a) for (q, a) in pairs if (q.category or "") == _BEHAVIORAL_CATEGORY + ] + # PERSONALITY 인데 카테고리 태깅이 없으면 비자기소개 답변 전체로 폴백. INTEGRATED 는 + # BEHAVIORAL 이 하나도 없으면 평가할 인성 답변이 없는 것으로 보고 건너뛴다. + selected = behavioral or ( + pairs if (req.mode or "") == "PERSONALITY" else [] + ) + if not selected: + return None + transcript = _qa_transcript(selected) + try: + ev = await self._personality_evaluator.evaluate( + job_category=req.job_category, + mode=req.mode, + transcript=transcript, + ) + except Exception as exc: # noqa: BLE001 + log.warning( + "feedback.personality.failed", + error=str(exc), + session_id=req.session_id, + ) + return None + return _to_panel_item(PERSONALITY_EVALUATOR_LABEL, PERSONALITY_DIMENSION, ev) + async def _coach_answers( self, req: GenerateFeedbackRequest ) -> list[AnswerCoachingItem]: @@ -427,6 +476,19 @@ def _panel_has_content(item: PanelBreakdownItem) -> bool: ) +def _qa_transcript( + pairs: list[tuple[FeedbackMessageItem, FeedbackMessageItem]], +) -> str: + """(질문, 답변) 쌍 목록을 인성 평가위원 입력용 전사 문자열로.""" + lines: list[str] = [] + for q, a in pairs: + lines.append(f"면접관: {q.content}") + if q.expected_signal: + lines.append(f" └ 기대 신호: {q.expected_signal}") + lines.append(f"지원자: {a.content}") + return "\n".join(lines) + + def _to_panel_item( label: str, dimension: str, ev: EvaluatorResult ) -> PanelBreakdownItem: diff --git a/ai/src/ai_server/messaging/runner.py b/ai/src/ai_server/messaging/runner.py index 7aaf7a4..d295c9e 100644 --- a/ai/src/ai_server/messaging/runner.py +++ b/ai/src/ai_server/messaging/runner.py @@ -23,12 +23,14 @@ from ai_server.chain.feedback_generation_chain import ( LlmAnswerCoach, LlmJobFitEvaluator, + LlmPersonalityEvaluator, LlmSelfIntroEvaluator, PanelFeedbackGenerator, build_answer_coaching_chain, build_feedback_synthesis_chain, build_job_fit_evaluation_chain, build_panel_evaluator_chain, + build_personality_evaluation_chain, build_self_intro_evaluation_chain, ) from ai_server.chain.question_generation_chain import ( @@ -261,6 +263,10 @@ def __init__(self, settings: Settings) -> None: job_fit_evaluator=LlmJobFitEvaluator( build_job_fit_evaluation_chain(settings, core_client=core_client) ), + # 인성·자소서 평가(Flash). PERSONALITY·INTEGRATED 에서만 동작, 종합 점수엔 미포함. + personality_evaluator=LlmPersonalityEvaluator( + build_personality_evaluation_chain(settings, core_client=core_client) + ), # 질문별 복기(Flash, 답변 수만큼 병렬). 모범 답안+리라이트+한 줄 코칭. answer_coach=LlmAnswerCoach( build_answer_coaching_chain(settings, core_client=core_client) diff --git a/ai/tests/test_feedback_consumer.py b/ai/tests/test_feedback_consumer.py index cf5e7df..43909cb 100644 --- a/ai/tests/test_feedback_consumer.py +++ b/ai/tests/test_feedback_consumer.py @@ -854,3 +854,116 @@ def test_build_transcript_annotates_interviewee_evaluation(): assert "correctness=1" in out # 면접관 줄엔 평가 주석 없음 assert out.splitlines()[0].endswith("질문?") + + +def _personality_envelope(mode: str) -> bytes: + env = { + "messageId": "fb-pers", + "messageType": "generate.feedback", + "version": "v1", + "traceId": "t-pers", + "publishedAt": "2026-05-30T00:00:00Z", + "publisher": "core-server", + "payload": { + "sessionId": 61, + "mode": mode, + "jobCategory": "BACKEND", + "totalQuestionCount": 2, + "endReason": "POOL_EXHAUSTED", + "messages": [ + { + "id": 300, + "sequenceNumber": 1, + "role": "INTERVIEWER", + "content": "리더십을 발휘한 경험을 말해보세요.", + "category": "BEHAVIORAL", + }, + { + "id": 301, + "sequenceNumber": 2, + "role": "INTERVIEWEE", + "content": "배포 지연 문제에서 제가 온콜 로테이션을 제안해 장애 대응 시간을 절반으로 줄였습니다.", + "parentMessageId": 300, + }, + ], + "contextDocumentIds": [], + }, + "context": {"userId": 1, "sessionId": 61}, + } + return json.dumps(env).encode() + + +def _personality_evaluator(): + ev = MagicMock() + ev.evaluate = AsyncMock( + return_value=EvaluatorResult( + score=72.0, + strength="구체적 행동·결과 제시", + weakness="STAR 상황 설명 부족", + detail="온콜 로테이션 제안과 대응시간 절반 감소를 구체적으로 언급.", + score_rationale="본인 행동·결과는 명확하나 상황 맥락이 약함", + ) + ) + return ev + + +def _make_consumer(publisher, generator, evaluator): + return FeedbackConsumer( + generator=generator, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=10), + callback_routing_key="callback.feedback", + core_client=MagicMock(), + embedder=None, + personality_evaluator=evaluator, + ) + + +@pytest.mark.asyncio +async def test_consumer_appends_personality_panel_item(): + generator = _generator() + publisher = MagicMock() + publisher.publish = AsyncMock() + evaluator = _personality_evaluator() + consumer = _make_consumer(publisher, generator, evaluator) + + await consumer.handle(_StubMessage(_personality_envelope("PERSONALITY"))) + + evaluator.evaluate.assert_awaited_once() + assert "온콜" in evaluator.evaluate.await_args.kwargs["transcript"] + payload: FeedbackCallbackPayload = publisher.publish.await_args.kwargs["payload"] + items = [b for b in payload.panel_breakdown if b.evaluator == "인성·자소서"] + assert len(items) == 1 + assert items[0].score == 72.0 + # 가산·비집계: 종합 점수는 불변. + assert payload.overall_score == 85.0 + + +@pytest.mark.asyncio +async def test_consumer_appends_personality_in_integrated(): + generator = _generator() + publisher = MagicMock() + publisher.publish = AsyncMock() + evaluator = _personality_evaluator() + consumer = _make_consumer(publisher, generator, evaluator) + + await consumer.handle(_StubMessage(_personality_envelope("INTEGRATED"))) + + evaluator.evaluate.assert_awaited_once() + payload: FeedbackCallbackPayload = publisher.publish.await_args.kwargs["payload"] + assert any(b.evaluator == "인성·자소서" for b in payload.panel_breakdown) + + +@pytest.mark.asyncio +async def test_consumer_skips_personality_in_technical_mode(): + generator = _generator() + publisher = MagicMock() + publisher.publish = AsyncMock() + evaluator = _personality_evaluator() + consumer = _make_consumer(publisher, generator, evaluator) + + await consumer.handle(_StubMessage(_personality_envelope("TECHNICAL"))) + + evaluator.evaluate.assert_not_awaited() + payload: FeedbackCallbackPayload = publisher.publish.await_args.kwargs["payload"] + assert all(b.evaluator != "인성·자소서" for b in payload.panel_breakdown)