Skip to content

Commit 9b00e7c

Browse files
i3monthsi3months
authored
Merge pull request #64 from Team-StackUp/feature/followup-quality
feat(followup): RAG 부활 + correctness 평가 + 카테고리 루브릭 + 대화 히스토리
2 parents a1d30f0 + 6906a68 commit 9b00e7c

8 files changed

Lines changed: 167 additions & 21 deletions

File tree

‎ai/src/ai_server/chain/followup_generation_chain.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ async def generate(
2828
previous_question: str,
2929
answer_text: str,
3030
context: str = "(none)",
31+
parent_category: str = "UNKNOWN",
32+
history: str = "(none)",
3133
) -> FollowupResult: ...
3234

3335

@@ -43,6 +45,8 @@ async def generate(
4345
previous_question: str,
4446
answer_text: str,
4547
context: str = "(none)",
48+
parent_category: str = "UNKNOWN",
49+
history: str = "(none)",
4650
) -> FollowupResult:
4751
result = await self._chain.ainvoke(
4852
{
@@ -51,6 +55,8 @@ async def generate(
5155
"previous_question": previous_question,
5256
"answer_text": answer_text,
5357
"context": context,
58+
"parent_category": parent_category,
59+
"history": history,
5460
}
5561
)
5662
if not isinstance(result, FollowupResult):
@@ -60,7 +66,9 @@ async def generate(
6066
return result
6167

6268

63-
def build_followup_generation_chain(settings: Settings, core_client: CoreClient | None = None) -> Runnable:
69+
def build_followup_generation_chain(
70+
settings: Settings, core_client: CoreClient | None = None
71+
) -> Runnable:
6472
from langchain_openai import ChatOpenAI
6573

6674
parser = PydanticOutputParser(pydantic_object=FollowupResult)
@@ -73,11 +81,13 @@ def build_followup_generation_chain(settings: Settings, core_client: CoreClient
7381

7482
callbacks = []
7583
if core_client is not None:
76-
callbacks.append(CoreAiLogCallback(
77-
core_client=core_client,
78-
request_type="generate.followup",
79-
default_model=settings.llm_flash_model,
80-
))
84+
callbacks.append(
85+
CoreAiLogCallback(
86+
core_client=core_client,
87+
request_type="generate.followup",
88+
default_model=settings.llm_flash_model,
89+
)
90+
)
8191

8292
llm = ChatOpenAI(
8393
model=settings.llm_flash_model,

‎ai/src/ai_server/chain/prompts/followup_generation.py‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,34 @@
11
# 꼬리질문 생성 + 답변 평가 (US-19)
2-
# Flash 모델 + 저지연 (< 3s). 사용자 답변의 specificity·logic·structure 채점 + 부족 부분 파고드는 꼬리질문 1개.
2+
# Flash 모델 + 저지연 (< 3s). 답변을 4축으로 평가하고 가장 약한 축을 파고드는 꼬리질문 1개.
33

44
SYSTEM_PROMPT = (
55
"당신은 IT 직군 면접관입니다. 직전 질문에 대한 지원자의 답변을 평가하고, "
6-
"부족한 부분(구체성·논리·구조)을 파고드는 꼬리질문 1개를 한국어로 만드세요.\n"
6+
"가장 약한 부분을 파고드는 꼬리질문 1개를 한국어로 만드세요.\n"
77
"- 평가 항목:\n"
88
" - specificity (0~5): 답변에 구체적 수치/사례/기술 선택 근거가 있는가.\n"
99
" - logic (0~5): 인과관계와 trade-off 가 명확한가.\n"
1010
" - structure: STAR (Situation-Task-Action-Result) 구조 측면.\n"
11-
" - FULL_STAR: 네 요소 모두 명확.\n"
12-
" - PARTIAL_STAR: 일부 요소 누락.\n"
13-
" - NONE: 구조 부재.\n"
14-
"- 꼬리질문은 답변에서 가장 약한 축 (예: 구체성 낮음 → 수치/사례 요구) 을 겨냥합니다.\n"
11+
" - FULL_STAR / PARTIAL_STAR / NONE.\n"
12+
" - 단, STAR 는 **행동/경험형 답변에 적합**한 기준이다. 카테고리가 "
13+
"BEHAVIORAL/PERSONALITY 면 STAR 를 중시하고, CS_FUNDAMENTAL/TECH_CHOICE/"
14+
"PROJECT_DEEP_DIVE 같은 기술형이면 structure 는 참고만 하고 정확성·깊이를 우선한다.\n"
15+
" - correctness (0~5): 답변이 '검색 문서 컨텍스트'의 사실과 일치하는가. "
16+
"**컨텍스트가 '(none)' 이면 판단 불가 → correctness 는 null**. 추측으로 채우지 마세요.\n"
17+
"- 꼬리질문은 가장 약한 축을 겨냥합니다 (예: 구체성 낮음→수치/사례 요구, "
18+
"correctness 의심→자료와의 불일치 확인).\n"
19+
"- '이미 나눈 대화'에서 다룬 내용을 그대로 반복하지 말고 새로운 각도로 파고드세요.\n"
1520
"- 응답은 반드시 지정된 JSON 스키마를 따릅니다."
1621
)
1722

1823
HUMAN_PROMPT = (
1924
"직군: {job_category}\n"
20-
"면접 모드: {mode}\n\n"
25+
"면접 모드: {mode}\n"
26+
"직전 질문 카테고리: {parent_category}\n\n"
2127
"모드별 지침:\n"
2228
"- TECHNICAL: 기술 역량, 프로젝트 경험, 문제 해결을 중심으로 파고듭니다.\n"
2329
"- PERSONALITY: 협업, 갈등 해결, 성장 경험을 중심으로 파고듭니다.\n"
2430
"- INTEGRATED: 기술 질문과 인성 질문의 관점을 균형 있게 반영합니다.\n\n"
31+
"이미 나눈 대화 (반복 금지):\n{history}\n\n"
2532
"직전 질문:\n{previous_question}\n\n"
2633
"지원자 답변:\n{answer_text}\n\n"
2734
"검색 문서 컨텍스트:\n---\n{context}\n---\n\n"

‎ai/src/ai_server/messaging/consumers/followup_consumer.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ async def handle(self, message: AbstractIncomingMessage) -> None:
7979
previous_question=req.previous_question,
8080
answer_text=req.answer_text,
8181
context=await self._build_rag_context(req),
82+
parent_category=req.parent_category or "UNKNOWN",
83+
history=_format_history(req.history),
8284
)
8385

8486
payload = FollowupCallbackPayload(
@@ -131,3 +133,12 @@ async def _build_rag_context(self, req: GenerateFollowupRequest) -> str:
131133
return "\n---\n".join(
132134
f"[doc#{h.document_id} chunk#{h.chunk_index}] {h.chunk_text}" for h in hits
133135
)
136+
137+
138+
def _format_history(history: list) -> str:
139+
"""대화 히스토리를 '화자: 내용' 라인으로. 비면 '(none)'."""
140+
if not history:
141+
return "(none)"
142+
speaker = {"INTERVIEWER": "면접관", "INTERVIEWEE": "지원자"}
143+
lines = [f"{speaker.get(h.role, h.role)}: {h.content}" for h in history]
144+
return "\n".join(lines)

‎ai/src/ai_server/model/messages/followup.py‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,27 +7,42 @@
77
InterviewMode = Literal["PERSONALITY", "TECHNICAL", "INTEGRATED"]
88

99

10+
class HistoryItem(BaseModel):
11+
"""대화 히스토리 한 줄 (중복 질문 회피용)."""
12+
13+
model_config = camel_config()
14+
15+
role: str # INTERVIEWER | INTERVIEWEE | SYSTEM
16+
content: str
17+
18+
1019
class GenerateFollowupRequest(BaseModel):
1120
"""Core 가 답변 commit 후 발행."""
21+
1222
model_config = camel_config()
1323

1424
session_id: int
15-
parent_message_id: int # 직전 질문 메시지 ID
16-
answer_message_id: int # 답변 메시지 ID
25+
parent_message_id: int # 직전 질문 메시지 ID
26+
answer_message_id: int # 답변 메시지 ID
1727
previous_question: str
1828
answer_text: str
1929
mode: InterviewMode
2030
job_category: Literal["FRONTEND", "BACKEND", "INFRA", "DBA"]
2131
context_document_ids: list[int] = Field(default_factory=list)
32+
parent_category: str | None = None # 직전 질문 카테고리 (루브릭 선택)
33+
history: list[HistoryItem] = Field(default_factory=list) # 최근 대화 (중복 회피)
2234

2335

2436
class AnswerEvaluation(BaseModel):
25-
"""답변 평가 (US-19). LLM 이 specificity/logic/structure 채움."""
37+
"""답변 평가 (US-19). LLM 이 specificity/logic/structure/correctness 채움."""
38+
2639
model_config = camel_config()
2740

28-
specificity: float # 0~5
29-
logic: float # 0~5
41+
specificity: float # 0~5
42+
logic: float # 0~5
3043
structure: Literal["FULL_STAR", "PARTIAL_STAR", "NONE"]
44+
# 답변이 자료(RAG)와 사실적으로 맞는지. 검색 컨텍스트 없으면 null.
45+
correctness: float | None = None # 0~5
3146

3247

3348
class FollowupCallbackPayload(BaseModel):

‎ai/tests/test_followup_consumer.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,3 +185,76 @@ async def test_consumer_idempotent_skip():
185185
await consumer.handle(_StubMessage(_envelope()))
186186
generator.generate.assert_not_awaited()
187187
publisher.publish.assert_not_awaited()
188+
189+
190+
def test_format_history_formats_and_empty():
191+
from ai_server.messaging.consumers.followup_consumer import _format_history
192+
from ai_server.model.messages.followup import HistoryItem
193+
194+
assert _format_history([]) == "(none)"
195+
out = _format_history(
196+
[
197+
HistoryItem(role="INTERVIEWER", content="Q1?"),
198+
HistoryItem(role="INTERVIEWEE", content="A1"),
199+
]
200+
)
201+
assert out == "면접관: Q1?\n지원자: A1"
202+
203+
204+
def test_answer_evaluation_correctness_defaults_none_and_parses():
205+
e1 = AnswerEvaluation(specificity=1.0, logic=2.0, structure="NONE")
206+
assert e1.correctness is None
207+
e2 = AnswerEvaluation.model_validate(
208+
{"specificity": 4, "logic": 4, "structure": "FULL_STAR", "correctness": 3.5}
209+
)
210+
assert e2.correctness == 3.5
211+
212+
213+
@pytest.mark.asyncio
214+
async def test_consumer_passes_parent_category_and_history_to_generator():
215+
generator = MagicMock()
216+
generator.generate = AsyncMock(
217+
return_value=FollowupResult(
218+
followup_question="새 각도 질문",
219+
answer_evaluation=AnswerEvaluation(
220+
specificity=2.0, logic=2.0, structure="NONE"
221+
),
222+
)
223+
)
224+
publisher = MagicMock()
225+
publisher.publish = AsyncMock()
226+
consumer = FollowupConsumer(
227+
generator=generator,
228+
publisher=publisher,
229+
idempotency=LruIdempotencyStore(max_size=10),
230+
callback_routing_key="callback.questions",
231+
)
232+
env = {
233+
"messageId": "m-9",
234+
"messageType": "generate.followup",
235+
"version": "v1",
236+
"traceId": "t-9",
237+
"publishedAt": "2026-05-29T00:00:00Z",
238+
"publisher": "core-server",
239+
"payload": {
240+
"sessionId": 99,
241+
"parentMessageId": 501,
242+
"answerMessageId": 502,
243+
"previousQuestion": "Q?",
244+
"answerText": "A.",
245+
"mode": "TECHNICAL",
246+
"jobCategory": "BACKEND",
247+
"parentCategory": "PROJECT_DEEP_DIVE",
248+
"history": [
249+
{"role": "INTERVIEWER", "content": "이전 질문"},
250+
{"role": "INTERVIEWEE", "content": "이전 답변"},
251+
],
252+
},
253+
"context": {"userId": 42, "sessionId": 99},
254+
}
255+
await consumer.handle(_StubMessage(json.dumps(env).encode()))
256+
257+
kwargs = generator.generate.await_args.kwargs
258+
assert kwargs["parent_category"] == "PROJECT_DEEP_DIVE"
259+
assert "이전 질문" in kwargs["history"]
260+
assert "면접관:" in kwargs["history"]

‎backend/src/main/java/com/stackup/stackup/session/application/SessionFollowupRequester.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@
44
import com.stackup.stackup.common.messaging.MessageContext;
55
import com.stackup.stackup.common.messaging.RabbitMessagePublisher;
66
import com.stackup.stackup.session.application.dto.GenerateFollowupPayload;
7+
import com.stackup.stackup.session.application.dto.GenerateFollowupPayload.HistoryItem;
78
import com.stackup.stackup.session.application.event.AnswerSubmittedEvent;
89
import com.stackup.stackup.session.domain.InterviewMessage;
910
import com.stackup.stackup.session.domain.InterviewMessageRepository;
1011
import com.stackup.stackup.session.domain.InterviewSession;
12+
import com.stackup.stackup.session.domain.SessionContextRepository;
13+
import java.util.List;
1114
import lombok.RequiredArgsConstructor;
1215
import org.slf4j.Logger;
1316
import org.slf4j.LoggerFactory;
@@ -24,9 +27,12 @@ public class SessionFollowupRequester {
2427

2528
private static final Logger log = LoggerFactory.getLogger(SessionFollowupRequester.class);
2629

30+
private static final int HISTORY_MAX = 6;
31+
2732
private final RabbitMessagePublisher publisher;
2833
private final RabbitMqProperties properties;
2934
private final InterviewMessageRepository messageRepository;
35+
private final SessionContextRepository contextRepository;
3036

3137
@Transactional(propagation = Propagation.REQUIRES_NEW)
3238
@TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT)
@@ -39,14 +45,31 @@ public void onAnswerSubmitted(AnswerSubmittedEvent event) {
3945
return;
4046
}
4147
InterviewSession session = parent.getSession();
48+
49+
// RAG(자료 근거/correctness) 용 세션 컨텍스트 문서 — 피드백 발행부와 동일 패턴.
50+
List<Long> contextDocumentIds = contextRepository.findBySession_Id(session.getId()).stream()
51+
.map(c -> c.getDocument().getId())
52+
.toList();
53+
54+
// 최근 대화 히스토리 (중복 질문 회피). 시퀀스 순 → 마지막 N개.
55+
List<InterviewMessage> ordered =
56+
messageRepository.findBySession_IdOrderBySequenceNumberAsc(session.getId());
57+
List<HistoryItem> history = ordered.stream()
58+
.skip(Math.max(0, ordered.size() - HISTORY_MAX))
59+
.map(m -> new HistoryItem(m.getRole().name(), m.getContent()))
60+
.toList();
61+
4262
GenerateFollowupPayload payload = new GenerateFollowupPayload(
4363
session.getId(),
4464
parent.getId(),
4565
answer.getId(),
4666
parent.getContent(),
4767
answer.getContent(),
4868
session.getMode(),
49-
session.getJobCategory()
69+
session.getJobCategory(),
70+
contextDocumentIds,
71+
parent.getCategory(),
72+
history
5073
);
5174
publisher.publishToAi(
5275
properties.routingKeys().generateFollowup(),

‎backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFollowupPayload.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.stackup.stackup.session.domain.JobCategory;
44
import com.stackup.stackup.session.domain.SessionMode;
5+
import java.util.List;
56

67
public record GenerateFollowupPayload(
78
Long sessionId,
@@ -10,6 +11,11 @@ public record GenerateFollowupPayload(
1011
String previousQuestion,
1112
String answerText,
1213
SessionMode mode,
13-
JobCategory jobCategory
14+
JobCategory jobCategory,
15+
List<Long> contextDocumentIds, // RAG(자료 근거/correctness) 용
16+
String parentCategory, // 직전 질문 카테고리 (루브릭 선택)
17+
List<HistoryItem> history // 최근 대화 (중복 회피)
1418
) {
19+
public record HistoryItem(String role, String content) {
20+
}
1521
}

‎backend/src/main/java/com/stackup/stackup/session/application/dto/QuestionsCallbackPayload.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ public record GeneratedQuestion(
3131
public record AnswerEvaluation(
3232
Double specificity,
3333
Double logic,
34-
String structure
34+
String structure,
35+
Double correctness // 자료 근거 사실성(0~5). RAG 컨텍스트 없으면 null.
3536
) {
3637
}
3738
}

0 commit comments

Comments
 (0)