Skip to content

Commit 3f43979

Browse files
authored
Merge pull request #71 from Team-StackUp/feature/voice-interview
Feature/voice interview
2 parents 2965193 + 2d02d7f commit 3f43979

15 files changed

Lines changed: 407 additions & 31 deletions

File tree

backend/openapi.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2613,6 +2613,12 @@
26132613
},
26142614
"targetEvidence" : {
26152615
"type" : "string"
2616+
},
2617+
"ttsAudioUrl" : {
2618+
"type" : "string"
2619+
},
2620+
"audioFileUrl" : {
2621+
"type" : "string"
26162622
}
26172623
}
26182624
},

backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.stackup.stackup.common.exception.ApiErrorCode;
44
import com.stackup.stackup.common.exception.DomainException;
5+
import com.stackup.stackup.common.storage.ObjectStorageClient;
56
import com.stackup.stackup.session.application.dto.MessageResult;
67
import com.stackup.stackup.session.application.event.AnswerSubmittedEvent;
78
import com.stackup.stackup.session.domain.InterviewMessage;
@@ -11,8 +12,13 @@
1112
import com.stackup.stackup.session.domain.MessageRole;
1213
import com.stackup.stackup.session.domain.MessageStatus;
1314
import com.stackup.stackup.session.domain.SessionStatus;
15+
import com.stackup.stackup.session.domain.TtsStatus;
16+
import java.net.URI;
17+
import java.time.Duration;
1418
import java.util.List;
1519
import lombok.RequiredArgsConstructor;
20+
import org.slf4j.Logger;
21+
import org.slf4j.LoggerFactory;
1622
import org.springframework.context.ApplicationEventPublisher;
1723
import org.springframework.stereotype.Service;
1824
import org.springframework.transaction.annotation.Transactional;
@@ -24,17 +30,43 @@
2430
@Transactional(readOnly = true)
2531
public class InterviewMessageService {
2632

33+
private static final Logger log = LoggerFactory.getLogger(InterviewMessageService.class);
34+
private static final Duration AUDIO_URL_TTL = Duration.ofMinutes(30);
35+
2736
private final InterviewSessionRepository sessionRepository;
2837
private final InterviewMessageRepository messageRepository;
38+
private final ObjectStorageClient storage;
2939
private final ApplicationEventPublisher events;
3040

3141
public List<MessageResult> list(Long userId, Long sessionId) {
3242
ownedSession(userId, sessionId);
3343
return messageRepository.findBySession_IdOrderBySequenceNumberAsc(sessionId).stream()
34-
.map(MessageResult::of)
44+
.map(this::toResultWithAudioUrls)
3545
.toList();
3646
}
3747

48+
// 재생용 presigned URL 동봉: 질문 TTS(SUCCEEDED) + 음성 답변 원본.
49+
// presign 실패가 메시지 조회 전체를 깨뜨리지 않도록 개별 try/catch.
50+
private MessageResult toResultWithAudioUrls(InterviewMessage m) {
51+
String ttsUrl = m.getTtsStatus() == TtsStatus.SUCCEEDED
52+
? presign(m.getTtsAudioPath()) : null;
53+
String audioUrl = presign(m.getAudioFilePath());
54+
return MessageResult.of(m, ttsUrl, audioUrl);
55+
}
56+
57+
private String presign(String key) {
58+
if (key == null || key.isBlank()) {
59+
return null;
60+
}
61+
try {
62+
URI url = storage.createPresignedGetUrl(key, AUDIO_URL_TTL);
63+
return url == null ? null : url.toString();
64+
} catch (RuntimeException e) {
65+
log.warn("presigned audio URL creation failed. key={}", key, e);
66+
return null;
67+
}
68+
}
69+
3870
@Transactional
3971
public MessageResult submitAnswer(Long userId, Long sessionId, String content, String idempotencyKey) {
4072
InterviewSession session = ownedSession(userId, sessionId);

backend/src/main/java/com/stackup/stackup/session/application/dto/MessageResult.java

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,17 @@ public record MessageResult(
2020
MessageStatus status,
2121
Instant createdAt,
2222
String category,
23-
String targetEvidence
23+
String targetEvidence,
24+
// 재생용 presigned URL. 조회(list) 경로에서만 채우고, 그 외에는 null.
25+
String ttsAudioUrl,
26+
String audioFileUrl
2427
// expectedSignal 은 의도적으로 제외 — 정답 유출 방지(라이브 비노출).
2528
) {
2629
public static MessageResult of(InterviewMessage m) {
30+
return of(m, null, null);
31+
}
32+
33+
public static MessageResult of(InterviewMessage m, String ttsAudioUrl, String audioFileUrl) {
2734
return new MessageResult(
2835
m.getId(),
2936
m.getSession().getId(),
@@ -38,7 +45,9 @@ public static MessageResult of(InterviewMessage m) {
3845
m.getStatus(),
3946
m.getCreatedAt(),
4047
m.getCategory(),
41-
m.getTargetEvidence()
48+
m.getTargetEvidence(),
49+
ttsAudioUrl,
50+
audioFileUrl
4251
);
4352
}
4453
}

backend/src/main/java/com/stackup/stackup/session/presentation/dto/MessageResponse.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ public record MessageResponse(
2020
MessageStatus status,
2121
Instant createdAt,
2222
String category,
23-
String targetEvidence
23+
String targetEvidence,
24+
// 재생용 presigned URL (질문 TTS / 음성 답변 원본). 조회 응답에서만 채움.
25+
String ttsAudioUrl,
26+
String audioFileUrl
2427
// expectedSignal 은 노출하지 않음 — 자기연습 시 정답 유출 방지.
2528
) {
2629
public static MessageResponse from(MessageResult r) {
@@ -38,7 +41,9 @@ public static MessageResponse from(MessageResult r) {
3841
r.status(),
3942
r.createdAt(),
4043
r.category(),
41-
r.targetEvidence()
44+
r.targetEvidence(),
45+
r.ttsAudioUrl(),
46+
r.audioFileUrl()
4247
);
4348
}
4449
}

frontend/src/domain/session/index.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,13 @@ export type {
77
JobCategory,
88
MessageRole,
99
} from './model/types'
10-
export { isQuestion, isAnswer, currentTurn, canSubmitAnswer, sessionProgress } from './lib/turn'
10+
export {
11+
isQuestion,
12+
isAnswer,
13+
isTranscribing,
14+
VOICE_TRANSCRIBING_TEXT,
15+
currentTurn,
16+
canSubmitAnswer,
17+
sessionProgress,
18+
} from './lib/turn'
1119
export type { Turn } from './lib/turn'

frontend/src/domain/session/lib/turn.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,17 @@ export function isAnswer(message: Message): boolean {
1010
return message.role === 'INTERVIEWEE'
1111
}
1212

13+
// 음성 답변 placeholder 의 임시 content (백엔드 InterviewMessage.VOICE_TRANSCRIPTION_PENDING_TEXT 와 동일).
14+
// STT 완료 전까지 이 값이 들어 있고, 완료되면 실제 transcript 로 교체된다.
15+
export const VOICE_TRANSCRIBING_TEXT = '(transcribing)'
16+
17+
// STT 대기 중(transcript 아직 없음)인 음성 답변인지.
18+
export function isTranscribing(message: Message): boolean {
19+
if (!isAnswer(message)) return false
20+
const c = (message.content ?? '').trim()
21+
return c === '' || c === VOICE_TRANSCRIBING_TEXT
22+
}
23+
1324
export function currentTurn(messages: Message[]): Turn {
1425
const last = messages[messages.length - 1]
1526
// 마지막이 '내용이 있는' 면접관 질문일 때만 답변 차례. 질문 생성 중(빈 content)

frontend/src/features/interview/api/messageApi.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,26 @@ type S = components['schemas']
66
export async function listMessages(sessionId: number): Promise<S['MessageResponse'][]> {
77
return (await apiClient.get<S['MessageResponse'][]>(`/api/sessions/${sessionId}/messages`)).data
88
}
9+
10+
// 음성 답변 업로드(multipart). 응답은 transcribing placeholder 메시지.
11+
// STT 완료 후 SESSION_MESSAGE SSE 로 transcript 가 도착한다.
12+
export async function submitVoiceAnswer(
13+
sessionId: number,
14+
audio: Blob,
15+
idempotencyKey?: string,
16+
): Promise<S['MessageResponse']> {
17+
const form = new FormData()
18+
const ext = audio.type.includes('ogg') ? 'ogg' : 'webm'
19+
form.append('audio', audio, `answer.${ext}`)
20+
const { data } = await apiClient.post<S['MessageResponse']>(
21+
`/api/sessions/${sessionId}/messages/voice`,
22+
form,
23+
{
24+
headers: {
25+
'Content-Type': 'multipart/form-data',
26+
...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}),
27+
},
28+
},
29+
)
30+
return data
31+
}
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { useCallback, useEffect, useRef, useState } from 'react'
2+
3+
export type RecorderStatus = 'idle' | 'requesting' | 'recording' | 'denied' | 'unsupported'
4+
5+
// MediaRecorder 가 만들 수 있는 후보. 백엔드 허용 목록(webm/ogg/mpeg/mp4/wav)과 교집합.
6+
const PREFERRED_MIME = ['audio/webm;codecs=opus', 'audio/webm', 'audio/ogg;codecs=opus', 'audio/ogg']
7+
8+
function pickMimeType(): string | undefined {
9+
if (typeof MediaRecorder === 'undefined') return undefined
10+
return PREFERRED_MIME.find((t) => MediaRecorder.isTypeSupported(t))
11+
}
12+
13+
// 라이브 면접 음성 답변 녹음. start → 마이크 권한 요청 + 녹음, stop → 오디오 Blob 반환.
14+
// 권한 거부/미지원 시 status 로 알려서 호출부가 텍스트 입력으로 fallback 한다.
15+
export function useVoiceRecorder() {
16+
const [status, setStatus] = useState<RecorderStatus>(() =>
17+
typeof MediaRecorder === 'undefined' || !navigator.mediaDevices?.getUserMedia
18+
? 'unsupported'
19+
: 'idle',
20+
)
21+
22+
const recorderRef = useRef<MediaRecorder | null>(null)
23+
const streamRef = useRef<MediaStream | null>(null)
24+
const chunksRef = useRef<Blob[]>([])
25+
26+
const cleanup = useCallback(() => {
27+
streamRef.current?.getTracks().forEach((t) => t.stop())
28+
streamRef.current = null
29+
recorderRef.current = null
30+
chunksRef.current = []
31+
}, [])
32+
33+
useEffect(() => cleanup, [cleanup])
34+
35+
const start = useCallback(async (): Promise<boolean> => {
36+
if (status === 'unsupported' || status === 'recording') return false
37+
setStatus('requesting')
38+
let stream: MediaStream
39+
try {
40+
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
41+
} catch {
42+
setStatus('denied')
43+
return false
44+
}
45+
const mimeType = pickMimeType()
46+
const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined)
47+
chunksRef.current = []
48+
recorder.ondataavailable = (e) => {
49+
if (e.data.size > 0) chunksRef.current.push(e.data)
50+
}
51+
recorder.start()
52+
streamRef.current = stream
53+
recorderRef.current = recorder
54+
setStatus('recording')
55+
return true
56+
}, [status])
57+
58+
// 녹음 중지 후 Blob 반환. 녹음 중이 아니면 null.
59+
const stop = useCallback((): Promise<Blob | null> => {
60+
const recorder = recorderRef.current
61+
if (!recorder || recorder.state === 'inactive') {
62+
cleanup()
63+
setStatus('idle')
64+
return Promise.resolve(null)
65+
}
66+
return new Promise((resolve) => {
67+
recorder.onstop = () => {
68+
const type = recorder.mimeType || chunksRef.current[0]?.type || 'audio/webm'
69+
const blob = new Blob(chunksRef.current, { type })
70+
cleanup()
71+
setStatus('idle')
72+
resolve(blob.size > 0 ? blob : null)
73+
}
74+
recorder.stop()
75+
})
76+
}, [cleanup])
77+
78+
// 녹음 폐기(업로드 안 함).
79+
const cancel = useCallback(() => {
80+
const recorder = recorderRef.current
81+
if (recorder && recorder.state !== 'inactive') {
82+
recorder.onstop = null
83+
recorder.stop()
84+
}
85+
cleanup()
86+
setStatus('idle')
87+
}, [cleanup])
88+
89+
return { status, start, stop, cancel }
90+
}

frontend/src/features/interview/model/useLiveInterview.ts

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
import { useCallback, useEffect, useState } from 'react'
22
import { useNavigate } from 'react-router-dom'
3-
import { useQueryClient } from '@tanstack/react-query'
3+
import { useMutation, useQueryClient } from '@tanstack/react-query'
44
import { currentTurn } from '@/domain/session'
55
import type { Message } from '@/domain/session'
6+
import { submitVoiceAnswer } from '../api/messageApi'
67
import { sessionKeys, useSession } from './useSession'
78
import { messageKeys, useSessionMessages } from './useSessionMessages'
89
import { useSessionLifecycle } from './useSessionLifecycle'
@@ -68,13 +69,30 @@ export function useLiveInterview(sessionId: number) {
6869
[socketSubmit],
6970
)
7071

72+
// 음성 답변은 REST 업로드(multipart). 성공 시 placeholder 메시지를
73+
// 받으므로 목록을 무효화해 "음성 인식 중…" 버블을 띄운다.
74+
// 이후 STT 완료는 callback.voice → SESSION_MESSAGE SSE 로 갱신된다.
75+
const voiceMutation = useMutation({
76+
mutationFn: (audio: Blob) => submitVoiceAnswer(sessionId, audio, crypto.randomUUID()),
77+
onSuccess: () =>
78+
queryClient.invalidateQueries({ queryKey: messageKeys.list(sessionId) }),
79+
})
80+
81+
const submitVoice = useCallback(
82+
(audio: Blob) => voiceMutation.mutate(audio),
83+
[voiceMutation],
84+
)
85+
7186
return {
7287
session: sessionQuery.data,
7388
status,
7489
items,
7590
turn: currentTurn(items),
7691
connection,
7792
submitAnswer,
93+
submitVoice,
94+
voiceUploading: voiceMutation.isPending,
95+
voiceError: voiceMutation.isError,
7896
endSession: () => end.mutate(),
7997
isLoading: sessionQuery.isLoading,
8098
}
Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,31 @@
1+
import { isTranscribing } from '@/domain/session'
12
import type { Message } from '@/domain/session'
23

34
export function AnswerBubble({ message }: { message: Message }) {
5+
const transcribing = isTranscribing(message)
6+
const failed = transcribing && message.status === 'FAILED'
7+
48
return (
5-
<div className="flex justify-end">
9+
<div className="flex flex-col items-end gap-1">
610
<div className="max-w-[80%] whitespace-pre-wrap rounded-lg rounded-tr-sm bg-primary px-4 py-3 text-body text-fg-on-primary shadow-sm">
7-
{message.content}
11+
{!transcribing ? (
12+
message.content
13+
) : (
14+
<span className="inline-flex items-center gap-2 text-fg-on-primary/80">
15+
{failed ? (
16+
'음성 인식에 실패했어요. 다시 답변해 주세요.'
17+
) : (
18+
<>
19+
<span className="inline-block h-2 w-2 animate-pulse rounded-full bg-fg-on-primary" />
20+
음성 인식 중…
21+
</>
22+
)}
23+
</span>
24+
)}
825
</div>
26+
{message.audioFileUrl && (
27+
<audio controls src={message.audioFileUrl} className="max-w-[80%]" preload="none" />
28+
)}
929
</div>
1030
)
1131
}

0 commit comments

Comments
 (0)