Skip to content

Commit 1ad604e

Browse files
i3monthsclaude
i3months
and
claude
committed
fix(voice): 음성 답변 content-type 거부 + 오디오 재생 불가 수정
증상(prod 로그/설정으로 확인): - 음성 답변 업로드가 VOICE_INVALID_CONTENT_TYPE 로 거부 — MediaRecorder 가 "audio/webm;codecs=opus" 를 보내는데 허용 목록은 base MIME 만 가짐. - "음성 듣기"/음성 재생 불가 — presigned URL 이 내부 호스트 (S3_ENDPOINT=http://minio:38060)를 가리켜 브라우저가 도달 불가. 백엔드: - VoiceAnswerUploadService: content-type 의 코덱 파라미터를 떼고 base MIME 으로 허용/확장자 판정(baseContentType) - InterviewMessageService.streamAudio + GET /api/sessions/{sid}/messages/{mid}/audio: Core 가 인증 거쳐 오디오 바이트 스트리밍(질문=TTS, 답변=원본). presigned 내부호스트 우회. 프론트: - submitVoiceAnswer: 업로드 전 코덱 파라미터 제거(방어) - useMessageAudio: 프록시로 blob 받아 object URL 캐싱 - QuestionBubble/AnswerBubble: presigned URL 대신 프록시로 재생 (질문 자동재생/음성 듣기, 내 답변 듣기) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent ab3f53e commit 1ad604e

9 files changed

Lines changed: 306 additions & 41 deletions

File tree

‎backend/openapi.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1801,6 +1801,66 @@
18011801
}
18021802
}
18031803
},
1804+
"/api/sessions/{sessionId}/messages/{messageId}/audio" : {
1805+
"get" : {
1806+
"tags" : [ "Session Messages" ],
1807+
"summary" : "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)",
1808+
"description" : "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다.",
1809+
"operationId" : "streamMessageAudio",
1810+
"parameters" : [ {
1811+
"name" : "sessionId",
1812+
"in" : "path",
1813+
"required" : true,
1814+
"schema" : {
1815+
"type" : "integer",
1816+
"format" : "int64"
1817+
}
1818+
}, {
1819+
"name" : "messageId",
1820+
"in" : "path",
1821+
"required" : true,
1822+
"schema" : {
1823+
"type" : "integer",
1824+
"format" : "int64"
1825+
}
1826+
} ],
1827+
"responses" : {
1828+
"200" : {
1829+
"description" : "오디오 바이트",
1830+
"content" : {
1831+
"*/*" : {
1832+
"schema" : {
1833+
"type" : "string",
1834+
"format" : "binary"
1835+
}
1836+
}
1837+
}
1838+
},
1839+
"401" : {
1840+
"description" : "인증 실패",
1841+
"content" : {
1842+
"*/*" : {
1843+
"schema" : {
1844+
"type" : "string",
1845+
"format" : "binary"
1846+
}
1847+
}
1848+
}
1849+
},
1850+
"404" : {
1851+
"description" : "세션/메시지/오디오 없음",
1852+
"content" : {
1853+
"*/*" : {
1854+
"schema" : {
1855+
"type" : "string",
1856+
"format" : "binary"
1857+
}
1858+
}
1859+
}
1860+
}
1861+
}
1862+
}
1863+
},
18041864
"/api/sessions/{sessionId}/feedback" : {
18051865
"get" : {
18061866
"tags" : [ "Session Feedback" ],

‎backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,32 @@ private String presign(String key) {
6767
}
6868
}
6969

70+
// 오디오 스트리밍 프록시. MinIO presigned URL 이 내부 호스트라 브라우저가 못 가므로,
71+
// Core 가 인증을 거쳐 바이트를 그대로 흘려준다. 질문=TTS, 답변=음성 원본.
72+
public AudioStream streamAudio(Long userId, Long sessionId, Long messageId) {
73+
ownedSession(userId, sessionId);
74+
InterviewMessage m = messageRepository.findById(messageId)
75+
.filter(x -> x.getSession().getId().equals(sessionId))
76+
.orElseThrow(() -> new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND));
77+
String key = m.getRole() == MessageRole.INTERVIEWER ? m.getTtsAudioPath() : m.getAudioFilePath();
78+
if (key == null || key.isBlank()) {
79+
throw new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND);
80+
}
81+
return new AudioStream(storage.get(key), contentTypeForKey(key));
82+
}
83+
84+
public record AudioStream(java.io.InputStream stream, String contentType) {}
85+
86+
private static String contentTypeForKey(String key) {
87+
String k = key.toLowerCase();
88+
if (k.endsWith(".mp3")) return "audio/mpeg";
89+
if (k.endsWith(".wav")) return "audio/wav";
90+
if (k.endsWith(".ogg")) return "audio/ogg";
91+
if (k.endsWith(".m4a") || k.endsWith(".mp4")) return "audio/mp4";
92+
if (k.endsWith(".webm")) return "audio/webm";
93+
return "application/octet-stream";
94+
}
95+
7096
@Transactional
7197
public MessageResult submitAnswer(Long userId, Long sessionId, String content, String idempotencyKey) {
7298
InterviewSession session = ownedSession(userId, sessionId);

‎backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,13 +116,24 @@ private void validate(VoiceAnswerUploadCommand cmd) {
116116
if (cmd.size() > MAX_BYTES) {
117117
throw new DomainException(ApiErrorCode.VOICE_FILE_TOO_LARGE);
118118
}
119-
if (cmd.contentType() == null || !ALLOWED_CONTENT_TYPES.contains(cmd.contentType().trim().toLowerCase())) {
119+
if (baseContentType(cmd.contentType()) == null) {
120120
throw new DomainException(ApiErrorCode.VOICE_INVALID_CONTENT_TYPE);
121121
}
122122
}
123123

124+
// 브라우저 MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙인다.
125+
// 파라미터를 떼고 base MIME 만으로 허용 여부를 판단한다. 허용 외면 null.
126+
private static String baseContentType(String contentType) {
127+
if (contentType == null) {
128+
return null;
129+
}
130+
String base = contentType.split(";", 2)[0].trim().toLowerCase();
131+
return ALLOWED_CONTENT_TYPES.contains(base) ? base : null;
132+
}
133+
124134
private static String buildKey(Long sessionId, Long messageId, String contentType) {
125-
String ext = switch (contentType.trim().toLowerCase()) {
135+
String base = baseContentType(contentType);
136+
String ext = switch (base == null ? "" : base) {
126137
case "audio/webm" -> "webm";
127138
case "audio/ogg" -> "ogg";
128139
case "audio/mpeg" -> "mp3";

‎backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,21 @@
44
import com.stackup.stackup.session.application.InterviewMessageService;
55
import com.stackup.stackup.session.presentation.dto.MessageResponse;
66
import com.stackup.stackup.session.presentation.dto.MessageSubmitRequest;
7+
import com.stackup.stackup.session.application.InterviewMessageService.AudioStream;
78
import io.swagger.v3.oas.annotations.Operation;
89
import io.swagger.v3.oas.annotations.responses.ApiResponse;
910
import io.swagger.v3.oas.annotations.responses.ApiResponses;
1011
import io.swagger.v3.oas.annotations.tags.Tag;
1112
import jakarta.validation.Valid;
13+
import java.time.Duration;
1214
import java.util.List;
1315
import lombok.RequiredArgsConstructor;
16+
import org.springframework.core.io.InputStreamResource;
17+
import org.springframework.core.io.Resource;
18+
import org.springframework.http.CacheControl;
1419
import org.springframework.http.HttpStatus;
20+
import org.springframework.http.MediaType;
21+
import org.springframework.http.ResponseEntity;
1522
import org.springframework.security.core.annotation.AuthenticationPrincipal;
1623
import org.springframework.web.bind.annotation.GetMapping;
1724
import org.springframework.web.bind.annotation.PathVariable;
@@ -70,4 +77,28 @@ public MessageResponse submit(
7077
messageService.submitAnswer(principal.userId(), sessionId, request.content(), idempotencyKey)
7178
);
7279
}
80+
81+
@Operation(
82+
operationId = "streamMessageAudio",
83+
summary = "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)",
84+
description = "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 "
85+
+ "Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다."
86+
)
87+
@ApiResponses({
88+
@ApiResponse(responseCode = "200", description = "오디오 바이트"),
89+
@ApiResponse(responseCode = "401", description = "인증 실패"),
90+
@ApiResponse(responseCode = "404", description = "세션/메시지/오디오 없음")
91+
})
92+
@GetMapping("/{messageId}/audio")
93+
public ResponseEntity<Resource> audio(
94+
@AuthenticationPrincipal UserPrincipal principal,
95+
@PathVariable Long sessionId,
96+
@PathVariable Long messageId
97+
) {
98+
AudioStream audio = messageService.streamAudio(principal.userId(), sessionId, messageId);
99+
return ResponseEntity.ok()
100+
.contentType(MediaType.parseMediaType(audio.contentType()))
101+
.cacheControl(CacheControl.maxAge(Duration.ofHours(1)).cachePrivate())
102+
.body(new InputStreamResource(audio.stream()));
103+
}
73104
}

‎frontend/src/features/interview/api/messageApi.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,13 @@ export async function submitVoiceAnswer(
1414
audio: Blob,
1515
idempotencyKey?: string,
1616
): Promise<S['MessageResponse']> {
17+
// MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙이는데
18+
// 일부 검증은 base MIME 만 허용하므로 파라미터를 떼고 업로드한다.
19+
const baseType = (audio.type || 'audio/webm').split(';')[0]
20+
const clean = audio.type === baseType ? audio : new Blob([audio], { type: baseType })
21+
const ext = baseType.includes('ogg') ? 'ogg' : 'webm'
1722
const form = new FormData()
18-
const ext = audio.type.includes('ogg') ? 'ogg' : 'webm'
19-
form.append('audio', audio, `answer.${ext}`)
23+
form.append('audio', clean, `answer.${ext}`)
2024
const { data } = await apiClient.post<S['MessageResponse']>(
2125
`/api/sessions/${sessionId}/messages/voice`,
2226
form,
@@ -29,3 +33,16 @@ export async function submitVoiceAnswer(
2933
)
3034
return data
3135
}
36+
37+
// 오디오 바이트를 Core 프록시로 받아 object URL 로 변환.
38+
// MinIO presigned URL 이 내부 호스트라 브라우저가 직접 못 가므로 이 경로를 쓴다.
39+
export async function fetchMessageAudioObjectUrl(
40+
sessionId: number,
41+
messageId: number,
42+
): Promise<string> {
43+
const { data } = await apiClient.get<Blob>(
44+
`/api/sessions/${sessionId}/messages/${messageId}/audio`,
45+
{ responseType: 'blob' },
46+
)
47+
return URL.createObjectURL(data)
48+
}

‎frontend/src/features/interview/lib/media/useMessageAudio.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import { useCallback, useEffect, useRef, useState } from 'react'
2+
import { fetchMessageAudioObjectUrl } from '../../api/messageApi'
3+
4+
// 메시지 오디오를 Core 프록시로 1회 받아 object URL 로 캐싱(언마운트 시 revoke).
5+
// load() 는 처음 호출 시 fetch, 이후 캐시 반환. id 가 없으면 비활성.
6+
export function useMessageAudio(sessionId?: number, messageId?: number) {
7+
const [url, setUrl] = useState<string>()
8+
const loadingRef = useRef(false)
9+
10+
const load = useCallback(async (): Promise<string | undefined> => {
11+
if (url) return url
12+
if (loadingRef.current || sessionId == null || messageId == null) return undefined
13+
loadingRef.current = true
14+
try {
15+
const u = await fetchMessageAudioObjectUrl(sessionId, messageId)
16+
setUrl(u)
17+
return u
18+
} catch {
19+
return undefined
20+
} finally {
21+
loadingRef.current = false
22+
}
23+
}, [url, sessionId, messageId])
24+
25+
useEffect(
26+
() => () => {
27+
if (url) URL.revokeObjectURL(url)
28+
},
29+
[url],
30+
)
31+
32+
return { url, load }
33+
}

‎frontend/src/features/interview/ui/live/AnswerBubble.tsx‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
import { isTranscribing } from '@/domain/session'
22
import type { Message } from '@/domain/session'
3+
import { useMessageAudio } from '../../lib/media/useMessageAudio'
34

45
export function AnswerBubble({ message }: { message: Message }) {
56
const transcribing = isTranscribing(message)
67
const failed = transcribing && message.status === 'FAILED'
8+
const hasVoice = Boolean(message.audioFilePath)
9+
10+
const { url, load } = useMessageAudio(message.sessionId, message.id)
711

812
return (
913
<div className="flex flex-col items-end gap-1">
@@ -23,9 +27,18 @@ export function AnswerBubble({ message }: { message: Message }) {
2327
</span>
2428
)}
2529
</div>
26-
{message.audioFileUrl && (
27-
<audio controls src={message.audioFileUrl} className="max-w-[80%]" preload="none" />
28-
)}
30+
{hasVoice &&
31+
(url ? (
32+
<audio controls src={url} autoPlay className="h-9 max-w-[80%]" />
33+
) : (
34+
<button
35+
type="button"
36+
onClick={() => void load()}
37+
className="rounded-pill px-2.5 py-1 text-caption text-fg-muted transition-colors hover:bg-surface hover:text-fg"
38+
>
39+
▶ 내 답변 듣기
40+
</button>
41+
))}
2942
</div>
3043
)
3144
}

‎frontend/src/features/interview/ui/live/QuestionBubble.tsx‎

Lines changed: 47 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { useEffect, useRef, useState } from 'react'
22
import type { Message } from '@/domain/session'
33
import { StatusBadge } from '@/shared/ui/StatusBadge'
4+
import { useMessageAudio } from '../../lib/media/useMessageAudio'
45

56
const CATEGORY_LABEL: Record<string, string> = {
67
CS_FUNDAMENTAL: 'CS 기초',
@@ -17,29 +18,6 @@ function PlayIcon({ playing }: { playing: boolean }) {
1718
)
1819
}
1920

20-
// ttsAudioUrl 이 생기면(최신 질문일 때) 한 번 자동재생 시도.
21-
// 브라우저 자동재생 차단 시 조용히 실패 → 사용자가 버튼으로 재생.
22-
function useTtsPlayer(url: string | undefined, autoPlay: boolean) {
23-
const audioRef = useRef<HTMLAudioElement | null>(null)
24-
const [playing, setPlaying] = useState(false)
25-
const autoPlayedFor = useRef<string | null>(null)
26-
27-
useEffect(() => {
28-
if (!url || !autoPlay || autoPlayedFor.current === url) return
29-
autoPlayedFor.current = url
30-
audioRef.current?.play().catch(() => {})
31-
}, [url, autoPlay])
32-
33-
const toggle = () => {
34-
const el = audioRef.current
35-
if (!el) return
36-
if (el.paused) el.play().catch(() => {})
37-
else el.pause()
38-
}
39-
40-
return { audioRef, playing, setPlaying, toggle }
41-
}
42-
4321
export function QuestionBubble({
4422
message,
4523
autoPlay = false,
@@ -51,8 +29,41 @@ export function QuestionBubble({
5129
? (CATEGORY_LABEL[message.category] ?? message.category)
5230
: null
5331
const hasMeta = Boolean(categoryLabel || message.targetEvidence)
54-
const ttsUrl = message.ttsStatus === 'SUCCEEDED' ? message.ttsAudioUrl : undefined
55-
const { audioRef, playing, setPlaying, toggle } = useTtsPlayer(ttsUrl, autoPlay)
32+
const ttsReady = message.ttsStatus === 'SUCCEEDED'
33+
34+
const { url, load } = useMessageAudio(message.sessionId, message.id)
35+
const audioRef = useRef<HTMLAudioElement | null>(null)
36+
const [playing, setPlaying] = useState(false)
37+
const wantPlay = useRef(false)
38+
const autoTried = useRef(false)
39+
40+
// 최신 질문이면 오디오를 받아 자동재생 시도(차단 시 조용히 폴백).
41+
useEffect(() => {
42+
if (!autoPlay || !ttsReady || autoTried.current) return
43+
autoTried.current = true
44+
wantPlay.current = true
45+
void load()
46+
}, [autoPlay, ttsReady, load])
47+
48+
// object URL 이 준비되면 재생 요청을 반영.
49+
useEffect(() => {
50+
if (url && wantPlay.current) {
51+
wantPlay.current = false
52+
audioRef.current?.play().catch(() => {})
53+
}
54+
}, [url])
55+
56+
const toggle = async () => {
57+
const el = audioRef.current
58+
if (!url) {
59+
wantPlay.current = true
60+
await load()
61+
return
62+
}
63+
if (!el) return
64+
if (el.paused) el.play().catch(() => {})
65+
else el.pause()
66+
}
5667

5768
return (
5869
<div className="flex justify-start">
@@ -67,7 +78,7 @@ export function QuestionBubble({
6778
)}
6879
<div className="rounded-lg rounded-tl-sm bg-surface-raised px-4 py-3 text-body text-fg shadow-sm">
6980
<p className="whitespace-pre-wrap">{message.content}</p>
70-
{ttsUrl && (
81+
{ttsReady && (
7182
<div className="mt-2 flex items-center gap-2 border-t border-border pt-2">
7283
<button
7384
type="button"
@@ -78,14 +89,16 @@ export function QuestionBubble({
7889
<PlayIcon playing={playing} />
7990
{playing ? '일시정지' : '음성 듣기'}
8091
</button>
81-
<audio
82-
ref={audioRef}
83-
src={ttsUrl}
84-
preload="none"
85-
onPlay={() => setPlaying(true)}
86-
onPause={() => setPlaying(false)}
87-
onEnded={() => setPlaying(false)}
88-
/>
92+
{url && (
93+
<audio
94+
ref={audioRef}
95+
src={url}
96+
preload="none"
97+
onPlay={() => setPlaying(true)}
98+
onPause={() => setPlaying(false)}
99+
onEnded={() => setPlaying(false)}
100+
/>
101+
)}
89102
</div>
90103
)}
91104
</div>

0 commit comments

Comments
 (0)