From 1ad604ed0d55fb36369419aeded4111d708de3e2 Mon Sep 17 00:00:00 2001 From: jmj Date: Wed, 3 Jun 2026 12:51:27 +0900 Subject: [PATCH] =?UTF-8?q?fix(voice):=20=EC=9D=8C=EC=84=B1=20=EB=8B=B5?= =?UTF-8?q?=EB=B3=80=20content-type=20=EA=B1=B0=EB=B6=80=20+=20=EC=98=A4?= =?UTF-8?q?=EB=94=94=EC=98=A4=20=EC=9E=AC=EC=83=9D=20=EB=B6=88=EA=B0=80=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 증상(prod 로그/설정으로 확인): - 음성 답변 업로드가 VOICE_INVALID_CONTENT_TYPE 로 거부 — MediaRecorder 가 "audio/webm;codecs=opus" 를 보내는데 허용 목록은 base MIME 만 가짐. - "음성 듣기"/음성 재생 불가 — presigned URL 이 내부 호스트 (S3_ENDPOINT=http://minio:38060)를 가리켜 브라우저가 도달 불가. 백엔드: - VoiceAnswerUploadService: content-type 의 코덱 파라미터를 떼고 base MIME 으로 허용/확장자 판정(baseContentType) - InterviewMessageService.streamAudio + GET /api/sessions/{sid}/messages/{mid}/audio: Core 가 인증 거쳐 오디오 바이트 스트리밍(질문=TTS, 답변=원본). presigned 내부호스트 우회. 프론트: - submitVoiceAnswer: 업로드 전 코덱 파라미터 제거(방어) - useMessageAudio: 프록시로 blob 받아 object URL 캐싱 - QuestionBubble/AnswerBubble: presigned URL 대신 프록시로 재생 (질문 자동재생/음성 듣기, 내 답변 듣기) Co-Authored-By: Claude Opus 4.8 --- backend/openapi.json | 60 ++++++++++++++ .../application/InterviewMessageService.java | 26 ++++++ .../application/VoiceAnswerUploadService.java | 15 +++- .../InterviewMessageController.java | 31 +++++++ .../src/features/interview/api/messageApi.ts | 21 ++++- .../interview/lib/media/useMessageAudio.ts | 33 ++++++++ .../interview/ui/live/AnswerBubble.tsx | 19 ++++- .../interview/ui/live/QuestionBubble.tsx | 81 +++++++++++-------- frontend/src/shared/api/generated.ts | 61 ++++++++++++++ 9 files changed, 306 insertions(+), 41 deletions(-) create mode 100644 frontend/src/features/interview/lib/media/useMessageAudio.ts diff --git a/backend/openapi.json b/backend/openapi.json index 4772afe0..48edb13b 100644 --- a/backend/openapi.json +++ b/backend/openapi.json @@ -1801,6 +1801,66 @@ } } }, + "/api/sessions/{sessionId}/messages/{messageId}/audio" : { + "get" : { + "tags" : [ "Session Messages" ], + "summary" : "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)", + "description" : "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다.", + "operationId" : "streamMessageAudio", + "parameters" : [ { + "name" : "sessionId", + "in" : "path", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int64" + } + }, { + "name" : "messageId", + "in" : "path", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int64" + } + } ], + "responses" : { + "200" : { + "description" : "오디오 바이트", + "content" : { + "*/*" : { + "schema" : { + "type" : "string", + "format" : "binary" + } + } + } + }, + "401" : { + "description" : "인증 실패", + "content" : { + "*/*" : { + "schema" : { + "type" : "string", + "format" : "binary" + } + } + } + }, + "404" : { + "description" : "세션/메시지/오디오 없음", + "content" : { + "*/*" : { + "schema" : { + "type" : "string", + "format" : "binary" + } + } + } + } + } + } + }, "/api/sessions/{sessionId}/feedback" : { "get" : { "tags" : [ "Session Feedback" ], diff --git a/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java b/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java index 16bb145b..9100511a 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java @@ -67,6 +67,32 @@ private String presign(String key) { } } + // 오디오 스트리밍 프록시. MinIO presigned URL 이 내부 호스트라 브라우저가 못 가므로, + // Core 가 인증을 거쳐 바이트를 그대로 흘려준다. 질문=TTS, 답변=음성 원본. + public AudioStream streamAudio(Long userId, Long sessionId, Long messageId) { + ownedSession(userId, sessionId); + InterviewMessage m = messageRepository.findById(messageId) + .filter(x -> x.getSession().getId().equals(sessionId)) + .orElseThrow(() -> new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND)); + String key = m.getRole() == MessageRole.INTERVIEWER ? m.getTtsAudioPath() : m.getAudioFilePath(); + if (key == null || key.isBlank()) { + throw new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND); + } + return new AudioStream(storage.get(key), contentTypeForKey(key)); + } + + public record AudioStream(java.io.InputStream stream, String contentType) {} + + private static String contentTypeForKey(String key) { + String k = key.toLowerCase(); + if (k.endsWith(".mp3")) return "audio/mpeg"; + if (k.endsWith(".wav")) return "audio/wav"; + if (k.endsWith(".ogg")) return "audio/ogg"; + if (k.endsWith(".m4a") || k.endsWith(".mp4")) return "audio/mp4"; + if (k.endsWith(".webm")) return "audio/webm"; + return "application/octet-stream"; + } + @Transactional public MessageResult submitAnswer(Long userId, Long sessionId, String content, String idempotencyKey) { InterviewSession session = ownedSession(userId, sessionId); diff --git a/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java b/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java index 5287ba5e..ce1501e7 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java @@ -116,13 +116,24 @@ private void validate(VoiceAnswerUploadCommand cmd) { if (cmd.size() > MAX_BYTES) { throw new DomainException(ApiErrorCode.VOICE_FILE_TOO_LARGE); } - if (cmd.contentType() == null || !ALLOWED_CONTENT_TYPES.contains(cmd.contentType().trim().toLowerCase())) { + if (baseContentType(cmd.contentType()) == null) { throw new DomainException(ApiErrorCode.VOICE_INVALID_CONTENT_TYPE); } } + // 브라우저 MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙인다. + // 파라미터를 떼고 base MIME 만으로 허용 여부를 판단한다. 허용 외면 null. + private static String baseContentType(String contentType) { + if (contentType == null) { + return null; + } + String base = contentType.split(";", 2)[0].trim().toLowerCase(); + return ALLOWED_CONTENT_TYPES.contains(base) ? base : null; + } + private static String buildKey(Long sessionId, Long messageId, String contentType) { - String ext = switch (contentType.trim().toLowerCase()) { + String base = baseContentType(contentType); + String ext = switch (base == null ? "" : base) { case "audio/webm" -> "webm"; case "audio/ogg" -> "ogg"; case "audio/mpeg" -> "mp3"; diff --git a/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java b/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java index 04233f93..73767fe6 100644 --- a/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java +++ b/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java @@ -4,14 +4,21 @@ import com.stackup.stackup.session.application.InterviewMessageService; import com.stackup.stackup.session.presentation.dto.MessageResponse; import com.stackup.stackup.session.presentation.dto.MessageSubmitRequest; +import com.stackup.stackup.session.application.InterviewMessageService.AudioStream; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; +import java.time.Duration; import java.util.List; import lombok.RequiredArgsConstructor; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.CacheControl; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -70,4 +77,28 @@ public MessageResponse submit( messageService.submitAnswer(principal.userId(), sessionId, request.content(), idempotencyKey) ); } + + @Operation( + operationId = "streamMessageAudio", + summary = "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)", + description = "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 " + + "Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다." + ) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "오디오 바이트"), + @ApiResponse(responseCode = "401", description = "인증 실패"), + @ApiResponse(responseCode = "404", description = "세션/메시지/오디오 없음") + }) + @GetMapping("/{messageId}/audio") + public ResponseEntity audio( + @AuthenticationPrincipal UserPrincipal principal, + @PathVariable Long sessionId, + @PathVariable Long messageId + ) { + AudioStream audio = messageService.streamAudio(principal.userId(), sessionId, messageId); + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType(audio.contentType())) + .cacheControl(CacheControl.maxAge(Duration.ofHours(1)).cachePrivate()) + .body(new InputStreamResource(audio.stream())); + } } diff --git a/frontend/src/features/interview/api/messageApi.ts b/frontend/src/features/interview/api/messageApi.ts index a6785975..1b676d13 100644 --- a/frontend/src/features/interview/api/messageApi.ts +++ b/frontend/src/features/interview/api/messageApi.ts @@ -14,9 +14,13 @@ export async function submitVoiceAnswer( audio: Blob, idempotencyKey?: string, ): Promise { + // MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙이는데 + // 일부 검증은 base MIME 만 허용하므로 파라미터를 떼고 업로드한다. + const baseType = (audio.type || 'audio/webm').split(';')[0] + const clean = audio.type === baseType ? audio : new Blob([audio], { type: baseType }) + const ext = baseType.includes('ogg') ? 'ogg' : 'webm' const form = new FormData() - const ext = audio.type.includes('ogg') ? 'ogg' : 'webm' - form.append('audio', audio, `answer.${ext}`) + form.append('audio', clean, `answer.${ext}`) const { data } = await apiClient.post( `/api/sessions/${sessionId}/messages/voice`, form, @@ -29,3 +33,16 @@ export async function submitVoiceAnswer( ) return data } + +// 오디오 바이트를 Core 프록시로 받아 object URL 로 변환. +// MinIO presigned URL 이 내부 호스트라 브라우저가 직접 못 가므로 이 경로를 쓴다. +export async function fetchMessageAudioObjectUrl( + sessionId: number, + messageId: number, +): Promise { + const { data } = await apiClient.get( + `/api/sessions/${sessionId}/messages/${messageId}/audio`, + { responseType: 'blob' }, + ) + return URL.createObjectURL(data) +} diff --git a/frontend/src/features/interview/lib/media/useMessageAudio.ts b/frontend/src/features/interview/lib/media/useMessageAudio.ts new file mode 100644 index 00000000..d4fd7219 --- /dev/null +++ b/frontend/src/features/interview/lib/media/useMessageAudio.ts @@ -0,0 +1,33 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { fetchMessageAudioObjectUrl } from '../../api/messageApi' + +// 메시지 오디오를 Core 프록시로 1회 받아 object URL 로 캐싱(언마운트 시 revoke). +// load() 는 처음 호출 시 fetch, 이후 캐시 반환. id 가 없으면 비활성. +export function useMessageAudio(sessionId?: number, messageId?: number) { + const [url, setUrl] = useState() + const loadingRef = useRef(false) + + const load = useCallback(async (): Promise => { + if (url) return url + if (loadingRef.current || sessionId == null || messageId == null) return undefined + loadingRef.current = true + try { + const u = await fetchMessageAudioObjectUrl(sessionId, messageId) + setUrl(u) + return u + } catch { + return undefined + } finally { + loadingRef.current = false + } + }, [url, sessionId, messageId]) + + useEffect( + () => () => { + if (url) URL.revokeObjectURL(url) + }, + [url], + ) + + return { url, load } +} diff --git a/frontend/src/features/interview/ui/live/AnswerBubble.tsx b/frontend/src/features/interview/ui/live/AnswerBubble.tsx index 407ecb1f..207f59c2 100644 --- a/frontend/src/features/interview/ui/live/AnswerBubble.tsx +++ b/frontend/src/features/interview/ui/live/AnswerBubble.tsx @@ -1,9 +1,13 @@ import { isTranscribing } from '@/domain/session' import type { Message } from '@/domain/session' +import { useMessageAudio } from '../../lib/media/useMessageAudio' export function AnswerBubble({ message }: { message: Message }) { const transcribing = isTranscribing(message) const failed = transcribing && message.status === 'FAILED' + const hasVoice = Boolean(message.audioFilePath) + + const { url, load } = useMessageAudio(message.sessionId, message.id) return (
@@ -23,9 +27,18 @@ export function AnswerBubble({ message }: { message: Message }) { )}
- {message.audioFileUrl && ( -