diff --git a/backend/openapi.json b/backend/openapi.json index 4772afe0..48edb13b 100644 --- a/backend/openapi.json +++ b/backend/openapi.json @@ -1801,6 +1801,66 @@ } } }, + "/api/sessions/{sessionId}/messages/{messageId}/audio" : { + "get" : { + "tags" : [ "Session Messages" ], + "summary" : "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)", + "description" : "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다.", + "operationId" : "streamMessageAudio", + "parameters" : [ { + "name" : "sessionId", + "in" : "path", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int64" + } + }, { + "name" : "messageId", + "in" : "path", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int64" + } + } ], + "responses" : { + "200" : { + "description" : "오디오 바이트", + "content" : { + "*/*" : { + "schema" : { + "type" : "string", + "format" : "binary" + } + } + } + }, + "401" : { + "description" : "인증 실패", + "content" : { + "*/*" : { + "schema" : { + "type" : "string", + "format" : "binary" + } + } + } + }, + "404" : { + "description" : "세션/메시지/오디오 없음", + "content" : { + "*/*" : { + "schema" : { + "type" : "string", + "format" : "binary" + } + } + } + } + } + } + }, "/api/sessions/{sessionId}/feedback" : { "get" : { "tags" : [ "Session Feedback" ], diff --git a/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java b/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java index 16bb145b..9100511a 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java @@ -67,6 +67,32 @@ private String presign(String key) { } } + // 오디오 스트리밍 프록시. MinIO presigned URL 이 내부 호스트라 브라우저가 못 가므로, + // Core 가 인증을 거쳐 바이트를 그대로 흘려준다. 질문=TTS, 답변=음성 원본. + public AudioStream streamAudio(Long userId, Long sessionId, Long messageId) { + ownedSession(userId, sessionId); + InterviewMessage m = messageRepository.findById(messageId) + .filter(x -> x.getSession().getId().equals(sessionId)) + .orElseThrow(() -> new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND)); + String key = m.getRole() == MessageRole.INTERVIEWER ? m.getTtsAudioPath() : m.getAudioFilePath(); + if (key == null || key.isBlank()) { + throw new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND); + } + return new AudioStream(storage.get(key), contentTypeForKey(key)); + } + + public record AudioStream(java.io.InputStream stream, String contentType) {} + + private static String contentTypeForKey(String key) { + String k = key.toLowerCase(); + if (k.endsWith(".mp3")) return "audio/mpeg"; + if (k.endsWith(".wav")) return "audio/wav"; + if (k.endsWith(".ogg")) return "audio/ogg"; + if (k.endsWith(".m4a") || k.endsWith(".mp4")) return "audio/mp4"; + if (k.endsWith(".webm")) return "audio/webm"; + return "application/octet-stream"; + } + @Transactional public MessageResult submitAnswer(Long userId, Long sessionId, String content, String idempotencyKey) { InterviewSession session = ownedSession(userId, sessionId); diff --git a/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java b/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java index 5287ba5e..ce1501e7 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java @@ -116,13 +116,24 @@ private void validate(VoiceAnswerUploadCommand cmd) { if (cmd.size() > MAX_BYTES) { throw new DomainException(ApiErrorCode.VOICE_FILE_TOO_LARGE); } - if (cmd.contentType() == null || !ALLOWED_CONTENT_TYPES.contains(cmd.contentType().trim().toLowerCase())) { + if (baseContentType(cmd.contentType()) == null) { throw new DomainException(ApiErrorCode.VOICE_INVALID_CONTENT_TYPE); } } + // 브라우저 MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙인다. + // 파라미터를 떼고 base MIME 만으로 허용 여부를 판단한다. 허용 외면 null. + private static String baseContentType(String contentType) { + if (contentType == null) { + return null; + } + String base = contentType.split(";", 2)[0].trim().toLowerCase(); + return ALLOWED_CONTENT_TYPES.contains(base) ? base : null; + } + private static String buildKey(Long sessionId, Long messageId, String contentType) { - String ext = switch (contentType.trim().toLowerCase()) { + String base = baseContentType(contentType); + String ext = switch (base == null ? "" : base) { case "audio/webm" -> "webm"; case "audio/ogg" -> "ogg"; case "audio/mpeg" -> "mp3"; diff --git a/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java b/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java index 04233f93..73767fe6 100644 --- a/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java +++ b/backend/src/main/java/com/stackup/stackup/session/presentation/InterviewMessageController.java @@ -4,14 +4,21 @@ import com.stackup.stackup.session.application.InterviewMessageService; import com.stackup.stackup.session.presentation.dto.MessageResponse; import com.stackup.stackup.session.presentation.dto.MessageSubmitRequest; +import com.stackup.stackup.session.application.InterviewMessageService.AudioStream; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; +import java.time.Duration; import java.util.List; import lombok.RequiredArgsConstructor; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.CacheControl; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -70,4 +77,28 @@ public MessageResponse submit( messageService.submitAnswer(principal.userId(), sessionId, request.content(), idempotencyKey) ); } + + @Operation( + operationId = "streamMessageAudio", + summary = "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)", + description = "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 " + + "Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다." + ) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "오디오 바이트"), + @ApiResponse(responseCode = "401", description = "인증 실패"), + @ApiResponse(responseCode = "404", description = "세션/메시지/오디오 없음") + }) + @GetMapping("/{messageId}/audio") + public ResponseEntity audio( + @AuthenticationPrincipal UserPrincipal principal, + @PathVariable Long sessionId, + @PathVariable Long messageId + ) { + AudioStream audio = messageService.streamAudio(principal.userId(), sessionId, messageId); + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType(audio.contentType())) + .cacheControl(CacheControl.maxAge(Duration.ofHours(1)).cachePrivate()) + .body(new InputStreamResource(audio.stream())); + } } diff --git a/frontend/src/features/interview/api/messageApi.ts b/frontend/src/features/interview/api/messageApi.ts index a6785975..1b676d13 100644 --- a/frontend/src/features/interview/api/messageApi.ts +++ b/frontend/src/features/interview/api/messageApi.ts @@ -14,9 +14,13 @@ export async function submitVoiceAnswer( audio: Blob, idempotencyKey?: string, ): Promise { + // MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙이는데 + // 일부 검증은 base MIME 만 허용하므로 파라미터를 떼고 업로드한다. + const baseType = (audio.type || 'audio/webm').split(';')[0] + const clean = audio.type === baseType ? audio : new Blob([audio], { type: baseType }) + const ext = baseType.includes('ogg') ? 'ogg' : 'webm' const form = new FormData() - const ext = audio.type.includes('ogg') ? 'ogg' : 'webm' - form.append('audio', audio, `answer.${ext}`) + form.append('audio', clean, `answer.${ext}`) const { data } = await apiClient.post( `/api/sessions/${sessionId}/messages/voice`, form, @@ -29,3 +33,16 @@ export async function submitVoiceAnswer( ) return data } + +// 오디오 바이트를 Core 프록시로 받아 object URL 로 변환. +// MinIO presigned URL 이 내부 호스트라 브라우저가 직접 못 가므로 이 경로를 쓴다. +export async function fetchMessageAudioObjectUrl( + sessionId: number, + messageId: number, +): Promise { + const { data } = await apiClient.get( + `/api/sessions/${sessionId}/messages/${messageId}/audio`, + { responseType: 'blob' }, + ) + return URL.createObjectURL(data) +} diff --git a/frontend/src/features/interview/lib/media/useMessageAudio.ts b/frontend/src/features/interview/lib/media/useMessageAudio.ts new file mode 100644 index 00000000..d4fd7219 --- /dev/null +++ b/frontend/src/features/interview/lib/media/useMessageAudio.ts @@ -0,0 +1,33 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { fetchMessageAudioObjectUrl } from '../../api/messageApi' + +// 메시지 오디오를 Core 프록시로 1회 받아 object URL 로 캐싱(언마운트 시 revoke). +// load() 는 처음 호출 시 fetch, 이후 캐시 반환. id 가 없으면 비활성. +export function useMessageAudio(sessionId?: number, messageId?: number) { + const [url, setUrl] = useState() + const loadingRef = useRef(false) + + const load = useCallback(async (): Promise => { + if (url) return url + if (loadingRef.current || sessionId == null || messageId == null) return undefined + loadingRef.current = true + try { + const u = await fetchMessageAudioObjectUrl(sessionId, messageId) + setUrl(u) + return u + } catch { + return undefined + } finally { + loadingRef.current = false + } + }, [url, sessionId, messageId]) + + useEffect( + () => () => { + if (url) URL.revokeObjectURL(url) + }, + [url], + ) + + return { url, load } +} diff --git a/frontend/src/features/interview/ui/live/AnswerBubble.tsx b/frontend/src/features/interview/ui/live/AnswerBubble.tsx index 407ecb1f..207f59c2 100644 --- a/frontend/src/features/interview/ui/live/AnswerBubble.tsx +++ b/frontend/src/features/interview/ui/live/AnswerBubble.tsx @@ -1,9 +1,13 @@ import { isTranscribing } from '@/domain/session' import type { Message } from '@/domain/session' +import { useMessageAudio } from '../../lib/media/useMessageAudio' export function AnswerBubble({ message }: { message: Message }) { const transcribing = isTranscribing(message) const failed = transcribing && message.status === 'FAILED' + const hasVoice = Boolean(message.audioFilePath) + + const { url, load } = useMessageAudio(message.sessionId, message.id) return (
@@ -23,9 +27,18 @@ export function AnswerBubble({ message }: { message: Message }) { )}
- {message.audioFileUrl && ( -