Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions backend/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -1801,6 +1801,66 @@
}
}
},
"/api/sessions/{sessionId}/messages/{messageId}/audio" : {
"get" : {
"tags" : [ "Session Messages" ],
"summary" : "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)",
"description" : "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다.",
"operationId" : "streamMessageAudio",
"parameters" : [ {
"name" : "sessionId",
"in" : "path",
"required" : true,
"schema" : {
"type" : "integer",
"format" : "int64"
}
}, {
"name" : "messageId",
"in" : "path",
"required" : true,
"schema" : {
"type" : "integer",
"format" : "int64"
}
} ],
"responses" : {
"200" : {
"description" : "오디오 바이트",
"content" : {
"*/*" : {
"schema" : {
"type" : "string",
"format" : "binary"
}
}
}
},
"401" : {
"description" : "인증 실패",
"content" : {
"*/*" : {
"schema" : {
"type" : "string",
"format" : "binary"
}
}
}
},
"404" : {
"description" : "세션/메시지/오디오 없음",
"content" : {
"*/*" : {
"schema" : {
"type" : "string",
"format" : "binary"
}
}
}
}
}
}
},
"/api/sessions/{sessionId}/feedback" : {
"get" : {
"tags" : [ "Session Feedback" ],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,32 @@ private String presign(String key) {
}
}

// 오디오 스트리밍 프록시. MinIO presigned URL 이 내부 호스트라 브라우저가 못 가므로,
// Core 가 인증을 거쳐 바이트를 그대로 흘려준다. 질문=TTS, 답변=음성 원본.
public AudioStream streamAudio(Long userId, Long sessionId, Long messageId) {
ownedSession(userId, sessionId);
InterviewMessage m = messageRepository.findById(messageId)
.filter(x -> x.getSession().getId().equals(sessionId))
.orElseThrow(() -> new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND));
String key = m.getRole() == MessageRole.INTERVIEWER ? m.getTtsAudioPath() : m.getAudioFilePath();
if (key == null || key.isBlank()) {
throw new DomainException(ApiErrorCode.VOICE_MESSAGE_NOT_FOUND);
}
return new AudioStream(storage.get(key), contentTypeForKey(key));
}

public record AudioStream(java.io.InputStream stream, String contentType) {}

private static String contentTypeForKey(String key) {
String k = key.toLowerCase();
if (k.endsWith(".mp3")) return "audio/mpeg";
if (k.endsWith(".wav")) return "audio/wav";
if (k.endsWith(".ogg")) return "audio/ogg";
if (k.endsWith(".m4a") || k.endsWith(".mp4")) return "audio/mp4";
if (k.endsWith(".webm")) return "audio/webm";
return "application/octet-stream";
}

@Transactional
public MessageResult submitAnswer(Long userId, Long sessionId, String content, String idempotencyKey) {
InterviewSession session = ownedSession(userId, sessionId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,13 +116,24 @@ private void validate(VoiceAnswerUploadCommand cmd) {
if (cmd.size() > MAX_BYTES) {
throw new DomainException(ApiErrorCode.VOICE_FILE_TOO_LARGE);
}
if (cmd.contentType() == null || !ALLOWED_CONTENT_TYPES.contains(cmd.contentType().trim().toLowerCase())) {
if (baseContentType(cmd.contentType()) == null) {
throw new DomainException(ApiErrorCode.VOICE_INVALID_CONTENT_TYPE);
}
}

// 브라우저 MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙인다.
// 파라미터를 떼고 base MIME 만으로 허용 여부를 판단한다. 허용 외면 null.
private static String baseContentType(String contentType) {
if (contentType == null) {
return null;
}
String base = contentType.split(";", 2)[0].trim().toLowerCase();
return ALLOWED_CONTENT_TYPES.contains(base) ? base : null;
}

private static String buildKey(Long sessionId, Long messageId, String contentType) {
String ext = switch (contentType.trim().toLowerCase()) {
String base = baseContentType(contentType);
String ext = switch (base == null ? "" : base) {
case "audio/webm" -> "webm";
case "audio/ogg" -> "ogg";
case "audio/mpeg" -> "mp3";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,21 @@
import com.stackup.stackup.session.application.InterviewMessageService;
import com.stackup.stackup.session.presentation.dto.MessageResponse;
import com.stackup.stackup.session.presentation.dto.MessageSubmitRequest;
import com.stackup.stackup.session.application.InterviewMessageService.AudioStream;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.Valid;
import java.time.Duration;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
import org.springframework.http.CacheControl;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
Expand Down Expand Up @@ -70,4 +77,28 @@ public MessageResponse submit(
messageService.submitAnswer(principal.userId(), sessionId, request.content(), idempotencyKey)
);
}

@Operation(
operationId = "streamMessageAudio",
summary = "메시지 오디오 스트리밍 (질문 TTS / 음성 답변)",
description = "MinIO presigned URL 이 내부 호스트라 브라우저가 직접 접근 불가하므로 "
+ "Core 가 인증을 거쳐 오디오 바이트를 스트리밍한다."
)
@ApiResponses({
@ApiResponse(responseCode = "200", description = "오디오 바이트"),
@ApiResponse(responseCode = "401", description = "인증 실패"),
@ApiResponse(responseCode = "404", description = "세션/메시지/오디오 없음")
})
@GetMapping("/{messageId}/audio")
public ResponseEntity<Resource> audio(
@AuthenticationPrincipal UserPrincipal principal,
@PathVariable Long sessionId,
@PathVariable Long messageId
) {
AudioStream audio = messageService.streamAudio(principal.userId(), sessionId, messageId);
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(audio.contentType()))
.cacheControl(CacheControl.maxAge(Duration.ofHours(1)).cachePrivate())
.body(new InputStreamResource(audio.stream()));
}
}
21 changes: 19 additions & 2 deletions frontend/src/features/interview/api/messageApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,13 @@ export async function submitVoiceAnswer(
audio: Blob,
idempotencyKey?: string,
): Promise<S['MessageResponse']> {
// MediaRecorder 는 "audio/webm;codecs=opus" 처럼 코덱 파라미터를 붙이는데
// 일부 검증은 base MIME 만 허용하므로 파라미터를 떼고 업로드한다.
const baseType = (audio.type || 'audio/webm').split(';')[0]
const clean = audio.type === baseType ? audio : new Blob([audio], { type: baseType })
const ext = baseType.includes('ogg') ? 'ogg' : 'webm'
const form = new FormData()
const ext = audio.type.includes('ogg') ? 'ogg' : 'webm'
form.append('audio', audio, `answer.${ext}`)
form.append('audio', clean, `answer.${ext}`)
const { data } = await apiClient.post<S['MessageResponse']>(
`/api/sessions/${sessionId}/messages/voice`,
form,
Expand All @@ -29,3 +33,16 @@ export async function submitVoiceAnswer(
)
return data
}

// 오디오 바이트를 Core 프록시로 받아 object URL 로 변환.
// MinIO presigned URL 이 내부 호스트라 브라우저가 직접 못 가므로 이 경로를 쓴다.
export async function fetchMessageAudioObjectUrl(
sessionId: number,
messageId: number,
): Promise<string> {
const { data } = await apiClient.get<Blob>(
`/api/sessions/${sessionId}/messages/${messageId}/audio`,
{ responseType: 'blob' },
)
return URL.createObjectURL(data)
}
33 changes: 33 additions & 0 deletions frontend/src/features/interview/lib/media/useMessageAudio.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { fetchMessageAudioObjectUrl } from '../../api/messageApi'

// 메시지 오디오를 Core 프록시로 1회 받아 object URL 로 캐싱(언마운트 시 revoke).
// load() 는 처음 호출 시 fetch, 이후 캐시 반환. id 가 없으면 비활성.
export function useMessageAudio(sessionId?: number, messageId?: number) {
const [url, setUrl] = useState<string>()
const loadingRef = useRef(false)

const load = useCallback(async (): Promise<string | undefined> => {
if (url) return url
if (loadingRef.current || sessionId == null || messageId == null) return undefined
loadingRef.current = true
try {
const u = await fetchMessageAudioObjectUrl(sessionId, messageId)
setUrl(u)
return u
} catch {
return undefined
} finally {
loadingRef.current = false
}
}, [url, sessionId, messageId])

useEffect(
() => () => {
if (url) URL.revokeObjectURL(url)
},
[url],
)

return { url, load }
}
19 changes: 16 additions & 3 deletions frontend/src/features/interview/ui/live/AnswerBubble.tsx
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
import { isTranscribing } from '@/domain/session'
import type { Message } from '@/domain/session'
import { useMessageAudio } from '../../lib/media/useMessageAudio'

export function AnswerBubble({ message }: { message: Message }) {
const transcribing = isTranscribing(message)
const failed = transcribing && message.status === 'FAILED'
const hasVoice = Boolean(message.audioFilePath)

const { url, load } = useMessageAudio(message.sessionId, message.id)

return (
<div className="flex flex-col items-end gap-1">
Expand All @@ -23,9 +27,18 @@ export function AnswerBubble({ message }: { message: Message }) {
</span>
)}
</div>
{message.audioFileUrl && (
<audio controls src={message.audioFileUrl} className="max-w-[80%]" preload="none" />
)}
{hasVoice &&
(url ? (
<audio controls src={url} autoPlay className="h-9 max-w-[80%]" />
) : (
<button
type="button"
onClick={() => void load()}
className="rounded-pill px-2.5 py-1 text-caption text-fg-muted transition-colors hover:bg-surface hover:text-fg"
>
▶ 내 답변 듣기
</button>
))}
</div>
)
}
81 changes: 47 additions & 34 deletions frontend/src/features/interview/ui/live/QuestionBubble.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { useEffect, useRef, useState } from 'react'
import type { Message } from '@/domain/session'
import { StatusBadge } from '@/shared/ui/StatusBadge'
import { useMessageAudio } from '../../lib/media/useMessageAudio'

const CATEGORY_LABEL: Record<string, string> = {
CS_FUNDAMENTAL: 'CS 기초',
Expand All @@ -17,29 +18,6 @@ function PlayIcon({ playing }: { playing: boolean }) {
)
}

// ttsAudioUrl 이 생기면(최신 질문일 때) 한 번 자동재생 시도.
// 브라우저 자동재생 차단 시 조용히 실패 → 사용자가 버튼으로 재생.
function useTtsPlayer(url: string | undefined, autoPlay: boolean) {
const audioRef = useRef<HTMLAudioElement | null>(null)
const [playing, setPlaying] = useState(false)
const autoPlayedFor = useRef<string | null>(null)

useEffect(() => {
if (!url || !autoPlay || autoPlayedFor.current === url) return
autoPlayedFor.current = url
audioRef.current?.play().catch(() => {})
}, [url, autoPlay])

const toggle = () => {
const el = audioRef.current
if (!el) return
if (el.paused) el.play().catch(() => {})
else el.pause()
}

return { audioRef, playing, setPlaying, toggle }
}

export function QuestionBubble({
message,
autoPlay = false,
Expand All @@ -51,8 +29,41 @@ export function QuestionBubble({
? (CATEGORY_LABEL[message.category] ?? message.category)
: null
const hasMeta = Boolean(categoryLabel || message.targetEvidence)
const ttsUrl = message.ttsStatus === 'SUCCEEDED' ? message.ttsAudioUrl : undefined
const { audioRef, playing, setPlaying, toggle } = useTtsPlayer(ttsUrl, autoPlay)
const ttsReady = message.ttsStatus === 'SUCCEEDED'

const { url, load } = useMessageAudio(message.sessionId, message.id)
const audioRef = useRef<HTMLAudioElement | null>(null)
const [playing, setPlaying] = useState(false)
const wantPlay = useRef(false)
const autoTried = useRef(false)

// 최신 질문이면 오디오를 받아 자동재생 시도(차단 시 조용히 폴백).
useEffect(() => {
if (!autoPlay || !ttsReady || autoTried.current) return
autoTried.current = true
wantPlay.current = true
void load()
}, [autoPlay, ttsReady, load])

// object URL 이 준비되면 재생 요청을 반영.
useEffect(() => {
if (url && wantPlay.current) {
wantPlay.current = false
audioRef.current?.play().catch(() => {})
}
}, [url])

const toggle = async () => {
const el = audioRef.current
if (!url) {
wantPlay.current = true
await load()
return
}
if (!el) return
if (el.paused) el.play().catch(() => {})
else el.pause()
}

return (
<div className="flex justify-start">
Expand All @@ -67,7 +78,7 @@ export function QuestionBubble({
)}
<div className="rounded-lg rounded-tl-sm bg-surface-raised px-4 py-3 text-body text-fg shadow-sm">
<p className="whitespace-pre-wrap">{message.content}</p>
{ttsUrl && (
{ttsReady && (
<div className="mt-2 flex items-center gap-2 border-t border-border pt-2">
<button
type="button"
Expand All @@ -78,14 +89,16 @@ export function QuestionBubble({
<PlayIcon playing={playing} />
{playing ? '일시정지' : '음성 듣기'}
</button>
<audio
ref={audioRef}
src={ttsUrl}
preload="none"
onPlay={() => setPlaying(true)}
onPause={() => setPlaying(false)}
onEnded={() => setPlaying(false)}
/>
{url && (
<audio
ref={audioRef}
src={url}
preload="none"
onPlay={() => setPlaying(true)}
onPause={() => setPlaying(false)}
onEnded={() => setPlaying(false)}
/>
)}
</div>
)}
</div>
Expand Down
Loading