diff --git a/README.md b/README.md index 71568ac..65fd132 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,7 @@ const client = createGenerationClient({ - `z-image-turbo` - `qwen-image-edit` - `qwen-audio-3.1-tts-flash` +- `scene3` - `gemini-3.1-flash-image-preview` - `kling-text-to-video` - `kling-image-to-video` @@ -412,6 +413,7 @@ Each TTS request accepts exactly one non-empty text block and returns one URL au | Use a default voice, including a delegated choice expressed only as any, random, suitable, or natural | `qwen-audio-3.1-tts-flash` with a short generic `voice_prompt`, such as a natural, neutral narrator | - Qwen: `voice_prompt` design OR one-reference clone via `qwen-audio-3.1-tts-flash`; requires at least 15 Unicode code points. +- Scene 3: built-in voice or one-reference clone, with inline `[bracketed]` delivery cues; use `scene3` when the user or an external policy names it. - Conflict: reference + redesign requires user choice before generation. - Dependency: clone prior generated audio. - Short text: input under 15 Unicode code points is rejected; ask the user to lengthen it. @@ -436,6 +438,22 @@ await client.generate({ TTS reference audio only supports HTTP(S) URLs. Qwen always uses the single text block as the speech input. +`scene3` takes exactly one non-empty text block, where inline `[bracketed]` cues such as `[sighs]` or `[laughs]` mark where a sound or a change of delivery lands. It also takes at most one HTTP(S) reference URL for a clone, and `meta.text` on that audio block can carry the transcript of the recording. + +```ts +await client.generate({ + model: "scene3", + content: [ + { type: "text", text: "[sighs] We missed the last train again. [laughs] Well, at least the night is warm." }, + { + type: "audio", + source: { type: "url", url: "https://example.com/reference.mp3" }, + meta: { text: "参考音频里实际说的话。" }, + }, + ], +}); +``` + ## Video generation ```ts diff --git a/models/scene3.yaml b/models/scene3.yaml new file mode 100644 index 0000000..5eccae9 --- /dev/null +++ b/models/scene3.yaml @@ -0,0 +1,49 @@ +schema: neta.generation.model.v1 +model: scene3 +title: Scene 3 +category: audio +description: 'Modes: built-in voice, or one-reference clone. Direction: inline [bracketed] cues such as [sighs] or [laughs] in the text mark where a sound or a change of delivery lands. Dependency: clone prior generated audio.' +adapter: + type: openai.audioSpeech +content: + input: + - type: text + required: true + min: 1 + max: 1 + description: Exactly one non-empty text block to speak; inline [bracketed] cues direct the delivery. + - type: audio + required: false + max: 1 + sources: + - url + description: 'Clone: one reference recording URL. meta.text: optional transcript of that recording, transcribed downstream when omitted. Dependency: use prior generated audio.' +examples: + - title: Built-in voice with directions + request: + model: scene3 + content: + - type: text + text: '[sighs] We missed the last train again. [laughs] Well, at least the night is warm.' + - title: Reference clone + request: + model: scene3 + content: + - type: text + text: 使用参考音频的声音朗读这段文本。 + - type: audio + source: + type: url + url: https://example.com/reference.mp3 + - title: Reference clone with its transcript + request: + model: scene3 + content: + - type: text + text: 使用参考音频的声音朗读这段文本。 + - type: audio + source: + type: url + url: https://example.com/reference.mp3 + meta: + text: 参考音频里实际说的话。 diff --git a/src/adapters/audio-speech.ts b/src/adapters/audio-speech.ts index 99d65b9..0374244 100644 --- a/src/adapters/audio-speech.ts +++ b/src/adapters/audio-speech.ts @@ -9,6 +9,7 @@ import type { const REQUEST_TIMEOUT_MS = 210_000; const QWEN_MODEL = "qwen-audio-3.1-tts-flash"; +const SCENE3_MODEL = "scene3"; type TextBlock = Extract; type AudioBlock = Extract; @@ -95,12 +96,32 @@ function validateQwen(input: ResolvedGenerationRequest, text: TextBlock, audio: } } +function validateScene3(input: ResolvedGenerationRequest, text: TextBlock, audio: AudioBlock[]): void { + if (audio.length > 1) { + throw new GenerationValidationError(`${input.declaration.model} supports at most one reference audio`); + } + validateMetaKeys("request.metadata", input.request.metadata, new Set()); + validateMetaKeys("request.meta", input.request.meta, new Set()); + validateMetaKeys("text content meta", text.meta, new Set()); + for (const [index, block] of audio.entries()) { + validateMetaKeys("audio content meta", block.meta, new Set(["text"])); + const transcript = block.meta?.text; + if (transcript !== undefined && (typeof transcript !== "string" || !transcript.trim())) { + throw new GenerationValidationError(`audio content block ${index} meta.text must be a non-empty string`); + } + } +} + function validateAudioSpeechRequest(input: ResolvedGenerationRequest): void { const { text, audio } = validateCommonContent(input); if (input.declaration.model === QWEN_MODEL) { validateQwen(input, text, audio); return; } + if (input.declaration.model === SCENE3_MODEL) { + validateScene3(input, text, audio); + return; + } throw new GenerationValidationError(`Unsupported audio speech model: ${input.declaration.model}`); } @@ -108,10 +129,28 @@ function requestIdFromHeaders(headers: Headers): string | undefined { return headers.get("x-request-id")?.trim() || headers.get("x-oneapi-request-id")?.trim() || undefined; } +function buildScene3Payload(text: TextBlock, audio: AudioBlock[]): Record { + const payload: Record = { model: SCENE3_MODEL, input: text.text }; + const reference = audio[0]; + if (!reference) return payload; + if (reference.source.type !== "url") throw new GenerationValidationError("Audio speech input was not validated"); + const transcript = reference.meta?.text; + payload.metadata = { + references: [ + { + url: reference.source.url.trim(), + ...(typeof transcript === "string" ? { text: transcript.trim() } : {}), + }, + ], + }; + return payload; +} + function buildPayload(input: ResolvedGenerationRequest): Record { const text = textBlocks(input)[0]; if (!text) throw new GenerationValidationError("Audio speech input was not validated"); const audio = audioBlocks(input); + if (input.declaration.model === SCENE3_MODEL) return buildScene3Payload(text, audio); const payload: Record = { model: input.declaration.model, input: text.text, diff --git a/src/builtins.ts b/src/builtins.ts index ea719a4..c8b0add 100644 --- a/src/builtins.ts +++ b/src/builtins.ts @@ -764,6 +764,72 @@ const audioSpeechModels = [ }, ], }, + { + schema: MODEL_SCHEMA, + model: "scene3", + title: "Scene 3", + category: "audio", + description: + "Modes: built-in voice, or one-reference clone. Direction: inline [bracketed] cues such as [sighs] or [laughs] in the text mark where a sound or a change of delivery lands. Dependency: clone prior generated audio.", + adapter: { type: "openai.audioSpeech" }, + content: { + input: [ + { + type: "text", + required: true, + min: 1, + max: 1, + description: "Exactly one non-empty text block to speak; inline [bracketed] cues direct the delivery.", + }, + { + type: "audio", + required: false, + max: 1, + sources: ["url"], + description: + "Clone: one reference recording URL. meta.text: optional transcript of that recording, transcribed downstream when omitted. Dependency: use prior generated audio.", + }, + ], + }, + examples: [ + { + title: "Built-in voice with directions", + request: { + model: "scene3", + content: [ + { + type: "text", + text: "[sighs] We missed the last train again. [laughs] Well, at least the night is warm.", + }, + ], + }, + }, + { + title: "Reference clone", + request: { + model: "scene3", + content: [ + { type: "text", text: "使用参考音频的声音朗读这段文本。" }, + { type: "audio", source: { type: "url", url: "https://example.com/reference.mp3" } }, + ], + }, + }, + { + title: "Reference clone with its transcript", + request: { + model: "scene3", + content: [ + { type: "text", text: "使用参考音频的声音朗读这段文本。" }, + { + type: "audio", + source: { type: "url", url: "https://example.com/reference.mp3" }, + meta: { text: "参考音频里实际说的话。" }, + }, + ], + }, + }, + ], + }, ] satisfies GenerationModelDeclaration[]; const meshyTaskParameters = { diff --git a/test/adapters/audio-speech.test.ts b/test/adapters/audio-speech.test.ts index 8d5d93d..cab14d4 100644 --- a/test/adapters/audio-speech.test.ts +++ b/test/adapters/audio-speech.test.ts @@ -301,6 +301,133 @@ describe("openai.audioSpeech adapter validation", () => { }); }); +describe("openai.audioSpeech scene3 requests", () => { + it("sends a built-in voice request with the text exactly as given", async () => { + const { client, calls } = recordingClient(); + const input = " [sighs] We missed the last train again. [laughs] Well, at least the night is warm.\n"; + + await client.generate({ model: "scene3", content: [{ type: "text", text: input }] }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.url).toBe("https://router.neta.art/v1/audio/speech"); + expect(requestBody(calls[0])).toEqual({ model: "scene3", input }); + }); + + it("sends one clone reference as a trimmed URL without a transcript", async () => { + const { client, calls } = recordingClient(); + + await client.generate({ + model: "scene3", + content: [{ type: "text", text: "使用参考音频的声音朗读这段文本。" }, audio(` ${REFERENCE_URL}\n`)], + }); + + expect(requestBody(calls[0])).toEqual({ + model: "scene3", + input: "使用参考音频的声音朗读这段文本。", + metadata: { references: [{ url: REFERENCE_URL }] }, + }); + }); + + it("sends one clone reference with its trimmed transcript", async () => { + const { client, calls } = recordingClient(); + + await client.generate({ + model: "scene3", + content: [ + { type: "text", text: "使用参考音频的声音朗读这段文本。" }, + audio(` ${REFERENCE_URL}\n`, { text: " 参考音频里实际说的话。\n" }), + ], + }); + + expect(requestBody(calls[0])).toEqual({ + model: "scene3", + input: "使用参考音频的声音朗读这段文本。", + metadata: { references: [{ url: REFERENCE_URL, text: "参考音频里实际说的话。" }] }, + }); + }); +}); + +describe("openai.audioSpeech scene3 validation", () => { + function scene3Request(content: GenerationContentBlock[], overrides: Partial = {}): GenerateRequest { + return { model: "scene3", content, ...overrides }; + } + + it.each<{ name: string; request: GenerateRequest; message: string }>([ + { name: "zero text blocks", request: scene3Request([]), message: "Missing required text content block" }, + { + name: "two text blocks", + request: scene3Request([ + { type: "text", text: "一" }, + { type: "text", text: "二" }, + ]), + message: "Expected at most 1 text content block(s)", + }, + { + name: "blank text", + request: scene3Request([{ type: "text", text: " \n " }]), + message: "scene3 requires exactly one non-empty text content block as input", + }, + { + name: "two audio blocks", + request: scene3Request([{ type: "text", text: "文本" }, audio(), audio(SECOND_REFERENCE_URL)]), + message: "Expected at most 1 audio content block(s)", + }, + { + name: "base64 audio source", + request: scene3Request([ + { type: "text", text: "文本" }, + { type: "audio", source: { type: "base64", mediaType: "audio/mpeg", data: "abc" } }, + ]), + message: "audio source is not supported by scene3: base64", + }, + { + name: "ftp reference URL", + request: scene3Request([{ type: "text", text: "文本" }, audio("ftp://example.com/reference.mp3")]), + message: "audio content block 0 URL must be a valid HTTP(S) URL", + }, + { + name: "request meta voice_prompt", + request: scene3Request([{ type: "text", text: "文本" }], { meta: { voice_prompt: "沉稳清晰的男性播音员声音" } }), + message: "Unknown request.meta field: voice_prompt", + }, + { + name: "audio meta weight", + request: scene3Request([{ type: "text", text: "文本" }, audio(REFERENCE_URL, { weight: 1 })]), + message: "Unknown audio content meta field: weight", + }, + { + name: "empty audio meta.text", + request: scene3Request([{ type: "text", text: "文本" }, audio(REFERENCE_URL, { text: "" })]), + message: "audio content block 0 meta.text must be a non-empty string", + }, + { + name: "numeric audio meta.text", + request: scene3Request([{ type: "text", text: "文本" }, audio(REFERENCE_URL, { text: 5 })]), + message: "audio content block 0 meta.text must be a non-empty string", + }, + ])("rejects $name before fetch", async ({ request, message }) => { + const { client, calls } = recordingClient(); + const error = await client.generate(request).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(GenerationValidationError); + expect((error as Error).message).toBe(message); + expect(calls).toHaveLength(0); + }); + + it("enforces scene3 reference limits inside the adapter hook when a declaration is overridden", () => { + const declaration = getBuiltinGenerationModel("scene3"); + if (!declaration) throw new Error("scene3 declaration is unavailable"); + const audioSpec = declaration.content.input.find((spec) => spec.type === "audio"); + if (!audioSpec) throw new Error("scene3 audio spec is unavailable"); + audioSpec.max = 2; + const client = createGenerationClient({ models: [declaration], includeBuiltinModels: false, apiKey: "key" }); + + expect(() => + client.validate(scene3Request([{ type: "text", text: "文本" }, audio(), audio(SECOND_REFERENCE_URL)])), + ).toThrow("scene3 supports at most one reference audio"); + }); +}); + describe("openai.audioSpeech adapter success responses", () => { it("captures request ID headers in result metadata with the documented priority", async () => { const fetchMock = async () => diff --git a/test/config.test.ts b/test/config.test.ts index 285e8d1..0721535 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -56,7 +56,7 @@ describe("config", () => { "seedance-2-5", "video-upscale-native", ], - audio: ["song2", "suno_music_chirp_fenix"], + audio: ["scene3", "song2", "suno_music_chirp_fenix"], "3d": ["meshy"], decision: ["jev-1.13.0", "jev-latest", "openai/gpt-6-luna-decisions"], } as const;