Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ const client = createGenerationClient({
- `z-image-turbo`
- `qwen-image-edit`
- `qwen-audio-3.1-tts-flash`
- `scene3`
- `gemini-3.1-flash-image-preview`
- `kling-text-to-video`
- `kling-image-to-video`
Expand Down Expand Up @@ -412,6 +413,7 @@ Each TTS request accepts exactly one non-empty text block and returns one URL au
| Use a default voice, including a delegated choice expressed only as any, random, suitable, or natural | `qwen-audio-3.1-tts-flash` with a short generic `voice_prompt`, such as a natural, neutral narrator |

- Qwen: `voice_prompt` design OR one-reference clone via `qwen-audio-3.1-tts-flash`; requires at least 15 Unicode code points.
- Scene 3: built-in voice or one-reference clone, with inline `[bracketed]` delivery cues; use `scene3` when the user or an external policy names it.
- Conflict: reference + redesign requires user choice before generation.
- Dependency: clone prior generated audio.
- Short text: input under 15 Unicode code points is rejected; ask the user to lengthen it.
Expand All @@ -436,6 +438,22 @@ await client.generate({

TTS reference audio only supports HTTP(S) URLs. Qwen always uses the single text block as the speech input.

`scene3` takes exactly one non-empty text block, where inline `[bracketed]` cues such as `[sighs]` or `[laughs]` mark where a sound or a change of delivery lands. It also takes at most one HTTP(S) reference URL for a clone, and `meta.text` on that audio block can carry the transcript of the recording.

```ts
await client.generate({
model: "scene3",
content: [
{ type: "text", text: "[sighs] We missed the last train again. [laughs] Well, at least the night is warm." },
{
type: "audio",
source: { type: "url", url: "https://example.com/reference.mp3" },
meta: { text: "参考音频里实际说的话。" },
},
],
});
```

## Video generation

```ts
Expand Down
49 changes: 49 additions & 0 deletions models/scene3.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
schema: neta.generation.model.v1
model: scene3
title: Scene 3
category: audio
description: 'Modes: built-in voice, or one-reference clone. Direction: inline [bracketed] cues such as [sighs] or [laughs] in the text mark where a sound or a change of delivery lands. Dependency: clone prior generated audio.'
adapter:
type: openai.audioSpeech
content:
input:
- type: text
required: true
min: 1
max: 1
description: Exactly one non-empty text block to speak; inline [bracketed] cues direct the delivery.
- type: audio
required: false
max: 1
sources:
- url
description: 'Clone: one reference recording URL. meta.text: optional transcript of that recording, transcribed downstream when omitted. Dependency: use prior generated audio.'
examples:
- title: Built-in voice with directions
request:
model: scene3
content:
- type: text
text: '[sighs] We missed the last train again. [laughs] Well, at least the night is warm.'
- title: Reference clone
request:
model: scene3
content:
- type: text
text: 使用参考音频的声音朗读这段文本。
- type: audio
source:
type: url
url: https://example.com/reference.mp3
- title: Reference clone with its transcript
request:
model: scene3
content:
- type: text
text: 使用参考音频的声音朗读这段文本。
- type: audio
source:
type: url
url: https://example.com/reference.mp3
meta:
text: 参考音频里实际说的话。
39 changes: 39 additions & 0 deletions src/adapters/audio-speech.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type {

const REQUEST_TIMEOUT_MS = 210_000;
const QWEN_MODEL = "qwen-audio-3.1-tts-flash";
const SCENE3_MODEL = "scene3";

type TextBlock = Extract<GenerationContentBlock, { type: "text" }>;
type AudioBlock = Extract<GenerationContentBlock, { type: "audio" }>;
Expand Down Expand Up @@ -95,23 +96,61 @@ function validateQwen(input: ResolvedGenerationRequest, text: TextBlock, audio:
}
}

function validateScene3(input: ResolvedGenerationRequest, text: TextBlock, audio: AudioBlock[]): void {
if (audio.length > 1) {
throw new GenerationValidationError(`${input.declaration.model} supports at most one reference audio`);
}
validateMetaKeys("request.metadata", input.request.metadata, new Set());
validateMetaKeys("request.meta", input.request.meta, new Set());
validateMetaKeys("text content meta", text.meta, new Set());
for (const [index, block] of audio.entries()) {
validateMetaKeys("audio content meta", block.meta, new Set(["text"]));
const transcript = block.meta?.text;
if (transcript !== undefined && (typeof transcript !== "string" || !transcript.trim())) {
throw new GenerationValidationError(`audio content block ${index} meta.text must be a non-empty string`);
}
}
}

function validateAudioSpeechRequest(input: ResolvedGenerationRequest): void {
const { text, audio } = validateCommonContent(input);
if (input.declaration.model === QWEN_MODEL) {
validateQwen(input, text, audio);
return;
}
if (input.declaration.model === SCENE3_MODEL) {
validateScene3(input, text, audio);
return;
}
throw new GenerationValidationError(`Unsupported audio speech model: ${input.declaration.model}`);
}

function requestIdFromHeaders(headers: Headers): string | undefined {
return headers.get("x-request-id")?.trim() || headers.get("x-oneapi-request-id")?.trim() || undefined;
}

function buildScene3Payload(text: TextBlock, audio: AudioBlock[]): Record<string, unknown> {
const payload: Record<string, unknown> = { model: SCENE3_MODEL, input: text.text };
const reference = audio[0];
if (!reference) return payload;
if (reference.source.type !== "url") throw new GenerationValidationError("Audio speech input was not validated");
const transcript = reference.meta?.text;
payload.metadata = {
references: [
{
url: reference.source.url.trim(),
...(typeof transcript === "string" ? { text: transcript.trim() } : {}),
},
],
};
return payload;
}

function buildPayload(input: ResolvedGenerationRequest): Record<string, unknown> {
const text = textBlocks(input)[0];
if (!text) throw new GenerationValidationError("Audio speech input was not validated");
const audio = audioBlocks(input);
if (input.declaration.model === SCENE3_MODEL) return buildScene3Payload(text, audio);
const payload: Record<string, unknown> = {
model: input.declaration.model,
input: text.text,
Expand Down
66 changes: 66 additions & 0 deletions src/builtins.ts
Original file line number Diff line number Diff line change
Expand Up @@ -764,6 +764,72 @@ const audioSpeechModels = [
},
],
},
{
schema: MODEL_SCHEMA,
model: "scene3",
title: "Scene 3",
category: "audio",
description:
"Modes: built-in voice, or one-reference clone. Direction: inline [bracketed] cues such as [sighs] or [laughs] in the text mark where a sound or a change of delivery lands. Dependency: clone prior generated audio.",
adapter: { type: "openai.audioSpeech" },
content: {
input: [
{
type: "text",
required: true,
min: 1,
max: 1,
description: "Exactly one non-empty text block to speak; inline [bracketed] cues direct the delivery.",
},
{
type: "audio",
required: false,
max: 1,
sources: ["url"],
description:
"Clone: one reference recording URL. meta.text: optional transcript of that recording, transcribed downstream when omitted. Dependency: use prior generated audio.",
},
],
},
examples: [
{
title: "Built-in voice with directions",
request: {
model: "scene3",
content: [
{
type: "text",
text: "[sighs] We missed the last train again. [laughs] Well, at least the night is warm.",
},
],
},
},
{
title: "Reference clone",
request: {
model: "scene3",
content: [
{ type: "text", text: "使用参考音频的声音朗读这段文本。" },
{ type: "audio", source: { type: "url", url: "https://example.com/reference.mp3" } },
],
},
},
{
title: "Reference clone with its transcript",
request: {
model: "scene3",
content: [
{ type: "text", text: "使用参考音频的声音朗读这段文本。" },
{
type: "audio",
source: { type: "url", url: "https://example.com/reference.mp3" },
meta: { text: "参考音频里实际说的话。" },
},
],
},
},
],
},
] satisfies GenerationModelDeclaration[];

const meshyTaskParameters = {
Expand Down
127 changes: 127 additions & 0 deletions test/adapters/audio-speech.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,133 @@ describe("openai.audioSpeech adapter validation", () => {
});
});

describe("openai.audioSpeech scene3 requests", () => {
it("sends a built-in voice request with the text exactly as given", async () => {
const { client, calls } = recordingClient();
const input = " [sighs] We missed the last train again. [laughs] Well, at least the night is warm.\n";

await client.generate({ model: "scene3", content: [{ type: "text", text: input }] });

expect(calls).toHaveLength(1);
expect(calls[0]?.url).toBe("https://router.neta.art/v1/audio/speech");
expect(requestBody(calls[0])).toEqual({ model: "scene3", input });
});

it("sends one clone reference as a trimmed URL without a transcript", async () => {
const { client, calls } = recordingClient();

await client.generate({
model: "scene3",
content: [{ type: "text", text: "使用参考音频的声音朗读这段文本。" }, audio(` ${REFERENCE_URL}\n`)],
});

expect(requestBody(calls[0])).toEqual({
model: "scene3",
input: "使用参考音频的声音朗读这段文本。",
metadata: { references: [{ url: REFERENCE_URL }] },
});
});

it("sends one clone reference with its trimmed transcript", async () => {
const { client, calls } = recordingClient();

await client.generate({
model: "scene3",
content: [
{ type: "text", text: "使用参考音频的声音朗读这段文本。" },
audio(` ${REFERENCE_URL}\n`, { text: " 参考音频里实际说的话。\n" }),
],
});

expect(requestBody(calls[0])).toEqual({
model: "scene3",
input: "使用参考音频的声音朗读这段文本。",
metadata: { references: [{ url: REFERENCE_URL, text: "参考音频里实际说的话。" }] },
});
});
});

describe("openai.audioSpeech scene3 validation", () => {
function scene3Request(content: GenerationContentBlock[], overrides: Partial<GenerateRequest> = {}): GenerateRequest {
return { model: "scene3", content, ...overrides };
}

it.each<{ name: string; request: GenerateRequest; message: string }>([
{ name: "zero text blocks", request: scene3Request([]), message: "Missing required text content block" },
{
name: "two text blocks",
request: scene3Request([
{ type: "text", text: "一" },
{ type: "text", text: "二" },
]),
message: "Expected at most 1 text content block(s)",
},
{
name: "blank text",
request: scene3Request([{ type: "text", text: " \n " }]),
message: "scene3 requires exactly one non-empty text content block as input",
},
{
name: "two audio blocks",
request: scene3Request([{ type: "text", text: "文本" }, audio(), audio(SECOND_REFERENCE_URL)]),
message: "Expected at most 1 audio content block(s)",
},
{
name: "base64 audio source",
request: scene3Request([
{ type: "text", text: "文本" },
{ type: "audio", source: { type: "base64", mediaType: "audio/mpeg", data: "abc" } },
]),
message: "audio source is not supported by scene3: base64",
},
{
name: "ftp reference URL",
request: scene3Request([{ type: "text", text: "文本" }, audio("ftp://example.com/reference.mp3")]),
message: "audio content block 0 URL must be a valid HTTP(S) URL",
},
{
name: "request meta voice_prompt",
request: scene3Request([{ type: "text", text: "文本" }], { meta: { voice_prompt: "沉稳清晰的男性播音员声音" } }),
message: "Unknown request.meta field: voice_prompt",
},
{
name: "audio meta weight",
request: scene3Request([{ type: "text", text: "文本" }, audio(REFERENCE_URL, { weight: 1 })]),
message: "Unknown audio content meta field: weight",
},
{
name: "empty audio meta.text",
request: scene3Request([{ type: "text", text: "文本" }, audio(REFERENCE_URL, { text: "" })]),
message: "audio content block 0 meta.text must be a non-empty string",
},
{
name: "numeric audio meta.text",
request: scene3Request([{ type: "text", text: "文本" }, audio(REFERENCE_URL, { text: 5 })]),
message: "audio content block 0 meta.text must be a non-empty string",
},
])("rejects $name before fetch", async ({ request, message }) => {
const { client, calls } = recordingClient();
const error = await client.generate(request).catch((caught: unknown) => caught);

expect(error).toBeInstanceOf(GenerationValidationError);
expect((error as Error).message).toBe(message);
expect(calls).toHaveLength(0);
});

it("enforces scene3 reference limits inside the adapter hook when a declaration is overridden", () => {
const declaration = getBuiltinGenerationModel("scene3");
if (!declaration) throw new Error("scene3 declaration is unavailable");
const audioSpec = declaration.content.input.find((spec) => spec.type === "audio");
if (!audioSpec) throw new Error("scene3 audio spec is unavailable");
audioSpec.max = 2;
const client = createGenerationClient({ models: [declaration], includeBuiltinModels: false, apiKey: "key" });

expect(() =>
client.validate(scene3Request([{ type: "text", text: "文本" }, audio(), audio(SECOND_REFERENCE_URL)])),
).toThrow("scene3 supports at most one reference audio");
});
});

describe("openai.audioSpeech adapter success responses", () => {
it("captures request ID headers in result metadata with the documented priority", async () => {
const fetchMock = async () =>
Expand Down
2 changes: 1 addition & 1 deletion test/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ describe("config", () => {
"seedance-2-5",
"video-upscale-native",
],
audio: ["song2", "suno_music_chirp_fenix"],
audio: ["scene3", "song2", "suno_music_chirp_fenix"],
"3d": ["meshy"],
decision: ["jev-1.13.0", "jev-latest", "openai/gpt-6-luna-decisions"],
} as const;
Expand Down