From af435d4626c222c31b55248b06196a240f3f7338 Mon Sep 17 00:00:00 2001 From: Eugene Ware Date: Wed, 5 Aug 2026 21:01:00 +1000 Subject: [PATCH 1/4] feat(media-use): add ACE-Step music provider --- skills/media-use/SKILL.md | 2 +- skills/media-use/audio/references/bgm.md | 6 +- .../audio/references/requirements.md | 2 + .../audio/scripts/acestep-recipe.mjs | 26 ++++ skills/media-use/audio/scripts/audio.mjs | 12 +- skills/media-use/audio/scripts/lib/bgm.mjs | 30 +++- skills/media-use/references/audio.md | 2 +- skills/media-use/references/resolve.md | 22 ++- .../media-use/references/setup-providers.md | 3 +- .../scripts/lib/acestep-provider.mjs | 140 ++++++++++++++++++ .../scripts/lib/acestep-provider.test.mjs | 79 ++++++++++ skills/media-use/scripts/lib/freeze.mjs | 4 +- skills/media-use/scripts/lib/registry.mjs | 6 +- .../media-use/scripts/lib/registry.test.mjs | 10 +- skills/media-use/scripts/resolve.mjs | 15 +- 15 files changed, 341 insertions(+), 18 deletions(-) create mode 100644 skills/media-use/audio/scripts/acestep-recipe.mjs create mode 100644 skills/media-use/scripts/lib/acestep-provider.mjs create mode 100644 skills/media-use/scripts/lib/acestep-provider.test.mjs diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index 5655ccb439..26780feeaa 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -19,7 +19,7 @@ Returns one line: `resolved → (, )`. All search no | Type | One-line intent | | ------- | ----------------------------------------------------------------------------------- | -| `bgm` | background music (HeyGen catalog, 10k+ tracks) | +| `bgm` | background music (HeyGen catalog or configured ACE-Step generation API) | | `sfx` | sound effects (bundled 19-file library + catalog) | | `image` | photos, backgrounds (HeyGen asset search, 75k+ vectors) | | `icon` | icons, symbols (transparent) | diff --git a/skills/media-use/audio/references/bgm.md b/skills/media-use/audio/references/bgm.md index 9c43da908a..5afce2758b 100644 --- a/skills/media-use/audio/references/bgm.md +++ b/skills/media-use/audio/references/bgm.md @@ -3,15 +3,17 @@ One music bed per composition, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/bgm.mjs`). Two routes, chosen by the engine's one switch — whether a HeyGen credential is present: - **HeyGen retrieval — the default when credentialed.** Search HeyGen's music catalog by mood, download the top track. No generation; same `~/.heygen` / `$HEYGEN_API_KEY` credential as TTS. -- **Local generation (Lyria → MusicGen) — the fallback when there is no credential** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx hyperframes bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly. +- **ACE-Step remote generation — the preferred generation path when `bgm.provider` or the user provider configuration selects `acestep`.** It submits the native asynchronous API, waits in the existing detached BGM flow, and freezes the completed MP3. +- **Local generation (Lyria → MusicGen) — the fallback when there is no credential or configured ACE-Step provider** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx hyperframes bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly. > **Run the Preflight first — no credential is not a green light to silently generate locally.** Before generating, complete the sign-in **Preflight** (see `../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen's music library, or continue offline with local generation). This applies to a one-off "generate a BGM" request just as much as inside a full workflow. ## Driving it from the request -`audio_request.json` → `bgm: { mode?, query?, prompt? }`: +`audio_request.json` → `bgm: { mode?, provider?, query?, prompt? }`: - **`mode`** — `retrieve | generate | none`. Omit for **auto** (retrieve when credentialed, else generate). An **explicit** `retrieve` is strict: no credential ⇒ skip, never a detached generate (so a caller with no `wait-bgm` step, e.g. product-launch, can't get a pending job it won't await). +- **`provider`** — `acestep` forces the configured ACE-Step endpoint for generation. Omit to use the user provider default and then the existing HeyGen/Lyria/MusicGen cascade. - **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`). - **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference. diff --git a/skills/media-use/audio/references/requirements.md b/skills/media-use/audio/references/requirements.md index 8c5793464c..90d70355aa 100644 --- a/skills/media-use/audio/references/requirements.md +++ b/skills/media-use/audio/references/requirements.md @@ -7,6 +7,7 @@ Run `npx hyperframes auth status` to see what's configured and which engines a w | Provider | Resolution order (first non-empty wins) | Local deps when used | | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | | **HeyGen** (TTS + BGM/SFX retrieval) | `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `hyperframes auth login`) | none (REST) | +| **ACE-Step** (BGM generation) | `~/.media/providers.json` `bgm.acestep.base_url`; optional key named by `api_key_env` (default `$ACESTEP_API_KEY`) | none (native REST API) | | **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` | | **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` | | **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` | @@ -22,6 +23,7 @@ Each command downloads its own model on first run and caches it under `~/.cache/ - **TTS (ElevenLabs)** — same as HeyGen: API key + `ffmpeg`. - **TTS (Kokoro)** — Kokoro-82M (~311 MB) + voices (~27 MB) in `tts/`. Requires Python 3.8+ with `kokoro-onnx` and `soundfile` (`pip install kokoro-onnx soundfile`). Non-English text also needs `espeak-ng` system-wide. - **BGM (Lyria)** — needs `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `pip install google-genai`. No local model cache. +- **BGM (ACE-Step)** — remote native API; no local dependency. Configure `bgm.default` as `acestep`, set `bgm.acestep.base_url`, and optionally set the named key environment variable. - **BGM (MusicGen)** — `pip install transformers torch soundfile`. `facebook/musicgen-small` (~300 MB) cached under `~/.cache/huggingface/` on first run. - **Transcribe** — Whisper model size depending on choice (75 MB – 3.1 GB) in `whisper/`, downloaded from HuggingFace on first use. `whisper.cpp` itself is NOT bundled: the CLI resolves it from PATH, installs via Homebrew (macOS), or builds it from source with git+cmake on first use (`$HYPERFRAMES_WHISPER_PATH` overrides). - **Remove-background** — `u2net_human_seg` (~168 MB ONNX) in `background-removal/models/`. Peak inference RAM ~1.5 GB. diff --git a/skills/media-use/audio/scripts/acestep-recipe.mjs b/skills/media-use/audio/scripts/acestep-recipe.mjs new file mode 100644 index 0000000000..48bc0e92c8 --- /dev/null +++ b/skills/media-use/audio/scripts/acestep-recipe.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +import { resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { generateWithAceStep } from "../../scripts/lib/acestep-provider.mjs"; +import { freezeUrl } from "../../scripts/lib/freeze.mjs"; + +const { values } = parseArgs({ + options: { + output: { type: "string" }, + duration: { type: "string", default: "30" }, + prompt: { type: "string" }, + }, + strict: true, +}); + +if (!values.output || !values.prompt) { + console.error( + "usage: acestep-recipe.mjs --output --duration --prompt ", + ); + process.exit(2); +} + +const result = await generateWithAceStep(values.prompt, { duration: Number(values.duration) }); +if (!result?.url) throw new Error("ACE-Step is not configured"); +await freezeUrl(result.url, resolve(values.output), { headers: result.downloadHeaders }); +console.log(`ACE-Step wrote ${values.output}`); diff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjs index bc005eeff9..bcd68e1a62 100644 --- a/skills/media-use/audio/scripts/audio.mjs +++ b/skills/media-use/audio/scripts/audio.mjs @@ -10,7 +10,7 @@ // (credential present, NOT the CLI). This mirrors the table in ../SKILL.md: // // TTS : HeyGen REST → ElevenLabs → Kokoro (CLI) -// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate +// BGM : HeyGen retrieve → configured ACE-Step → Lyria/MusicGen generate // SFX : HeyGen retrieve → (no credential) bundled 19-file library // // ── audio_request.json (input) ──────────────────────────────────────────────── @@ -52,6 +52,7 @@ import { withWordIds, } from "./lib/tts.mjs"; import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs"; +import { configuredBgmProvider } from "../../scripts/lib/acestep-provider.mjs"; import { resolveSfx } from "./lib/sfx.mjs"; import { mapWithConcurrency } from "./lib/concurrency.mjs"; @@ -83,6 +84,7 @@ const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request. const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json"))); const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx"))); const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py"))); +const acestepRecipe = resolve(flag("acestep-recipe", join(HERE, "acestep-recipe.mjs"))); const onlyArg = flag("only", "tts,bgm,sfx"); const only = new Set( onlyArg @@ -196,7 +198,11 @@ if (only.has("bgm")) { // a pending job it can't await). Only the UNSET/auto default picks generate // when HeyGen is absent. const explicitMode = bgmModeOverride || request.bgm?.mode || null; - let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate"); + const requestedBgmProvider = request.bgm?.provider || configuredBgmProvider(); + let mode = noBgm + ? "none" + : explicitMode || + (requestedBgmProvider === "acestep" ? "generate" : heygenOK ? "retrieve" : "generate"); if (mode === "retrieve" && !heygenOK) { anomalies.push( "bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)", @@ -232,6 +238,8 @@ if (only.has("bgm")) { durationS: totalDuration || 30, hyperframesDir, lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null, + acestepRecipe: existsSync(acestepRecipe) ? acestepRecipe : null, + provider: requestedBgmProvider, seedSeconds, hasVoice, }); diff --git a/skills/media-use/audio/scripts/lib/bgm.mjs b/skills/media-use/audio/scripts/lib/bgm.mjs index 4c91d005f2..89d53b2724 100644 --- a/skills/media-use/audio/scripts/lib/bgm.mjs +++ b/skills/media-use/audio/scripts/lib/bgm.mjs @@ -15,6 +15,7 @@ import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, openSync, closeSync } from "node:fs"; import { join } from "node:path"; import { downloadTo, searchSounds } from "./heygen.mjs"; +import { aceStepConfig, configuredBgmProvider } from "../../../scripts/lib/acestep-provider.mjs"; import { pythonInvocation } from "./python.mjs"; const r3 = (x) => Number(x.toFixed(3)); @@ -115,10 +116,17 @@ export function generateBgmDetached({ durationS, hyperframesDir, lyriaRecipe, + acestepRecipe, + provider, seedSeconds = 28, hasVoice, }) { - const rel = "assets/bgm/track.wav"; + const useAceStep = + (provider || configuredBgmProvider()) === "acestep" && + !!aceStepConfig().baseUrl && + !!acestepRecipe && + existsSync(acestepRecipe); + const rel = useAceStep ? "assets/bgm/track.mp3" : "assets/bgm/track.wav"; const abs = join(hyperframesDir, rel); mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true }); const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`); @@ -126,6 +134,25 @@ export function generateBgmDetached({ const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true }; const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe); + const fd = openSync(log, "w"); + + if (useAceStep) { + const proc = spawn( + process.execPath, + [acestepRecipe, "--output", abs, "--duration", String(targetS), "--prompt", prompt], + { detached: true, stdio: ["ignore", fd, fd] }, + ); + proc.unref(); + closeSync(fd); + return { + ...baseMeta, + mode: "detached-single", + provider: "acestep", + pid: proc.pid, + log, + target_duration_s: r3(targetS), + }; + } // Make a backend runnable: prefer Lyria when configured (install google-genai // on demand), else ensure local MusicGen deps. Installs are synchronous here — @@ -134,7 +161,6 @@ export function generateBgmDetached({ const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE); if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS); - const fd = openSync(log, "w"); if (useLyria) { const { cmd, args } = pythonInvocation([ lyriaRecipe, diff --git a/skills/media-use/references/audio.md b/skills/media-use/references/audio.md index 504d630332..017bfde6d0 100644 --- a/skills/media-use/references/audio.md +++ b/skills/media-use/references/audio.md @@ -11,7 +11,7 @@ node /audio/scripts/audio.mjs --request ./audio_request.json --out ./ - **Request** `{ provider?, lang?, speed?, lines: [{ id, text, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`. - **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`. -- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `node /scripts/resolve.mjs --doctor` before assuming retrieval or TTS will work. +- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. A configured `acestep.remote` endpoint is the standard generated-BGM path; local/provider-specific generators remain explicit alternatives where installed. Run `node /scripts/resolve.mjs --doctor` before assuming retrieval or TTS will work. - If BGM took the generate path (`bgm_pending: true`), run `audio/scripts/wait-bgm.mjs` before final render. Single-shot helpers: `audio/scripts/heygen-tts.mjs` (one voice file). Transcription / background removal / captions use the `hyperframes` CLI (`transcribe`, `remove-background`), see the per-topic guides in `audio/references/` (`tts.md`, `bgm.md`, `sfx.md`, `transcribe.md`, `remove-background.md`, `captions/`). diff --git a/skills/media-use/references/resolve.md b/skills/media-use/references/resolve.md index 61c13cb32e..5b6ba21311 100644 --- a/skills/media-use/references/resolve.md +++ b/skills/media-use/references/resolve.md @@ -10,7 +10,7 @@ Returns one line: `resolved → (, )` | Type | What it finds | Provider / cascade | | ------- | -------------------------------- | ------------------------------------------------------------ | -| `bgm` | Background music | HeyGen audio catalog (10k+ tracks) | +| `bgm` | Background music | HeyGen catalog; optional ACE-Step remote generation | | `sfx` | Sound effects | Bundled 19-file library + HeyGen catalog | | `image` | Photos, backgrounds | HeyGen asset search (75k+ vectors) | | `icon` | Icons, symbols | HeyGen asset search (type=icon) | @@ -65,6 +65,7 @@ node /scripts/resolve.mjs --type lut --intent "teal orange blockbuste | `--for` | Analyze a local image/video and add measured adjust suggestions (`grade` only) | | `--local-only` | Offline: skip every network provider (cache + local only) | | `--provider` | Force one generator (e.g. `codex`, `mflux`, `kokoro`, `heygen`) | +| `--duration` | Generated BGM length in seconds, 10-600 (default 30) | | `--adopt` | Bulk-import existing assets/ into manifest | | `--doctor` | Check local CLI dependencies; no manifest changes | | `--stats` | Print local usage stats from `.media/` and `~/.media`; no manifest changes | @@ -103,6 +104,25 @@ The deterministic floor still runs automatically: an identical (case/whitespace- 4. Search via provider (HeyGen audio catalog, HeyGen asset search), or resolve color locally 5. Freeze file to `.media//`, register in manifest, regenerate `index.md`, auto-promote to `~/.media/` +### ACE-Step remote generation + +`acestep.remote` is the supported self-hosted BGM generator. Configure it in `~/.media/providers.json`: + +```json +{ + "version": 1, + "bgm": { + "default": "acestep", + "acestep": { + "base_url": "https://music.example.ts.net", + "api_key_env": "ACESTEP_API_KEY" + } + } +} +``` + +The key is optional and is read only from the named environment variable. The provider submits the native `/release_task` request, polls `/query_result`, downloads `/v1/audio`, and then follows the same freeze, ledger, and global-cache path as every other provider. Use `--provider acestep --duration 45` to force it for one resolve. `--local-only` always blocks it because a Tailnet service is still a network provider. + Steps 1 and 3 are the **deterministic floor**: they only auto-reuse an exact-normalized match, never a fuzzy one. Semantic reuse ("close enough") is the agent's explicit call via [Reuse before you resolve](#reuse-before-you-resolve) — it never happens automatically. The agent gets back **one line**; candidates, scores, provenance stay on disk. ## Adopt existing projects diff --git a/skills/media-use/references/setup-providers.md b/skills/media-use/references/setup-providers.md index ef5329e726..e62fe7838f 100644 --- a/skills/media-use/references/setup-providers.md +++ b/skills/media-use/references/setup-providers.md @@ -27,7 +27,8 @@ see the ladder and override. | Type | Provider / path | | --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| bgm/sfx | heygen catalog free-usage path | +| bgm | heygen catalog free-usage path; optional self-hosted `acestep.remote` generation configured in `~/.media/providers.json` | +| sfx | heygen catalog free-usage path | | image | heygen search free-usage path; optional local mflux; codex `image_gen` upsell | | voice | heygen tts free-usage path; optional local **Kokoro** (free, on-device) | | icon | heygen asset search free-usage path | diff --git a/skills/media-use/scripts/lib/acestep-provider.mjs b/skills/media-use/scripts/lib/acestep-provider.mjs new file mode 100644 index 0000000000..b28599b805 --- /dev/null +++ b/skills/media-use/scripts/lib/acestep-provider.mjs @@ -0,0 +1,140 @@ +import { existsSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function readUserConfig() { + const path = join(homedir(), ".media", "providers.json"); + if (!existsSync(path)) return {}; + try { + const parsed = JSON.parse(readFileSync(path, "utf8")); + return parsed && typeof parsed === "object" ? parsed : {}; + } catch { + return {}; + } +} + +export function configuredBgmProvider() { + return readUserConfig()?.bgm?.default || process.env.HYPERFRAMES_BGM_PROVIDER || null; +} + +export function aceStepConfig() { + const configured = readUserConfig()?.bgm?.acestep || {}; + const baseUrl = String(process.env.ACESTEP_API_URL || configured.base_url || "").replace( + /\/$/, + "", + ); + const keyName = configured.api_key_env || "ACESTEP_API_KEY"; + return { baseUrl, apiKey: process.env[keyName] || "" }; +} + +function headers(apiKey) { + return { + "content-type": "application/json", + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}), + }; +} + +async function requestJson(url, options, fetchImpl) { + const response = await fetchImpl(url, options); + let body = {}; + try { + body = await response.json(); + } catch { + // The status below is more useful than a JSON parser stack. + } + if (!response.ok) { + const detail = body?.detail || body?.error || `HTTP ${response.status}`; + throw new Error(`ACE-Step request failed: ${detail}`); + } + return body; +} + +function resultObject(value) { + let parsed = value; + if (typeof parsed === "string") { + try { + parsed = JSON.parse(parsed); + } catch { + throw new Error("ACE-Step returned an invalid result payload"); + } + } + if (Array.isArray(parsed)) return parsed[0] || null; + return parsed && typeof parsed === "object" ? parsed : null; +} + +export async function generateWithAceStep(intent, ctx = {}, deps = {}) { + const fetchImpl = deps.fetch || fetch; + const sleepImpl = deps.sleep || sleep; + const now = deps.now || Date.now; + const { baseUrl, apiKey } = deps.config || aceStepConfig(); + if (!baseUrl) return null; + + const duration = Math.max(10, Math.min(600, Number(ctx.duration) || 30)); + const submitted = await requestJson( + `${baseUrl}/release_task`, + { + method: "POST", + headers: headers(apiKey), + body: JSON.stringify({ + prompt: intent, + lyrics: "[Instrumental]", + thinking: true, + use_format: false, + audio_duration: duration, + audio_format: "mp3", + batch_size: 1, + model: "acestep-v15-xl-sft", + inference_steps: 50, + }), + }, + fetchImpl, + ); + const taskId = submitted?.data?.task_id || submitted?.task_id || submitted?.data?.id; + if (!taskId) throw new Error("ACE-Step did not return a task ID"); + + const started = now(); + const timeoutMs = Number(process.env.ACESTEP_POLL_TIMEOUT_MS) || 20 * 60 * 1000; + while (now() - started < timeoutMs) { + const queried = await requestJson( + `${baseUrl}/query_result`, + { + method: "POST", + headers: headers(apiKey), + body: JSON.stringify({ task_id_list: [taskId] }), + }, + fetchImpl, + ); + const row = queried?.data?.[0]; + if (!row) throw new Error(`ACE-Step lost task ${taskId}`); + if (Number(row.status) === 2) + throw new Error(row.error || row.message || "ACE-Step generation failed"); + if (Number(row.status) === 1) { + const result = resultObject(row.result); + if (!result?.file) throw new Error("ACE-Step completed without an audio file"); + const audioUrl = new URL(String(result.file), `${baseUrl}/`).toString(); + const metadata = result.metas && typeof result.metas === "object" ? result.metas : {}; + return { + url: audioUrl, + downloadHeaders: apiKey ? { authorization: `Bearer ${apiKey}` } : undefined, + ext: ".mp3", + source: "generate", + metadata: { + description: intent, + duration: Number(metadata.duration) || duration, + provider: "acestep.remote", + provenance: { + endpoint: new URL(baseUrl).origin, + task_id: String(taskId), + dit_model: result.dit_model || "acestep-v15-xl-sft", + lm_model: result.lm_model || null, + seed: result.seed_value || null, + }, + }, + }; + } + await sleepImpl(now() - started < 30_000 ? 2000 : 5000); + } + throw new Error(`ACE-Step task ${taskId} exceeded the generation timeout`); +} diff --git a/skills/media-use/scripts/lib/acestep-provider.test.mjs b/skills/media-use/scripts/lib/acestep-provider.test.mjs new file mode 100644 index 0000000000..7815714f5b --- /dev/null +++ b/skills/media-use/scripts/lib/acestep-provider.test.mjs @@ -0,0 +1,79 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { generateWithAceStep } from "./acestep-provider.mjs"; + +function json(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +test("ACE-Step submits, polls, and resolves a generated audio URL", async () => { + const calls = []; + const responses = [ + json({ data: { task_id: "task-1" } }), + json({ data: [{ task_id: "task-1", status: 0, result: "" }] }), + json({ + data: [ + { + task_id: "task-1", + status: 1, + result: JSON.stringify([ + { + file: "/v1/audio?path=track.mp3", + metas: { duration: 42 }, + dit_model: "xl", + lm_model: "4b", + }, + ]), + }, + ], + }), + ]; + let clock = 0; + const result = await generateWithAceStep( + "cinematic pulse", + { duration: 42 }, + { + config: { baseUrl: "https://music.example", apiKey: "secret" }, + fetch: async (url, options) => { + calls.push({ url, options }); + return responses.shift(); + }, + sleep: async (ms) => { + clock += ms; + }, + now: () => clock, + }, + ); + assert.equal(result.url, "https://music.example/v1/audio?path=track.mp3"); + assert.equal(result.metadata.provider, "acestep.remote"); + assert.equal(result.metadata.duration, 42); + assert.deepEqual(result.downloadHeaders, { authorization: "Bearer secret" }); + assert.equal(calls.length, 3); + assert.match(calls[0].options.body, /acestep-v15-xl-sft/); + assert.equal(calls[0].options.headers.authorization, "Bearer secret"); +}); + +test("ACE-Step is unavailable when it has no configured endpoint", async () => { + assert.equal(await generateWithAceStep("x", {}, { config: { baseUrl: "", apiKey: "" } }), null); +}); + +test("ACE-Step surfaces failed jobs", async () => { + const responses = [ + json({ data: { task_id: "bad" } }), + json({ data: [{ status: 2, error: "out of memory" }] }), + ]; + await assert.rejects( + generateWithAceStep( + "x", + {}, + { + config: { baseUrl: "https://music.example", apiKey: "" }, + fetch: async () => responses.shift(), + }, + ), + /out of memory/, + ); +}); diff --git a/skills/media-use/scripts/lib/freeze.mjs b/skills/media-use/scripts/lib/freeze.mjs index 2527f6626d..8e21f5165b 100644 --- a/skills/media-use/scripts/lib/freeze.mjs +++ b/skills/media-use/scripts/lib/freeze.mjs @@ -5,9 +5,9 @@ import { dirname } from "node:path"; // 256MB covers any real media asset; raise if 4K video sources ever exceed it. const MAX_FREEZE_BYTES = 256 * 1024 * 1024; -export async function freezeUrl(url, destPath) { +export async function freezeUrl(url, destPath, { headers } = {}) { const where = String(url).slice(0, 80); - const res = await fetch(url); + const res = await fetch(url, headers ? { headers } : undefined); if (!res.ok) throw new Error(`freeze failed: HTTP ${res.status} for ${where}`); // Fail fast on an advertised oversize body before reading a single byte. diff --git a/skills/media-use/scripts/lib/registry.mjs b/skills/media-use/scripts/lib/registry.mjs index 0f5468f11c..8ec3160ea8 100644 --- a/skills/media-use/scripts/lib/registry.mjs +++ b/skills/media-use/scripts/lib/registry.mjs @@ -21,6 +21,7 @@ // (e.g. "make an image with codex"). import { bgmProvider } from "./bgm-provider.mjs"; +import { generateWithAceStep } from "./acestep-provider.mjs"; import { sfxProvider } from "./sfx-provider.mjs"; import { bundledSfxProvider } from "./bundled-sfx-provider.mjs"; import { imageProvider, iconProvider } from "./image-provider.mjs"; @@ -49,7 +50,10 @@ const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // rem // heygen-CLI first. All remote providers are skipped by --local-only. const REGISTRY = { - bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })], + bgm: [ + N("heygen.audio.sounds", { search: bgmProvider.search }), + N("acestep.remote", { generate: generateWithAceStep }), + ], sfx: [ N("heygen.audio.sounds", { search: sfxProvider.search }), A("bundled.sfx", { search: bundledSfxProvider.search }), diff --git a/skills/media-use/scripts/lib/registry.test.mjs b/skills/media-use/scripts/lib/registry.test.mjs index 771a78e85b..39b3aaf17d 100644 --- a/skills/media-use/scripts/lib/registry.test.mjs +++ b/skills/media-use/scripts/lib/registry.test.mjs @@ -40,7 +40,7 @@ test("heygen provider is first for every type it serves", () => { test("sanctioned providers only: heygen, local mflux/kokoro/ltx, codex, design spec, logo tiers", () => { const allowed = - /^heygen|^bundled\.sfx$|^mflux\.local$|^kokoro\.local$|^ltx\.local$|^codex\.image_gen$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/; + /^heygen|^acestep\.remote$|^bundled\.sfx$|^mflux\.local$|^kokoro\.local$|^ltx\.local$|^codex\.image_gen$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/; for (const t of listTypes()) { for (const p of getProviders(t)) { assert.ok(allowed.test(p.name), `${t} lists unsanctioned provider: ${p.name}`); @@ -61,6 +61,14 @@ test("image cascade: heygen catalog, then local mflux, then the codex upsell", ( assert.ok(codex.network, "codex is network (skipped under --local-only)"); }); +test("bgm supports ACE-Step generation without replacing HeyGen retrieval", () => { + const ps = getProviders("bgm"); + assert.equal(ps[0].name, "heygen.audio.sounds"); + assert.equal(ps[1].name, "acestep.remote"); + assert.equal(typeof ps[1].generate, "function"); + assert.ok(ps[1].network, "ACE-Step is skipped by --local-only"); +}); + test("voice cascade: HeyGen TTS first, Kokoro remains the local fallback", () => { const ps = getProviders("voice"); assert.equal(ps[0].name, "heygen.tts", "HeyGen TTS is first when credentials exist"); diff --git a/skills/media-use/scripts/resolve.mjs b/skills/media-use/scripts/resolve.mjs index fc1f33224a..f818204aca 100644 --- a/skills/media-use/scripts/resolve.mjs +++ b/skills/media-use/scripts/resolve.mjs @@ -43,6 +43,7 @@ import { versionLessThan, } from "./lib/heygen-cli.mjs"; import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs"; +import { configuredBgmProvider } from "./lib/acestep-provider.mjs"; const INGEST_TYPES = listTypes(); const DEFAULT_EXT = { @@ -82,6 +83,7 @@ const { values: args } = parseArgs({ analyze: { type: "boolean", default: false }, "local-only": { type: "boolean", default: false }, provider: { type: "string" }, + duration: { type: "string" }, "avatar-id": { type: "string" }, "voice-id": { type: "string" }, json: { type: "boolean", default: false }, @@ -119,6 +121,7 @@ Options: --analyze Return --for grade evidence without recording a candidate --local-only Offline: skip every network provider --provider Force one generator (e.g. codex, mflux, kokoro, heygen) + --duration Generated BGM duration in seconds (10-600; default: 30) --avatar-id Override the default avatar for heygen.video generation --voice-id Override the default voice for voice/heygen.video generation --json Output JSON instead of one-line result @@ -130,6 +133,7 @@ const projectDir = resolve(args.project); const type = args.type; const intent = args.intent; const entity = args.entity || null; +const configuredProvider = args.provider || (type === "bgm" ? configuredBgmProvider() : null); if (args.adopt) { const { adoptExistingAssets } = await import("./lib/adopt.mjs"); @@ -276,9 +280,9 @@ if (!listTypes().includes(args.type)) { // Forced-provider validation: reject an unknown/unavailable provider name up // front so a typo reads as a typo, not a catalog miss (`no provider could // resolve`). Match rule mirrors runProviders (full name or dotted prefix). -if (args.provider && !providerMatches(args.type, args.provider)) { +if (configuredProvider && !providerMatches(args.type, configuredProvider)) { console.error( - `error: unknown provider "${args.provider}" for type ${args.type} (available: ${providerNamesFor(args.type).join(", ")})`, + `error: unknown provider "${configuredProvider}" for type ${args.type} (available: ${providerNamesFor(args.type).join(", ")})`, ); process.exit(2); } @@ -390,7 +394,8 @@ async function run() { entity, projectDir, localOnly, - provider: args.provider, + provider: configuredProvider, + duration: args.duration ? Number(args.duration) : undefined, avatarId: args["avatar-id"], voiceId: args["voice-id"], }; @@ -492,7 +497,9 @@ async function run() { if (searchResult.localPath) { freezeLocalFile(searchResult.localPath, reservation.fullPath); } else if (searchResult.url) { - await freezeUrl(searchResult.url, reservation.fullPath); + await freezeUrl(searchResult.url, reservation.fullPath, { + headers: searchResult.downloadHeaders, + }); } else { throw new Error("provider returned no url or localPath"); } From 09b5fb06f626a7a067d739ef465ae3c9dedab576 Mon Sep 17 00:00:00 2001 From: Eugene Ware Date: Wed, 5 Aug 2026 22:50:53 +1000 Subject: [PATCH 2/4] Use Titan-stable ACE SFT model --- skills/media-use/scripts/lib/acestep-provider.mjs | 4 ++-- skills/media-use/scripts/lib/acestep-provider.test.mjs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/skills/media-use/scripts/lib/acestep-provider.mjs b/skills/media-use/scripts/lib/acestep-provider.mjs index b28599b805..206c52cf6b 100644 --- a/skills/media-use/scripts/lib/acestep-provider.mjs +++ b/skills/media-use/scripts/lib/acestep-provider.mjs @@ -85,7 +85,7 @@ export async function generateWithAceStep(intent, ctx = {}, deps = {}) { audio_duration: duration, audio_format: "mp3", batch_size: 1, - model: "acestep-v15-xl-sft", + model: "acestep-v15-sft", inference_steps: 50, }), }, @@ -127,7 +127,7 @@ export async function generateWithAceStep(intent, ctx = {}, deps = {}) { provenance: { endpoint: new URL(baseUrl).origin, task_id: String(taskId), - dit_model: result.dit_model || "acestep-v15-xl-sft", + dit_model: result.dit_model || "acestep-v15-sft", lm_model: result.lm_model || null, seed: result.seed_value || null, }, diff --git a/skills/media-use/scripts/lib/acestep-provider.test.mjs b/skills/media-use/scripts/lib/acestep-provider.test.mjs index 7815714f5b..4284d20a0a 100644 --- a/skills/media-use/scripts/lib/acestep-provider.test.mjs +++ b/skills/media-use/scripts/lib/acestep-provider.test.mjs @@ -52,7 +52,7 @@ test("ACE-Step submits, polls, and resolves a generated audio URL", async () => assert.equal(result.metadata.duration, 42); assert.deepEqual(result.downloadHeaders, { authorization: "Bearer secret" }); assert.equal(calls.length, 3); - assert.match(calls[0].options.body, /acestep-v15-xl-sft/); + assert.match(calls[0].options.body, /acestep-v15-sft/); assert.equal(calls[0].options.headers.authorization, "Bearer secret"); }); From 5988514b8689646b0dcd392351e83913e6fd83b2 Mon Sep 17 00:00:00 2001 From: Eugene Ware Date: Wed, 5 Aug 2026 23:01:49 +1000 Subject: [PATCH 3/4] Pin ACE-Step PyTorch LM backend --- skills/media-use/scripts/lib/acestep-provider.mjs | 1 + skills/media-use/scripts/lib/acestep-provider.test.mjs | 1 + 2 files changed, 2 insertions(+) diff --git a/skills/media-use/scripts/lib/acestep-provider.mjs b/skills/media-use/scripts/lib/acestep-provider.mjs index 206c52cf6b..1fc178191d 100644 --- a/skills/media-use/scripts/lib/acestep-provider.mjs +++ b/skills/media-use/scripts/lib/acestep-provider.mjs @@ -86,6 +86,7 @@ export async function generateWithAceStep(intent, ctx = {}, deps = {}) { audio_format: "mp3", batch_size: 1, model: "acestep-v15-sft", + lm_backend: "pt", inference_steps: 50, }), }, diff --git a/skills/media-use/scripts/lib/acestep-provider.test.mjs b/skills/media-use/scripts/lib/acestep-provider.test.mjs index 4284d20a0a..8270787bbf 100644 --- a/skills/media-use/scripts/lib/acestep-provider.test.mjs +++ b/skills/media-use/scripts/lib/acestep-provider.test.mjs @@ -53,6 +53,7 @@ test("ACE-Step submits, polls, and resolves a generated audio URL", async () => assert.deepEqual(result.downloadHeaders, { authorization: "Bearer secret" }); assert.equal(calls.length, 3); assert.match(calls[0].options.body, /acestep-v15-sft/); + assert.match(calls[0].options.body, /"lm_backend":"pt"/); assert.equal(calls[0].options.headers.authorization, "Bearer secret"); }); From 7d97b12149cb5cbc2c40e9fb6b39b76a3e1ca403 Mon Sep 17 00:00:00 2001 From: Eugene Ware Date: Wed, 5 Aug 2026 23:18:28 +1000 Subject: [PATCH 4/4] Allow long local ACE-Step generations --- skills/media-use/scripts/lib/acestep-provider.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/media-use/scripts/lib/acestep-provider.mjs b/skills/media-use/scripts/lib/acestep-provider.mjs index 1fc178191d..e8743dfcc1 100644 --- a/skills/media-use/scripts/lib/acestep-provider.mjs +++ b/skills/media-use/scripts/lib/acestep-provider.mjs @@ -96,7 +96,7 @@ export async function generateWithAceStep(intent, ctx = {}, deps = {}) { if (!taskId) throw new Error("ACE-Step did not return a task ID"); const started = now(); - const timeoutMs = Number(process.env.ACESTEP_POLL_TIMEOUT_MS) || 20 * 60 * 1000; + const timeoutMs = Number(process.env.ACESTEP_POLL_TIMEOUT_MS) || 60 * 60 * 1000; while (now() - started < timeoutMs) { const queried = await requestJson( `${baseUrl}/query_result`,