diff --git a/packages/cli/src/media-use/lib/local-models.mjs b/packages/cli/src/media-use/lib/local-models.mjs index 28b8715bebf..65892345b69 100644 --- a/packages/cli/src/media-use/lib/local-models.mjs +++ b/packages/cli/src/media-use/lib/local-models.mjs @@ -27,10 +27,10 @@ const MODELS = { tier: "medium", sizeMB: 330, needs: { ramMB: 2048, gpu: false }, - wordTimestamps: "native", + wordTimestamps: "transcribe", // `hyperframes tts` returns none; run `hyperframes transcribe` over output install: "pip install kokoro", invoke: "python -m kokoro --text {text} --voice {voice} --out {out}", - notes: "CPU, faster-than-realtime, native per-word timestamps. Default floor.", + notes: "CPU, faster-than-realtime. Word timing via `hyperframes transcribe`. Default floor.", }, { id: "fish-speech", diff --git a/packages/cli/src/media-use/lib/local-models.test.mjs b/packages/cli/src/media-use/lib/local-models.test.mjs index f1e8d0204f7..27687bbb1a2 100644 --- a/packages/cli/src/media-use/lib/local-models.test.mjs +++ b/packages/cli/src/media-use/lib/local-models.test.mjs @@ -62,7 +62,7 @@ test("selectModel picks the large tier on a strong machine", () => { test("selectModel falls back to medium on a CPU-only machine", () => { const r = selectModel("tts", cpuOnly); assert.equal(r.tier, "medium"); - assert.equal(r.model.id, "kokoro", "Kokoro is the CPU/medium default (native word timestamps)"); + assert.equal(r.model.id, "kokoro", "Kokoro is the CPU/medium default"); }); test("selectModel recommends the CLI path when no tier fits", () => { diff --git a/packages/cli/src/media-use/lib/tts-local-provider.mjs b/packages/cli/src/media-use/lib/tts-local-provider.mjs index d5e9416a73f..d1137b83d3c 100644 --- a/packages/cli/src/media-use/lib/tts-local-provider.mjs +++ b/packages/cli/src/media-use/lib/tts-local-provider.mjs @@ -7,9 +7,10 @@ import { resolveSpawnCommand } from "../../audio/scripts/lib/tts.mjs"; // Local voiceover via the packaged Kokoro-82M TTS (the `hyperframes tts` CLI), // the free/private default now that HeyGen TTS costs wallet credits. Kokoro runs -// on-device (CPU, faster-than-realtime, bundled voices, native word timestamps), -// so no key and no per-call charge. When Kokoro is not set up, this returns null -// and the registry falls through to the HeyGen TTS upsell. +// on-device (CPU, faster-than-realtime, bundled voices), so no key and no +// per-call charge. `hyperframes tts` returns no word timestamps: word timing +// comes from `hyperframes transcribe` over the output. When Kokoro is not set +// up, this returns null and the registry falls through to the HeyGen TTS upsell. // // Delegated to the hyperframes CLI (same as transcribe / remove-background), not // re-implemented here. ffprobe reads the duration back for the ledger. diff --git a/skills-manifest.json b/skills-manifest.json index 3a0e039cc68..b7873c91119 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 13 }, "media-use": { - "hash": "f3e4cc75041371b3", + "hash": "12fba5673f8eb60c", "files": 118 }, "remotion-to-hyperframes": { diff --git a/skills/media-use/audio/scripts/audio.mjs b/skills/media-use/audio/scripts/audio.mjs index 06cd7665e56..ece3ca72c2e 100644 --- a/skills/media-use/audio/scripts/audio.mjs +++ b/skills/media-use/audio/scripts/audio.mjs @@ -34,9 +34,11 @@ // total_duration_s } // // --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a -// caller can do TTS+BGM early, then SFX later once cues exist). When BGM uses -// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs -// before assembling. +// caller can do TTS+BGM early, then SFX later once cues exist). With --only, +// voices merge by id: a request that lists only some lines re-voices those and +// keeps the rest. Without --only, the voices are replaced, as before. +// When BGM uses the generate path it is spawned detached (bgm_pending:true) — +// run wait-bgm.mjs before assembling. import { existsSync, mkdirSync, readFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -53,7 +55,7 @@ import { import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs"; import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs"; import { mapWithConcurrency } from "./lib/concurrency.mjs"; -import { openAudioMeta } from "./lib/audio-meta.mjs"; +import { mergeVoices, openAudioMeta } from "./lib/audio-meta.mjs"; import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -124,6 +126,7 @@ const anomalies = []; // ── TTS ─────────────────────────────────────────────────────────────────────── let voices = prev.voices ?? []; +let freshVoices = []; let ttsProvider = prev.tts_provider ?? null; let voiceId = prev.voice_id ?? null; if (only.has("tts") && lines.length) { @@ -175,9 +178,15 @@ if (only.has("tts") && lines.length) { return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) }; }; const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine); - voices = results.filter(Boolean); - for (const v of voices) + freshVoices = results.filter(Boolean); + for (const v of freshVoices) console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`); + voices = has("only") + ? mergeVoices(voices, freshVoices, { + requested: lines.map((line) => String(line.id)), + exists: (rel) => existsSync(join(hyperframesDir, rel)), + }) + : freshVoices; } const hasVoice = voices.length > 0; const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0)); @@ -295,7 +304,15 @@ const meta = { }; mkdirSync(dirname(outPath), { recursive: true }); audioMeta.write(meta); -const written = writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx }); +const written = writtenAssets({ + only, + lines, + voices: freshVoices, + ttsProvider, + bgm, + bgmFields, + sfx, +}); anomalies.push(...recordInManifest(hyperframesDir, written)); console.log(`✓ audio engine → ${outPath}`); diff --git a/skills/media-use/audio/scripts/audio.test.mjs b/skills/media-use/audio/scripts/audio.test.mjs index 3b82ae9a62d..53315c64980 100644 --- a/skills/media-use/audio/scripts/audio.test.mjs +++ b/skills/media-use/audio/scripts/audio.test.mjs @@ -1,10 +1,19 @@ import { strict as assert } from "node:assert"; import { test } from "node:test"; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs"; +import { + chmodSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + existsSync, + writeFileSync, +} from "node:fs"; import { spawnSync } from "node:child_process"; import { join, dirname } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; +import { readManifest } from "../../scripts/lib/manifest.mjs"; import { recordInManifest } from "./lib/media-record.mjs"; import { resolveSfx } from "./lib/sfx.mjs"; @@ -164,3 +173,89 @@ for (const [label, request, args, shown] of [ test("a numeric string speed, as the adapters forward it, passes the check", () => { assert.doesNotMatch(runEngineWith({ speed: "0.8" }, "--only", "tts").stderr, /speed must/); }); + +// Stand-ins on PATH for the Kokoro path: `tts` copies the line's text into the wav, `transcribe` +// reads it back as one word, and ffprobe reports a tenth of a second per byte. +const npxStub = `#!${process.execPath} +const fs = require("node:fs"); +const path = require("node:path"); +const [, command, input, ...rest] = process.argv.slice(2); +const after = (name) => rest[rest.indexOf(name) + 1]; +if (command === "tts") { + fs.mkdirSync(path.dirname(after("--output")), { recursive: true }); + fs.copyFileSync(input, after("--output")); +} else { + const text = fs.readFileSync(input, "utf8"); + const words = [{ text, start: 0, end: 1 }]; + fs.writeFileSync(path.join(after("--dir"), "transcript.json"), JSON.stringify(words)); +} +`; +const ffprobeStub = `#!${process.execPath} +console.log(require("node:fs").statSync(process.argv.at(-1)).size / 10); +`; + +test("re-voicing one line keeps every other voice", { skip: process.platform === "win32" }, () => { + const dir = mkdtempSync(join(tmpdir(), "audio-partial-tts-")); + try { + const bin = join(dir, "bin"); + const project = join(dir, "project"); + mkdirSync(bin); + mkdirSync(project); + for (const [name, source] of [ + ["npx", npxStub], + ["ffprobe", ffprobeStub], + ]) { + writeFileSync(join(bin, name), source); + chmodSync(join(bin, name), 0o755); + } + const run = (lines, args = ["--only", "tts"]) => { + const request = { provider: "kokoro", lines }; + writeFileSync(join(project, "audio_request.json"), JSON.stringify(request)); + const r = spawnSync( + process.execPath, + [join(HERE, "audio.mjs"), "--hyperframes", project, ...args], + { + encoding: "utf8", + timeout: 15000, + env: { + ...process.env, + PATH: bin, + HEYGEN_CONFIG_DIR: dir, + HEYGEN_API_KEY: "", + HYPERFRAMES_API_KEY: "", + HYPERFRAMES_MEDIA_HOME: dir, + }, + }, + ); + assert.equal(r.status, 0, r.stderr); + return JSON.parse(readFileSync(join(project, "audio_meta.json"), "utf8")); + }; + run(["a", "b", "c"].map((id) => ({ id, text: `line ${id}` }))); + const meta = run([{ id: "b", text: "line b, again" }]); + assert.deepEqual( + meta.voices.map((v) => [v.id, v.path, v.duration_s, v.words[0].text]), + [ + ["a", "assets/voice/a.wav", 0.6, "line a"], + ["b", "assets/voice/b.wav", 1.3, "line b, again"], + ["c", "assets/voice/c.wav", 0.6, "line c"], + ], + ); + assert.equal(meta.total_duration_s, 2.5); + // Only the re-voiced line is recorded again; the kept voices keep their own records. + assert.deepEqual( + readManifest(project).map((record) => record.description), + ["line a", "line b", "line c", "line b, again"], + ); + // A full run without --only still replaces the voices, so a removed line goes away. + const full = run( + ["a", "c"].map((id) => ({ id, text: `line ${id}` })), + [], + ); + assert.deepEqual( + full.voices.map((v) => v.id), + ["a", "c"], + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/audio/scripts/lib/audio-meta.mjs b/skills/media-use/audio/scripts/lib/audio-meta.mjs index f3c032a9c43..d40040344a5 100644 --- a/skills/media-use/audio/scripts/lib/audio-meta.mjs +++ b/skills/media-use/audio/scripts/lib/audio-meta.mjs @@ -37,3 +37,21 @@ export function openAudioMeta(path) { }, }; } + +// A TTS run that re-voices only some lines must not drop the others. Voices merge by id: an +// unrequested voice stays in place, a re-voiced line replaces its entry and a new line is appended. +// A requested line that failed keeps its earlier entry only while that file exists, so the list +// never names a missing file. When every earlier id is requested, `fresh` is the whole result. +export function mergeVoices(previous, fresh, { requested, exists }) { + const asked = new Set(requested); + if (previous.every((voice) => asked.has(voice.id))) return fresh; + const freshById = new Map(fresh.map((voice) => [voice.id, voice])); + const merged = previous.flatMap((voice) => { + if (!asked.has(voice.id)) return [voice]; + const next = freshById.get(voice.id); + freshById.delete(voice.id); + if (next) return [next]; + return exists(voice.path) ? [voice] : []; + }); + return [...merged, ...freshById.values()]; +} diff --git a/skills/media-use/audio/scripts/lib/audio-meta.test.mjs b/skills/media-use/audio/scripts/lib/audio-meta.test.mjs index 3eb80dfee66..b249baaf70f 100644 --- a/skills/media-use/audio/scripts/lib/audio-meta.test.mjs +++ b/skills/media-use/audio/scripts/lib/audio-meta.test.mjs @@ -13,7 +13,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { openAudioMeta } from "./audio-meta.mjs"; +import { mergeVoices, openAudioMeta } from "./audio-meta.mjs"; function fixture(t) { const dir = mkdtempSync(join(tmpdir(), "audio-meta-")); @@ -134,3 +134,31 @@ test("CLI partial run retains unselected voices, BGM and SFX", (t) => { const actual = JSON.parse(readFileSync(path, "utf8")); for (const key of Object.keys(previous)) assert.deepEqual(actual[key], previous[key]); }); + +const voice = (id, duration_s = 1) => ({ id, path: `assets/voice/${id}.wav`, duration_s }); +const [a, b, c] = ["a", "b", "c"].map((id) => voice(id)); +const noFiles = () => false; + +test("mergeVoices keeps unrequested voices and replaces a re-voiced line in place", () => { + const b2 = voice("b", 2); + assert.deepEqual(mergeVoices([a, b, c], [b2], { requested: ["b"], exists: noFiles }), [a, b2, c]); +}); + +test("mergeVoices appends a line that had no voice yet", () => { + const d = voice("d"); + assert.deepEqual(mergeVoices([a, b], [d], { requested: ["d"], exists: noFiles }), [a, b, d]); +}); + +test("mergeVoices keeps a failed line's voice only while its file exists", () => { + const onlyB = (path) => path === b.path; + assert.deepEqual(mergeVoices([a, b, c], [], { requested: ["b"], exists: noFiles }), [a, c]); + assert.deepEqual(mergeVoices([a, b, c], [], { requested: ["b"], exists: onlyB }), [a, b, c]); +}); + +test("mergeVoices returns a full run as synthesized, in request order", () => { + const fresh = [voice("c", 3), voice("a", 4), voice("b", 5)]; + const requested = ["c", "a", "b"]; + assert.equal(mergeVoices([a, b, c], fresh, { requested, exists: () => true }), fresh); + const failedB = mergeVoices([a, b], [fresh[1]], { requested: ["a", "b"], exists: () => true }); + assert.deepEqual(failedB, [fresh[1]]); +}); diff --git a/skills/media-use/references/audio.md b/skills/media-use/references/audio.md index 350185512b9..02c270eb4f2 100644 --- a/skills/media-use/references/audio.md +++ b/skills/media-use/references/audio.md @@ -9,7 +9,7 @@ shot), use the shared engine at `audio/scripts/audio.mjs`. It takes a neutral node /audio/scripts/audio.mjs --request ./audio_request.json --out ./audio_meta.json ``` -- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`. +- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`; with `--only tts`, voices merge by line `id`, so a request listing some lines re-voices those and keeps the rest. - **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged. - **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`. - **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music retrieval; SFX come from Openverse with no account (`audio/references/sfx.md`). Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.