fix(media-use): keep the other voices when a TTS run re-voices some lines

packages/cli/src/media-use/lib/local-models.mjs

@@ -27,10 +27,10 @@ const MODELS = {
       tier: "medium",
       sizeMB: 330,
       needs: { ramMB: 2048, gpu: false },
-      wordTimestamps: "native",
+      wordTimestamps: "transcribe", // `hyperframes tts` returns none; run `hyperframes transcribe` over output
       install: "pip install kokoro",
       invoke: "python -m kokoro --text {text} --voice {voice} --out {out}",
-      notes: "CPU, faster-than-realtime, native per-word timestamps. Default floor.",
+      notes: "CPU, faster-than-realtime. Word timing via `hyperframes transcribe`. Default floor.",
     },
     {
       id: "fish-speech",

packages/cli/src/media-use/lib/local-models.test.mjs

@@ -62,7 +62,7 @@ test("selectModel picks the large tier on a strong machine", () => {
 test("selectModel falls back to medium on a CPU-only machine", () => {
   const r = selectModel("tts", cpuOnly);
   assert.equal(r.tier, "medium");
-  assert.equal(r.model.id, "kokoro", "Kokoro is the CPU/medium default (native word timestamps)");
+  assert.equal(r.model.id, "kokoro", "Kokoro is the CPU/medium default");
 });
 
 test("selectModel recommends the CLI path when no tier fits", () => {

packages/cli/src/media-use/lib/tts-local-provider.mjs

@@ -7,9 +7,10 @@ import { resolveSpawnCommand } from "../../audio/scripts/lib/tts.mjs";
 
 // Local voiceover via the packaged Kokoro-82M TTS (the `hyperframes tts` CLI),
 // the free/private default now that HeyGen TTS costs wallet credits. Kokoro runs
-// on-device (CPU, faster-than-realtime, bundled voices, native word timestamps),
-// so no key and no per-call charge. When Kokoro is not set up, this returns null
-// and the registry falls through to the HeyGen TTS upsell.
+// on-device (CPU, faster-than-realtime, bundled voices), so no key and no
+// per-call charge. `hyperframes tts` returns no word timestamps: word timing
+// comes from `hyperframes transcribe` over the output. When Kokoro is not set
+// up, this returns null and the registry falls through to the HeyGen TTS upsell.
 //
 // Delegated to the hyperframes CLI (same as transcribe / remove-background), not
 // re-implemented here. ffprobe reads the duration back for the ledger.

skills-manifest.json

@@ -18,7 +18,7 @@
       "files": 13
     },
     "media-use": {
-      "hash": "f3e4cc75041371b3",
+      "hash": "12fba5673f8eb60c",
       "files": 118
     },
     "remotion-to-hyperframes": {

skills/media-use/audio/scripts/audio.mjs

@@ -34,9 +34,11 @@
 //     total_duration_s }
 //
 // --only tts,bgm,sfx  runs a subset and MERGES into an existing --out (so a
-// caller can do TTS+BGM early, then SFX later once cues exist). When BGM uses
-// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
-// before assembling.
+// caller can do TTS+BGM early, then SFX later once cues exist). With --only,
+// voices merge by id: a request that lists only some lines re-voices those and
+// keeps the rest. Without --only, the voices are replaced, as before.
+// When BGM uses the generate path it is spawned detached (bgm_pending:true) —
+// run wait-bgm.mjs before assembling.
 
 import { existsSync, mkdirSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
@@ -53,7 +55,7 @@ import {
 import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
 import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
 import { mapWithConcurrency } from "./lib/concurrency.mjs";
-import { openAudioMeta } from "./lib/audio-meta.mjs";
+import { mergeVoices, openAudioMeta } from "./lib/audio-meta.mjs";
 import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
@@ -124,6 +126,7 @@ const anomalies = [];
 
 // ── TTS ───────────────────────────────────────────────────────────────────────
 let voices = prev.voices ?? [];
+let freshVoices = [];
 let ttsProvider = prev.tts_provider ?? null;
 let voiceId = prev.voice_id ?? null;
 if (only.has("tts") && lines.length) {
@@ -175,9 +178,15 @@ if (only.has("tts") && lines.length) {
     return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
   };
   const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
-  voices = results.filter(Boolean);
-  for (const v of voices)
+  freshVoices = results.filter(Boolean);
+  for (const v of freshVoices)
     console.error(`  voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
+  voices = has("only")
+    ? mergeVoices(voices, freshVoices, {
+        requested: lines.map((line) => String(line.id)),
+        exists: (rel) => existsSync(join(hyperframesDir, rel)),
+      })
+    : freshVoices;
 }
 const hasVoice = voices.length > 0;
 const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
@@ -295,7 +304,15 @@ const meta = {
 };
 mkdirSync(dirname(outPath), { recursive: true });
 audioMeta.write(meta);
-const written = writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx });
+const written = writtenAssets({
+  only,
+  lines,
+  voices: freshVoices,
+  ttsProvider,
+  bgm,
+  bgmFields,
+  sfx,
+});
 anomalies.push(...recordInManifest(hyperframesDir, written));
 
 console.log(`✓ audio engine → ${outPath}`);

skills/media-use/audio/scripts/audio.test.mjs

@@ -1,10 +1,19 @@
 import { strict as assert } from "node:assert";
 import { test } from "node:test";
-import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs";
+import {
+  chmodSync,
+  mkdtempSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  existsSync,
+  writeFileSync,
+} from "node:fs";
 import { spawnSync } from "node:child_process";
 import { join, dirname } from "node:path";
 import { tmpdir } from "node:os";
 import { fileURLToPath } from "node:url";
+import { readManifest } from "../../scripts/lib/manifest.mjs";
 import { recordInManifest } from "./lib/media-record.mjs";
 import { resolveSfx } from "./lib/sfx.mjs";
 
@@ -164,3 +173,89 @@ for (const [label, request, args, shown] of [
 test("a numeric string speed, as the adapters forward it, passes the check", () => {
   assert.doesNotMatch(runEngineWith({ speed: "0.8" }, "--only", "tts").stderr, /speed must/);
 });
+
+// Stand-ins on PATH for the Kokoro path: `tts` copies the line's text into the wav, `transcribe`
+// reads it back as one word, and ffprobe reports a tenth of a second per byte.
+const npxStub = `#!${process.execPath}
+const fs = require("node:fs");
+const path = require("node:path");
+const [, command, input, ...rest] = process.argv.slice(2);
+const after = (name) => rest[rest.indexOf(name) + 1];
+if (command === "tts") {
+  fs.mkdirSync(path.dirname(after("--output")), { recursive: true });
+  fs.copyFileSync(input, after("--output"));
+} else {
+  const text = fs.readFileSync(input, "utf8");
+  const words = [{ text, start: 0, end: 1 }];
+  fs.writeFileSync(path.join(after("--dir"), "transcript.json"), JSON.stringify(words));
+}
+`;
+const ffprobeStub = `#!${process.execPath}
+console.log(require("node:fs").statSync(process.argv.at(-1)).size / 10);
+`;
+
+test("re-voicing one line keeps every other voice", { skip: process.platform === "win32" }, () => {
+  const dir = mkdtempSync(join(tmpdir(), "audio-partial-tts-"));
+  try {
+    const bin = join(dir, "bin");
+    const project = join(dir, "project");
+    mkdirSync(bin);
+    mkdirSync(project);
+    for (const [name, source] of [
+      ["npx", npxStub],
+      ["ffprobe", ffprobeStub],
+    ]) {
+      writeFileSync(join(bin, name), source);
+      chmodSync(join(bin, name), 0o755);
+    }
+    const run = (lines, args = ["--only", "tts"]) => {
+      const request = { provider: "kokoro", lines };
+      writeFileSync(join(project, "audio_request.json"), JSON.stringify(request));
+      const r = spawnSync(
+        process.execPath,
+        [join(HERE, "audio.mjs"), "--hyperframes", project, ...args],
+        {
+          encoding: "utf8",
+          timeout: 15000,
+          env: {
+            ...process.env,
+            PATH: bin,
+            HEYGEN_CONFIG_DIR: dir,
+            HEYGEN_API_KEY: "",
+            HYPERFRAMES_API_KEY: "",
+            HYPERFRAMES_MEDIA_HOME: dir,
+          },
+        },
+      );
+      assert.equal(r.status, 0, r.stderr);
+      return JSON.parse(readFileSync(join(project, "audio_meta.json"), "utf8"));
+    };
+    run(["a", "b", "c"].map((id) => ({ id, text: `line ${id}` })));
+    const meta = run([{ id: "b", text: "line b, again" }]);
+    assert.deepEqual(
+      meta.voices.map((v) => [v.id, v.path, v.duration_s, v.words[0].text]),
+      [
+        ["a", "assets/voice/a.wav", 0.6, "line a"],
+        ["b", "assets/voice/b.wav", 1.3, "line b, again"],
+        ["c", "assets/voice/c.wav", 0.6, "line c"],
+      ],
+    );
+    assert.equal(meta.total_duration_s, 2.5);
+    // Only the re-voiced line is recorded again; the kept voices keep their own records.
+    assert.deepEqual(
+      readManifest(project).map((record) => record.description),
+      ["line a", "line b", "line c", "line b, again"],
+    );
+    // A full run without --only still replaces the voices, so a removed line goes away.
+    const full = run(
+      ["a", "c"].map((id) => ({ id, text: `line ${id}` })),
+      [],
+    );
+    assert.deepEqual(
+      full.voices.map((v) => v.id),
+      ["a", "c"],
+    );
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});

skills/media-use/audio/scripts/lib/audio-meta.mjs

@@ -37,3 +37,21 @@ export function openAudioMeta(path) {
     },
   };
 }
+
+// A TTS run that re-voices only some lines must not drop the others. Voices merge by id: an
+// unrequested voice stays in place, a re-voiced line replaces its entry and a new line is appended.
+// A requested line that failed keeps its earlier entry only while that file exists, so the list
+// never names a missing file. When every earlier id is requested, `fresh` is the whole result.
+export function mergeVoices(previous, fresh, { requested, exists }) {
+  const asked = new Set(requested);
+  if (previous.every((voice) => asked.has(voice.id))) return fresh;
+  const freshById = new Map(fresh.map((voice) => [voice.id, voice]));
+  const merged = previous.flatMap((voice) => {
+    if (!asked.has(voice.id)) return [voice];
+    const next = freshById.get(voice.id);
+    freshById.delete(voice.id);
+    if (next) return [next];
+    return exists(voice.path) ? [voice] : [];
+  });
+  return [...merged, ...freshById.values()];
+}

skills/media-use/audio/scripts/lib/audio-meta.test.mjs

@@ -13,7 +13,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { spawnSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
-import { openAudioMeta } from "./audio-meta.mjs";
+import { mergeVoices, openAudioMeta } from "./audio-meta.mjs";
 
 function fixture(t) {
   const dir = mkdtempSync(join(tmpdir(), "audio-meta-"));
@@ -134,3 +134,31 @@ test("CLI partial run retains unselected voices, BGM and SFX", (t) => {
   const actual = JSON.parse(readFileSync(path, "utf8"));
   for (const key of Object.keys(previous)) assert.deepEqual(actual[key], previous[key]);
 });
+
+const voice = (id, duration_s = 1) => ({ id, path: `assets/voice/${id}.wav`, duration_s });
+const [a, b, c] = ["a", "b", "c"].map((id) => voice(id));
+const noFiles = () => false;
+
+test("mergeVoices keeps unrequested voices and replaces a re-voiced line in place", () => {
+  const b2 = voice("b", 2);
+  assert.deepEqual(mergeVoices([a, b, c], [b2], { requested: ["b"], exists: noFiles }), [a, b2, c]);
+});
+
+test("mergeVoices appends a line that had no voice yet", () => {
+  const d = voice("d");
+  assert.deepEqual(mergeVoices([a, b], [d], { requested: ["d"], exists: noFiles }), [a, b, d]);
+});
+
+test("mergeVoices keeps a failed line's voice only while its file exists", () => {
+  const onlyB = (path) => path === b.path;
+  assert.deepEqual(mergeVoices([a, b, c], [], { requested: ["b"], exists: noFiles }), [a, c]);
+  assert.deepEqual(mergeVoices([a, b, c], [], { requested: ["b"], exists: onlyB }), [a, b, c]);
+});
+
+test("mergeVoices returns a full run as synthesized, in request order", () => {
+  const fresh = [voice("c", 3), voice("a", 4), voice("b", 5)];
+  const requested = ["c", "a", "b"];
+  assert.equal(mergeVoices([a, b, c], fresh, { requested, exists: () => true }), fresh);
+  const failedB = mergeVoices([a, b], [fresh[1]], { requested: ["a", "b"], exists: () => true });
+  assert.deepEqual(failedB, [fresh[1]]);
+});

skills/media-use/references/audio.md

@@ -9,7 +9,7 @@ shot), use the shared engine at `audio/scripts/audio.mjs`. It takes a neutral
 node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./audio_meta.json
 ```
 
-- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
+- **Request** `{ provider?, voice?, lang?, speed?, tts_model?, style?, lines: [{ id, text, style?, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`; with `--only tts`, voices merge by line `id`, so a request listing some lines re-voices those and keeps the rest.
 - **Gemini narration** is opt-in with `provider: "gemini"` and an API key or service-account credentials. `tts_model` defaults to `gemini-3.8-flash-tts`; 3.8 Flash-Lite, 3.1 Flash Preview, and 2.5 Pro/Flash Preview TTS are also supported. `style` directs delivery; a line's style overrides the request's style. CLI overrides: `--tts-model`, `--style`. See `audio/references/tts.md` for a complete request. The existing automatic provider order is unchanged.
 - **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
 - **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music retrieval; SFX come from Openverse with no account (`audio/references/sfx.md`). Local/provider-specific generators are explicit alternatives where installed; run `npx hyperframes media-use resolve --doctor` before assuming retrieval or TTS will work.