feat(core): preview health names the media element that held play

packages/core/src/index.ts

@@ -1,5 +1,5 @@
 // Types
-export type { RuntimeTimelineClipIdentity } from "./runtime/types.js";
+export type { PlaybackHeldBy, RuntimeTimelineClipIdentity } from "./runtime/types.js";
 export type {
   ExecutionMode,
   Orientation,

packages/core/src/runtime/init.buffering.test.ts

@@ -228,7 +228,7 @@ it("a hold for data does not reload a later clip", () => {
 });
 
 it("a film held for data reads as choppy, so the app can lower it", () => {
-  const reports: Array<{ choppy: boolean; waitingShare: number }> = [];
+  const reports: Array<{ choppy: boolean; waitingShare: number; heldBy: unknown }> = [];
   const { voiceReady, playFor } = playPodcast(0, "video");
   voiceReady.value = 4;
   playFor(0.5);
@@ -239,12 +239,13 @@ it("a film held for data reads as choppy, so the app can lower it", () => {
   for (const [message] of post.mock.calls) {
     const m = message as {
       type?: string;
-      health: { choppy: boolean; waitingShare: number } | null;
+      health: { choppy: boolean; waitingShare: number; heldBy: unknown } | null;
     };
     if (m?.type === "preview-health" && m.health) reports.push(m.health);
   }
   expect(reports.at(-1)?.choppy).toBe(true);
   expect(reports.at(-1)?.waitingShare).toBeGreaterThan(0.9);
+  expect(reports.at(-1)?.heldBy).toMatchObject({ kind: "video", readyState: 1 });
 });
 
 it("a video far behind the film lands on it when it starves", () => {

packages/core/src/runtime/init.ts

@@ -26,7 +26,12 @@ import {
   lengthIsAuthored,
   stopMediaDownload,
 } from "./preloadMedia";
-import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth";
+import {
+  heldByMedia,
+  notePlaybackFrame,
+  noteDisplayScale,
+  watchMediaFetches,
+} from "./playbackHealth";
 import { hasFrozenPicture } from "./frozenPicture";
 import { createPreviewSeek } from "./previewSeek";
 import { createPausedVideoFrames, releasePausedVideoFrame } from "./pausedVideoFrames";
@@ -3819,6 +3824,7 @@ export function initSandboxRuntimeModular(): void {
     document.removeEventListener("play", onMediaPlayWakeTransport, true);
   });
   runtimeCleanupCallbacks.push(watchAudioLoadErrors(document));
+  runtimeCleanupCallbacks.push(watchMediaFetches(document));
 
   let lastVisibilityPassSeconds: number | null = null;
   const syncMediaForCurrentState = (timingRevision?: number) => {
@@ -6237,18 +6243,21 @@ export function initSandboxRuntimeModular(): void {
       const t = clock.now();
       state.currentTime = t;
       if (!window.__HF_EXPORT_RENDER_SEEK_CONFIG) {
+        const nowMs = performance.now();
+        const holder =
+          bufferingFor[0] ??
+          mediaClipsInWindow.find(
+            ({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
+          )?.el;
         notePlaybackFrame({
-          nowMs: performance.now(),
+          nowMs,
           filmTime: t,
           fps: state.canonicalFps,
           rate: state.playbackRate,
           playing: playing || bufferingFor.length > 0,
           hidden: document.visibilityState === "hidden",
-          waiting:
-            bufferingFor.length > 0 ||
-            mediaClipsInWindow.some(
-              ({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
-            ),
+          waiting: holder !== undefined,
+          heldBy: holder ? heldByMedia(holder, nowMs) : null,
           frozen: hasFrozenPicture(mediaClipsInWindow, t),
           clock: clock.followedClock(),
           layoutKey: `${window.innerWidth}x${window.innerHeight}@${window.devicePixelRatio}`,

packages/core/src/runtime/playbackHealth.test.ts

@@ -4,11 +4,13 @@ vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() }));
 
 import { postRuntimeMessage } from "./bridge";
 import {
+  heldByMedia,
   notePlaybackFrame,
   noteDisplayScale,
   PLAYBACK_HEALTH,
   restartPlaybackHealth,
   type PlaybackHealthInput,
+  watchMediaFetches,
 } from "./playbackHealth";
 
 const posted = vi.mocked(postRuntimeMessage);
@@ -25,6 +27,7 @@ type Health = {
   gapP50Ms: number;
   gapP95Ms: number;
   clockSource: string;
+  heldBy: unknown;
 } | null;
 const healths = (): Health[] =>
   posted.mock.calls.map(([message]) =>
@@ -116,6 +119,50 @@ describe("playback health", () => {
     expect(waiting?.gapP95Ms).toBeLessThan(20);
   });
 
+  it("names the element that last held play in the window, and none when nothing held it", () => {
+    const voice = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 4000 } as const;
+    const clip = { kind: "video", readyState: 2, networkState: 2, fetchMs: null } as const;
+    play(1000, 3, { waiting: true, heldBy: clip });
+    play(4000 + 1000 / 60, 3, { waiting: true, heldBy: voice });
+    expect(healths().at(-1)?.heldBy).toEqual(voice);
+    restartPlaybackHealth();
+    posted.mockClear();
+    play(20_000, 6);
+    expect(healths().at(-1)?.heldBy).toBeNull();
+  });
+
+  it("times an element's fetch from its loadstart, else its resource timing, and says null for one never seen", () => {
+    const now = vi.spyOn(performance, "now").mockReturnValue(1000);
+    const stop = watchMediaFetches(document);
+    const voice = document.createElement("audio");
+    const clip = document.createElement("video");
+    document.body.append(voice, clip);
+    voice.dispatchEvent(new Event("loadstart"));
+    expect(heldByMedia(voice, 4500)).toEqual({
+      kind: "audio",
+      readyState: voice.readyState,
+      networkState: voice.networkState,
+      fetchMs: 3500,
+    });
+    expect(heldByMedia(clip, 4500)).toMatchObject({ kind: "video", fetchMs: null });
+    const early = document.createElement("audio");
+    Object.defineProperty(early, "currentSrc", { value: "http://film/voice.mp3" });
+    const entries = vi
+      .spyOn(performance, "getEntriesByName")
+      .mockImplementation((name) =>
+        name === "http://film/voice.mp3" ? [{ startTime: 500 } as PerformanceEntry] : [],
+      );
+    expect(heldByMedia(early, 4500).fetchMs).toBe(4000);
+    entries.mockRestore();
+    stop();
+    now.mockReturnValue(2000);
+    clip.dispatchEvent(new Event("loadstart"));
+    expect(heldByMedia(clip, 4500).fetchMs).toBeNull();
+    voice.remove();
+    clip.remove();
+    now.mockRestore();
+  });
+
   it("says which clock the film's time came from", () => {
     play(1000, 6);
     expect(healths().at(-1)?.clockSource).toBe("monotonic");

packages/core/src/runtime/playbackHealth.ts

@@ -1,5 +1,5 @@
 import { postRuntimeMessage } from "./bridge";
-import type { RuntimePreviewHealthMessage } from "./types";
+import type { PlaybackHeldBy, RuntimePreviewHealthMessage } from "./types";
 
 /** What counts as a choppy preview; every number of the rule lives here. */
 export const PLAYBACK_HEALTH = {
@@ -18,13 +18,21 @@ export type PlaybackHealthInput = {
   hidden: boolean;
   /** A playing video is waiting for data. */
   waiting: boolean;
+  heldBy?: PlaybackHeldBy | null;
   frozen?: boolean;
   clock: "audio" | "webaudio" | "monotonic";
   /** Anything that changes how heavy a frame is to draw (window size, zoom). */
   layoutKey: string;
 };
 
-type Sample = { atMs: number; gapMs: number; drawn: boolean; waiting: boolean; frozen: boolean };
+type Sample = {
+  atMs: number;
+  gapMs: number;
+  drawn: boolean;
+  waiting: boolean;
+  frozen: boolean;
+  heldBy: PlaybackHeldBy | null;
+};
 type Last = { ms: number; film: number; frame: number };
 
 const FORWARD_JUMP_SECONDS = 0.5;
@@ -35,6 +43,28 @@ let key = "";
 let posted = false;
 let postedAtMs = 0;
 let clockSource: PlaybackHealthInput["clock"] = "monotonic";
+const fetchStartedAt = new WeakMap<EventTarget, number>();
+
+export function watchMediaFetches(doc: Document): () => void {
+  const onStart = (event: Event) => {
+    if (event.target) fetchStartedAt.set(event.target, performance.now());
+  };
+  doc.addEventListener("loadstart", onStart, true);
+  return () => doc.removeEventListener("loadstart", onStart, true);
+}
+
+const resourceFetchStart = (src: string) => performance.getEntriesByName(src).at(-1)?.startTime;
+
+export function heldByMedia(el: HTMLMediaElement, nowMs: number): PlaybackHeldBy {
+  const started =
+    fetchStartedAt.get(el) ?? (el.currentSrc ? resourceFetchStart(el.currentSrc) : undefined);
+  return {
+    kind: el.tagName === "VIDEO" ? "video" : "audio",
+    readyState: el.readyState,
+    networkState: el.networkState,
+    fetchMs: started === undefined ? null : Math.round(nowMs - started),
+  };
+}
 
 function post(health: RuntimePreviewHealthMessage["health"]): void {
   postRuntimeMessage({ source: "hf-preview", type: "preview-health", health });
@@ -75,6 +105,8 @@ function measure(nowMs: number, fps: number, rate: number): void {
   const waitingMs = samples.reduce((sum, s) => sum + (s.waiting ? s.gapMs : 0), 0);
   const frozenMs = samples.reduce((sum, s) => sum + (s.frozen ? s.gapMs : 0), 0);
   const droppedShare = Math.max(skipped, waitingMs / spanMs);
+  let heldBy: PlaybackHeldBy | null = null;
+  for (const sample of samples) heldBy = sample.heldBy ?? heldBy;
   const gaps = samples.map((sample) => sample.gapMs).sort((a, b) => a - b);
   const gapAt = (share: number) => Math.round(gaps[Math.ceil(share * gaps.length) - 1] ?? 0);
   posted = true;
@@ -91,6 +123,7 @@ function measure(nowMs: number, fps: number, rate: number): void {
     gapP50Ms: gapAt(0.5),
     gapP95Ms: gapAt(0.95),
     clockSource,
+    heldBy,
     mode: "play",
   });
 }
@@ -130,6 +163,7 @@ export function notePlaybackFrame(input: PlaybackHealthInput): void {
     drawn: frame !== previous.frame,
     waiting: input.waiting,
     frozen: input.frozen === true,
+    heldBy: input.heldBy ?? null,
   });
   measure(nowMs, fps, rate);
 }

packages/core/src/runtime/types.ts

@@ -269,6 +269,15 @@ export type RuntimePreviewCopiesMessage = {
   } | null;
 };
 
+export type FetchNotSeenToStart = null;
+
+export type PlaybackHeldBy = {
+  kind: "audio" | "video";
+  readyState: number;
+  networkState: number;
+  fetchMs: number | FetchNotSeenToStart;
+};
+
 export type RuntimePreviewHealthMessage = {
   source: "hf-preview";
   type: "preview-health";
@@ -285,6 +294,7 @@ export type RuntimePreviewHealthMessage = {
     gapP50Ms: number;
     gapP95Ms: number;
     clockSource: "audio" | "webaudio" | "monotonic";
+    heldBy: PlaybackHeldBy | null;
     mode: "play";
   } | null;
 };

packages/studio/src/player/store/previewQualitySlice.test.ts

@@ -87,4 +87,31 @@ describe("preview quality in the player store", () => {
     ).toBeUndefined();
     expect(parsePlaybackHealth(health)?.clockSource).toBeUndefined();
   });
+
+  it("keeps what held play when the runtime names it in its own shape, and drops any other shape", () => {
+    const health = {
+      droppedShare: 1,
+      seconds: 3,
+      choppy: true,
+      fps: 0,
+      worstFrameMs: 20,
+      mode: "play",
+    };
+    const heldBy = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 3500 };
+    expect(parsePlaybackHealth({ ...health, heldBy })?.heldBy).toEqual(heldBy);
+    expect(
+      parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: null } })?.heldBy,
+    ).toEqual({
+      ...heldBy,
+      fetchMs: null,
+    });
+    expect(parsePlaybackHealth({ ...health, heldBy: null })?.heldBy).toBeNull();
+    expect(
+      parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: "3500" } })?.heldBy,
+    ).toBeUndefined();
+    expect(
+      parsePlaybackHealth({ ...health, heldBy: { ...heldBy, kind: "/a/b.mp3" } })?.heldBy,
+    ).toBeUndefined();
+    expect(parsePlaybackHealth(health)?.heldBy).toBeUndefined();
+  });
 });

packages/studio/src/player/store/previewQualitySlice.ts

@@ -1,5 +1,6 @@
 import type { StoreApi } from "zustand";
 import { isPreviewQuality, type PreviewQuality } from "@hyperframes/core/preview-quality";
+import type { PlaybackHeldBy } from "@hyperframes/core";
 import type { MediaLoadFailure } from "@hyperframes/core/runtime/protocol";
 import { readStudioUiPreferences, writeStudioUiPreferences } from "../../utils/studioUiPreferences";
 
@@ -25,6 +26,7 @@ export interface PlaybackHealth {
   gapP50Ms?: number;
   gapP95Ms?: number;
   clockSource?: "audio" | "webaudio" | "monotonic";
+  heldBy?: PlaybackHeldBy | null;
   mode: "play";
 }
 
@@ -118,6 +120,15 @@ export function parsePreviewCopies(value: unknown): PreviewCopies | null {
 const optionalNumber = (n: unknown) => (typeof n === "number" ? n : undefined);
 const CLOCK_SOURCES = ["audio", "webaudio", "monotonic"] as const;
 const optionalClock = (clock: unknown) => CLOCK_SOURCES.find((known) => known === clock);
+function parseHeldBy(value: unknown): PlaybackHealth["heldBy"] {
+  if (value === null) return null;
+  if (typeof value !== "object") return undefined;
+  const { kind, readyState, networkState, fetchMs } = value as Record<string, unknown>;
+  if (kind !== "audio" && kind !== "video") return undefined;
+  if (typeof readyState !== "number" || typeof networkState !== "number") return undefined;
+  if (typeof fetchMs !== "number" && fetchMs !== null) return undefined;
+  return { kind, readyState, networkState, fetchMs };
+}
 
 export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
   if (typeof value !== "object" || value === null) return null;
@@ -137,6 +148,7 @@ export function parsePlaybackHealth(value: unknown): PlaybackHealth | null {
     gapP50Ms: optionalNumber(extra.gapP50Ms),
     gapP95Ms: optionalNumber(extra.gapP95Ms),
     clockSource: optionalClock(extra.clockSource),
+    heldBy: parseHeldBy(extra.heldBy),
     mode: "play",
   };
 }