diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index fd13af9f2a9..de1eba65dec 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,5 +1,5 @@ // Types -export type { RuntimeTimelineClipIdentity } from "./runtime/types.js"; +export type { PlaybackHeldBy, RuntimeTimelineClipIdentity } from "./runtime/types.js"; export type { ExecutionMode, Orientation, diff --git a/packages/core/src/runtime/init.buffering.test.ts b/packages/core/src/runtime/init.buffering.test.ts index eb7e892b783..f70b604c595 100644 --- a/packages/core/src/runtime/init.buffering.test.ts +++ b/packages/core/src/runtime/init.buffering.test.ts @@ -228,7 +228,7 @@ it("a hold for data does not reload a later clip", () => { }); it("a film held for data reads as choppy, so the app can lower it", () => { - const reports: Array<{ choppy: boolean; waitingShare: number }> = []; + const reports: Array<{ choppy: boolean; waitingShare: number; heldBy: unknown }> = []; const { voiceReady, playFor } = playPodcast(0, "video"); voiceReady.value = 4; playFor(0.5); @@ -239,12 +239,13 @@ it("a film held for data reads as choppy, so the app can lower it", () => { for (const [message] of post.mock.calls) { const m = message as { type?: string; - health: { choppy: boolean; waitingShare: number } | null; + health: { choppy: boolean; waitingShare: number; heldBy: unknown } | null; }; if (m?.type === "preview-health" && m.health) reports.push(m.health); } expect(reports.at(-1)?.choppy).toBe(true); expect(reports.at(-1)?.waitingShare).toBeGreaterThan(0.9); + expect(reports.at(-1)?.heldBy).toMatchObject({ kind: "video", readyState: 1 }); }); it("a video far behind the film lands on it when it starves", () => { diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index af3d73800f6..85238cef457 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -26,7 +26,12 @@ import { lengthIsAuthored, stopMediaDownload, } from "./preloadMedia"; -import { notePlaybackFrame, noteDisplayScale } from "./playbackHealth"; +import { + heldByMedia, + notePlaybackFrame, + noteDisplayScale, + watchMediaFetches, +} from "./playbackHealth"; import { hasFrozenPicture } from "./frozenPicture"; import { createPreviewSeek } from "./previewSeek"; import { createPausedVideoFrames, releasePausedVideoFrame } from "./pausedVideoFrames"; @@ -3819,6 +3824,7 @@ export function initSandboxRuntimeModular(): void { document.removeEventListener("play", onMediaPlayWakeTransport, true); }); runtimeCleanupCallbacks.push(watchAudioLoadErrors(document)); + runtimeCleanupCallbacks.push(watchMediaFetches(document)); let lastVisibilityPassSeconds: number | null = null; const syncMediaForCurrentState = (timingRevision?: number) => { @@ -6237,18 +6243,21 @@ export function initSandboxRuntimeModular(): void { const t = clock.now(); state.currentTime = t; if (!window.__HF_EXPORT_RENDER_SEEK_CONFIG) { + const nowMs = performance.now(); + const holder = + bufferingFor[0] ?? + mediaClipsInWindow.find( + ({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3, + )?.el; notePlaybackFrame({ - nowMs: performance.now(), + nowMs, filmTime: t, fps: state.canonicalFps, rate: state.playbackRate, playing: playing || bufferingFor.length > 0, hidden: document.visibilityState === "hidden", - waiting: - bufferingFor.length > 0 || - mediaClipsInWindow.some( - ({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3, - ), + waiting: holder !== undefined, + heldBy: holder ? heldByMedia(holder, nowMs) : null, frozen: hasFrozenPicture(mediaClipsInWindow, t), clock: clock.followedClock(), layoutKey: `${window.innerWidth}x${window.innerHeight}@${window.devicePixelRatio}`, diff --git a/packages/core/src/runtime/playbackHealth.test.ts b/packages/core/src/runtime/playbackHealth.test.ts index 977a07d4331..f741e0f46a2 100644 --- a/packages/core/src/runtime/playbackHealth.test.ts +++ b/packages/core/src/runtime/playbackHealth.test.ts @@ -4,11 +4,13 @@ vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() })); import { postRuntimeMessage } from "./bridge"; import { + heldByMedia, notePlaybackFrame, noteDisplayScale, PLAYBACK_HEALTH, restartPlaybackHealth, type PlaybackHealthInput, + watchMediaFetches, } from "./playbackHealth"; const posted = vi.mocked(postRuntimeMessage); @@ -25,6 +27,7 @@ type Health = { gapP50Ms: number; gapP95Ms: number; clockSource: string; + heldBy: unknown; } | null; const healths = (): Health[] => posted.mock.calls.map(([message]) => @@ -116,6 +119,50 @@ describe("playback health", () => { expect(waiting?.gapP95Ms).toBeLessThan(20); }); + it("names the element that last held play in the window, and none when nothing held it", () => { + const voice = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 4000 } as const; + const clip = { kind: "video", readyState: 2, networkState: 2, fetchMs: null } as const; + play(1000, 3, { waiting: true, heldBy: clip }); + play(4000 + 1000 / 60, 3, { waiting: true, heldBy: voice }); + expect(healths().at(-1)?.heldBy).toEqual(voice); + restartPlaybackHealth(); + posted.mockClear(); + play(20_000, 6); + expect(healths().at(-1)?.heldBy).toBeNull(); + }); + + it("times an element's fetch from its loadstart, else its resource timing, and says null for one never seen", () => { + const now = vi.spyOn(performance, "now").mockReturnValue(1000); + const stop = watchMediaFetches(document); + const voice = document.createElement("audio"); + const clip = document.createElement("video"); + document.body.append(voice, clip); + voice.dispatchEvent(new Event("loadstart")); + expect(heldByMedia(voice, 4500)).toEqual({ + kind: "audio", + readyState: voice.readyState, + networkState: voice.networkState, + fetchMs: 3500, + }); + expect(heldByMedia(clip, 4500)).toMatchObject({ kind: "video", fetchMs: null }); + const early = document.createElement("audio"); + Object.defineProperty(early, "currentSrc", { value: "http://film/voice.mp3" }); + const entries = vi + .spyOn(performance, "getEntriesByName") + .mockImplementation((name) => + name === "http://film/voice.mp3" ? [{ startTime: 500 } as PerformanceEntry] : [], + ); + expect(heldByMedia(early, 4500).fetchMs).toBe(4000); + entries.mockRestore(); + stop(); + now.mockReturnValue(2000); + clip.dispatchEvent(new Event("loadstart")); + expect(heldByMedia(clip, 4500).fetchMs).toBeNull(); + voice.remove(); + clip.remove(); + now.mockRestore(); + }); + it("says which clock the film's time came from", () => { play(1000, 6); expect(healths().at(-1)?.clockSource).toBe("monotonic"); diff --git a/packages/core/src/runtime/playbackHealth.ts b/packages/core/src/runtime/playbackHealth.ts index 1730a1238f9..0639d9a3d15 100644 --- a/packages/core/src/runtime/playbackHealth.ts +++ b/packages/core/src/runtime/playbackHealth.ts @@ -1,5 +1,5 @@ import { postRuntimeMessage } from "./bridge"; -import type { RuntimePreviewHealthMessage } from "./types"; +import type { PlaybackHeldBy, RuntimePreviewHealthMessage } from "./types"; /** What counts as a choppy preview; every number of the rule lives here. */ export const PLAYBACK_HEALTH = { @@ -18,13 +18,21 @@ export type PlaybackHealthInput = { hidden: boolean; /** A playing video is waiting for data. */ waiting: boolean; + heldBy?: PlaybackHeldBy | null; frozen?: boolean; clock: "audio" | "webaudio" | "monotonic"; /** Anything that changes how heavy a frame is to draw (window size, zoom). */ layoutKey: string; }; -type Sample = { atMs: number; gapMs: number; drawn: boolean; waiting: boolean; frozen: boolean }; +type Sample = { + atMs: number; + gapMs: number; + drawn: boolean; + waiting: boolean; + frozen: boolean; + heldBy: PlaybackHeldBy | null; +}; type Last = { ms: number; film: number; frame: number }; const FORWARD_JUMP_SECONDS = 0.5; @@ -35,6 +43,28 @@ let key = ""; let posted = false; let postedAtMs = 0; let clockSource: PlaybackHealthInput["clock"] = "monotonic"; +const fetchStartedAt = new WeakMap(); + +export function watchMediaFetches(doc: Document): () => void { + const onStart = (event: Event) => { + if (event.target) fetchStartedAt.set(event.target, performance.now()); + }; + doc.addEventListener("loadstart", onStart, true); + return () => doc.removeEventListener("loadstart", onStart, true); +} + +const resourceFetchStart = (src: string) => performance.getEntriesByName(src).at(-1)?.startTime; + +export function heldByMedia(el: HTMLMediaElement, nowMs: number): PlaybackHeldBy { + const started = + fetchStartedAt.get(el) ?? (el.currentSrc ? resourceFetchStart(el.currentSrc) : undefined); + return { + kind: el.tagName === "VIDEO" ? "video" : "audio", + readyState: el.readyState, + networkState: el.networkState, + fetchMs: started === undefined ? null : Math.round(nowMs - started), + }; +} function post(health: RuntimePreviewHealthMessage["health"]): void { postRuntimeMessage({ source: "hf-preview", type: "preview-health", health }); @@ -75,6 +105,8 @@ function measure(nowMs: number, fps: number, rate: number): void { const waitingMs = samples.reduce((sum, s) => sum + (s.waiting ? s.gapMs : 0), 0); const frozenMs = samples.reduce((sum, s) => sum + (s.frozen ? s.gapMs : 0), 0); const droppedShare = Math.max(skipped, waitingMs / spanMs); + let heldBy: PlaybackHeldBy | null = null; + for (const sample of samples) heldBy = sample.heldBy ?? heldBy; const gaps = samples.map((sample) => sample.gapMs).sort((a, b) => a - b); const gapAt = (share: number) => Math.round(gaps[Math.ceil(share * gaps.length) - 1] ?? 0); posted = true; @@ -91,6 +123,7 @@ function measure(nowMs: number, fps: number, rate: number): void { gapP50Ms: gapAt(0.5), gapP95Ms: gapAt(0.95), clockSource, + heldBy, mode: "play", }); } @@ -130,6 +163,7 @@ export function notePlaybackFrame(input: PlaybackHealthInput): void { drawn: frame !== previous.frame, waiting: input.waiting, frozen: input.frozen === true, + heldBy: input.heldBy ?? null, }); measure(nowMs, fps, rate); } diff --git a/packages/core/src/runtime/types.ts b/packages/core/src/runtime/types.ts index 18eebbeb17a..64d61fe04a7 100644 --- a/packages/core/src/runtime/types.ts +++ b/packages/core/src/runtime/types.ts @@ -269,6 +269,15 @@ export type RuntimePreviewCopiesMessage = { } | null; }; +export type FetchNotSeenToStart = null; + +export type PlaybackHeldBy = { + kind: "audio" | "video"; + readyState: number; + networkState: number; + fetchMs: number | FetchNotSeenToStart; +}; + export type RuntimePreviewHealthMessage = { source: "hf-preview"; type: "preview-health"; @@ -285,6 +294,7 @@ export type RuntimePreviewHealthMessage = { gapP50Ms: number; gapP95Ms: number; clockSource: "audio" | "webaudio" | "monotonic"; + heldBy: PlaybackHeldBy | null; mode: "play"; } | null; }; diff --git a/packages/studio/src/player/store/previewQualitySlice.test.ts b/packages/studio/src/player/store/previewQualitySlice.test.ts index 4623e49025d..bf1ff3d6d23 100644 --- a/packages/studio/src/player/store/previewQualitySlice.test.ts +++ b/packages/studio/src/player/store/previewQualitySlice.test.ts @@ -87,4 +87,31 @@ describe("preview quality in the player store", () => { ).toBeUndefined(); expect(parsePlaybackHealth(health)?.clockSource).toBeUndefined(); }); + + it("keeps what held play when the runtime names it in its own shape, and drops any other shape", () => { + const health = { + droppedShare: 1, + seconds: 3, + choppy: true, + fps: 0, + worstFrameMs: 20, + mode: "play", + }; + const heldBy = { kind: "audio", readyState: 1, networkState: 2, fetchMs: 3500 }; + expect(parsePlaybackHealth({ ...health, heldBy })?.heldBy).toEqual(heldBy); + expect( + parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: null } })?.heldBy, + ).toEqual({ + ...heldBy, + fetchMs: null, + }); + expect(parsePlaybackHealth({ ...health, heldBy: null })?.heldBy).toBeNull(); + expect( + parsePlaybackHealth({ ...health, heldBy: { ...heldBy, fetchMs: "3500" } })?.heldBy, + ).toBeUndefined(); + expect( + parsePlaybackHealth({ ...health, heldBy: { ...heldBy, kind: "/a/b.mp3" } })?.heldBy, + ).toBeUndefined(); + expect(parsePlaybackHealth(health)?.heldBy).toBeUndefined(); + }); }); diff --git a/packages/studio/src/player/store/previewQualitySlice.ts b/packages/studio/src/player/store/previewQualitySlice.ts index 37a48bf1a7a..5d0ce9608bb 100644 --- a/packages/studio/src/player/store/previewQualitySlice.ts +++ b/packages/studio/src/player/store/previewQualitySlice.ts @@ -1,5 +1,6 @@ import type { StoreApi } from "zustand"; import { isPreviewQuality, type PreviewQuality } from "@hyperframes/core/preview-quality"; +import type { PlaybackHeldBy } from "@hyperframes/core"; import type { MediaLoadFailure } from "@hyperframes/core/runtime/protocol"; import { readStudioUiPreferences, writeStudioUiPreferences } from "../../utils/studioUiPreferences"; @@ -25,6 +26,7 @@ export interface PlaybackHealth { gapP50Ms?: number; gapP95Ms?: number; clockSource?: "audio" | "webaudio" | "monotonic"; + heldBy?: PlaybackHeldBy | null; mode: "play"; } @@ -118,6 +120,15 @@ export function parsePreviewCopies(value: unknown): PreviewCopies | null { const optionalNumber = (n: unknown) => (typeof n === "number" ? n : undefined); const CLOCK_SOURCES = ["audio", "webaudio", "monotonic"] as const; const optionalClock = (clock: unknown) => CLOCK_SOURCES.find((known) => known === clock); +function parseHeldBy(value: unknown): PlaybackHealth["heldBy"] { + if (value === null) return null; + if (typeof value !== "object") return undefined; + const { kind, readyState, networkState, fetchMs } = value as Record; + if (kind !== "audio" && kind !== "video") return undefined; + if (typeof readyState !== "number" || typeof networkState !== "number") return undefined; + if (typeof fetchMs !== "number" && fetchMs !== null) return undefined; + return { kind, readyState, networkState, fetchMs }; +} export function parsePlaybackHealth(value: unknown): PlaybackHealth | null { if (typeof value !== "object" || value === null) return null; @@ -137,6 +148,7 @@ export function parsePlaybackHealth(value: unknown): PlaybackHealth | null { gapP50Ms: optionalNumber(extra.gapP50Ms), gapP95Ms: optionalNumber(extra.gapP95Ms), clockSource: optionalClock(extra.clockSource), + heldBy: parseHeldBy(extra.heldBy), mode: "play", }; }