diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 843b64ebf55..1e8914dd343 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -176,7 +176,7 @@ import { forgetPositionEdit, installPositionEditsSeekReapply, } from "./positionEdits"; -import { stillStandsIn, unheldPoster, unproxiedSrc } from "./proxySrc"; +import { isOpenedAtInPoint, stillStandsIn, unheldPoster, unproxiedSrc } from "./proxySrc"; import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBindings"; import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading"; import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading"; @@ -517,12 +517,16 @@ const joinsWebAudio = (el: Element): el is HTMLMediaElement => })); const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]"; +const DECODE_FIRST_MAX_SECONDS = 120; + /** WebKit restarts a media element's audio 150 to 400 ms late; a decoded buffer on the context clock starts on time. */ function decodesFirst(el: HTMLMediaElement, webAudio: WebAudioTransport): boolean { return ( isWebKit() && isAudioElement(el) && !el.loop && + // A long source's whole-file fetch takes the link from the picture; its audio plays from the element. + el.duration <= DECODE_FIRST_MAX_SECONDS && webAudio.fitsDecodeBudget(el) && typeof readElementRateSpec(el) === "number" && readElementPlaybackRate(el) === 1 @@ -1314,7 +1318,7 @@ export function initSandboxRuntimeModular(): void { ? { from: "media", tag: media.tagName.toLowerCase(), - src: media.getAttribute("src") ?? media.currentSrc ?? "", + src: unproxiedSrc(media) ?? media.currentSrc ?? "", trimmed: media.hasAttribute("data-duration") || media.hasAttribute("data-end"), } : { from }; @@ -2530,7 +2534,7 @@ export function initSandboxRuntimeModular(): void { } const tagName = node.tagName.toLowerCase(); const assetUrl = - node.getAttribute("src") ?? + unproxiedSrc(node) ?? node.getAttribute("href") ?? node.getAttribute("poster") ?? null; @@ -3053,7 +3057,7 @@ export function initSandboxRuntimeModular(): void { const NEXT_TAKE_SPAN_SECONDS = 0.5; let nextTakes: { t: number; bound: number; clips: Set } | null = null; const fileOf = (el: HTMLMediaElement) => - el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? ""; + unproxiedSrc(el) ?? el.querySelector("source")?.getAttribute("src") ?? ""; const nextTakeClips = (t: number): Set => { if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips; const byFile = new Map(); @@ -3116,7 +3120,7 @@ export function initSandboxRuntimeModular(): void { const hostOf = (el: Element) => el.parentElement?.closest("[data-composition-id]"); // Clips due soon fetch a few at a time, earliest first: a fast cut sequence has dozens inside the // look-ahead, and fetching them all queues the clip that is due behind clips that are not. - const PREFETCH_SLOTS = 3; + const PREFETCH_SLOTS = 8; const prefetching = new Set(); type WaitingPrefetch = { element: HTMLMediaElement; start: number }; const waitingForSlot = new Map(); @@ -3475,9 +3479,11 @@ export function initSandboxRuntimeModular(): void { rawNode, isVisibleNow, loadsAhead && - (playerFrame - ? nextTakeClips(currentTime).has(rawNode) - : upcoming || mediaStaysDue(rawNode, visibleAt, currentTime)), + // A clip opened at its in-point reads little ahead of it: every one is fetched ahead, not just the next take. + (isOpenedAtInPoint(rawNode) || + (playerFrame + ? nextTakeClips(currentTime).has(rawNode) + : upcoming || mediaStaysDue(rawNode, visibleAt, currentTime))), currentTime, loadsAhead && dueAtLoopStart(rawNode, visibleAt), ); diff --git a/packages/core/src/runtime/mediaLoadFailure.test.ts b/packages/core/src/runtime/mediaLoadFailure.test.ts index 2a391b1326d..959b87886fe 100644 --- a/packages/core/src/runtime/mediaLoadFailure.test.ts +++ b/packages/core/src/runtime/mediaLoadFailure.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { postRuntimeMessage } from "./bridge"; import { watchAudioLoadErrors } from "./mediaLoadFailure"; import { RUNTIME_MEDIA_LOAD_FAILED } from "./protocol"; +import { recordOpenedAtInPoint } from "./proxySrc"; import { WebAudioTransport } from "./webAudioTransport"; vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() })); @@ -37,6 +38,15 @@ afterEach(() => { }); describe("a preview media load that fails is reported, not swallowed", () => { + it("names the src the author wrote, not the in-point fragment the runtime added", async () => { + vi.stubGlobal("fetch", vi.fn(async () => ({ ok: false, status: 404 }))); + const el = audio("assets/take.wav", "take"); + recordOpenedAtInPoint(el, "assets/take.wav", "assets/take.wav#t=30"); + el.setAttribute("src", "assets/take.wav#t=30"); + await transport(async () => ({}) as AudioBuffer).decodeAudioElement(el); + expect(failures()[0]).toMatchObject({ asset: "assets/take.wav" }); + }); + it("a fetch the server refuses says its status, once however often playback retries", async () => { vi.stubGlobal( "fetch", diff --git a/packages/core/src/runtime/mediaProxy.ts b/packages/core/src/runtime/mediaProxy.ts index d315f074d81..292ae44bbfb 100644 --- a/packages/core/src/runtime/mediaProxy.ts +++ b/packages/core/src/runtime/mediaProxy.ts @@ -10,7 +10,7 @@ import { import { findInjectedRenderFrame } from "./renderFrameSibling"; import type { RuntimeJson } from "./types"; import { isVideoElement } from "./domRealm"; -import { heldPosters, swappedElements } from "./proxySrc"; +import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc"; import { restartPlaybackHealth } from "./playbackHealth"; import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies"; import { waitForServedProxy } from "./proxyWait"; @@ -523,7 +523,7 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null return; } noteCopy(el, "ready", null, light); - swappedElements.set(el, originalAttr); + swappedElements.set(el, unproxiedSrc(el)); if (lightFacts) lightElements.add(el); loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined); // The shown size may have grown while this copy was being made. diff --git a/packages/core/src/runtime/picker.ts b/packages/core/src/runtime/picker.ts index 5e658425c04..b05564d5784 100644 --- a/packages/core/src/runtime/picker.ts +++ b/packages/core/src/runtime/picker.ts @@ -3,6 +3,7 @@ import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark"; import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading"; import { swallow } from "./diagnostics"; import { isElementNode } from "./domRealm"; +import { unproxiedSrc } from "./proxySrc"; import { createDrawnProbe } from "./pickerDrawn"; type PickerModuleDeps = { @@ -311,7 +312,7 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule { label: buildElementLabel(el), boundingBox: { x: rect.left, y: rect.top, width: rect.width, height: rect.height }, textContent: el.textContent ? el.textContent.trim().slice(0, 200) : null, - src: el.getAttribute("src") || el.getAttribute("data-composition-src") || null, + src: unproxiedSrc(el) || el.getAttribute("data-composition-src") || null, dataAttributes, }; } diff --git a/packages/core/src/runtime/preloadMedia.test.ts b/packages/core/src/runtime/preloadMedia.test.ts index 08e08a10957..8ae74cef5fc 100644 --- a/packages/core/src/runtime/preloadMedia.test.ts +++ b/packages/core/src/runtime/preloadMedia.test.ts @@ -1,6 +1,8 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { mediaElementTime } from "./mediaClock"; +import { unproxiedSrc } from "./proxySrc"; import { + deferMediaUntilDue, isRenderFrameVideo, preloadMedia, prepareUpcomingMedia, @@ -261,3 +263,90 @@ describe("preparing upcoming video playback", () => { element.remove(); }); }); + +describe("a clip opening at its in-point on Safari", () => { + const SAFARI = + "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1"; + const CHROME = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0 Safari/537.36"; + let meta: HTMLMetaElement; + const inPlayer = (on: boolean) => { + const frame = document.createElement("iframe"); + if (on) frame.setAttribute("data-hf-player-frame", ""); + Object.defineProperty(window, "frameElement", { value: frame, configurable: true }); + }; + beforeEach(() => { + meta = document.createElement("meta"); + meta.name = "hyperframes-studio-preview"; + document.head.appendChild(meta); + vi.spyOn(navigator, "userAgent", "get").mockReturnValue(SAFARI); + inPlayer(true); + deferMediaUntilDue(); + }); + afterEach(() => { + (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect(); + vi.restoreAllMocks(); + Reflect.deleteProperty(window, "frameElement"); + meta.remove(); + document.body.replaceChildren(); + }); + + async function added(html: string): Promise { + document.body.insertAdjacentHTML("beforeend", html); + await Promise.resolve(); + return document.body.lastElementChild as HTMLVideoElement; + } + + it("gets a start fragment and keeps its authored src for everything that reads it", async () => { + const v = await added( + '', + ); + expect(v.getAttribute("src")).toBe("/m/film.mp4#t=30"); + expect(unproxiedSrc(v)).toBe("/m/film.mp4"); + }); + + it("stops reporting the authored src once something else sets the src", async () => { + const v = await added( + '', + ); + v.setAttribute("src", "/m/other.mp4"); + expect(unproxiedSrc(v)).toBe("/m/other.mp4"); + }); + + it("does so for a clip already parsed when the runtime arrives", () => { + (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect(); + document.body.insertAdjacentHTML( + "beforeend", + '', + ); + deferMediaUntilDue(); + expect(document.body.lastElementChild!.getAttribute("src")).toBe("/m/film.mp4#t=30"); + }); + + it.each([ + ["starts at the head of the file", "/m/film.mp4", 'data-media-start="0"'], + ["loops", "/m/film.mp4", 'data-media-start="30" loop'], + ["already carries a fragment", "/m/film.mp4#t=1", 'data-media-start="30"'], + ])("is left alone when it %s", async (_name, src, attrs) => { + const v = await added( + ``, + ); + expect(v.getAttribute("src")).toBe(src); + expect(unproxiedSrc(v)).toBe(src); + }); + + it("is left alone in Studio's preview, which copies and saves what it reads from the page", async () => { + inPlayer(false); + const v = await added( + '', + ); + expect(v.getAttribute("src")).toBe("/m/film.mp4"); + }); + + it("is left alone off Safari", async () => { + vi.spyOn(navigator, "userAgent", "get").mockReturnValue(CHROME); + const v = await added( + '', + ); + expect(v.getAttribute("src")).toBe("/m/film.mp4"); + }); +}); diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts index 91eaaf30936..99727f611fe 100644 --- a/packages/core/src/runtime/preloadMedia.ts +++ b/packages/core/src/runtime/preloadMedia.ts @@ -1,7 +1,9 @@ import { parseStrictFiniteTimingNumber, readMediaStart } from "./playbackRate"; +import { isOpenedAtInPoint, recordOpenedAtInPoint } from "./proxySrc"; import { createRuntimeStartTimeResolver } from "./startResolver"; import { readMediaRenderId } from "./renderFrameSibling"; import { defersMedia } from "./timedClipHide"; +import { inPlayerFrame } from "../studioPreviewMark"; import { swallow } from "./diagnostics"; import { isWebKit, mediaElementTime } from "./mediaClock"; @@ -97,6 +99,21 @@ export function lengthIsAuthored(media: Element): boolean { return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null; } +/** WebKit reads about 40 MB from the head of the file for every fresh element however late its in-point + * seek lands; a start fragment makes it open at the in-point, so many clips of one long file stop pulling + * many times its size, and each is cheap enough to fetch ahead. */ +export function openAtInPoint(el: HTMLMediaElement): void { + const src = el.getAttribute("src"); + const start = readMediaStart(el); + // Only the player: Studio copies and saves what it reads from the page, and must never see the fragment. + if (!inPlayerFrame(document) || !src || src.includes("#") || el.loop) return; + if (el.querySelector("source")) return; + if (!(start > 0)) return; + const written = `${src}#t=${start}`; + recordOpenedAtInPoint(el, src, written); + el.setAttribute("src", written); +} + type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver }; /** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch. @@ -108,8 +125,10 @@ export function deferMediaUntilDue(): void { if (!defersMedia()) return; const defer = (el: Element, startInFilm: (el: Element) => number) => { const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start")); - if (start != null && lengthIsAuthored(el) && startInFilm(el) > 0) - (el as HTMLMediaElement).preload = "none"; + if (start == null || !lengthIsAuthored(el)) return; + const media = el as HTMLMediaElement; + openAtInPoint(media); + if (startInFilm(el) > 0 && !isOpenedAtInPoint(media)) media.preload = "none"; }; win.__hfMediaDeferral = new MutationObserver((records) => { const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement; @@ -122,4 +141,9 @@ export function deferMediaUntilDue(): void { } }); win.__hfMediaDeferral.observe(document.documentElement, { childList: true, subtree: true }); + // A runtime loaded after the clips parsed missed them; WebKit has already begun their head reads. + if (isWebKit()) { + const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement; + for (const media of document.querySelectorAll("video, audio")) defer(media, startInFilm); + } } diff --git a/packages/core/src/runtime/proxySrc.ts b/packages/core/src/runtime/proxySrc.ts index 82f62315c90..450ef3a4c1a 100644 --- a/packages/core/src/runtime/proxySrc.ts +++ b/packages/core/src/runtime/proxySrc.ts @@ -1,8 +1,21 @@ /** Media elements swapped to their proxy, each with the src attribute it had before the swap. */ export const swappedElements = new WeakMap(); +/** Media elements the runtime opened at their in-point: the src the author wrote and the one written in its place. */ +const openedAtInPoint = new WeakMap(); + +export function recordOpenedAtInPoint(el: Element, authored: string, written: string): void { + openedAtInPoint.set(el, { authored, written }); +} + +/** True while the src is still the one the runtime wrote: a script or binding that sets another ends it. */ +export function isOpenedAtInPoint(el: Element): boolean { + return openedAtInPoint.get(el)?.written === el.getAttribute("src"); +} + export function unproxiedSrc(el: Element): string | null { - return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src"); + if (swappedElements.has(el)) return swappedElements.get(el) ?? null; + return isOpenedAtInPoint(el) ? openedAtInPoint.get(el)!.authored : el.getAttribute("src"); } /** Videos showing a still of their last frame as poster across a copy swap: the poster they had before, and when. */ diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.ts index aac2d0bd466..f37af327c14 100644 --- a/packages/core/src/runtime/timeline.ts +++ b/packages/core/src/runtime/timeline.ts @@ -4,6 +4,7 @@ import type { RuntimeTimelineScene, RuntimeTimelineLike, } from "./types"; +import { unproxiedSrc } from "./proxySrc"; import { stableClipId } from "./clipTree"; import { findRootCompositionElement, parseCompositionDimension } from "./compositionDimension"; import { @@ -166,13 +167,13 @@ function toAbsoluteAssetUrl(rawValue: string | null | undefined): string | null } function resolveNodeAssetUrl(node: Element): string | null { - const src = node.getAttribute("src") ?? node.getAttribute("data-src"); + const src = unproxiedSrc(node) ?? node.getAttribute("data-src"); if (src) return toAbsoluteAssetUrl(src); const compositionSrc = node.getAttribute("data-composition-src"); if (compositionSrc) return toAbsoluteAssetUrl(compositionSrc); const mediaDescendant = node.querySelector("img[src], video[src], audio[src], source[src]"); if (!mediaDescendant) return null; - return toAbsoluteAssetUrl(mediaDescendant.getAttribute("src")); + return toAbsoluteAssetUrl(unproxiedSrc(mediaDescendant)); } function getFirstClassToken(node: Element): string | null { diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index 1e97fbeffa5..da96a07dd2d 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -1868,6 +1868,22 @@ describe("WebAudioTransport", () => { vi.unstubAllGlobals(); }); + it("decodes a file once for clips that open at different in-points of it", async () => { + const transport = transportWithDecode(async () => ({}) as AudioBuffer); + const fetchMock = vi.fn(async () => ({ + ok: true, + arrayBuffer: async () => new ArrayBuffer(8), + })); + vi.stubGlobal("fetch", fetchMock); + await Promise.all([ + transport.decodeAudioElement(el("take.wav#t=10")), + transport.decodeAudioElement(el("take.wav#t=90")), + ]); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0]![0]).toBe(key("take.wav")); + vi.unstubAllGlobals(); + }); + it("DOES blacklist genuinely undecodable bytes — not retried", async () => { const transport = transportWithDecode(async () => { throw new Error("unsupported codec"); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index 6f9f721ef87..f404d79b620 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -210,11 +210,14 @@ function isBufferSource( return source.sourceKind === "buffer"; } +// The fragment narrowing a clip's span is not part of the file: every clip of it shares one decode. const decodeKey = (el: HTMLMediaElement): string | null => { const src = el.currentSrc || el.getAttribute("src"); if (!src) return null; try { - return assetUrl(src); + const url = new URL(assetUrl(src)); + url.hash = ""; + return url.href; } catch { return src; }