perf(core): Safari plays a long source's audio from its element, not a whole-file decode

packages/core/src/runtime/init.ts

@@ -176,7 +176,7 @@ import {
   forgetPositionEdit,
   installPositionEditsSeekReapply,
 } from "./positionEdits";
-import { stillStandsIn, unheldPoster, unproxiedSrc } from "./proxySrc";
+import { isOpenedAtInPoint, stillStandsIn, unheldPoster, unproxiedSrc } from "./proxySrc";
 import { applyVariableBindings, unproxiedMediaSrc } from "./applyVariableBindings";
 import { createColorGradingRuntime, type RuntimeColorGradingApi } from "./colorGrading";
 import { COLOR_GRADING_AUTHORED_OPACITY_ATTR } from "../colorGrading";
@@ -517,12 +517,16 @@ const joinsWebAudio = (el: Element): el is HTMLMediaElement =>
     }));
 const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";
 
+const DECODE_FIRST_MAX_SECONDS = 120;
+
 /** WebKit restarts a media element's audio 150 to 400 ms late; a decoded buffer on the context clock starts on time. */
 function decodesFirst(el: HTMLMediaElement, webAudio: WebAudioTransport): boolean {
   return (
     isWebKit() &&
     isAudioElement(el) &&
     !el.loop &&
+    // A long source's whole-file fetch takes the link from the picture; its audio plays from the element.
+    el.duration <= DECODE_FIRST_MAX_SECONDS &&
     webAudio.fitsDecodeBudget(el) &&
     typeof readElementRateSpec(el) === "number" &&
     readElementPlaybackRate(el) === 1
@@ -1314,7 +1318,7 @@ export function initSandboxRuntimeModular(): void {
       ? {
           from: "media",
           tag: media.tagName.toLowerCase(),
-          src: media.getAttribute("src") ?? media.currentSrc ?? "",
+          src: unproxiedSrc(media) ?? media.currentSrc ?? "",
           trimmed: media.hasAttribute("data-duration") || media.hasAttribute("data-end"),
         }
       : { from };
@@ -2530,7 +2534,7 @@ export function initSandboxRuntimeModular(): void {
         }
         const tagName = node.tagName.toLowerCase();
         const assetUrl =
-          node.getAttribute("src") ??
+          unproxiedSrc(node) ??
           node.getAttribute("href") ??
           node.getAttribute("poster") ??
           null;
@@ -3053,7 +3057,7 @@ export function initSandboxRuntimeModular(): void {
   const NEXT_TAKE_SPAN_SECONDS = 0.5;
   let nextTakes: { t: number; bound: number; clips: Set<HTMLMediaElement> } | null = null;
   const fileOf = (el: HTMLMediaElement) =>
-    el.getAttribute("src") ?? el.querySelector("source")?.getAttribute("src") ?? "";
+    unproxiedSrc(el) ?? el.querySelector("source")?.getAttribute("src") ?? "";
   const nextTakeClips = (t: number): Set<HTMLMediaElement> => {
     if (nextTakes?.t === t && nextTakes.bound === metadataBoundMedia.size) return nextTakes.clips;
     const byFile = new Map<string, { el: HTMLMediaElement; start: number }[]>();
@@ -3116,7 +3120,7 @@ export function initSandboxRuntimeModular(): void {
   const hostOf = (el: Element) => el.parentElement?.closest("[data-composition-id]");
   // Clips due soon fetch a few at a time, earliest first: a fast cut sequence has dozens inside the
   // look-ahead, and fetching them all queues the clip that is due behind clips that are not.
-  const PREFETCH_SLOTS = 3;
+  const PREFETCH_SLOTS = 8;
   const prefetching = new Set<HTMLMediaElement>();
   type WaitingPrefetch = { element: HTMLMediaElement; start: number };
   const waitingForSlot = new Map<HTMLMediaElement, WaitingPrefetch>();
@@ -3475,9 +3479,11 @@ export function initSandboxRuntimeModular(): void {
           rawNode,
           isVisibleNow,
           loadsAhead &&
-            (playerFrame
-              ? nextTakeClips(currentTime).has(rawNode)
-              : upcoming || mediaStaysDue(rawNode, visibleAt, currentTime)),
+            // A clip opened at its in-point reads little ahead of it: every one is fetched ahead, not just the next take.
+            (isOpenedAtInPoint(rawNode) ||
+              (playerFrame
+                ? nextTakeClips(currentTime).has(rawNode)
+                : upcoming || mediaStaysDue(rawNode, visibleAt, currentTime))),
           currentTime,
           loadsAhead && dueAtLoopStart(rawNode, visibleAt),
         );

packages/core/src/runtime/mediaLoadFailure.test.ts

@@ -2,6 +2,7 @@
 import { postRuntimeMessage } from "./bridge";
 import { watchAudioLoadErrors } from "./mediaLoadFailure";
 import { RUNTIME_MEDIA_LOAD_FAILED } from "./protocol";
+import { recordOpenedAtInPoint } from "./proxySrc";
 import { WebAudioTransport } from "./webAudioTransport";
 
 vi.mock("./bridge", () => ({ postRuntimeMessage: vi.fn() }));
@@ -37,6 +38,15 @@
 });
 
 describe("a preview media load that fails is reported, not swallowed", () => {
+  it("names the src the author wrote, not the in-point fragment the runtime added", async () => {
+    vi.stubGlobal("fetch", vi.fn(async () => ({ ok: false, status: 404 })));
+    const el = audio("assets/take.wav", "take");
+    recordOpenedAtInPoint(el, "assets/take.wav", "assets/take.wav#t=30");
+    el.setAttribute("src", "assets/take.wav#t=30");
+    await transport(async () => ({}) as AudioBuffer).decodeAudioElement(el);
+    expect(failures()[0]).toMatchObject({ asset: "assets/take.wav" });
+  });
+
   it("a fetch the server refuses says its status, once however often playback retries", async () => {
     vi.stubGlobal(
       "fetch",

packages/core/src/runtime/mediaProxy.ts

@@ -10,7 +10,7 @@ import {
 import { findInjectedRenderFrame } from "./renderFrameSibling";
 import type { RuntimeJson } from "./types";
 import { isVideoElement } from "./domRealm";
-import { heldPosters, swappedElements } from "./proxySrc";
+import { heldPosters, swappedElements, unproxiedSrc } from "./proxySrc";
 import { restartPlaybackHealth } from "./playbackHealth";
 import { failedCopyElements, noteCopy, setCopyReportingLevel } from "./previewCopies";
 import { waitForServedProxy } from "./proxyWait";
@@ -523,7 +523,7 @@ async function makeCopy(el: HTMLMediaElement, job: CopyJob, scale: number | null
     return;
   }
   noteCopy(el, "ready", null, light);
-  swappedElements.set(el, originalAttr);
+  swappedElements.set(el, unproxiedSrc(el));
   if (lightFacts) lightElements.add(el);
   loadProxy(el, proxiedSrc, originalSrc, variant, box, lightFacts !== undefined);
   // The shown size may have grown while this copy was being made.

packages/core/src/runtime/picker.ts

@@ -3,6 +3,7 @@ import { PREVIEW_VIDEO_FRAME_ATTR } from "../studioPreviewMark";
 import { COLOR_GRADING_SOURCE_HIDDEN_ATTR } from "../colorGrading";
 import { swallow } from "./diagnostics";
 import { isElementNode } from "./domRealm";
+import { unproxiedSrc } from "./proxySrc";
 import { createDrawnProbe } from "./pickerDrawn";
 
 type PickerModuleDeps = {
@@ -311,7 +312,7 @@ export function createPickerModule(deps: PickerModuleDeps): PickerModule {
       label: buildElementLabel(el),
       boundingBox: { x: rect.left, y: rect.top, width: rect.width, height: rect.height },
       textContent: el.textContent ? el.textContent.trim().slice(0, 200) : null,
-      src: el.getAttribute("src") || el.getAttribute("data-composition-src") || null,
+      src: unproxiedSrc(el) || el.getAttribute("data-composition-src") || null,
       dataAttributes,
     };
   }

packages/core/src/runtime/preloadMedia.test.ts

@@ -1,6 +1,8 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { mediaElementTime } from "./mediaClock";
+import { unproxiedSrc } from "./proxySrc";
 import {
+  deferMediaUntilDue,
   isRenderFrameVideo,
   preloadMedia,
   prepareUpcomingMedia,
@@ -261,3 +263,90 @@ describe("preparing upcoming video playback", () => {
     element.remove();
   });
 });
+
+describe("a clip opening at its in-point on Safari", () => {
+  const SAFARI =
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1";
+  const CHROME = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0 Safari/537.36";
+  let meta: HTMLMetaElement;
+  const inPlayer = (on: boolean) => {
+    const frame = document.createElement("iframe");
+    if (on) frame.setAttribute("data-hf-player-frame", "");
+    Object.defineProperty(window, "frameElement", { value: frame, configurable: true });
+  };
+  beforeEach(() => {
+    meta = document.createElement("meta");
+    meta.name = "hyperframes-studio-preview";
+    document.head.appendChild(meta);
+    vi.spyOn(navigator, "userAgent", "get").mockReturnValue(SAFARI);
+    inPlayer(true);
+    deferMediaUntilDue();
+  });
+  afterEach(() => {
+    (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect();
+    vi.restoreAllMocks();
+    Reflect.deleteProperty(window, "frameElement");
+    meta.remove();
+    document.body.replaceChildren();
+  });
+
+  async function added(html: string): Promise<HTMLVideoElement> {
+    document.body.insertAdjacentHTML("beforeend", html);
+    await Promise.resolve();
+    return document.body.lastElementChild as HTMLVideoElement;
+  }
+
+  it("gets a start fragment and keeps its authored src for everything that reads it", async () => {
+    const v = await added(
+      '<video src="/m/film.mp4" data-start="4" data-duration="2" data-media-start="30"></video>',
+    );
+    expect(v.getAttribute("src")).toBe("/m/film.mp4#t=30");
+    expect(unproxiedSrc(v)).toBe("/m/film.mp4");
+  });
+
+  it("stops reporting the authored src once something else sets the src", async () => {
+    const v = await added(
+      '<video src="/m/film.mp4" data-start="4" data-duration="2" data-media-start="30"></video>',
+    );
+    v.setAttribute("src", "/m/other.mp4");
+    expect(unproxiedSrc(v)).toBe("/m/other.mp4");
+  });
+
+  it("does so for a clip already parsed when the runtime arrives", () => {
+    (window as Window & { __hfMediaDeferral?: MutationObserver }).__hfMediaDeferral?.disconnect();
+    document.body.insertAdjacentHTML(
+      "beforeend",
+      '<audio src="/m/film.mp4" data-start="4" data-duration="2" data-media-start="30"></audio>',
+    );
+    deferMediaUntilDue();
+    expect(document.body.lastElementChild!.getAttribute("src")).toBe("/m/film.mp4#t=30");
+  });
+
+  it.each([
+    ["starts at the head of the file", "/m/film.mp4", 'data-media-start="0"'],
+    ["loops", "/m/film.mp4", 'data-media-start="30" loop'],
+    ["already carries a fragment", "/m/film.mp4#t=1", 'data-media-start="30"'],
+  ])("is left alone when it %s", async (_name, src, attrs) => {
+    const v = await added(
+      `<video src="${src}" data-start="4" data-duration="2" ${attrs}></video>`,
+    );
+    expect(v.getAttribute("src")).toBe(src);
+    expect(unproxiedSrc(v)).toBe(src);
+  });
+
+  it("is left alone in Studio's preview, which copies and saves what it reads from the page", async () => {
+    inPlayer(false);
+    const v = await added(
+      '<video src="/m/film.mp4" data-start="4" data-duration="2" data-media-start="30"></video>',
+    );
+    expect(v.getAttribute("src")).toBe("/m/film.mp4");
+  });
+
+  it("is left alone off Safari", async () => {
+    vi.spyOn(navigator, "userAgent", "get").mockReturnValue(CHROME);
+    const v = await added(
+      '<video src="/m/film.mp4" data-start="4" data-duration="2" data-media-start="30"></video>',
+    );
+    expect(v.getAttribute("src")).toBe("/m/film.mp4");
+  });
+});

packages/core/src/runtime/preloadMedia.ts

@@ -1,7 +1,9 @@
 import { parseStrictFiniteTimingNumber, readMediaStart } from "./playbackRate";
+import { isOpenedAtInPoint, recordOpenedAtInPoint } from "./proxySrc";
 import { createRuntimeStartTimeResolver } from "./startResolver";
 import { readMediaRenderId } from "./renderFrameSibling";
 import { defersMedia } from "./timedClipHide";
+import { inPlayerFrame } from "../studioPreviewMark";
 import { swallow } from "./diagnostics";
 import { isWebKit, mediaElementTime } from "./mediaClock";
 
@@ -97,6 +99,21 @@ export function lengthIsAuthored(media: Element): boolean {
   return parseStrictFiniteTimingNumber(media.getAttribute("data-duration")) != null;
 }
 
+/** WebKit reads about 40 MB from the head of the file for every fresh element however late its in-point
+ * seek lands; a start fragment makes it open at the in-point, so many clips of one long file stop pulling
+ * many times its size, and each is cheap enough to fetch ahead. */
+export function openAtInPoint(el: HTMLMediaElement): void {
+  const src = el.getAttribute("src");
+  const start = readMediaStart(el);
+  // Only the player: Studio copies and saves what it reads from the page, and must never see the fragment.
+  if (!inPlayerFrame(document) || !src || src.includes("#") || el.loop) return;
+  if (el.querySelector("source")) return;
+  if (!(start > 0)) return;
+  const written = `${src}#t=${start}`;
+  recordOpenedAtInPoint(el, src, written);
+  el.setAttribute("src", written);
+}
+
 type DeferralWindow = Window & { __hfMediaDeferral?: MutationObserver };
 
 /** Preview only: later clips parse at preload none, as Chromium ignores a none set mid-fetch.
@@ -108,8 +125,10 @@ export function deferMediaUntilDue(): void {
   if (!defersMedia()) return;
   const defer = (el: Element, startInFilm: (el: Element) => number) => {
     const start = parseStrictFiniteTimingNumber(el.getAttribute("data-start"));
-    if (start != null && lengthIsAuthored(el) && startInFilm(el) > 0)
-      (el as HTMLMediaElement).preload = "none";
+    if (start == null || !lengthIsAuthored(el)) return;
+    const media = el as HTMLMediaElement;
+    openAtInPoint(media);
+    if (startInFilm(el) > 0 && !isOpenedAtInPoint(media)) media.preload = "none";
   };
   win.__hfMediaDeferral = new MutationObserver((records) => {
     const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement;
@@ -122,4 +141,9 @@ export function deferMediaUntilDue(): void {
       }
   });
   win.__hfMediaDeferral.observe(document.documentElement, { childList: true, subtree: true });
+  // A runtime loaded after the clips parsed missed them; WebKit has already begun their head reads.
+  if (isWebKit()) {
+    const startInFilm = createRuntimeStartTimeResolver({}).resolveMediaStartForElement;
+    for (const media of document.querySelectorAll("video, audio")) defer(media, startInFilm);
+  }
 }

packages/core/src/runtime/proxySrc.ts

@@ -1,8 +1,21 @@
 /** Media elements swapped to their proxy, each with the src attribute it had before the swap. */
 export const swappedElements = new WeakMap<Element, string | null>();
 
+/** Media elements the runtime opened at their in-point: the src the author wrote and the one written in its place. */
+const openedAtInPoint = new WeakMap<Element, { authored: string; written: string }>();
+
+export function recordOpenedAtInPoint(el: Element, authored: string, written: string): void {
+  openedAtInPoint.set(el, { authored, written });
+}
+
+/** True while the src is still the one the runtime wrote: a script or binding that sets another ends it. */
+export function isOpenedAtInPoint(el: Element): boolean {
+  return openedAtInPoint.get(el)?.written === el.getAttribute("src");
+}
+
 export function unproxiedSrc(el: Element): string | null {
-  return swappedElements.has(el) ? (swappedElements.get(el) ?? null) : el.getAttribute("src");
+  if (swappedElements.has(el)) return swappedElements.get(el) ?? null;
+  return isOpenedAtInPoint(el) ? openedAtInPoint.get(el)!.authored : el.getAttribute("src");
 }
 
 /** Videos showing a still of their last frame as poster across a copy swap: the poster they had before, and when. */

packages/core/src/runtime/timeline.ts

@@ -4,6 +4,7 @@ import type {
   RuntimeTimelineScene,
   RuntimeTimelineLike,
 } from "./types";
+import { unproxiedSrc } from "./proxySrc";
 import { stableClipId } from "./clipTree";
 import { findRootCompositionElement, parseCompositionDimension } from "./compositionDimension";
 import {
@@ -166,13 +167,13 @@ function toAbsoluteAssetUrl(rawValue: string | null | undefined): string | null
 }
 
 function resolveNodeAssetUrl(node: Element): string | null {
-  const src = node.getAttribute("src") ?? node.getAttribute("data-src");
+  const src = unproxiedSrc(node) ?? node.getAttribute("data-src");
   if (src) return toAbsoluteAssetUrl(src);
   const compositionSrc = node.getAttribute("data-composition-src");
   if (compositionSrc) return toAbsoluteAssetUrl(compositionSrc);
   const mediaDescendant = node.querySelector("img[src], video[src], audio[src], source[src]");
   if (!mediaDescendant) return null;
-  return toAbsoluteAssetUrl(mediaDescendant.getAttribute("src"));
+  return toAbsoluteAssetUrl(unproxiedSrc(mediaDescendant));
 }
 
 function getFirstClassToken(node: Element): string | null {

packages/core/src/runtime/webAudioTransport.test.ts

@@ -1868,6 +1868,22 @@ describe("WebAudioTransport", () => {
       vi.unstubAllGlobals();
     });
 
+    it("decodes a file once for clips that open at different in-points of it", async () => {
+      const transport = transportWithDecode(async () => ({}) as AudioBuffer);
+      const fetchMock = vi.fn(async () => ({
+        ok: true,
+        arrayBuffer: async () => new ArrayBuffer(8),
+      }));
+      vi.stubGlobal("fetch", fetchMock);
+      await Promise.all([
+        transport.decodeAudioElement(el("take.wav#t=10")),
+        transport.decodeAudioElement(el("take.wav#t=90")),
+      ]);
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+      expect(fetchMock.mock.calls[0]![0]).toBe(key("take.wav"));
+      vi.unstubAllGlobals();
+    });
+
     it("DOES blacklist genuinely undecodable bytes — not retried", async () => {
       const transport = transportWithDecode(async () => {
         throw new Error("unsupported codec");

packages/core/src/runtime/webAudioTransport.ts

@@ -210,11 +210,14 @@ function isBufferSource(
   return source.sourceKind === "buffer";
 }
 
+// The fragment narrowing a clip's span is not part of the file: every clip of it shares one decode.
 const decodeKey = (el: HTMLMediaElement): string | null => {
   const src = el.currentSrc || el.getAttribute("src");
   if (!src) return null;
   try {
-    return assetUrl(src);
+    const url = new URL(assetUrl(src));
+    url.hash = "";
+    return url.href;
   } catch {
     return src;
   }