feat(core): WebKit plays bare audio from a decoded buffer on the audio clock

packages/core/src/runtime/init.test.ts

@@ -5923,6 +5923,57 @@ describe("initSandboxRuntimeModular", () => {
   // returns a node that outputs silence per the Web Audio spec rather than
   // throwing, so the composition played through with visuals animating and no
   // sound, and nothing was logged.
+  describe("bare audio in WebKit", () => {
+    useMockAudioContext();
+
+    const SAFARI =
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15";
+
+    const startRouted = async (userAgent?: string) => {
+      if (userAgent) vi.spyOn(navigator, "userAgent", "get").mockReturnValue(userAgent);
+      const audio = mountAudio("/assets/track.mp3");
+      Object.defineProperty(audio, "duration", { value: 10 });
+      const capture = vi
+        .spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback")
+        .mockResolvedValue({ el: audio } as never);
+      vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement").mockResolvedValue(
+        {} as AudioBuffer,
+      );
+      const schedule = vi
+        .spyOn(WebAudioTransport.prototype, "schedulePlayback")
+        .mockResolvedValue({} as never);
+      const release = vi.spyOn(WebAudioTransport.prototype, "releaseRouted");
+      await startPlayback();
+      return { audio, schedule, release, capture };
+    };
+
+    it("hands the sound to a decoded buffer on the context clock once it is decoded", async () => {
+      const { audio, schedule, release } = await startRouted(SAFARI);
+      await vi.waitFor(() => expect(release).toHaveBeenCalledWith(audio));
+      expect(schedule).toHaveBeenCalledTimes(1);
+    });
+
+    it("returns a handed-over track to its element when the speed changes, to hold its pitch", async () => {
+      const { audio, schedule, release, capture } = await startRouted(SAFARI);
+      await vi.waitFor(() => expect(release).toHaveBeenCalledWith(audio));
+      vi.spyOn(WebAudioTransport.prototype, "hasHandedOverSources").mockReturnValue(true);
+      schedule.mockClear();
+      capture.mockClear();
+
+      setRate(1.5);
+
+      await vi.waitFor(() => expect(capture).toHaveBeenCalled());
+      expect(schedule).not.toHaveBeenCalled();
+    });
+
+    it("leaves the element on its route elsewhere", async () => {
+      const { schedule, release } = await startRouted();
+      await Promise.resolve();
+      expect(schedule).not.toHaveBeenCalled();
+      expect(release).not.toHaveBeenCalled();
+    });
+  });
+
   describe("cross-origin audio without a CORS opt-in", () => {
     useMockAudioContext();
 

packages/core/src/runtime/init.ts

@@ -31,7 +31,7 @@ import { copiesReady } from "./previewCopies";
 import { installRuntimeControlBridge, postRuntimeMessage, setRuntimeProtocolFps } from "./bridge";
 import { instantTolerance } from "../clipFacts";
 import { isInClipWindow } from "./clipWindow";
-import { mediaElementTime } from "./mediaClock";
+import { isWebKit, mediaElementTime } from "./mediaClock";
 import {
   CLIP_HIDDEN_ATTR,
   revealTimedClipsAfterFirstPass,
@@ -477,6 +477,19 @@ const joinsWebAudio = (el: Element): el is HTMLMediaElement =>
       masterChain: readMasterFxChain(el.ownerDocument),
     }));
 const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";
+const DECODE_FIRST_MAX_SECONDS = 300;
+
+/** WebKit restarts a media element's audio 150 to 400 ms late; a decoded buffer on the context clock starts on time. */
+function decodesFirst(el: HTMLMediaElement): boolean {
+  return (
+    isWebKit() &&
+    isAudioElement(el) &&
+    !el.loop &&
+    el.duration <= DECODE_FIRST_MAX_SECONDS &&
+    typeof readElementRateSpec(el) === "number" &&
+    readElementPlaybackRate(el) === 1
+  );
+}
 const webAudioMediaIn = (root: ParentNode): HTMLMediaElement[] =>
   Array.from(root.querySelectorAll(WEB_AUDIO_MEDIA)).filter(joinsWebAudio);
 
@@ -5973,6 +5986,25 @@ export function initSandboxRuntimeModular(): void {
           : Promise.resolve(null);
       void capture.then((scheduled) => {
         const replacedByNewerPass = gen !== webAudio.currentGeneration();
+        if (scheduled && state.playbackRate === 1 && decodesFirst(rawEl)) {
+          void webAudio.decodeAudioElement(rawEl).then(async (buffer) => {
+            if (!buffer || !clock.isPlaying() || gen !== webAudio.currentGeneration()) return;
+            const played = await webAudio.schedulePlayback(
+              rawEl,
+              buffer,
+              compStart,
+              mediaStart,
+              () => clock.now(),
+              vol,
+              gen,
+              state.playbackRate,
+              clipDuration,
+              kept.start,
+            );
+            if (played) webAudio.releaseRouted(rawEl);
+          });
+          return;
+        }
         // A video's picture must keep playing from the element, so it has no decode fallback.
         if (scheduled || !isAudioElement(rawEl) || !clock.isPlaying() || replacedByNewerPass)
           return;
@@ -6032,7 +6064,7 @@ export function initSandboxRuntimeModular(): void {
       !state.webAudioMediaDisabled &&
       webAudioReady &&
       clock.isPlaying() &&
-      webAudio.hasBoundedActiveSources()
+      (webAudio.hasBoundedActiveSources() || webAudio.hasHandedOverSources())
     ) {
       webAudio.stopAll();
       scheduleWebAudioForActiveClips();

packages/core/src/runtime/webAudioTransport.ts

@@ -236,6 +236,7 @@ export class WebAudioTransport {
   private _decoding = new Map<string, Promise<AudioBuffer | null>>();
   private _failedSrcs = new Set<string>();
   private _mediaElementSources = new WeakMap<HTMLMediaElement, MediaElementAudioSourceNode>();
+  private _handedOver = new WeakSet<HTMLMediaElement>();
   private _activeSources: ScheduledSource[] = [];
   private _masterGain: GainNode | null = null;
   /** Where the master bus chain lands; the program meter taps here. */
@@ -1021,6 +1022,20 @@ export class WebAudioTransport {
     }
   }
 
+  /** Drops the routed element's source once a decoded buffer plays in its place; the element then sounds natively. */
+  releaseRouted(el: HTMLMediaElement): void {
+    const routed = this._activeSources.find((s) => s.el === el && s.sourceKind === "media-element");
+    if (!routed) return;
+    this._activeSources.splice(this._activeSources.indexOf(routed), 1);
+    this.dispose(routed);
+    this._handedOver.add(el);
+  }
+
+  /** A decoded buffer plays what an element carried: it cannot hold pitch through a speed change. */
+  hasHandedOverSources(): boolean {
+    return this._activeSources.some((s) => isBufferSource(s) && this._handedOver.has(s.el));
+  }
+
   /**
    * Rebases the composition-time reference frame before swapping rate so
    * `getTime()` stays continuous across the change. Sources scheduled to