fix(core): Safari keeps playing edits' audio without seeks or rate changes

.github/workflows/player-safari-audio.yml

@@ -0,0 +1,65 @@
+name: Player Safari audio
+
+permissions:
+  contents: read
+
+env:
+  HYPERFRAMES_NO_TELEMETRY: "1"
+
+on:
+  pull_request:
+    paths:
+      - "packages/core/src/runtime/**"
+      - "packages/player/tests/playbench/**"
+      - ".github/workflows/player-safari-audio.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: player-safari-audio-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  audio:
+    name: "Audio in real Safari: no playhead stops, no seek loop"
+    runs-on: macos-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+
+      - run: bun install --frozen-lockfile
+
+      - run: |
+          bun run --filter @hyperframes/parsers build
+          bun run --filter @hyperframes/core build
+          bun run --filter @hyperframes/player build
+
+      - name: Generate media
+        run: |
+          brew install ffmpeg
+          bash packages/player/tests/playbench/make-media.sh
+
+      - name: Enable safaridriver
+        run: sudo safaridriver --enable
+
+      - name: Play both edits in Safari
+        working-directory: packages/player/tests/playbench
+        shell: bash # -o pipefail, so the tee below cannot hide a failed --assert
+        run: |
+          sw_vers
+          # a failed edit must not hide the other one's numbers
+          status=0
+          for edit in primary tentake; do
+            bun webkit-stops.ts --browser safari --edit "$edit" --runs 3 --assert | tee "safari-$edit.log" || status=1
+          done
+          exit $status
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: player-safari-audio
+          path: packages/player/tests/playbench/safari-*.log
+          if-no-files-found: warn
+          retention-days: 30

packages/core/src/runtime/clock.safariAudio.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { TransportClock } from "./clock";
 import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";
 
@@ -83,14 +83,21 @@ function playAudioClock(seconds: number) {
 }
 
 describe("an audio clock whose output starts late, as Safari's does", () => {
+  beforeEach(() => {
+    vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15",
+    );
+  });
+
   afterEach(() => {
+    vi.restoreAllMocks();
     document.body.innerHTML = "";
   });
 
   it("plays through without seeking the audio it follows", () => {
     const run = playAudioClock(10);
     expect(run.seeks).toBe(0);
     expect(run.time).toBeGreaterThan(9.5);
-    expect(Math.abs(run.time - run.audioTime)).toBeLessThan(0.1);
+    expect(Math.abs(run.time - run.audioTime)).toBeLessThan(0.65);
   });
 });

packages/core/src/runtime/mediaClock.ts

@@ -8,3 +8,8 @@ export function startsSoundOnlyInsideATap(): boolean {
   const { userAgent, maxTouchPoints } = navigator;
   return /iP(hone|ad|od)/.test(userAgent) || (/Macintosh/.test(userAgent) && maxTouchPoints > 1);
 }
+
+export function isWebKit(): boolean {
+  const { userAgent } = navigator;
+  return /AppleWebKit/.test(userAgent) && !/Chrom(e|ium)\/|Edg\/|Android/.test(userAgent);
+}

packages/core/src/runtime/mediaSteer.test.ts

@@ -1,9 +1,29 @@
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import { forgetSteering, isSteering, markMediaRestarted, steerMedia } from "./mediaSteer";
 
 const media = (tag: "audio" | "video") => document.createElement(tag);
 
 describe("steerMedia", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it("treats an embedded WKWebView, whose user agent has no Safari token, as WebKit", () => {
+    vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
+      "Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",
+    );
+    expect(steerMedia(media("audio"), 0.35, 1, true)).toEqual({ rate: 1, seek: false });
+  });
+
+  it("never steers WebKit audio: it is seeked only once its lag holds past 0.65 s", () => {
+    vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15",
+    );
+    const el = media("audio");
+    expect(steerMedia(el, 0.35, 1, true)).toEqual({ rate: 1, seek: false });
+    expect(steerMedia(el, 0.8, 1, false)).toEqual({ rate: 1, seek: false });
+    expect(steerMedia(el, 0.8, 1, true)).toEqual({ rate: 1, seek: true });
+    expect(isSteering(el)).toBe(false);
+  });
+
   it("leaves media inside the tolerance at its rate", () => {
     expect(steerMedia(media("audio"), 0.03, 1, true)).toEqual({ rate: 1, seek: false });
   });

packages/core/src/runtime/mediaSteer.ts

@@ -1,3 +1,5 @@
+import { isWebKit } from "./mediaClock";
+
 /** Drift a playing element may carry before sync pulls it back onto the playhead. */
 export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
 export const MEDIA_HARD_SYNC_SECONDS = 0.5;
@@ -12,6 +14,7 @@ const MEDIA_STEER_RELEASE_SECONDS = 0.01;
 // Audio never steers past 10%, where speech changes tempo; a lag held past 100 ms is seeked instead.
 const AUDIO_MAX_STEER = 0.1;
 const AUDIO_STEER_LIMIT_SECONDS = 0.1;
+const WEBKIT_AUDIO_START_LAG_SECONDS = 0.65;
 
 /** Direction (+1 fast, -1 slow) an element is being steered in; absent when it plays at its authored rate. */
 const steering = new WeakMap<HTMLMediaElement, number>();
@@ -31,6 +34,10 @@ export function steerMedia(
   audioOnly = el.tagName === "AUDIO",
 ): { rate: number; seek: boolean } {
   const lag = Math.abs(offset);
+  if (audioOnly && isWebKit()) {
+    steering.delete(el);
+    return { rate: baseRate, seek: settled && lag > WEBKIT_AUDIO_START_LAG_SECONDS };
+  }
   if (settled && lag <= MEDIA_SYNC_TOLERANCE_SECONDS) catchingUp.delete(el);
   const seekable = !catchingUp.has(el) || lag > MEDIA_HARD_SYNC_SECONDS;
   if (audioOnly && settled && lag > AUDIO_STEER_LIMIT_SECONDS && seekable) {

packages/core/vitest.config.ts

@@ -4,6 +4,7 @@ export default defineConfig({
   test: {
     include: ["src/**/*.test.ts"],
     environment: "jsdom",
+    setupFiles: ["./vitest.setup.ts"],
     coverage: {
       provider: "v8",
       include: ["src/runtime/**/*.ts"],

packages/core/vitest.setup.ts

@@ -0,0 +1,5 @@
+Object.defineProperty(navigator, "userAgent", {
+  configurable: true,
+  get: () =>
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36",
+});

packages/player/tests/playbench/webkit-stops.ts

@@ -1,7 +1,9 @@
 #!/usr/bin/env bun
 // Plays an edit in WebKit and counts how often the playhead stands still at the cuts, how often audio is seeked,
 // and how much of the time sounding audio stands still.
-// Flags: --edit tentake|primary, --runs 3, --browser webkit|chromium|manual, --port 8099.
+// Flags: --edit tentake|primary, --runs 3, --browser webkit|chromium|safari|manual, --port 8099, --assert.
+// safari drives the real Safari through safaridriver (`sudo safaridriver --enable`, macOS only). --assert exits 1
+// unless every run has at most MAX_AUDIO_SEEKS audio seeks and under MAX_FROZEN_PCT frozen audio.
 // webkit drives Playwright's WebKit (`playwright-core` and `playwright-core install webkit`); manual only serves
 // /host.html and waits for a real Safari to open it, click Play and post the result.
 import { existsSync } from "node:fs";
@@ -166,8 +168,57 @@ async function drivePlaywright(engine: "webkit" | "chromium") {
   await browser.close();
 }
 
+async function driveSafari() {
+  const base = "http://127.0.0.1:4455";
+  const driver = Bun.spawn(["safaridriver", "-p", "4455"], {
+    stdout: "inherit",
+    stderr: "inherit",
+  });
+  const wd = async (method: string, path: string, body?: unknown) => {
+    const res = await fetch(base + path, {
+      method,
+      headers: { "Content-Type": "application/json" },
+      body: body === undefined ? undefined : JSON.stringify(body),
+    });
+    const json = (await res.json()) as { value: any }; // oxlint-disable-line no-explicit-any
+    if (json.value?.error) throw new Error(`${path}: ${json.value.message}`);
+    return json.value;
+  };
+  for (
+    let i = 0;
+    i < 50 &&
+    !(await fetch(`${base}/status`).then(
+      (r) => r.ok,
+      () => false,
+    ));
+    i++
+  )
+    await Bun.sleep(200);
+  try {
+    for (let i = 0; i < RUNS; i++) {
+      const { sessionId } = await wd("POST", "/session", {
+        capabilities: { alwaysMatch: { browserName: "safari" } },
+      });
+      const s = `/session/${sessionId}`;
+      try {
+        await wd("POST", `${s}/url`, { url: `${origin}/host.html` });
+        await Bun.sleep(8000); // media load
+        const done = new Promise<void>((r) => (resolveResult = r));
+        const btn = await wd("POST", `${s}/element`, { using: "css selector", value: "#go" });
+        await wd("POST", `${s}/element/${Object.values(btn)[0]}/click`, {});
+        await done;
+      } finally {
+        await wd("DELETE", s).catch(() => {});
+      }
+    }
+  } finally {
+    driver.kill();
+  }
+}
+
 const engine = arg("browser", "webkit");
 if (engine === "webkit" || engine === "chromium") await drivePlaywright(engine);
+else if (engine === "safari") await driveSafari();
 else {
   console.log(`open ${origin}/host.html in Safari, click Play (${RUNS} runs)`);
   while (results.length < RUNS) await new Promise<void>((r) => (resolveResult = r));
@@ -184,4 +235,14 @@ for (const [i, r] of results.entries()) {
   }
 }
 server.stop(true);
+const MAX_AUDIO_SEEKS = 15; // the loop made 50 to 190, a clean play 1 to 6
+const MAX_FROZEN_PCT = 15; // the loop froze 38 to 99%, a clean play 2 to 10%
+const bad = results.filter((r) => {
+  const m = measure(r, cuts);
+  return m.seeks > MAX_AUDIO_SEEKS || m.audioFrozenPct >= MAX_FROZEN_PCT;
+});
+if (process.argv.includes("--assert") && bad.length) {
+  console.error(`${bad.length} of ${results.length} runs looped on audio seeks or froze the sound`);
+  process.exit(1);
+}
 process.exit(0);

packages/player/vitest.config.ts

@@ -26,7 +26,15 @@ export default defineConfig({
   test: {
     environment: "happy-dom",
     environmentOptions: {
-      happyDOM: { settings: { handleDisabledFileLoadingAsSuccess: true } },
+      happyDOM: {
+        settings: {
+          handleDisabledFileLoadingAsSuccess: true,
+          navigator: {
+            userAgent:
+              "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.0.0 Safari/537.36",
+          },
+        },
+      },
     },
     setupFiles: ["./src/slideshow/test-setup.ts"],
   },