fix(core): long cut-list edits play every cut without dropped or late frames

packages/core/src/runtime/init.test.ts

@@ -612,6 +612,25 @@ describe("initSandboxRuntimeModular", () => {
       }
     });
 
+    it("leaves hidden clips alone on every jump when the page has no timeline", () => {
+      document.body.innerHTML =
+        `<div data-composition-id="main" data-root="true" data-start="0" data-duration="10" data-width="1920" data-height="1080">` +
+        `<video id="first" src="a.mp4" data-start="0" data-duration="2" data-track-index="0"></video>` +
+        `<video id="second" src="a.mp4" data-start="2" data-duration="8" data-track-index="0"></video>` +
+        `</div>`;
+      window.__timelines = {};
+      initSandboxRuntimeModular();
+      const query = vi.spyOn(document, "querySelectorAll");
+      const looked = () =>
+        query.mock.calls.filter(([selector]) => selector === "[data-hf-clip-hidden]").length;
+      try {
+        for (const t of [0.5, 1, 1.5, 3, 6]) window.__player?.seek(t);
+        expect(looked()).toBe(0);
+      } finally {
+        query.mockRestore();
+      }
+    });
+
     it("styles a hidden clip visible once while playing toward its start, far scene included", () => {
       document.body.innerHTML =
         `<div data-composition-id="main" data-root="true" data-start="0" data-duration="10" data-width="1920" data-height="1080">` +

packages/core/src/runtime/init.ts

@@ -5163,6 +5163,15 @@ export function initSandboxRuntimeModular(): void {
       return false;
     }
   };
+  const sceneTimelines = () => {
+    const timelines = (window.__timelines ?? {}) as Record<string, RuntimeTimelineLike | undefined>;
+    const rootCompositionId =
+      resolveRootCompositionElement()?.getAttribute("data-composition-id") ?? null;
+    return Object.entries(timelines).filter(
+      (entry): entry is [string, RuntimeTimelineLike] =>
+        !!entry[1] && typeof entry[1] === "object" && entry[0] !== rootCompositionId,
+    );
+  };
   const seekStandaloneRegisteredTimelines = (
     timeSeconds: number,
     revision: number,
@@ -5172,12 +5181,7 @@ export function initSandboxRuntimeModular(): void {
       sceneClocks = new WeakMap();
       sceneClocksRevision = revision;
     }
-    const timelines = (window.__timelines ?? {}) as Record<string, RuntimeTimelineLike | undefined>;
-    const rootCompositionId =
-      resolveRootCompositionElement()?.getAttribute("data-composition-id") ?? null;
-    for (const [compositionId, timeline] of Object.entries(timelines)) {
-      if (!timeline || typeof timeline !== "object" || compositionId === rootCompositionId)
-        continue;
+    for (const [compositionId, timeline] of sceneTimelines()) {
       const scene = sceneClockOf(compositionId, timeline);
       if (!scene) continue;
       const sourceTime =
@@ -5291,9 +5295,9 @@ export function initSandboxRuntimeModular(): void {
     // Only clips holding a tween that renders for the first time on this seek need their authored style read.
     const firstRendered =
       firstRenders && tl ? clipsFirstRenderedIn(tl, tweensRenderedThrough.time, t) : null;
-    // An empty set means no tween draws for the first time: nothing to show, so no page-wide query.
+    // An empty set, or a page with no timeline, means no tween draws for the first time: no page-wide query.
     const hideClipsAgain =
-      firstRenders && firstRendered?.size !== 0
+      firstRenders && firstRendered?.size !== 0 && (tl || sceneTimelines().length > 0)
         ? showHiddenClips(firstRendered ? (el) => firstRendered.has(el) : undefined)
         : null;
     // Scenes first, each paused on its own clock, so the root's pass skips them and its later

packages/core/src/runtime/media.test.ts

@@ -2070,6 +2070,83 @@ describe("syncRuntimeMedia", () => {
       expect(el.play).not.toHaveBeenCalled();
     });
 
+    describe("where the browser reports presented frames", () => {
+      // A playing 24 fps video whose presented frames the test hands to the runtime's frame callback.
+      const reportingVideo = () => {
+        const clip = videoClip({ start: 0, end: 10, mediaStart: 0 });
+        const el = clip.el as HTMLVideoElement;
+        Object.defineProperty(el, "paused", { value: false, writable: true, configurable: true });
+        el.pause = vi.fn(() =>
+          Object.defineProperty(el, "paused", { value: true, writable: true, configurable: true }),
+        );
+        const pending: VideoFrameRequestCallback[] = [];
+        el.requestVideoFrameCallback = vi.fn((callback: VideoFrameRequestCallback) => {
+          pending.push(callback);
+          return pending.length;
+        });
+        const present = (mediaTime: number) => {
+          el.currentTime = mediaTime;
+          const frame = { mediaTime } as VideoFrameCallbackMetadata;
+          pending.splice(0).forEach((callback) => callback(0, frame));
+        };
+        return { clip, el, present };
+      };
+
+      it("plays a video in step with the playhead up to the last frame before its out-point", () => {
+        const { clip, el, present } = reportingVideo();
+        el.currentTime = 9.6;
+        tick(clip, 9.6);
+        present(9.9167);
+        el.currentTime = 9.97; // within one sync tolerance of the out-point, its frame not shown yet
+        tick(clip, 9.965);
+        expect(el.pause).not.toHaveBeenCalled();
+        present(9.9583); // the next frame, at 10, is the out-point's
+        expect(el.pause).toHaveBeenCalledTimes(1);
+        tick(clip, 9.99);
+        expect(el.play).not.toHaveBeenCalled();
+      });
+
+      it("holds a video on its last frame before the out-point when that frame is dropped", () => {
+        const { clip, el, present } = reportingVideo();
+        el.currentTime = 9.6;
+        tick(clip, 9.6);
+        present(9.875);
+        present(9.9167); // the frame at 9.9583 is dropped; the next one would be the out-point's
+        expect(el.pause).not.toHaveBeenCalled();
+        el.currentTime = 9.975;
+        tick(clip, 9.975);
+        expect(el.pause).toHaveBeenCalledTimes(1);
+      });
+
+      it("holds a video with an uneven cadence before a long step crosses the out-point", () => {
+        const { clip, el, present } = reportingVideo();
+        el.currentTime = 9.6;
+        tick(clip, 9.6);
+        present(9.8);
+        present(9.8833);
+        present(9.925); // a step as long as the earlier one would land past 10
+        expect(el.pause).toHaveBeenCalledTimes(1);
+      });
+
+      it("plays a held video again once a seek takes the playhead back", () => {
+        const { clip, el, present } = reportingVideo();
+        el.currentTime = 9.6;
+        tick(clip, 9.6);
+        present(9.9167);
+        present(9.9583);
+        expect(el.pause).toHaveBeenCalledTimes(1);
+        el.play = vi.fn(() => {
+          Object.defineProperty(el, "paused", { value: false, writable: true, configurable: true });
+          return Promise.resolve();
+        });
+        tick(clip, 2);
+        expect(el.currentTime).toBe(mediaElementTime(2));
+        expect(el.play).toHaveBeenCalledTimes(1);
+        tick(clip, 2.016);
+        expect(el.pause).toHaveBeenCalledTimes(1);
+      });
+    });
+
     it("keeps a looping video playing when the playhead wraps a lap before it", () => {
       const clip = videoClip({ start: 0, end: 10, mediaStart: 0, loop: true, sourceDuration: 0.4 });
       const el = clip.el;

packages/core/src/runtime/media.ts

@@ -247,6 +247,7 @@ export function evictMediaSyncState(el: HTMLMediaElement): void {
   lastRuntimeAppliedVolume.delete(el);
   forgetSteering(el);
   heldAhead.delete(el);
+  forgetOutPoint(el);
   videoStarts.delete(el);
 }
 
@@ -319,6 +320,61 @@ const MIN_NATIVE_PLAYBACK_RATE = 1 / 16;
 const VIDEO_LEAD_HOLD_SECONDS = 0.1;
 const heldAhead = new WeakSet<HTMLMediaElement>();
 
+// Where a browser reports presented frames, a video nearing its out-point is paused on the last
+// frame before it, so a cut shows every frame up to the out-point and none past it.
+const OUT_POINT_WATCH_SECONDS = 0.5;
+const outPointWatch = new WeakMap<HTMLMediaElement, number>();
+const heldAtOutPoint = new WeakMap<HTMLMediaElement, number>();
+const lastFrameBefore = new WeakMap<HTMLMediaElement, { outPoint: number; at: number }>();
+const reportsFrames = (el: HTMLMediaElement): el is HTMLVideoElement =>
+  typeof (el as HTMLVideoElement).requestVideoFrameCallback === "function";
+
+function watchOutPoint(el: HTMLVideoElement, outPoint: number): void {
+  if (outPointWatch.get(el) === outPoint) return;
+  outPointWatch.set(el, outPoint);
+  let previous: number | null = null;
+  let shortestStep = Number.POSITIVE_INFINITY;
+  let longestStep = 0;
+  const onFrame = (_now: number, frame: VideoFrameCallbackMetadata) => {
+    if (outPointWatch.get(el) !== outPoint) return;
+    if (el.paused) {
+      outPointWatch.delete(el);
+      return;
+    }
+    // A seek between two frames is not a frame step; a dropped frame or an uneven cadence is a longer one.
+    const gap = previous === null ? Number.NaN : frame.mediaTime - previous;
+    if (gap > 0 && gap < VIDEO_LEAD_HOLD_SECONDS) {
+      shortestStep = Math.min(shortestStep, gap);
+      longestStep = Math.max(longestStep, gap);
+      const framesLeft = Math.floor((outPoint - 0.001 - frame.mediaTime) / shortestStep);
+      lastFrameBefore.set(el, { outPoint, at: frame.mediaTime + framesLeft * shortestStep });
+    }
+    previous = frame.mediaTime;
+    if (frame.mediaTime + (longestStep || MEDIA_SYNC_TOLERANCE_SECONDS) < outPoint - 0.001) {
+      el.requestVideoFrameCallback(onFrame);
+      return;
+    }
+    outPointWatch.delete(el);
+    heldAtOutPoint.set(el, outPoint);
+    heldAhead.add(el);
+    el.pause();
+  };
+  el.requestVideoFrameCallback(onFrame);
+}
+
+function forgetOutPoint(el: HTMLMediaElement): void {
+  outPointWatch.delete(el);
+  heldAtOutPoint.delete(el);
+  lastFrameBefore.delete(el);
+}
+
+/** Where a watched video is held: on the last frame before its out-point once its cadence is known. */
+function outPointHoldTime(el: HTMLMediaElement, outPoint: number, watched: boolean): number {
+  if (!watched) return outPoint - MEDIA_SYNC_TOLERANCE_SECONDS;
+  const last = lastFrameBefore.get(el);
+  return last?.outPoint === outPoint ? last.at : outPoint;
+}
+
 // A muted video's clock is its audio track, which starts tens of ms after play(), so an upcoming
 // video starts that early, hidden, by the delay learned from earlier starts.
 let videoStartLatency: number | null = null;
@@ -596,15 +652,27 @@ export function syncRuntimeMedia(params: {
       // strict and force sync and is steered by rate; hard sync (>0.5s) and audio past its steer
       // limit seek. A paused transport pauses it below, so a seek that pauses mid-playback lands it.
       if (el.tagName === "VIDEO" && !isHeldVideoTail) {
+        const outPoint = mediaSourceTimeAt(clip, clip.end);
+        const watched = reportsFrames(el);
+        if (
+          watched &&
+          playing &&
+          !clip.loop &&
+          !el.paused &&
+          outPoint - currentElTime <= OUT_POINT_WATCH_SECONDS * baseRate
+        )
+          watchOutPoint(el, outPoint);
+        const atOutPoint = heldAtOutPoint.get(el) === outPoint;
         // A loop's playhead wraps before its element, so its offset jumps by a lap: steering owns loops.
         const ahead =
           !clip.loop &&
           (offset < -VIDEO_LEAD_HOLD_SECONDS ||
-            currentElTime >= mediaSourceTimeAt(clip, clip.end) - MEDIA_SYNC_TOLERANCE_SECONDS);
+            atOutPoint ||
+            currentElTime >= outPointHoldTime(el, outPoint, watched));
         if (!playing || (!ahead && offset >= -MEDIA_SYNC_TOLERANCE_SECONDS)) heldAhead.delete(el);
-        else if (ahead && !el.paused) {
+        else if (ahead && (!el.paused || atOutPoint)) {
           heldAhead.add(el);
-          el.pause();
+          if (!el.paused) el.pause();
         }
       }
       const holdingAhead = heldAhead.has(el);
@@ -682,6 +750,7 @@ export function syncRuntimeMedia(params: {
         }
         playRequested.delete(el);
         heldAhead.delete(el);
+        forgetOutPoint(el);
         videoStarts.delete(el);
       } else if (!playing) {
         holdSeekBarrierUntilVideoLands(el);

packages/core/src/runtime/preloadMedia.test.ts

@@ -116,6 +116,36 @@ describe("releasing a clip's download", () => {
     expect(video.getAttribute("src")).toBe("a.mp4");
   });
 
+  it("keeps a clip whose whole file is buffered", () => {
+    const video = document.createElement("video");
+    video.setAttribute("src", "a.mp4");
+    Object.defineProperty(video, "duration", { value: 6.67 });
+    Object.defineProperty(video, "buffered", {
+      value: { length: 1, start: () => 0, end: () => 6.67 },
+    });
+    video.load = vi.fn();
+
+    releaseMedia(video);
+    expect(video.load).not.toHaveBeenCalled();
+    expect(video.getAttribute("src")).toBe("a.mp4");
+  });
+
+  it.each([
+    [0, 15],
+    [430, 2642],
+  ])("releases a clip with only %s-%s s of a long file buffered", (from, to) => {
+    const video = document.createElement("video");
+    video.setAttribute("src", "a.mp4");
+    Object.defineProperty(video, "duration", { value: 2642 });
+    Object.defineProperty(video, "buffered", {
+      value: { length: 1, start: () => from, end: () => to },
+    });
+    video.load = vi.fn();
+
+    releaseMedia(video);
+    expect(video.load).toHaveBeenCalledTimes(1);
+  });
+
   it("leaves a clip on <source> children, their order and its markup untouched", () => {
     const video = document.createElement("video");
     video.innerHTML =

packages/core/src/runtime/preloadMedia.ts

@@ -5,6 +5,8 @@ import { defersMedia } from "./timedClipHide";
 import { swallow } from "./diagnostics";
 import { mediaElementTime } from "./mediaClock";
 
+const MEDIA_EDGE_SECONDS = 0.1;
+
 type PreloadableMedia = Pick<
   HTMLMediaElement,
   "id" | "tagName" | "getAttribute" | "preload" | "readyState" | "networkState" | "load"
@@ -73,8 +75,19 @@ export function stopMediaDownload(media: HTMLMediaElement): void {
   media.load();
 }
 
+function wholeFileBuffered(media: HTMLMediaElement): boolean {
+  const { buffered, duration } = media;
+  return (
+    Number.isFinite(duration) &&
+    duration > 0 &&
+    buffered.length === 1 &&
+    buffered.start(0) <= MEDIA_EDGE_SECONDS &&
+    buffered.end(0) >= duration - MEDIA_EDGE_SECONDS
+  );
+}
+
 export function releaseMedia(media: HTMLMediaElement): void {
-  if (media.querySelector("source")) return;
+  if (media.querySelector("source") || wholeFileBuffered(media)) return;
   const src = media.getAttribute("src");
   stopMediaDownload(media);
   if (src !== null) media.setAttribute("src", src);