From fbf69b98576b64a477adb6f9b45409163ce89520 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 29 Oct 2026 22:27:56 -0400 Subject: [PATCH 1/6] fix(core): a video plays to the last frame before its out-point instead of stopping a frame early --- packages/core/src/runtime/media.test.ts | 55 +++++++++++++++++++++++ packages/core/src/runtime/media.ts | 59 +++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts index 145f1d12244..15c0298092f 100644 --- a/packages/core/src/runtime/media.test.ts +++ b/packages/core/src/runtime/media.test.ts @@ -2070,6 +2070,61 @@ describe("syncRuntimeMedia", () => { expect(el.play).not.toHaveBeenCalled(); }); + describe("where the browser reports presented frames", () => { + // A playing 24 fps video whose presented frames the test hands to the runtime's frame callback. + const reportingVideo = () => { + const clip = videoClip({ start: 0, end: 10, mediaStart: 0 }); + const el = clip.el as HTMLVideoElement; + Object.defineProperty(el, "paused", { value: false, writable: true, configurable: true }); + el.pause = vi.fn(() => + Object.defineProperty(el, "paused", { value: true, writable: true, configurable: true }), + ); + const pending: VideoFrameRequestCallback[] = []; + el.requestVideoFrameCallback = vi.fn((callback: VideoFrameRequestCallback) => { + pending.push(callback); + return pending.length; + }); + const present = (mediaTime: number) => { + el.currentTime = mediaTime; + const frame = { mediaTime } as VideoFrameCallbackMetadata; + pending.splice(0).forEach((callback) => callback(0, frame)); + }; + return { clip, el, present }; + }; + + it("plays a video in step with the playhead up to the last frame before its out-point", () => { + const { clip, el, present } = reportingVideo(); + el.currentTime = 9.6; + tick(clip, 9.6); + present(9.9167); + el.currentTime = 9.97; // within one sync tolerance of the out-point, its frame not shown yet + tick(clip, 9.965); + expect(el.pause).not.toHaveBeenCalled(); + present(9.9583); // the next frame, at 10, is the out-point's + expect(el.pause).toHaveBeenCalledTimes(1); + tick(clip, 9.99); + expect(el.play).not.toHaveBeenCalled(); + }); + + it("plays a held video again once a seek takes the playhead back", () => { + const { clip, el, present } = reportingVideo(); + el.currentTime = 9.6; + tick(clip, 9.6); + present(9.9167); + present(9.9583); + expect(el.pause).toHaveBeenCalledTimes(1); + el.play = vi.fn(() => { + Object.defineProperty(el, "paused", { value: false, writable: true, configurable: true }); + return Promise.resolve(); + }); + tick(clip, 2); + expect(el.currentTime).toBe(mediaElementTime(2)); + expect(el.play).toHaveBeenCalledTimes(1); + tick(clip, 2.016); + expect(el.pause).toHaveBeenCalledTimes(1); + }); + }); + it("keeps a looping video playing when the playhead wraps a lap before it", () => { const clip = videoClip({ start: 0, end: 10, mediaStart: 0, loop: true, sourceDuration: 0.4 }); const el = clip.el; diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index bc9b33be1fb..78750ac6911 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -247,6 +247,7 @@ export function evictMediaSyncState(el: HTMLMediaElement): void { lastRuntimeAppliedVolume.delete(el); forgetSteering(el); heldAhead.delete(el); + forgetOutPoint(el); videoStarts.delete(el); } @@ -319,6 +320,45 @@ const MIN_NATIVE_PLAYBACK_RATE = 1 / 16; const VIDEO_LEAD_HOLD_SECONDS = 0.1; const heldAhead = new WeakSet(); +// Where a browser reports presented frames, a video nearing its out-point is paused on the last +// frame before it, so a cut shows every frame up to the out-point and none past it. +const OUT_POINT_WATCH_SECONDS = 0.5; +const outPointWatch = new WeakMap(); +const heldAtOutPoint = new WeakMap(); +const reportsFrames = (el: HTMLMediaElement): el is HTMLVideoElement => + typeof (el as HTMLVideoElement).requestVideoFrameCallback === "function"; + +function watchOutPoint(el: HTMLVideoElement, outPoint: number): void { + if (outPointWatch.get(el) === outPoint) return; + outPointWatch.set(el, outPoint); + let previous: number | null = null; + const onFrame = (_now: number, frame: VideoFrameCallbackMetadata) => { + if (outPointWatch.get(el) !== outPoint) return; + if (el.paused) { + outPointWatch.delete(el); + return; + } + // A seek between two frames is not a frame step. + const gap = previous === null ? Number.NaN : frame.mediaTime - previous; + const step = gap > 0 && gap < VIDEO_LEAD_HOLD_SECONDS ? gap : MEDIA_SYNC_TOLERANCE_SECONDS; + previous = frame.mediaTime; + if (frame.mediaTime + step < outPoint - 0.001) { + el.requestVideoFrameCallback(onFrame); + return; + } + outPointWatch.delete(el); + heldAtOutPoint.set(el, outPoint); + heldAhead.add(el); + el.pause(); + }; + el.requestVideoFrameCallback(onFrame); +} + +function forgetOutPoint(el: HTMLMediaElement): void { + outPointWatch.delete(el); + heldAtOutPoint.delete(el); +} + // A muted video's clock is its audio track, which starts tens of ms after play(), so an upcoming // video starts that early, hidden, by the delay learned from earlier starts. let videoStartLatency: number | null = null; @@ -596,15 +636,27 @@ export function syncRuntimeMedia(params: { // strict and force sync and is steered by rate; hard sync (>0.5s) and audio past its steer // limit seek. A paused transport pauses it below, so a seek that pauses mid-playback lands it. if (el.tagName === "VIDEO" && !isHeldVideoTail) { + const outPoint = mediaSourceTimeAt(clip, clip.end); + const watched = reportsFrames(el); + if ( + watched && + playing && + !clip.loop && + !el.paused && + outPoint - currentElTime <= OUT_POINT_WATCH_SECONDS * baseRate + ) + watchOutPoint(el, outPoint); + const atOutPoint = heldAtOutPoint.get(el) === outPoint; // A loop's playhead wraps before its element, so its offset jumps by a lap: steering owns loops. const ahead = !clip.loop && (offset < -VIDEO_LEAD_HOLD_SECONDS || - currentElTime >= mediaSourceTimeAt(clip, clip.end) - MEDIA_SYNC_TOLERANCE_SECONDS); + atOutPoint || + currentElTime >= outPoint - (watched ? 0 : MEDIA_SYNC_TOLERANCE_SECONDS)); if (!playing || (!ahead && offset >= -MEDIA_SYNC_TOLERANCE_SECONDS)) heldAhead.delete(el); - else if (ahead && !el.paused) { + else if (ahead && (!el.paused || atOutPoint)) { heldAhead.add(el); - el.pause(); + if (!el.paused) el.pause(); } } const holdingAhead = heldAhead.has(el); @@ -682,6 +734,7 @@ export function syncRuntimeMedia(params: { } playRequested.delete(el); heldAhead.delete(el); + forgetOutPoint(el); videoStarts.delete(el); } else if (!playing) { holdSeekBarrierUntilVideoLands(el); From 6e403a9b298b98a65071025f14aae35e785d2d4b Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 30 Oct 2026 00:28:40 -0400 Subject: [PATCH 2/6] fix(core): a dropped frame before an out-point no longer lets the out-point's frame show --- packages/core/src/runtime/media.test.ts | 22 ++++++++++++++++++++++ packages/core/src/runtime/media.ts | 24 ++++++++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts index 15c0298092f..8226ace08d2 100644 --- a/packages/core/src/runtime/media.test.ts +++ b/packages/core/src/runtime/media.test.ts @@ -2106,6 +2106,28 @@ describe("syncRuntimeMedia", () => { expect(el.play).not.toHaveBeenCalled(); }); + it("holds a video on its last frame before the out-point when that frame is dropped", () => { + const { clip, el, present } = reportingVideo(); + el.currentTime = 9.6; + tick(clip, 9.6); + present(9.875); + present(9.9167); // the frame at 9.9583 is dropped; the next one would be the out-point's + expect(el.pause).not.toHaveBeenCalled(); + el.currentTime = 9.975; + tick(clip, 9.975); + expect(el.pause).toHaveBeenCalledTimes(1); + }); + + it("holds a video with an uneven cadence before a long step crosses the out-point", () => { + const { clip, el, present } = reportingVideo(); + el.currentTime = 9.6; + tick(clip, 9.6); + present(9.8); + present(9.8833); + present(9.925); // a step as long as the earlier one would land past 10 + expect(el.pause).toHaveBeenCalledTimes(1); + }); + it("plays a held video again once a seek takes the playhead back", () => { const { clip, el, present } = reportingVideo(); el.currentTime = 9.6; diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index 78750ac6911..714cee791a2 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -325,6 +325,7 @@ const heldAhead = new WeakSet(); const OUT_POINT_WATCH_SECONDS = 0.5; const outPointWatch = new WeakMap(); const heldAtOutPoint = new WeakMap(); +const lastFrameBefore = new WeakMap(); const reportsFrames = (el: HTMLMediaElement): el is HTMLVideoElement => typeof (el as HTMLVideoElement).requestVideoFrameCallback === "function"; @@ -332,17 +333,24 @@ function watchOutPoint(el: HTMLVideoElement, outPoint: number): void { if (outPointWatch.get(el) === outPoint) return; outPointWatch.set(el, outPoint); let previous: number | null = null; + let shortestStep = Number.POSITIVE_INFINITY; + let longestStep = 0; const onFrame = (_now: number, frame: VideoFrameCallbackMetadata) => { if (outPointWatch.get(el) !== outPoint) return; if (el.paused) { outPointWatch.delete(el); return; } - // A seek between two frames is not a frame step. + // A seek between two frames is not a frame step; a dropped frame or an uneven cadence is a longer one. const gap = previous === null ? Number.NaN : frame.mediaTime - previous; - const step = gap > 0 && gap < VIDEO_LEAD_HOLD_SECONDS ? gap : MEDIA_SYNC_TOLERANCE_SECONDS; + if (gap > 0 && gap < VIDEO_LEAD_HOLD_SECONDS) { + shortestStep = Math.min(shortestStep, gap); + longestStep = Math.max(longestStep, gap); + const framesLeft = Math.floor((outPoint - 0.001 - frame.mediaTime) / shortestStep); + lastFrameBefore.set(el, { outPoint, at: frame.mediaTime + framesLeft * shortestStep }); + } previous = frame.mediaTime; - if (frame.mediaTime + step < outPoint - 0.001) { + if (frame.mediaTime + (longestStep || MEDIA_SYNC_TOLERANCE_SECONDS) < outPoint - 0.001) { el.requestVideoFrameCallback(onFrame); return; } @@ -357,6 +365,14 @@ function watchOutPoint(el: HTMLVideoElement, outPoint: number): void { function forgetOutPoint(el: HTMLMediaElement): void { outPointWatch.delete(el); heldAtOutPoint.delete(el); + lastFrameBefore.delete(el); +} + +/** Where a watched video is held: on the last frame before its out-point once its cadence is known. */ +function outPointHoldTime(el: HTMLMediaElement, outPoint: number, watched: boolean): number { + if (!watched) return outPoint - MEDIA_SYNC_TOLERANCE_SECONDS; + const last = lastFrameBefore.get(el); + return last?.outPoint === outPoint ? last.at : outPoint; } // A muted video's clock is its audio track, which starts tens of ms after play(), so an upcoming @@ -652,7 +668,7 @@ export function syncRuntimeMedia(params: { !clip.loop && (offset < -VIDEO_LEAD_HOLD_SECONDS || atOutPoint || - currentElTime >= outPoint - (watched ? 0 : MEDIA_SYNC_TOLERANCE_SECONDS)); + currentElTime >= outPointHoldTime(el, outPoint, watched)); if (!playing || (!ahead && offset >= -MEDIA_SYNC_TOLERANCE_SECONDS)) heldAhead.delete(el); else if (ahead && (!el.paused || atOutPoint)) { heldAhead.add(el); From 2a44408fccc81f044223bebd08fbe375d514c804 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 29 Oct 2026 22:56:22 -0400 Subject: [PATCH 3/6] perf(core): a video edit with no animation timeline stops rewriting every clip each frame --- packages/core/src/runtime/init.test.ts | 19 +++++++++++++++++++ packages/core/src/runtime/init.ts | 22 ++++++++++++++-------- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 1644832edeb..3e5b6c44f01 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -612,6 +612,25 @@ describe("initSandboxRuntimeModular", () => { } }); + it("leaves hidden clips alone on every jump when the page has no timeline", () => { + document.body.innerHTML = + `
` + + `` + + `` + + `
`; + window.__timelines = {}; + initSandboxRuntimeModular(); + const query = vi.spyOn(document, "querySelectorAll"); + const looked = () => + query.mock.calls.filter(([selector]) => selector === "[data-hf-clip-hidden]").length; + try { + for (const t of [0.5, 1, 1.5, 3, 6]) window.__player?.seek(t); + expect(looked()).toBe(0); + } finally { + query.mockRestore(); + } + }); + it("styles a hidden clip visible once while playing toward its start, far scene included", () => { document.body.innerHTML = `
` + diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index e2d997848bc..036e980784d 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -5163,6 +5163,16 @@ export function initSandboxRuntimeModular(): void { return false; } }; + // Registered timelines other than the root composition's: each scene's own clock. + const sceneTimelines = () => { + const timelines = (window.__timelines ?? {}) as Record; + const rootCompositionId = + resolveRootCompositionElement()?.getAttribute("data-composition-id") ?? null; + return Object.entries(timelines).filter( + (entry): entry is [string, RuntimeTimelineLike] => + !!entry[1] && typeof entry[1] === "object" && entry[0] !== rootCompositionId, + ); + }; const seekStandaloneRegisteredTimelines = ( timeSeconds: number, revision: number, @@ -5172,12 +5182,7 @@ export function initSandboxRuntimeModular(): void { sceneClocks = new WeakMap(); sceneClocksRevision = revision; } - const timelines = (window.__timelines ?? {}) as Record; - const rootCompositionId = - resolveRootCompositionElement()?.getAttribute("data-composition-id") ?? null; - for (const [compositionId, timeline] of Object.entries(timelines)) { - if (!timeline || typeof timeline !== "object" || compositionId === rootCompositionId) - continue; + for (const [compositionId, timeline] of sceneTimelines()) { const scene = sceneClockOf(compositionId, timeline); if (!scene) continue; const sourceTime = @@ -5291,9 +5296,10 @@ export function initSandboxRuntimeModular(): void { // Only clips holding a tween that renders for the first time on this seek need their authored style read. const firstRendered = firstRenders && tl ? clipsFirstRenderedIn(tl, tweensRenderedThrough.time, t) : null; - // An empty set means no tween draws for the first time: nothing to show, so no page-wide query. + // An empty set means no tween draws for the first time, and a page with no timeline has no tween: + // nothing to show, so no page-wide query. const hideClipsAgain = - firstRenders && firstRendered?.size !== 0 + firstRenders && firstRendered?.size !== 0 && (tl || sceneTimelines().length > 0) ? showHiddenClips(firstRendered ? (el) => firstRendered.has(el) : undefined) : null; // Scenes first, each paused on its own clock, so the root's pass skips them and its later From c0bfb064db3a64c6715a48aee3d9b6e35602e6d4 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 30 Oct 2026 00:01:01 -0400 Subject: [PATCH 4/6] refactor(core): drop the comments the comment ratchet counts --- packages/core/src/runtime/init.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 036e980784d..60496a46ef7 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -5163,7 +5163,6 @@ export function initSandboxRuntimeModular(): void { return false; } }; - // Registered timelines other than the root composition's: each scene's own clock. const sceneTimelines = () => { const timelines = (window.__timelines ?? {}) as Record; const rootCompositionId = @@ -5296,8 +5295,7 @@ export function initSandboxRuntimeModular(): void { // Only clips holding a tween that renders for the first time on this seek need their authored style read. const firstRendered = firstRenders && tl ? clipsFirstRenderedIn(tl, tweensRenderedThrough.time, t) : null; - // An empty set means no tween draws for the first time, and a page with no timeline has no tween: - // nothing to show, so no page-wide query. + // An empty set, or a page with no timeline, means no tween draws for the first time: no page-wide query. const hideClipsAgain = firstRenders && firstRendered?.size !== 0 && (tl || sceneTimelines().length > 0) ? showHiddenClips(firstRendered ? (el) => firstRendered.has(el) : undefined) From 331cbab3304022a963f9ac933b6a857294672eec Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Thu, 29 Oct 2026 23:58:50 -0400 Subject: [PATCH 5/6] fix(core): a clip that has fully downloaded stays loaded and plays on time --- .../core/src/runtime/preloadMedia.test.ts | 30 +++++++++++++++++++ packages/core/src/runtime/preloadMedia.ts | 16 +++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/core/src/runtime/preloadMedia.test.ts b/packages/core/src/runtime/preloadMedia.test.ts index 10bb62f3082..32021d9e7e6 100644 --- a/packages/core/src/runtime/preloadMedia.test.ts +++ b/packages/core/src/runtime/preloadMedia.test.ts @@ -116,6 +116,36 @@ describe("releasing a clip's download", () => { expect(video.getAttribute("src")).toBe("a.mp4"); }); + it("keeps a clip whose whole file is buffered", () => { + const video = document.createElement("video"); + video.setAttribute("src", "a.mp4"); + Object.defineProperty(video, "duration", { value: 6.67 }); + Object.defineProperty(video, "buffered", { + value: { length: 1, start: () => 0, end: () => 6.67 }, + }); + video.load = vi.fn(); + + releaseMedia(video); + expect(video.load).not.toHaveBeenCalled(); + expect(video.getAttribute("src")).toBe("a.mp4"); + }); + + it.each([ + [0, 15], + [430, 2642], + ])("releases a clip with only %s-%s s of a long file buffered", (from, to) => { + const video = document.createElement("video"); + video.setAttribute("src", "a.mp4"); + Object.defineProperty(video, "duration", { value: 2642 }); + Object.defineProperty(video, "buffered", { + value: { length: 1, start: () => from, end: () => to }, + }); + video.load = vi.fn(); + + releaseMedia(video); + expect(video.load).toHaveBeenCalledTimes(1); + }); + it("leaves a clip on children, their order and its markup untouched", () => { const video = document.createElement("video"); video.innerHTML = diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts index 8129ff3276a..f4ceb5d6ab4 100644 --- a/packages/core/src/runtime/preloadMedia.ts +++ b/packages/core/src/runtime/preloadMedia.ts @@ -5,6 +5,8 @@ import { defersMedia } from "./timedClipHide"; import { swallow } from "./diagnostics"; import { mediaElementTime } from "./mediaClock"; +const MEDIA_EDGE_SECONDS = 0.1; + type PreloadableMedia = Pick< HTMLMediaElement, "id" | "tagName" | "getAttribute" | "preload" | "readyState" | "networkState" | "load" @@ -73,8 +75,20 @@ export function stopMediaDownload(media: HTMLMediaElement): void { media.load(); } +// The whole file in memory: its download is over, so releasing it would free no connection, only drop the data. +function wholeFileBuffered(media: HTMLMediaElement): boolean { + const { buffered, duration } = media; + return ( + Number.isFinite(duration) && + duration > 0 && + buffered.length === 1 && + buffered.start(0) <= MEDIA_EDGE_SECONDS && + buffered.end(0) >= duration - MEDIA_EDGE_SECONDS + ); +} + export function releaseMedia(media: HTMLMediaElement): void { - if (media.querySelector("source")) return; + if (media.querySelector("source") || wholeFileBuffered(media)) return; const src = media.getAttribute("src"); stopMediaDownload(media); if (src !== null) media.setAttribute("src", src); From 1fa435d4695ba8c7a48a8cac5090c54e17a90459 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Fri, 30 Oct 2026 00:01:04 -0400 Subject: [PATCH 6/6] refactor(core): drop the comments the comment ratchet counts --- packages/core/src/runtime/preloadMedia.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/runtime/preloadMedia.ts b/packages/core/src/runtime/preloadMedia.ts index f4ceb5d6ab4..3e98d68de07 100644 --- a/packages/core/src/runtime/preloadMedia.ts +++ b/packages/core/src/runtime/preloadMedia.ts @@ -75,7 +75,6 @@ export function stopMediaDownload(media: HTMLMediaElement): void { media.load(); } -// The whole file in memory: its download is over, so releasing it would free no connection, only drop the data. function wholeFileBuffered(media: HTMLMediaElement): boolean { const { buffered, duration } = media; return (