fix(engine): a PNG-sequence frame no longer captures DOM still pending in virtual rAF

packages/engine/src/services/frameCapture.postSeekSettle.test.ts

@@ -0,0 +1,209 @@
+/**
+ * Tests for `settlePostSeekDomUpdates` — the per-seek readiness barrier in
+ * prepareFrameForCapture (heygen-com/hyperframes#5888).
+ *
+ * The `page.evaluate` callbacks run in a synthetic page context so the
+ * browser-side body executes in Node against a controllable
+ * `__HF_VIRTUAL_TIME__` shim, `document`, and media stubs — same pattern as
+ * frameCapture-videoReady.test.ts.
+ */
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { settlePostSeekDomUpdates } from "./frameCapture.js";
+
+type EvalFn = (arg: unknown) => Promise<unknown>;
+
+interface FakeVt {
+  queue: Array<() => void>;
+  drainCalls: number;
+  paintCalls: number;
+  lastDrain: { turns: number; ran: number; pending: number };
+  drainAnimationFrames: () => { turns: number; ran: number; pending: number };
+  originalRequestAnimationFrame: (cb: (time: number) => void) => number;
+  originalSetTimeout: (cb: () => void, ms: number) => ReturnType<typeof setTimeout>;
+}
+
+function makeFakeVt({ paintImmediately = true } = {}): FakeVt {
+  const queue: Array<() => void> = [];
+  const fake: FakeVt = {
+    queue,
+    drainCalls: 0,
+    paintCalls: 0,
+    lastDrain: { turns: 0, ran: 0, pending: 0 },
+    drainAnimationFrames() {
+      fake.drainCalls++;
+      let turns = 0;
+      let ran = 0;
+      while (queue.length && turns < 64 && ran < 2048) {
+        const batch = queue.splice(0, queue.length);
+        ran += batch.length;
+        turns++;
+        for (const cb of batch) cb();
+      }
+      fake.lastDrain = { turns, ran, pending: queue.length };
+      return fake.lastDrain;
+    },
+    originalRequestAnimationFrame(cb) {
+      fake.paintCalls++;
+      if (paintImmediately) setTimeout(() => cb(0), 0);
+      return fake.paintCalls;
+    },
+    originalSetTimeout(cb, ms) {
+      return setTimeout(cb, ms);
+    },
+  };
+  return fake;
+}
+
+/** Installs synthetic page globals, runs the evaluate body, then restores. */
+function makePage(doc: object, extras: Record<string, unknown> = {}) {
+  return {
+    evaluate: async (fn: EvalFn, arg?: unknown) => {
+      const g = globalThis as Record<string, unknown>;
+      const saved: Record<string, unknown> = {};
+      for (const key of [
+        "document",
+        "__HF_VIRTUAL_TIME__",
+        "__hfDecodeDynamicCssBackgroundImages",
+      ]) {
+        saved[key] = g[key];
+      }
+      g.document = doc;
+      for (const [k, v] of Object.entries(extras)) g[k] = v;
+      try {
+        return await fn(arg);
+      } finally {
+        for (const key of Object.keys(saved)) {
+          if (saved[key] === undefined) delete g[key];
+          else g[key] = saved[key];
+        }
+        // __hf_seek_settle_decoded intentionally survives: the settle stores
+        // its decoded-image WeakMap on the page context across seeks.
+      }
+    },
+  } as never;
+}
+
+const cleanDoc = {
+  documentElement: { offsetHeight: 0 },
+  fonts: { status: "loaded", ready: Promise.resolve() },
+  querySelectorAll: () => [],
+};
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe("settlePostSeekDomUpdates", () => {
+  it("drains deferred work queued after the seek at the same virtual time", async () => {
+    const vt = makeFakeVt();
+    let applied = false;
+    let pending = 5;
+    const step = (): void => {
+      if (--pending > 0) vt.queue.push(step);
+      else applied = true;
+    };
+    vt.queue.push(step);
+    const stats = (await settlePostSeekDomUpdates(
+      makePage(cleanDoc, { __HF_VIRTUAL_TIME__: vt }),
+      true,
+    )) as { drainedRan: number; drainPending: number };
+    expect(applied).toBe(true);
+    expect(stats.drainedRan).toBe(5);
+    expect(stats.drainPending).toBe(0);
+  });
+
+  it("re-decodes dynamic CSS backgrounds only when the drain ran work", async () => {
+    const vt = makeFakeVt();
+    const bgDecode = vi.fn(() => Promise.resolve());
+    vt.queue.push(() => undefined);
+    await settlePostSeekDomUpdates(
+      makePage(cleanDoc, {
+        __HF_VIRTUAL_TIME__: vt,
+        __hfDecodeDynamicCssBackgroundImages: bgDecode,
+      }),
+      true,
+    );
+    expect(bgDecode).toHaveBeenCalledTimes(1);
+
+    // Simulate the next seek having drained nothing — seekToTime refreshes
+    // lastDrain before each settle in production.
+    vt.lastDrain = { turns: 0, ran: 0, pending: 0 };
+    const cleanStats = (await settlePostSeekDomUpdates(
+      makePage(cleanDoc, {
+        __HF_VIRTUAL_TIME__: vt,
+        __hfDecodeDynamicCssBackgroundImages: bgDecode,
+      }),
+      true,
+    )) as { paintsCommitted: number };
+    // A drained-clean queue: no second decode pass, and no paint commit.
+    expect(bgDecode).toHaveBeenCalledTimes(1);
+    expect(cleanStats.paintsCommitted).toBe(0);
+  });
+
+  it("commits two native paints when deferred work ran and the compositor free-runs", async () => {
+    const vt = makeFakeVt();
+    // Simulate the drain inside __hf.seek itself having run work.
+    vt.lastDrain = { turns: 3, ran: 3, pending: 0 };
+    const stats = (await settlePostSeekDomUpdates(
+      makePage(cleanDoc, { __HF_VIRTUAL_TIME__: vt }),
+      true,
+    )) as { paintsCommitted: number };
+    expect(stats.paintsCommitted).toBe(2);
+    expect(vt.paintCalls).toBe(2);
+  });
+
+  it("skips the paint wait for BeginFrame-driven capture sessions", async () => {
+    const vt = makeFakeVt();
+    vt.lastDrain = { turns: 3, ran: 3, pending: 0 };
+    const stats = (await settlePostSeekDomUpdates(
+      makePage(cleanDoc, { __HF_VIRTUAL_TIME__: vt }),
+      false,
+    )) as { paintsCommitted: number };
+    expect(stats.paintsCommitted).toBe(0);
+    expect(vt.paintCalls).toBe(0);
+  });
+
+  it("no-ops when the virtual-time shim is absent or predates the drain API", async () => {
+    const stats = await settlePostSeekDomUpdates(makePage(cleanDoc), true);
+    expect(stats).toEqual({
+      drainedRan: 0,
+      drainPending: 0,
+      fontsPending: false,
+      imagesDecoded: 0,
+      paintsCommitted: 0,
+    });
+  });
+
+  it("decodes newly active images once and remembers them across seeks", async () => {
+    const vt = makeFakeVt();
+    vt.lastDrain = { turns: 1, ran: 1, pending: 0 };
+    const decoded = vi.fn(() => Promise.resolve());
+    // A loaded img and a still-loading one both get decode() calls; the
+    // src-keyed WeakMap then skips both on the next seek.
+    const fresh = {
+      complete: true,
+      naturalWidth: 10,
+      src: "a.png",
+      getAttribute: () => "a.png",
+      decode: decoded,
+    };
+    const inflight = {
+      complete: false,
+      naturalWidth: 0,
+      src: "b.png",
+      getAttribute: () => "b.png",
+      decode: decoded,
+    };
+    const doc = {
+      ...cleanDoc,
+      querySelectorAll: (sel: string) => (sel === "img" ? [fresh, inflight] : []),
+    };
+    const page = makePage(doc, { __HF_VIRTUAL_TIME__: vt });
+    const first = (await settlePostSeekDomUpdates(page, true)) as { imagesDecoded: number };
+    const second = (await settlePostSeekDomUpdates(page, true)) as { imagesDecoded: number };
+    expect(first.imagesDecoded).toBe(2);
+    expect(decoded).toHaveBeenCalledTimes(2);
+    expect(second.imagesDecoded).toBe(0);
+  });
+});

packages/engine/src/services/frameCapture.ts

@@ -116,6 +116,7 @@ type CaptureStep =
   | "injecting video frames"
   | "waiting for page seek handlers"
   | "loading color-grading LUTs"
+  | "settling deferred DOM updates"
   | "compositing the page"
   | "capturing the frame";
 
@@ -2851,6 +2852,149 @@ async function seekPageTimeline(
   );
 }
 
+/**
+ * Shared wall-clock budget for {@link settlePostSeekDomUpdates}: fonts,
+ * image decode, and native paint waits all race a single real-time deadline
+ * (`Date`/`performance.now` are virtualized in-page, so the budget is armed
+ * through the shim's stored `originalSetTimeout`). Tune with
+ * `HF_POST_SEEK_SETTLE_MS`; 0 disables the settle entirely.
+ */
+const POST_SEEK_SETTLE_BUDGET_MS = Number(process.env.HF_POST_SEEK_SETTLE_MS ?? "1500");
+
+interface PostSeekSettleStats {
+  /** Callback executions the post-seek drain ran (0 when the queue settled in `__hf.seek`). */
+  drainedRan: number;
+  /** Deferred callbacks still queued when the drain caps stopped it. */
+  drainPending: number;
+  /** True when `document.fonts` reported a load pending at settle time. */
+  fontsPending: boolean;
+  /** Images whose decode completed inside this settle. */
+  imagesDecoded: number;
+  /** Native compositor paints committed before returning. */
+  paintsCommitted: number;
+}
+
+/**
+ * Per-seek readiness barrier for the capture path (heygen-com/hyperframes#5888).
+ * `__hf.seek` already drains virtual rAF callbacks at the authored time; this
+ * covers what the seek cannot guarantee — callbacks queued by post-seek hooks,
+ * fonts and image decodes that only start once the drained DOM forces layout,
+ * and, when the compositor free-runs (no BeginFrame drive), two native paint
+ * commits so the screenshot cannot race the commit. BeginFrame sessions skip
+ * that wait: the capture's own BeginFrame is the commit and a native rAF would
+ * stall on the locked compositor. All waits race one shared wall-clock budget;
+ * the paint wait only runs when the settle observed pending work.
+ */
+export async function settlePostSeekDomUpdates(
+  page: Pick<Page, "evaluate">,
+  allowNativePaintWait: boolean,
+): Promise<PostSeekSettleStats | null> {
+  if (!(POST_SEEK_SETTLE_BUDGET_MS > 0)) return null;
+  return page.evaluate(
+    async ({ allowPaint, budgetMs }): Promise<PostSeekSettleStats> => {
+      const root = globalThis as typeof globalThis & {
+        __HF_VIRTUAL_TIME__?: {
+          originalSetTimeout?: (cb: () => void, ms?: number) => unknown;
+          originalRequestAnimationFrame?: (cb: (time: number) => void) => unknown;
+          drainAnimationFrames?: (
+            maxTurns?: number,
+            maxCallbacks?: number,
+          ) => { turns: number; ran: number; pending: number };
+          readonly lastDrain?: { turns: number; ran: number; pending: number };
+        };
+        __hfDecodeDynamicCssBackgroundImages?: () => Promise<void>;
+        __hf_seek_settle_decoded?: WeakMap<HTMLImageElement, string>;
+      };
+      const empty: PostSeekSettleStats = {
+        drainedRan: 0,
+        drainPending: 0,
+        fontsPending: false,
+        imagesDecoded: 0,
+        paintsCommitted: 0,
+      };
+      const vt = root.__HF_VIRTUAL_TIME__;
+      if (!vt || typeof vt.drainAnimationFrames !== "function") return empty;
+
+      const armRealTimeout =
+        typeof vt.originalSetTimeout === "function"
+          ? (cb: () => void, ms: number) => {
+              vt.originalSetTimeout?.(cb, ms);
+            }
+          : (cb: () => void, ms: number) => {
+              setTimeout(cb, ms);
+            };
+      let expired = false;
+      const deadline = new Promise<void>((resolve) =>
+        armRealTimeout(() => {
+          expired = true;
+          resolve();
+        }, budgetMs),
+      );
+      const bounded = <T>(work: Promise<T>): Promise<T | void> => Promise.race([work, deadline]);
+      const stats: PostSeekSettleStats = { ...empty };
+
+      // Deferred work could have been queued after `__hf.seek` returned — by
+      // seek-completion handlers, video-frame injection, or LUT application.
+      // Drain it at the same virtual time so capture observes the settled DOM.
+      const seekDrainRan = vt.lastDrain?.ran ?? 0;
+      const drain = vt.drainAnimationFrames();
+      stats.drainedRan = drain.ran;
+      stats.drainPending = drain.pending;
+      if (drain.ran > 0 && root.__hfDecodeDynamicCssBackgroundImages) {
+        await bounded(root.__hfDecodeDynamicCssBackgroundImages());
+      }
+
+      // A synchronous reflow starts font/image loads that lazy layout would
+      // otherwise only request at the capture's implicit style pass — too late
+      // for the readiness check to see them.
+      void document.documentElement?.offsetHeight;
+      const fontSet = (document as { fonts?: FontFaceSet }).fonts;
+      stats.fontsPending = typeof fontSet?.status === "string" && fontSet.status !== "loaded";
+      if (stats.fontsPending && !expired && fontSet) {
+        await bounded(fontSet.ready.then(() => undefined));
+      }
+
+      // Keyed by currentSrc so a mutated src decodes again; a complete img with
+      // naturalWidth 0 already settled in error — waiting can't fix it (same
+      // rule pollImagesReady applies). decode() on a still-loading img simply
+      // outlives its frame's deadline and resolves in-page later.
+      const decoded = (root.__hf_seek_settle_decoded ??= new WeakMap<HTMLImageElement, string>());
+      const decodes: Promise<unknown>[] = [];
+      for (const img of Array.from(document.querySelectorAll("img"))) {
+        const src = img.currentSrc || img.src || img.getAttribute("src") || "";
+        if (!src || decoded.get(img) === src || typeof img.decode !== "function") continue;
+        if (img.complete && img.naturalWidth === 0) continue;
+        decodes.push(
+          img
+            .decode()
+            .then(() => decoded.set(img, src))
+            .catch(() => undefined),
+        );
+      }
+      if (decodes.length && !expired) {
+        await bounded(Promise.all(decodes));
+        stats.imagesDecoded = decodes.length;
+      }
+
+      const observedWork =
+        seekDrainRan > 0 || stats.drainedRan > 0 || stats.fontsPending || decodes.length > 0;
+      if (allowPaint && observedWork && typeof vt.originalRequestAnimationFrame === "function") {
+        for (let i = 0; i < 2 && !expired; i++) {
+          await bounded(
+            new Promise<void>((resolve) => {
+              vt.originalRequestAnimationFrame?.(() => resolve());
+            }),
+          );
+          if (expired) break;
+          stats.paintsCommitted++;
+        }
+      }
+      return stats;
+    },
+    { allowPaint: allowNativePaintWait, budgetMs: POST_SEEK_SETTLE_BUDGET_MS },
+  );
+}
+
 async function prepareFrameForCapture(
   session: CaptureSession,
   frameIndex: number,
@@ -2903,6 +3047,8 @@ async function prepareFrameForCapture(
     ).__hf?.colorGrading;
     await runtime?.waitForActiveLuts?.();
   });
+  session.captureStep = "settling deferred DOM updates";
+  await settlePostSeekDomUpdates(page, session.beginFrameTimeTicks === 0);
   const beforeCaptureMs = Date.now() - beforeCaptureStart;
 
   // Page-side compositing three-phase protocol: