From b95b63684630c8c20c7f3b3779dc9b94ea547971 Mon Sep 17 00:00:00 2001 From: Andreas Sundquist Date: Mon, 14 Sep 2026 13:35:30 -0700 Subject: [PATCH] Fix paged LoD churn races and add on-demand rendering support with a browser test suite - On-demand rendering: SparkRenderer.onDirty now fires for every async result (sort, LoD traverse, chunk landed, mesh initialized, callback exiting with pending work); new docs page, streaming example, and SplatMesh init watching. - Pager races: removed-but-in-flight chunks are retired instead of mapped (retiredSplats/activateSplats); removeSplats purges fetched/lodTreeUpdates/uploads; root-chunk eviction clears rootPage and indices; untraversed paged meshes render nothing. - LoD cleanup: lodDisposeTimeoutMs option (default 3000); visible meshes are never disposed, including ones re-added while a callback runs. - Sort robustness: a failed sort no longer leaves `sorting` stuck; sortDirty re-arms a retry. - Rust hardening: unknown lodId returns an error instead of panicking the worker; traversals bounds-check the root page; get_lod_tree_info/get_lod_tree_ids for introspection. - Test instrumentation: SparkHooks hook points in sort/LoD/pager paths; SplatPager.debugState() and checkInvariants(). - Browser tests: Playwright + SwiftShader suite (sanity, on-demand, churn, orderings, regressions, seeded pager fuzzer) with in-page harness, hook/network seams, pager-vs-Rust cross-check, and synthetic LoD fixtures; manual-only CI workflow. - Docs: on-demand-rendering.md, SparkRenderer onDirty row, LoD guide pointer, pager internals reference. - Fix SplatLoader fileBytes type narrowing so tsc passes. --- .github/workflows/ci-browser.yml | 70 +++ .gitignore | 3 + biome.json | 1 + docs/docs/index.md | 1 + docs/docs/lod-getting-started.md | 4 + docs/docs/on-demand-rendering.md | 99 ++++ docs/docs/spark-renderer.md | 1 + docs/internals/pager/fixes-2026-09.html | 674 +++++++++++++++++++++ docs/internals/pager/index.html | 532 +++++++++++++++++ docs/internals/pager/internals.css | 115 ++++ examples.html | 6 +- examples/on-demand/index.html | 143 +++++ index.html | 1 + mkdocs.yml | 1 + package-lock.json | 74 ++- package.json | 6 +- rust/spark-rs/src/lod_tree.rs | 80 ++- src/SparkHooks.ts | 36 ++ src/SparkRenderer.ts | 341 ++++++++--- src/SplatLoader.ts | 11 +- src/SplatPager.ts | 383 +++++++++++- src/index.ts | 1 + src/worker.ts | 22 + test/browser/churn.spec.ts | 239 ++++++++ test/browser/fuzz-shared.ts | 30 + test/browser/global-setup.ts | 19 + test/browser/helpers.ts | 293 +++++++++ test/browser/on-demand.spec.ts | 285 +++++++++ test/browser/orderings.spec.ts | 193 ++++++ test/browser/pager-fuzz.spec.ts | 39 ++ test/browser/pages/fuzz.ts | 324 ++++++++++ test/browser/pages/harness.html | 19 + test/browser/pages/harness.ts | 752 ++++++++++++++++++++++++ test/browser/regressions.spec.ts | 142 +++++ test/browser/sanity.spec.ts | 91 +++ test/fixtures/gen-fixture.mjs | 200 +++++++ test/playwright.config.ts | 60 ++ test/tsconfig.json | 22 + 38 files changed, 5165 insertions(+), 148 deletions(-) create mode 100644 .github/workflows/ci-browser.yml create mode 100644 docs/docs/on-demand-rendering.md create mode 100644 docs/internals/pager/fixes-2026-09.html create mode 100644 docs/internals/pager/index.html create mode 100644 docs/internals/pager/internals.css create mode 100644 examples/on-demand/index.html create mode 100644 src/SparkHooks.ts create mode 100644 test/browser/churn.spec.ts create mode 100644 test/browser/fuzz-shared.ts create mode 100644 test/browser/global-setup.ts create mode 100644 test/browser/helpers.ts create mode 100644 test/browser/on-demand.spec.ts create mode 100644 test/browser/orderings.spec.ts create mode 100644 test/browser/pager-fuzz.spec.ts create mode 100644 test/browser/pages/fuzz.ts create mode 100644 test/browser/pages/harness.html create mode 100644 test/browser/pages/harness.ts create mode 100644 test/browser/regressions.spec.ts create mode 100644 test/browser/sanity.spec.ts create mode 100644 test/fixtures/gen-fixture.mjs create mode 100644 test/playwright.config.ts create mode 100644 test/tsconfig.json diff --git a/.github/workflows/ci-browser.yml b/.github/workflows/ci-browser.yml new file mode 100644 index 00000000..f7bfe3a2 --- /dev/null +++ b/.github/workflows/ci-browser.yml @@ -0,0 +1,70 @@ +name: Spark CI Browser +# Manual-only: run from the Actions tab or with +# gh workflow run ci-browser.yml --ref +# To run automatically, add push / pull_request triggers here. +on: + workflow_dispatch: + inputs: + fuzz_iters: + description: "Fresh fuzz seeds to run in pager-fuzz.spec.ts" + required: false + default: "3" +permissions: + contents: read +jobs: + browser-tests: + name: playwright (paged LoD / on-demand) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Use Node.js 22.x + uses: actions/setup-node@v4 + with: + node-version: '22.x' + + # Needed for the spark-rs WASM module and for build-lod, which generates + # the synthetic LoD fixtures under test/fixtures/out/ (see globalSetup). + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + rust/spark-rs/target + rust/build-lod/target + key: ${{ runner.os }}-cargo-browser-${{ hashFiles('rust/**/Cargo.lock') }} + + - name: Install dependencies + run: npm install + + - name: Build spark-rs + run: npm run build:wasm + + - name: Install Playwright Chromium + run: npx playwright install --with-deps chromium + + - name: Generate LoD fixtures + run: npm run test:fixtures + + - name: Browser tests + run: npm run test:browser + env: + CI: true + FUZZ_ITERS: ${{ inputs.fuzz_iters || '3' }} + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-test-results + path: test-results/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index ac19000c..6694d8f6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ site/ site-repo/ *.zip *.gltf +test/fixtures/out/ +test-results/ +playwright-report/ diff --git a/biome.json b/biome.json index 28c30a89..93f632d9 100644 --- a/biome.json +++ b/biome.json @@ -17,6 +17,7 @@ "site-repo", "docs", "*.backup*", + "test-results/", "examples/**/spark.module.js", "examples/**/*.json", "examples/**/pkg" diff --git a/docs/docs/index.md b/docs/docs/index.md index e6d69c41..b9bc1601 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -65,6 +65,7 @@ This will run a Web server at [http://localhost:8080/](http://localhost:8080/) w - [Spark Overview](overview.md) - [System Design](system-design.md) - [SparkRenderer](spark-renderer.md) +- [On-demand rendering](on-demand-rendering.md) - [SplatMesh](splat-mesh.md) - [PackedSplats](packed-splats.md) - [ExtSplats](ext-splats.md) diff --git a/docs/docs/lod-getting-started.md b/docs/docs/lod-getting-started.md index bed0ca0f..d0188c00 100644 --- a/docs/docs/lod-getting-started.md +++ b/docs/docs/lod-getting-started.md @@ -87,6 +87,10 @@ The above parameters adjust the global splat LoD parameters, but you can also ad - `SplatMesh.behindFoveate` / `SplatMesh.coneFov0` / `SplatMesh.coneFov` / `SplatMesh.coneFoveate`: Override the global `SparkRenderer.behindFoveate` / `SparkRenderer.coneFov0` / `SparkRenderer.coneFov` / `SparkRenderer.coneFoveate` for this object. +## On-demand rendering with LoD and streaming + +If your app renders only when something changes rather than on every animation frame, Spark needs a way to request frames as sorts, LoD updates and streamed chunks complete. Pass an `onDirty` callback to `SparkRenderer` and render once whenever it fires. See [On-demand rendering](on-demand-rendering.md) for details and examples with vanilla Three.js and React Three Fiber. + ## `build-lod` command-line tool To pre-build an LoD tree for a splat file and output a `.RAD` that can be loaded faster in Spark and even streamed in, use the `build-lod` command-line tool: diff --git a/docs/docs/on-demand-rendering.md b/docs/docs/on-demand-rendering.md new file mode 100644 index 00000000..99d649de --- /dev/null +++ b/docs/docs/on-demand-rendering.md @@ -0,0 +1,99 @@ +# On-demand rendering + +By default a Three.js app renders continuously with `renderer.setAnimationLoop()`, and Spark's asynchronous work (splat sorting, LoD selection, fetching and paging in chunks of a `paged` `SplatMesh`) simply rides along with each frame. If your scene is mostly static, or you want to save power on mobile, you can instead render only when something changed. This page explains what Spark needs from you to make that work, with a vanilla Three.js example and a React Three Fiber example. + +## How Spark drives its own work + +Spark does its work from inside `renderer.render()`: that is when it checks the camera, kicks off sorts and LoD updates in its workers, and pages in newly streamed chunks. Results come back asynchronously, and each one needs another render to become visible. In a continuous animation loop the next frame is always coming, so this is invisible. In an on-demand app there is no next frame unless someone asks for one, so Spark needs a way to ask. + +That is what the `onDirty` option on `SparkRenderer` is for. Spark calls it (at most once per rendered frame) whenever it has something new to show or needs another frame to make progress: + +- A splat sort finished. +- A new LoD selection was computed. +- A `SplatMesh` finished loading. +- A streamed chunk of a `paged` `SplatMesh` landed and is waiting to be paged in. +- LoD work was requested while the LoD worker was still busy, so it needs to be retried. + +Do **not** call `render()` on a timer to "poll" for progress; that defeats the purpose and can leave gaps where loading appears to stall. Pass `onDirty` and render once whenever it fires. + +## Vanilla Three.js + +```javascript +let renderScheduled = false; +function requestRender() { + if (renderScheduled) return; + renderScheduled = true; + requestAnimationFrame(() => { + renderScheduled = false; + renderer.render(scene, camera); + }); +} + +const spark = new SparkRenderer({ renderer, onDirty: requestRender }); +scene.add(spark); + +const splats = new SplatMesh({ url: "./my-splats-lod.rad", paged: true }); +scene.add(splats); + +// Render once to kick things off; from here on Spark asks for frames. +requestRender(); +``` + +`requestRender()` coalesces multiple requests into a single frame, so it is safe to call it from anywhere, as often as you like. + +Your application must also call `requestRender()` for its own changes (camera moves, objects added/removed/transformed, material changes), since Spark only notices those during a render. If you use one of the Three.js controls, hook its `change` event: + +```javascript +controls.addEventListener("change", requestRender); +``` + +See `examples/on-demand/` for a complete example with a streamed `.rad` file and a frame counter showing how few frames are actually rendered. + +## React Three Fiber + +The same pattern maps directly onto React Three Fiber's on-demand mode: set `frameloop="demand"` on the `Canvas` and wire `onDirty` to R3F's `invalidate()`, which schedules exactly one frame. Add both objects to the scene with `` so R3F manages their lifetime: + +```jsx +import { Canvas, useThree } from "@react-three/fiber"; +import { OrbitControls } from "@react-three/drei"; +import { SparkRenderer, SplatMesh } from "@sparkjsdev/spark"; +import { useEffect, useMemo } from "react"; + +function Splats({ url }) { + const { gl, invalidate } = useThree(); + + const spark = useMemo( + () => new SparkRenderer({ renderer: gl, onDirty: invalidate }), + [gl, invalidate], + ); + const splats = useMemo(() => new SplatMesh({ url, paged: true }), [url]); + + useEffect(() => () => spark.dispose(), [spark]); + useEffect(() => () => splats.dispose(), [splats]); + + return ( + <> + + + + ); +} + +export function App() { + return ( + + + {/* drei controls call invalidate() on camera change in demand mode */} + + + ); +} +``` + +Nothing else is required: R3F renders once on mount, that render kicks off Spark's loading, sorting and LoD work, and each completed step calls `invalidate()` to request the next frame. Anything your own components change (props that move the camera or a `SplatMesh`, adding or removing meshes) should also call `invalidate()`, as usual in demand mode. + +## Related + +- [Spark Level-of-Detail](lod-getting-started.md) for building `.rad` files and enabling `paged` streaming. +- [SparkRenderer](spark-renderer.md) for the full list of constructor options, including `onDirty`. +- [Performance tuning](performance.md) for other ways to reduce GPU and CPU load. diff --git a/docs/docs/spark-renderer.md b/docs/docs/spark-renderer.md index 8a152e8e..d2c92bcd 100644 --- a/docs/docs/spark-renderer.md +++ b/docs/docs/spark-renderer.md @@ -31,6 +31,7 @@ const spark = new SparkRenderer({ | **Parameter** | Description | | ----------------- | ----------- | +| **onDirty** | Callback invoked (at most once per rendered frame) when Spark needs another render to show new results, e.g. a completed sort or LoD update, or a newly streamed chunk. Use this to drive [on-demand rendering](on-demand-rendering.md). (default: `undefined`) | **premultipliedAlpha** | Whether to use premultiplied alpha when accumulating splat RGB. (default: `true`) | **timer** | Pass in a `THREE.Timer` to synchronize time-based effects across different systems. (default: `new THREE.Timer`) | **autoUpdate** | Controls whether to check and automatically update splat collection each frame render. (default: `true`) diff --git a/docs/internals/pager/fixes-2026-09.html b/docs/internals/pager/fixes-2026-09.html new file mode 100644 index 00000000..cdb2882d --- /dev/null +++ b/docs/internals/pager/fixes-2026-09.html @@ -0,0 +1,674 @@ + + + + + +Spark: paged LoD churn and on-demand rendering fixes (September 2026) + + + + + +
+

What this is

+
+ Internal engineering notes, as of September 2026 (Spark 2.1.0). This page was written alongside + the paged-LoD add/remove churn and on-demand rendering fixes (the change that introduced + SparkHooks, SplatPager.checkInvariants() and the test/browser/ + Playwright suite). It describes the code, defects, fixes and tests as they were at that point and is + not kept in sync with later changes: hook names and test names may drift, and the "without fix" + behaviours described here no longer exist in the code. It is not linked from the main documentation + navigation. The source and the tests are authoritative. +
+

+ This is a record of the defects found in SparkRenderer / SplatPager during the + September 2026 work, how each was reproduced and fixed, and which browser test covers it. It assumes you + know how the pieces fit together; that is covered by the + internals overview (architecture, pager tables, invariants, test harness). +

+

Two problem classes were addressed:

+
    +
  • Add/remove churn corrupting the pager tables (P*): a paged SplatMesh removed from + the scene (and later re-added, hidden, or disposed) while fetches or LoD callbacks were in flight could + leave pages mapped to a dead mesh, leave pending tree updates pointing at freed pages, or leave a stale + rootPage so a mesh traversed another chunk's data.
    • +
  • Missed setDirty() in on-demand mode (D*): an application that renders only from + onDirty would stall because some asynchronous completion (chunk landing, mesh + initialization, a LoD change during a busy callback, a failed sort) had no path to request a render.
    • +
+

+ Each row in the catalog was first written as a failing spec against the unmodified code, then fixed with + the smallest change that made that spec pass. The step-throughs in section 2 are small scripted models + of the real tables (not the real code). Each replays one interleaving with and without the fix and runs a + simplified copy of SplatPager.checkInvariants() after every step. +

+

+ Key files: src/SparkRenderer.ts, src/SplatPager.ts, + src/SparkHooks.ts, src/worker.ts, + rust/spark-rs/src/lod_tree.rs, test/browser/*. +

+ + +

1. Defect catalog

+

+ Rows marked "guard" describe suspected defects that were not reachable; their specs were kept as + guards. Invariant ids (I1-I7) and setDirty sites refer to the + invariants table and the + setDirty table in the overview. +

+ + + + + + + + + + + + + + + + +
IdSymptomTrigger / orderingRoot causeFixTestStatus
D1On-demand app: a paged mesh never appears or never refines.App renders once; chunk lands after the last render.processFetched queues data in lodTreeUpdates; only a LoD callback consumes it and callbacks only run from render(). Nothing asked the app to render.pager.onUpdate = () => setDirty() set when the pager is created (lodCallback); processFetched calls onUpdate when it mapped anything. See setDirty sites.on-demand.spec.ts "D1: paged chunk landing triggers a render in on-demand mode"redgreen
D1bAfter a camera move the on-demand pipeline stops converging.Same as D1 with new chunks becoming relevant after one app render.Same as D1.Same as D1.on-demand.spec.ts "D1b: on-demand pipeline keeps converging after camera moves"redgreen
D2On-demand app: a SplatMesh that finished loading after the last render is never shown.Add mesh, render once before initialized resolves.Nothing observed initialized.initWatched WeakSet in updateInternal: first time an uninitialized mesh is seen, initialized.then(setDirty).on-demand.spec.ts "D2: SplatMesh async initialization triggers a render in on-demand mode"redgreen
D2bSame for a non-paged LoD mesh.Same.Same.Same.on-demand.spec.ts "D2b: LoD (non-paged) SplatMesh initialization ..."redgreen
D3LoD budget or view changed while a callback was running; the change never applies until the app renders again.Render A starts a long callback; render B sets lodDirty but tryExclusive skips; callback A exits.Callback A's exit path did not call setDirty; its own traverse had already done so earlier.try/finally around the tryExclusive body: setDirty() if lodDirty, lodInitQueue.length or pager.hasPendingUpdates(). See the LoD callback.on-demand.spec.ts "D3: LoD callback that exits with lodDirty pending requests a render"redgreen
D5After any sort failure, later mesh adds never display.Fault injected at sort.afterWorker.sorting = true was never reset on a rejection; updateInternal skips accumulator swaps while mappingUpdated && sorting.driveSort split into sortInternal + try/catch/finally: re-arm sortDirty, clear sorting, setDirty(). See sort pipeline.on-demand.spec.ts "D5: a failed sort does not leave `sorting` stuck"redgreen
P1aPages stay mapped for a mesh whose tree is gone (I4b); pool shrinks; fuzz runs stall.Child chunk fetch in flight; mesh removed; cleanupLodTrees disposes its tree; fetch lands.The landing fetch only checked abortController. A removed but not disposed mesh is not aborted, so processFetched mapped a page for a tree that no longer existed.retiredSplats WeakSet: removeSplats adds, initLodTree -> activateSplats deletes; both the fetch then and processFetched drop retired meshes (lifecycle transition T11 in the overview).churn.spec.ts "P1a: a chunk that lands for a removed mesh does not stay mapped"; regressions.spec.ts seeds 12, 17redgreen
P1bPending insert refers to a freed page (I7); when the mesh is re-added the stale insert lands in the new tree: Rust and pager disagree, wrong data traversed.Chunk lands between consumeLodTreeUpdates and cleanupLodTrees of the callback that disposes the mesh (held at lod.beforeCleanup), or naturally under churn.removeSplats freed pages but left fetched, lodTreeUpdates and uploads that referenced them.removeSplats purges fetched, lodTreeUpdates, newUploads, readyUploads for the mesh / freed pages, plus the P1a retirement (transitions T9 and T12).orderings.spec.ts "P1b (hooks): ..."; churn.spec.ts "P1b (natural): ..."redgreen
P2A hidden mesh whose root page was evicted keeps rootPage; when shown again it renders another chunk's data (foreign splats), rootPage != rust chunk_to_page[0].2-page pool; M1 resident; hide M1; add M2 whose chunks evict all of M1's pages incl. chunk 0; show M1.lodCallback only set rootPage on insert. Evicts of chunk 0 were forwarded to the worker but the JS record kept the page.On chunk-0 evict of the recorded root: rootPage = undefined and splats.update(0, EMPTY_INDICES); instance skipped in updateLodInstances; paged meshes not traversed are zeroed after the traverse; cleanupLodTrees also zeroes (transition T8).churn.spec.ts "P2: evicting a mesh's root chunk does not leave a stale rootPage"redgreen
P3(suspected) landed chunk dropped when no page is available.Full pool under sustained pressure.Not reachable: driveFetchers caps mapped + fetched + in-flight at maxPages.None. Harness counts allocateFreeable returning undefined (pagerDrops).orderings.spec.ts "P3: fetched chunks are never dropped when the pool is full"greenguard
P4(suspected) double insert orphans a pageLru entry; allocateFreeable throws.No known path.Not reproduced.None; covered by I4 in the fuzzer.fuzzer invariantsn/a
P5(suspected) dispose() then re-add spins on aborted fetches.Dispose a mesh and put the same object back.Not observed: dispose clears the mesh's splat source so the re-added mesh contributes no LoD source.None.churn.spec.ts "P5: dispose() then re-adding the same mesh does not spin on aborted fetches"greenguard
P6A disposed paged mesh gets its indices texture re-created by a traverse that was already in flight.Hold lod.afterTraverse, dispose the mesh, release.PagedSplats.update() did not check abortController.Guard at the top of update(): if aborted, numSplats = 0 and return.orderings.spec.ts "P6: disposing a paged mesh during a traverse does not recreate its indices texture"redgreen
R1A worker call with an unknown lodId panics the WASM module; every later LoD call fails with "unreachable".Direct worker call in the test; any future bookkeeping error in production.unwrap() on lod_trees.get(id).ok_or_else(unknown_lod_id)? everywhere, set_lod_tree_data returns Result, root bounds checks in both traversals. See Rust side.orderings.spec.ts "R1 (gated hardening): unknown lodId in a worker call rejects without poisoning the worker"redgreen
+ + +

2. Interactive step-throughs

+

+ Each box replays one interleaving. Use Next/Prev or click a step; toggle + without fix / with fix to see where the behaviours diverge. Cells that changed in the current + step are outlined. The verdict line runs the modelled invariants; the strings match what the real + checkInvariants() and harness cross-check emit. The Mermaid diagram under each box is a + static version of the same sequence. +

+ +

P1a: a chunk lands after its mesh was removed and its tree disposed

+
+
+sequenceDiagram
+  participant App
+  participant SR as SparkRenderer
+  participant Pg as SplatPager
+  participant Net
+  App->>SR: scene.add(A)#59; render()
+  SR->>Pg: initLodTree(A) -> newSharedLodTree#59; activateSplats(A)
+  Pg->>Net: fetch A:0
+  Net-->>Pg: A:0 lands -> page 0, insert queued
+  SR->>SR: callback: rootPage(A) = 0#59; traverse wants A:4
+  Pg->>Net: fetch A:4 (slow)
+  App->>SR: scene.remove(A)#59; render()
+  Note over SR: A untouched for lodDisposeTimeoutMs
+  SR->>Pg: cleanupLodTrees: removeSplats(A) [retire A], disposeLodTree
+  Net-->>Pg: A:4 lands
+  alt without fix
+    Pg->>Pg: A alive? (only abort checked) -> map page 1 for A:4
+    Note over Pg: I4b violated: page mapped for splats not in lodIds
+  else with fix
+    Pg->>Pg: retiredSplats.has(A) -> drop
+  end
+
+ +

P1b: a chunk lands while the disposing callback is held, then the mesh is re-added

+
+
+sequenceDiagram
+  participant App
+  participant SR as SparkRenderer
+  participant Pg as SplatPager
+  participant Rust as LoD worker
+  Note over SR,Pg: A: root on page 0#59; A:4 in flight#59; A removed and untouched past the timeout
+  App->>SR: render()
+  SR->>Pg: consumeLodTreeUpdates() (empty)
+  SR->>SR: ... held at [lod.beforeCleanup]
+  Pg->>Pg: A:4 lands: not retired yet -> page 1, insert A:4@1 queued, upload queued
+  SR->>Pg: resume: cleanupLodTrees -> removeSplats(A)
+  alt without fix
+    Note over Pg: pages 0,1 freed but insert A:4@1 and upload for page 1 remain (I7)
+    App->>SR: scene.add(A)#59; render()
+    SR->>Rust: newSharedLodTree -> lodId 3
+    SR->>Rust: updateLodTrees(insert A:4 @ page 1) applied to the new tree
+    Note over Rust,Pg: Rust chunk_to_page[4] = 1, pager: page 1 free or owned by another chunk
+  else with fix
+    Note over Pg: removeSplats purges fetched / lodTreeUpdates / uploads for A#59; retire A
+    App->>SR: scene.add(A)#59; render()
+    SR->>Rust: newSharedLodTree -> lodId 3 (empty)#59; activateSplats(A)#59; fetch A:0 anew
+  end
+
+ +

P2: a hidden mesh's root chunk is evicted

+
+
+sequenceDiagram
+  participant App
+  participant SR as SparkRenderer
+  participant Pg as SplatPager
+  participant Rust as LoD worker
+  Note over SR,Pg: maxPages = 2. M1 resident: chunk 0 on page 0, chunk 3 on page 1#59; rootPage(M1) = 0
+  App->>SR: M1.visible = false#59; scene.add(M2)
+  Pg->>Pg: M2:0 lands: allocateFreeable evicts M1:3 (page 1)
+  SR->>Rust: updateLodTrees(evict M1:3, insert M2:0 @ 1)#59; rootPage(M2) = 1
+  Pg->>Pg: M2:5 lands: allocateFreeable evicts M1:0 (page 0)
+  SR->>Rust: updateLodTrees(evict M1:0, insert M2:5 @ 0)
+  alt without fix
+    Note over SR: rootPage(M1) stays 0 although page 0 now holds M2:5
+    App->>SR: M1.visible = true#59; render()
+    SR->>Rust: traverse M1 from root page 0 -> M2's data -> M1 shows foreign splats
+  else with fix
+    Note over SR: chunk-0 evict of the recorded root -> rootPage(M1) = undefined#59; M1.update(0, EMPTY)
+    App->>SR: M1.visible = true#59; render()
+    SR->>SR: M1 skipped in updateLodInstances (numSplats 0)#59; fetchPriority gets M1:0 -> root refetched
+  end
+
+ +

D1: chunk data lands but no render is requested (on-demand)

+
+
+sequenceDiagram
+  participant App as App (renders only from onDirty)
+  participant SR as SparkRenderer
+  participant Pg as SplatPager
+  participant Net
+  App->>SR: scene.add(A)#59; render #1
+  SR->>SR: initLodTree(A)#59; fetchPriority [A:0]#59; setDirty (mapping changed)
+  SR-->>App: onDirty -> render #2 (sort settles) ... pipeline idle, dirty = false
+  Pg->>Net: fetch A:0
+  Net-->>Pg: A:0 lands -> processFetched maps page 0, insert queued
+  alt without fix
+    Note over Pg,App: lodTreeUpdates = 1, no render scheduled: stall until the user moves the camera
+  else with fix
+    Pg-->>SR: onUpdate -> setDirty()
+    SR-->>App: onDirty -> render #3
+    SR->>SR: callback consumes update, traverses, A displayed, setDirty -> render #4 ... converges
+  end
+
+ +

D3: a LoD change arrives while the callback is running (on-demand)

+
+
+sequenceDiagram
+  participant App as App (on-demand)
+  participant SR as SparkRenderer
+  participant Lod as LoD worker
+  App->>SR: camera moved#59; render #1
+  SR->>Lod: lodCallback: traverse, setDirty -> render #2 ... callback still running (long cleanup)
+  App->>SR: spark.lodSplatCount = 12000#59; render #3
+  SR->>SR: driveLod: lodDirty = true#59; tryExclusive -> busy, skipped
+  Note over SR: render #3 changed nothing else#59; no sort#59; dirty = false
+  Lod-->>SR: callback #1 finishes
+  alt without fix
+    Note over SR,App: lodDirty stays true#59; the new budget applies only on the next user-driven render
+  else with fix
+    SR->>SR: finally: lodDirty -> setDirty()
+    SR-->>App: onDirty -> render #4 -> callback #2 traverses with the new budget
+  end
+
+ +

D5: a sort fails

+
+
+sequenceDiagram
+  participant App
+  participant SR as SparkRenderer
+  participant Sort as Sort worker
+  App->>SR: camera moved#59; render()
+  SR->>SR: driveSort: sorting = true, sortDirty = false
+  SR->>Sort: readback, sortSplats32
+  Sort-->>SR: ordering
+  SR->>SR: [sort.afterWorker] throws (injected)
+  alt without fix
+    Note over SR: sorting stays true
+    App->>SR: scene.add(Q)#59; render()
+    SR->>SR: mappingUpdated && sorting -> doUpdate = false: Q never displayed
+  else with fix
+    SR->>SR: catch: sortDirty = true#59; finally: sorting = false#59; setDirty()
+    App->>SR: scene.add(Q)#59; render()
+    SR->>Sort: sort runs#59; Q displayed
+  end
+
+ + +

3. What changed and what each test covers

+ +

Code changes by file

+ + + + + + + + + + + + + + + + + +
FileChangeFor
src/SparkHooks.ts (new)SparkHooks interface and hookPoint(hooks, name): named awaits inside the sort, LoD callback and pager pipelines. In production hooks is undefined and each call is a single falsy check; the test harness installs a controller that can hold, release or fail any point.test seam
src/SparkRenderer.tspager.onUpdate = () => this.setDirty() when the pager is created in lodCallback.D1, D1b
initWatched WeakSet in updateInternal: the first time an uninitialized SplatMesh is seen, initialized.then(() => setDirty()).D2, D2b
try/finally around the tryExclusive(lodCallback) body in driveLod: setDirty() on exit if lodDirty, lodInitQueue.length or pager.hasPendingUpdates().D3
driveSort split into sortInternal plus try/catch/finally: catch re-arms sortDirty; finally clears sorting, calls setDirty() and re-drives.D5
In lodCallback, a chunk-0 evict of the recorded rootPage clears it and calls splats.update(0, EMPTY_INDICES); updateLodInstances skips paged meshes without a root and zeroes paged meshes not traversed; cleanupLodTrees zeroes too. initLodTree for a PagedSplats calls pager.activateSplats.P2, P1a
Hook points sort.start, sort.afterReadback, sort.afterWorker, lod.start, lod.afterInit, lod.afterUpdateTrees, lod.afterTraverse, lod.beforeCleanup; hooks constructor option.test seam
src/SplatPager.tsretiredSplats: WeakSet<PagedSplats>; removeSplats adds, activateSplats deletes. The fetch then handler and processFetched drop chunks for retired meshes.P1a
removeSplats also purges fetched, lodTreeUpdates, newUploads and readyUploads entries for the mesh and the freed pages, and filters freeablePages.P1b
onUpdate callback invoked from processFetched when at least one chunk was mapped; hasPendingUpdates().D1, D3
checkInvariants(liveSplats?) (I1-I7), debugState(), hook points pager.fetched and pager.beforeProcessFetched.test oracle
src/SplatPager.ts (PagedSplats.update)Guard at the top of update(): if abortController.signal.aborted, set numSplats = 0 and return instead of re-creating the indices texture.P6
rust/spark-rs/src/lod_tree.rsEvery lod_trees.get(id).unwrap() replaced by ok_or_else(unknown_lod_id)?; set_lod_tree_data returns Result; both traversals bounds-check root_index. New exports get_lod_tree_info(lodId) and get_lod_tree_ids().R1, test oracle
src/worker.tsRPC handlers getLodTreeInfo and getLodTreeIds so the harness can compare Rust chunk_to_page / page_to_chunk with the pager's tables.test oracle
test/browser/*, test/fixtures/gen-fixture.mjs, test/playwright.config.ts, .github/workflows/ci-browser.ymlPlaywright suite, in-page harness and fuzzer, synthetic fixture generator, CI job. Described in the test infrastructure section of the overview.all
+ +

What each browser test covers

+

+ The suite has two kinds of specs: those that reproduce a specific defect above (they failed on the + unmodified code) and those that guard a suspected defect or exercise many orderings at once. +

+ + + + + + + + +
Spec fileTestsCovers
sanity.spec.ts"WebGL2 + workers + WASM: paged mesh loads and renders in loop mode"; "non-paged LoD mesh and plain PLY load in loop mode".Baseline; a failure here means the fixture, WASM build, headless GPU or harness is broken, not a pager defect.
on-demand.spec.ts"D1: paged chunk landing triggers a render in on-demand mode"; "D1b: on-demand pipeline keeps converging after camera moves"; "D2: SplatMesh async initialization triggers a render in on-demand mode"; "D2b: LoD (non-paged) SplatMesh initialization ..."; "D3: LoD callback that exits with lodDirty pending requests a render"; "D5: a failed sort does not leave `sorting` stuck".Every setDirty site added by this work. Each test lets the application render exactly once and asserts the pipeline converges from onDirty alone.
churn.spec.ts"P1a: a chunk that lands for a removed mesh does not stay mapped"; "P1b (natural): ..."; "P2: evicting a mesh's root chunk does not leave a stale rootPage"; "P5: dispose() then re-adding the same mesh does not spin on aborted fetches".Orderings reachable by a real application through the network seam alone (held chunk requests via page.route), plus the P5 guard.
orderings.spec.ts"P1b (hooks): ..."; "P3: fetched chunks are never dropped when the pool is full"; "P6: disposing a paged mesh during a traverse does not recreate its indices texture"; "R1 (gated hardening): unknown lodId in a worker call rejects without poisoning the worker".Orderings that need a hook point inside one callback (lod.beforeCleanup, lod.afterTraverse), the P3 guard, and the Rust hardening via a direct worker RPC.
pager-fuzz.spec.tsRuns FUZZ_ITERS fresh seeds through the in-page fuzzer with invariant checks at every drain point and a final full-release check.The P* class as a whole; the only test that explores orderings not written by hand.
regressions.spec.tsReplays the PINNED list of { seed, steps } pairs with the exact harness config.Fuzz seeds that found P1a before the fix (next section).
+ +

Fuzz seeds and invariants that fired

+

+ Before the fixes, seeds 12 and 17 (120 steps each, FUZZ_HARNESS config) failed + with mapped pages for splats not in lodIds (I4b). One of the two runs also never reached idle + because the pool had shrunk to pages owned by dead meshes. Both are the P1a class. They are kept in + PINNED in regressions.spec.ts with the same config and step count so + the identical interleaving replays. After the retiredSplats fix both pass. +

+ + + + + + +
InvariantFired during this workHow
I4byesFuzz seeds 12 and 17; then reproduced deterministically as churn "P1a".
I7yesConstructed with the lod.beforeCleanup hook (orderings "P1b (hooks)"), then also observed naturally (churn "P1b (natural)").
Rust cross-check (rootPage vs chunk_to_page[0])yesConstructed in churn "P2" with a 2-page pool. Not one of I1-I7; it lives in the harness because it needs the worker.
I1, I2, I3, I4, I5, I6noNever violated by the fuzzer or the specs. I2 would have caught a P1b variant (page re-mapped while an insert for it was still pending) without the purge; I4 covers P4.
+

+ P3 (landed chunk dropped when the pool is full) and P5 (dispose then re-add spins on aborted fetches) + did not reproduce. driveFetchers caps mapped + fetched + in-flight at maxPages, + so P3 is unreachable; dispose() clears the mesh's splat source, so P5 is a no-op. Their + specs remain as guards. +

+ +
+ + + + + + diff --git a/docs/internals/pager/index.html b/docs/internals/pager/index.html new file mode 100644 index 00000000..40058243 --- /dev/null +++ b/docs/internals/pager/index.html @@ -0,0 +1,532 @@ + + + + + +Spark internals: paged LoD and on-demand rendering + + + + + +
+

Scope

+
+ Internal engineering notes, as of September 2026 (Spark 2.1.0). This page is not generated from + the source and is not linked from the main documentation. Hook names, field names and method names may + drift. The source and the tests are authoritative. +
+

+ This page describes the paged level-of-detail pipeline: the components involved, what happens in one + frame, the asynchronous flows that run alongside the application's renders, the tables the pager keeps + and the invariants that tie them to the Rust LoD tree, and the Playwright browser suite that exercises + all of it. Diagrams are Mermaid. +

+

+ Key files: src/SparkRenderer.ts, src/SplatPager.ts, + src/SparkHooks.ts, src/worker.ts, + rust/spark-rs/src/lod_tree.rs, test/browser/*. +

+ + +

1. Renderer pipeline

+ +

Components

+

+ The application owns the three.js scene and calls renderer.render(). Spark runs from the + onBeforeRender hook of the SparkRenderer object in the scene. Three asynchronous + flows run concurrently with the application's frames: the depth sort, the LoD callback (tree + updates and traversal) and the pager (fetch, decode, page assignment, texture upload). +

+
+flowchart LR
+  subgraph app [Application]
+    Scene[THREE.Scene]
+    RenderCall["renderer.render()"]
+  end
+  subgraph main [Main thread: Spark]
+    SR[SparkRenderer]
+    Acc[SplatAccumulator]
+    IdxTex[Indices texture]
+    PageTex[Page textures]
+    Pager[SplatPager]
+    Mesh[SplatMesh]
+    PagedS[PagedSplats]
+  end
+  subgraph workers [Web Workers]
+    SortW[Sort worker]
+    LodW[LoD worker]
+    Wasm[spark-rs WASM]
+  end
+  Cdn[("Chunked .rad files")]
+  RenderCall -->|onBeforeRender| SR
+  SR -->|sort| SortW --> Wasm
+  SR -->|LoD| LodW --> Wasm
+  SR -->|accumulate| Acc
+  SR -->|indices| IdxTex --> Acc
+  SR -->|drive| Pager
+  Pager -->|upload| PageTex --> Acc
+  Pager -->|fetch| PagedS --> Cdn
+  Scene --> Mesh --> PagedS
+
+ + + + + + + + + +
PieceRoleOwns
SplatMeshUser-facing object. With paged: true it holds a PagedSplats instead of a fully loaded PackedSplats.paged, version, visible, lodScale
PagedSplatsPer-mesh streaming source. Knows the chunk URLs, implements fetchDecodeChunk(chunk), and holds the mesh's LoD indices texture, written by update(numSplats, indices). dispose() aborts its abortController; after that, update() sets numSplats = 0 and returns.pager, numSplats, dynoIndices
SplatAccumulatorAggregates all visible generators into one splat list for the current view. Two accumulators alternate (current/display) so a sort can run against a stable mapping.version, mappingVersion, numSplats
SparkRendererDrives everything from onBeforeRender: accumulate, sort, LoD. Tracks one LoD tree id per LoD source in lodIds, and for each paged mesh the page holding its root chunk (rootPage).lodIds, lodIdToSplats, lodInitQueue, lodUpdates, lodDirty, sorting, dirty, pager, pagerId
SplatPagerOne global pool of maxPages pages of 65536 splats each, shared by every paged mesh. Maps (splats, chunk) to pages, runs fetchers, queues tree updates and texture uploads.see section 2
Sort workerSplatWorker running sortSplats32 from sort.rs.
LoD worker + lod_tree.rsSplatWorker holding one LodTree per lodId. RPCs: newLodTree, newSharedLodTree, initLodTree, updateLodTrees, traverseLodTrees, disposeLodTree. Paged meshes share one large tree (pagerId, capacity maxSplats); each mesh gets a newSharedLodTree view onto it with its own chunk_to_page / page_to_chunk. traverseLodTrees selects the splats to show per instance under a budget.lod_trees
+ +

One frame

+

+ Solid arrows are synchronous calls: the caller continues only after they return. Dashed arrows are + results that arrive on a later task: a worker reply, a GPU readback, a network response or a resolved + promise. Inside each shaded block, everything after the first dashed arrow runs outside the frame that + started it. The four asynchronous boundaries in the whole pipeline are the GPU depth readback, the + worker RPCs, the chunk fetch and decode, and SplatMesh.initialized; every other step is + synchronous. +

+
+sequenceDiagram
+  autonumber
+  participant App as Application
+  participant SR as SparkRenderer
+  participant Acc as SplatAccumulator
+  participant Sort as Sort worker
+  participant Lod as LoD worker
+  participant Pg as SplatPager
+  participant Net as Network + decode
+
+  App->>SR: renderer.render() -> onBeforeRender
+  SR->>SR: updateInternal
+  Note over SR: with preUpdate = false (WebXR) this runs from a 1 ms setTimeout instead
+  SR->>Acc: prepareGenerate / generate (if view or version changed)
+  SR->>SR: sortDirty = true#59; setDirty()
+  SR->>SR: driveLod: lodDirty, lodMeshes, lodInitQueue
+  SR->>SR: tryExclusive(lodCallback): runs until its first await [skipped if one is running]
+  SR->>SR: driveSort: sorting = true#59; runs until its first await [skipped if sorting or !sortDirty]
+  SR-->>SR: SplatMesh.initialized resolves -> setDirty() (loading meshes seen this frame)
+
+  rect rgb(245,247,251)
+    Note over SR,Sort: sort
+    SR->>SR: sortInternal: start GPU depth readback
+    SR-->>SR: readback complete
+    SR->>Sort: sortSplats32
+    Sort-->>SR: ordering, activeSplats
+    SR->>SR: upload orderingTexture#59; display = current
+    SR->>SR: finally: sorting = false#59; setDirty()#59; driveSort()
+  end
+
+  rect rgb(245,247,251)
+    Note over SR,Pg: LoD callback (one at a time)
+    SR->>Lod: newLodTree (first paged mesh), initLodTree / newSharedLodTree for lodInitQueue
+    Lod-->>SR: lodIds
+    SR->>Pg: activateSplats (paged meshes)
+    SR->>Pg: consumeLodTreeUpdates
+    SR->>Lod: updateLodTrees
+    Lod-->>SR: done
+    SR->>Lod: traverseLodTrees (if lodDirty)
+    Lod-->>SR: keyIndices per instance, wanted chunks
+    SR->>SR: updateLodIndices#59; setDirty()
+    SR->>Pg: processUploads#59; fetchPriority#59; driveFetchers
+    SR->>Pg: cleanupLodTrees: removeSplats (if a tree timed out)
+    SR->>Lod: disposeLodTree
+    Lod-->>SR: done
+    SR->>SR: finally: setDirty() if work is pending
+  end
+
+  rect rgb(245,247,251)
+    Note over Pg,Net: pager (per fetcher, started by driveFetchers)
+    Pg->>Net: fetchDecodeChunk(chunk)
+    Net-->>Pg: decoded chunk
+    Pg->>Pg: push to fetched#59; processFetched: allocate page, map, queue update + upload
+    Pg->>SR: onUpdate -> setDirty()
+    Pg->>Pg: driveFetchers (start the next fetch)
+  end
+
+ +

Sort pipeline

+
+flowchart TD
+  S0[driveSort] --> S1{sorting or !sortDirty?}
+  S1 -->|yes| S1a[return]
+  S1 -->|no| S2["sorting = true#59; sortDirty = false"]
+  S2 --> SI
+  subgraph SI [sortInternal]
+    direction TB
+    S3[readbackDepth] --> S4["sortSplats32 (worker)"]
+    S4 --> S5["upload orderingTexture#59; display = current"]
+  end
+  SI -->|ok| S7
+  SI -.->|throws| S6["catch: sortDirty = true"]
+  S6 --> S7["finally: sorting = false#59; setDirty()#59; driveSort()"]
+
+

+ driveSort honors minSortIntervalMs before starting. The sorting + flag guards the accumulator swap: updateInternal does not swap accumulators while + mappingUpdated && sorting. driveSort always clears sorting + in finally. If the sort throws, catch sets sortDirty again so the + next driveSort retries. +

+ +

LoD callback

+

+ driveLod runs synchronously inside the frame. It sets lodDirty if the view or + the LoD parameters changed, collects the LoD-capable meshes, and queues meshes that have no tree yet in + lodInitQueue. It then starts lodCallback through tryExclusive. + If a callback is already running, the flags stay set and the next callback picks them up. Because a + callback only starts from a render, the finally around it calls setDirty() + when it exits with work still pending. +

+
+flowchart TD
+  A[driveLod] --> B{callback running?}
+  B -->|yes| Bskip["skip#59; flags stay set"]
+  B -->|no| C[create pager if needed]
+  C --> D[init queued trees]
+  D --> E[consume pager updates]
+  E --> F[rootPage bookkeeping]
+  F --> G[updateLodTrees]
+  G --> H{lodDirty?}
+  H -->|yes| I[traverseLodTrees]
+  I --> J[updateLodIndices]
+  J --> K["processUploads#59; fetchPriority#59; driveFetchers"]
+  K --> L
+  H -->|no| L[cleanupLodTrees]
+  L --> M["finally: setDirty() if pending"]
+
+ + + + + + + + + + + + +
PhaseWhat happens
create pager if neededOn the first paged mesh: new SplatPager, pager.onUpdate = () => setDirty(), newLodTree(capacity) gives pagerId. Any mesh.paged without a pager is assigned this one.
init queued treesFor each source in lodInitQueue: a PackedSplats gets initLodTree; a PagedSplats gets newSharedLodTree and pager.activateSplats. Sets lodDirty.
consume pager updatespager.consumeLodTreeUpdates() returns the pending inserts and evicts and moves newUploads to readyUploads.
rootPage bookkeepingFor each update whose mesh still has a lodIds record: an insert of chunk 0 sets rootPage = page; an evict of the recorded rootPage clears it and calls splats.update(0, EMPTY_INDICES). The update is pushed to lodUpdates.
updateLodTreesIf lodUpdates is non-empty, send the ranges to the worker and set lodDirty.
traverseLodTreeslodDirty = false. Instances are the meshes with a record that are either non-paged or have a defined rootPage. The worker returns key indices per instance and the chunks it touched.
updateLodIndicesWrite each instance's indices texture. Paged meshes that were not traversed get update(0, EMPTY_INDICES). Then setDirty().
processUploads; fetchPriority; driveFetchersUpload readyUploads to the page textures. fetchPriority = every paged mesh's chunk 0 by camera distance, then the chunks the traverse touched. driveFetchers starts fetches (section 2).
cleanupLodTreesRecords backing a mesh in this frame's lodMeshes are skipped (their lastTouched predates the async callback). Of the rest, the oldest one untouched for lodDisposeTimeoutMs (at most one per callback): lodIds.delete, pager.removeSplats, splats.update(0, EMPTY_INDICES), disposeLodTree.
finallysetDirty() if lodDirty, lodInitQueue is non-empty, or pager.hasPendingUpdates().
+

Two rules hold throughout the callback:

+
    +
  • A paged mesh is traversed only while its rootPage is defined. The Rust side also bounds-checks root_index.
    • +
  • Every path that releases a mesh's pages (root evict, cleanup, dispose) also calls update(0, EMPTY_INDICES) on the mesh, so its indices never point into pages owned by another chunk.
    • +
+ +

The dirty flag

+

+ setDirty() sets dirty and calls the application's onDirty once; + the next onBeforeRender clears it. An on-demand application renders only from + onDirty and from its own input events, so every asynchronous completion that changes what + should be on screen calls setDirty(). +

+ + + + + + + + + +
SiteWhenWhat it makes visible
updateInternal after generate()The accumulator changed: generators added, removed, moved or toggled, or the view changed.Scene edits made by the application.
driveSort finallyA sort finished or failed.The new back-to-front ordering; display = current.
lodCallback after updateLodIndicesA traverse produced new indices.The new LoD selection.
pager.onUpdateprocessFetched mapped at least one chunk.Chunk data waiting in lodTreeUpdates; the next callback inserts it into the tree.
initWatched in updateInternalAn uninitialized SplatMesh was seen; its initialized promise resolved later.A mesh that finished loading after the last render.
tryExclusive(...) finally in driveLodA callback exited while lodDirty, lodInitQueue or pager.hasPendingUpdates().LoD work flagged by frames that ran while the callback was busy.
clearSplatsExplicit clear.The empty scene.
+ + +

2. Pager

+ +

SplatPager state

+

+ The pool is maxPages = ceil(maxSplats / 65536) layers of a few DataArrayTextures. + Every table below is indexed either by page or by (splats, chunk). The invariants in the + next section state how they must agree. +

+ + + + + + + + + + + + + +
FieldTypeMeaningWritten byRead by
splatsChunkToPageMap<PagedSplats, ({page, lru} | undefined)[]>Forward map: which page holds chunk c of a mesh. Arrays are trimmed of trailing undefined; a mesh with no pages has no entry.insertSplatsChunkPage, removeSplatsChunkPage, removeSplatsdriveFetchers, checkInvariants, harness cross-check
pageToSplatsChunk({splats, chunk, time} | undefined)[]Reverse map: owner of each page. Trailing undefined trimmed.sameallocateFreeable, debugState
pageFreelistnumber[]Unowned pages. allocatePage() shifts from the front; freed pages are pushed at the back.ctor, removeSplats, allocatePageprocessFetched
pageLruSet<{page, lru}>Insertion-ordered set of the same {page, lru} objects stored in splatsChunkToPage. Re-inserted on every driveFetchers, so iteration order is LRU.insertSplatsChunkPage, driveFetchersdriveFetchers
freeablePagesnumber[]Mapped pages that the last traverse did not want, oldest first. Eviction candidates.driveFetchers, filtered in removeSplatsallocateFreeable
fetchPriority{splats, chunk}[]What the last traverse wants, in priority order: every paged mesh's chunk 0 by camera distance, then the chunks the traverse touched.updateLodInstancesdriveFetchers
fetchers{splats, chunk}[]Fetches in flight (at most numFetchers).driveFetchersdriveFetchers, checkInvariants
fetched{splats, chunk, data}[]Decoded chunks waiting for a page.fetch then, processFetched, removeSplatsprocessFetched
lodTreeUpdates{splats, page, chunk, numSplats, lodTree?}[]Pending tree edits for the LoD worker. With lodTree: insert chunk data at page. Without: evict. Order matters; the renderer replays them in lodCallback.processFetched, allocateFreeable, removeSplatsconsumeLodTreeUpdates, hasPendingUpdates
newUploads / readyUploadsPageUpload[]Texture uploads for newly mapped pages. consumeLodTreeUpdates moves them from new to ready, so a page's tree entry and its texels become visible in the same callback. processUploads uploads them after the traverse.processFetched, consumeLodTreeUpdates, removeSplatsprocessUploads
retiredSplatsWeakSet<PagedSplats>Meshes whose pages were released by removeSplats and whose tree has not been re-created since. Fetches that land for them are dropped.removeSplats (add), activateSplats (delete)fetch then, processFetched
+

Related fields in SparkRenderer:

+ + + + + + + +
FieldMeaning
lodIds: Map<splats, {lodId, lastTouched, rootPage?}>One entry per LoD source that has a worker tree. rootPage is the page holding chunk 0 of a paged mesh, or undefined when the root is not resident. Only meshes with a defined root are traversed.
lodIdToSplatsReverse of lodIds. Turns the traverse's (lodId, chunk) pairs into fetchPriority.
lodInitQueueSources seen this frame without a record. The next callback creates their trees.
lodUpdatesConsumed pager updates translated into worker ranges (pageBase, chunkBase, count, lodTreeData).
pagerIdThe shared tree that every paged mesh's newSharedLodTree points into.
+ +

Lifecycle of one (splats, chunk)

+

+ The main path runs top to bottom: a chunk is wanted, fetched, given a page, made visible to the LoD tree + and the GPU, and later evicted or freed. Failed and Dropped are the early exits. Transition labels + (T1-T12) refer to the table below. +

+
+stateDiagram-v2
+  direction TB
+  state "In flight" as InFlight
+
+  [*] --> Wanted: T1 traverse
+  Wanted --> InFlight: T2 driveFetchers
+  InFlight --> Fetched: T3 then()
+  Fetched --> Mapped: T4 processFetched
+  Mapped --> Resident: T5 lodCallback
+  Resident --> Freeable: T6 unwanted
+  Freeable --> Resident: T7 wanted
+  Freeable --> [*]: T8 evicted
+  Resident --> [*]: T9 removeSplats
+
+  InFlight --> Failed: T10 reject
+  Failed --> Wanted: T10 retry
+  InFlight --> Dropped: T11 aborted or retired
+  Fetched --> Dropped: T11, T12
+  Dropped --> [*]
+
+ + + + + + + + + + + + + + +
IdTransitionWhat happensCode
T1start → WantedThe traverse returns the chunks it touched. updateLodInstances builds fetchPriority: every paged mesh's chunk 0 by camera distance, then the touched chunks.updateLodInstances
T2Wanted → In flight (fetchers)A fetch starts only while mapped + fetched + in-flight entries are below maxPages and fewer than numFetchers fetches are running. A wanted chunk that is already mapped only gets its LRU refreshed.driveFetchers
T3In flight → Fetched (fetched)The decoded chunk is pushed to fetched.fetch then()
T4Fetched → MappedallocatePage() takes a page from the freelist, or allocateFreeable() evicts the least recently wanted page and queues an evict update for its previous owner. Then insertSplatsChunkPage, an insert entry in lodTreeUpdates, an entry in newUploads, and onUpdate(). If no page is available the chunk is dropped and refetched later.processFetched
T5Mapped → Resident (in the Rust tree and the page texture)The next LoD callback consumes the update (rootPage bookkeeping for chunk 0), sends updateLodTrees to the worker, and after the traverse uploads the texels with processUploads.lodCallback, consumeLodTreeUpdates, processUploads
T6Resident → Freeable (freeablePages)A later traverse did not list the chunk in fetchPriority. The page stays mapped and resident and is added to freeablePages in LRU order.driveFetchers
T7Freeable → ResidentThe chunk is wanted again before it was evicted. Only its LRU timestamp is refreshed.driveFetchers
T8Freeable → end (evicted)Another chunk needed a page and none was free. removeSplatsChunkPage unmaps it, an evict entry is queued in lodTreeUpdates, and the page goes to the incoming chunk. If this was chunk 0 of a mesh, the consuming callback clears that mesh's rootPage and zeroes its indices; the mesh is traversed again once its root is refetched.allocateFreeable
T9Resident → end (freed)The mesh's tree is disposed (cleanupLodTrees after lodDisposeTimeoutMs, or the mesh was disposed). All of its pages return to the freelist. Its entries in fetched, lodTreeUpdates, newUploads and readyUploads are removed and the mesh is added to retiredSplats.removeSplats
T10In flight → Failed → WantedThe fetch rejected. AbortError is silent; anything else is logged and followed by a 250-750 ms backoff. If the next traverse still wants the chunk it is retried.fetch catch
T11In flight / Fetched → DroppedBefore the chunk is mapped, both the fetch handler and processFetched check abortController.signal.aborted (mesh disposed) and retiredSplats.has(splats) (tree disposed by removeSplats, not re-created yet). Either drops the data without touching the tables. Removing a mesh from the scene does not abort it; the retired check covers that case.fetch then(), processFetched
T12Fetched → DroppedremoveSplats removes the mesh's chunks from fetched.removeSplats
+ +

Invariants checked by SplatPager.checkInvariants(liveSplats)

+ + + + + + + + + + +
IdStatementStructuresMessage
I1Freelist entries are unique and in [0, maxPages).pageFreelistfreelist has duplicate page
I2Mapped pages and the freelist are disjoint and together cover every page.pageToSplatsChunk, pageFreelistpage N is both mapped and in freelist, freelist + mapped != maxPages
I3Forward and reverse maps agree: splatsChunkToPage[s][c] = p iff pageToSplatsChunk[p] = (s, c). Chunk arrays are trimmed and have no empty entries.both mapschunk c -> page p but page maps back to ...
I4pageLru contains exactly the {page, lru} objects referenced by the forward map.pageLrupageLru has N orphaned entries
I4bIf liveSplats is given, every mesh that owns pages is in it (the renderer still has a tree for it).forward map vs SparkRenderer.lodIdsmapped pages for splats not in lodIds
I5freeablePages is a unique subset of the mapped pages.freeablePagesfreeable page p is not mapped
I6A (splats, chunk) appears at most once across the mapping, fetched and fetchers.all threeduplicate (splats, chunk c) in ...
I7Pending uploads refer to mapped pages. Replaying lodTreeUpdates in order ends at the current mapping for every page it touches.newUploads, readyUploads, lodTreeUpdates vs pageToSplatsChunkpending lodTreeUpdates end with chunk c on page p but mapping has nothing, newUploads page p is not mapped
+

+ The test harness adds cross-checks that need the queues drained (lodTreeUpdates and + lodUpdates empty, no callback running): the set of Rust tree ids equals + {pagerId} + lodIds; for every paged mesh, Rust chunk_to_page equals the pager's + forward map, Rust page_to_chunk owners match pageToSplatsChunk, and + record.rootPage equals Rust chunk_to_page[0]. +

+ +

Rust side (rust/spark-rs/src/lod_tree.rs)

+
    +
  • lod_trees: HashMap<u32, LodTree>. Non-paged sources get a private tree from initLodTree. Paged sources share the pager tree: newLodTree(capacity) once, then newSharedLodTree(pagerId) per mesh. The shared tree's splats array is indexed by page << 16 | offset.
    • +
  • updateLodTrees(ranges) copies chunk data at pageBase and records chunk_to_page[chunk] = page / page_to_chunk[page] = chunk, or clears them for an evict.
    • +
  • traverseLodTrees starts each instance at root_index = rootPage << 16 and bounds-checks it. An instance whose root page has no data is skipped.
    • +
  • Unknown ids return an error (a rejected promise in JS) instead of panicking; a panic would leave the WASM instance unusable. Every lod_trees lookup uses ok_or_else(unknown_lod_id)? and set_lod_tree_data returns Result.
    • +
  • get_lod_tree_info(lodId) and get_lod_tree_ids() are test-facing exports (RPCs getLodTreeInfo, getLodTreeIds in src/worker.ts) used by the harness cross-check above.
    • +
+ + +

3. Test infrastructure

+ +

Pieces

+

+ Top: what runs in CI and Node. Bottom: what runs inside the browser page. Specs talk to the page through + page.evaluate and to the network through page.route. +

+
+flowchart TB
+  subgraph node [Node]
+    direction TB
+    CI[CI job]
+    Specs[Specs]
+    Helpers[helpers.ts]
+    PW[Playwright runner]
+    Gen[globalSetup: fixtures]
+    Vite[Vite]
+  end
+  subgraph browser [Browser page]
+    direction TB
+    Page[harness page]
+    Hooks[HooksController]
+    Fuzz[FetchGate + fuzzer]
+    Spark[Spark]
+  end
+  CI --> PW
+  Helpers --> Specs
+  Specs --> PW
+  PW --> Gen
+  PW --> Vite
+  Vite -->|serves| Page
+  Specs -->|page.evaluate| Page
+  Specs -->|page.route| Vite
+  Page --> Hooks
+  Page --> Fuzz
+  Page --> Spark
+  Hooks -->|spark.hooks| Spark
+  Fuzz --> Spark
+
+
    +
  • CI job: .github/workflows/ci-browser.yml. Manual only (workflow_dispatch): run it from the Actions tab or with gh workflow run ci-browser.yml --ref <branch>; it does not run on push or pull request. Installs Rust with the wasm32 target, runs build:wasm, installs Chromium, then test:fixtures and test:browser with FUZZ_ITERS from the fuzz_iters input (default 3).
    • +
  • Playwright runner: test/playwright.config.ts. Chromium with SwiftShader, one worker, 320x320 viewport. Starts Vite on port 8080 (or reuses a running one) and runs the global setup.
    • +
  • Fixtures: test/fixtures/gen-fixture.mjs writes deterministic synthetic scenes (seeded blobs) so LoD trees and chunk boundaries are stable across runs, then runs build-lod --rad-chunked into test/fixtures/out (gitignored). fixture.ply (300K splats) is used by paged/LoD tests; fixture-small.ply (20K) by non-paged tests where SwiftShader is slow. Outputs: fixture-lod.rad and chunked/fixture-lod.rad + *.radc.
    • +
  • Specs and helpers: test/browser/*.spec.ts (sanity: boot, fixture and GPU baseline; on-demand: every setDirty site in section 1; churn: add/remove/hide/dispose orderings through the network seam; orderings: interleavings that need hook points, plus a direct worker RPC test; pager-fuzz: fresh random seeds; regressions: pinned seeds). helpers.ts provides openHarness, harness(), waitForSnapshot, holdRequests, expectInvariants, expectNoErrors.
    • +
  • Harness page: test/browser/pages/harness.html + harness.ts expose window.harness (next section). HooksController implements SparkHooks with hold / release / failNext / rejectHeld. fuzz.ts holds the seeded scheduler and FetchGate, a patched fetch that can hold chunk requests.
    • +
  • Two ways to control timing: the network (Playwright page.route or the in-page FetchGate) reproduces everything a real application can hit; hook points reproduce interleavings inside a single callback (a chunk landing between consumeLodTreeUpdates and cleanupLodTrees, a dispose during a traverse, a fault injected into the sort).
    • +
  • Render modes: loop (rAF), manual (only harness.render()), ondemand (renders only from onDirty, plus explicit calls). On-demand tests let the application render once and assert that the pipeline converges from onDirty alone.
    • +
+ +

Hook points (src/SparkHooks.ts)

+

+ hookPoint(hooks, name) is called at fixed awaits inside the three asynchronous flows. In + production hooks is undefined and the call is a single falsy check. The harness + installs a controller that can hold, release or fail any point. +

+ + + + + + + + + + + + +
NameWhere
sort.startStart of sortInternal, before readbackDepth.
sort.afterReadbackAfter the GPU depth readback, before the worker call.
sort.afterWorkerAfter sortSplats32 returns, before the ordering texture upload.
lod.startStart of lodCallback.
lod.afterInitAfter the trees in lodInitQueue are created.
lod.afterUpdateTreesAfter updateLodTrees returns.
lod.afterTraverseAfter traverseLodTrees returns, before updateLodIndices.
lod.beforeCleanupBefore cleanupLodTrees.
pager.fetchedA chunk fetch and decode completed, before it is pushed to fetched.
pager.beforeProcessFetchedBefore processFetched assigns pages to queued chunks.
+ +

Harness API (test/browser/pages/harness.ts)

+ + + + + + + + + + + +
MethodPurpose
init({mode, maxPages, lodSplatCount, numLodFetchers, lodDisposeTimeoutMs, fetchPause})Create renderer, scene, camera and SparkRenderer with small pools and short timeouts so eviction and cleanup happen within a test.
addPaged(name, url, {position}), addMesh(name, url, opts)Add a paged or fully loaded SplatMesh. Names map to PagedSplats for readable snapshots.
remove, readd, dispose, destroy, setVisible, setPositionScene churn primitives used by specs and the fuzzer.
setMode, render, moveCamera(pos, look)Frame control.
snapshot()Renders nothing. Returns counters and tables: renders, dirtyEvents, pagerUpdates, pagerDrops, sorting, sortDirty, lodDirty, activeSplats, inFlight, held, lodIds[{name, lodId, rootPage}], pager.{maxPages, freelist, mapped, fetchers, fetched, lodTreeUpdates, ...}, meshes[name].{pagedNumSplats, pagedAborted, pagedHasIndicesTexture}, errors.
hooks.hold(name) / unhold / release(name, n) / releaseAll / failNext(name, msg) / rejectHeld / heldCount / heldNamesThe SparkHooks controller. A held point returns a pending promise; release resolves the oldest; failNext rejects the next arrival.
waitIdle, waitQuiet, isBusy, renderUntilIdleConvergence: no fetchers, no pending updates, no sort, no scheduled render, stable for settleMs.
checkInvariants()Pager invariants with the live set from spark.lodIds, accumulator freelist sanity, and (when drained) the Rust cross-check via lodWorkerCall("getLodTreeIds" | "getLodTreeInfo").
residentChunks(name), countLitPixels(), awaitInitialized(name), lodWorkerCall(name, args), fuzz(options)Residency, the rendered image, load completion, direct worker RPC, and the fuzzer entry point.
+ +

Fuzzer (test/browser/pages/fuzz.ts, pager-fuzz.spec.ts, regressions.spec.ts)

+

+ A seeded in-page scheduler (mulberry32 RNG) draws one weighted action per step against a small pool + (FUZZ_HARNESS in fuzz-shared.ts), yields a macrotask, and runs the + quick pager invariant check. Every drainEvery steps and at the end it releases every held + hook and request, renders until idle and runs the full check including the Rust cross-check. The final + phase removes every mesh, waits past lodDisposeTimeoutMs and requires the pool to be fully + released and lodIds empty. +

+ + + + + + + + + + + + + + +
ProbabilityActionExercises
0.10add a paged mesh at a random position (up to maxMeshes)tree creation, root fetch priority by distance
0.08remove a mesh from the scenecleanup after the timeout while fetches may be in flight
0.08re-add a removed meshre-created tree with in-flight or pending state
0.04dispose a meshabort path; dispose during a traverse
0.06toggle visibilityhidden meshes keep their tree but lose pages, including the root
0.08jump the camera to one of a few presetschanges wanted chunks, eviction pressure
0.18render 1-3 frameslets callbacks run; on-demand behaviour
0.10release one held chunk fetch (FetchGate)fetch landing at arbitrary points
0.08release one held hook pointresume a held callback or fetch
0.06hold a random hook pointhold future arrivals
0.04unhold + flush a hook point
0.10sleep 0-80 mslet timers (backoff, dispose timeout) fire
+

+ On failure the spec prints the seed, the step trace and the violation strings. Seeds that have failed + are kept in PINNED in regressions.spec.ts with the exact harness + config and step count so the same interleaving replays on every run. FUZZ_ITERS controls + how many fresh seeds pager-fuzz.spec.ts runs (3 in CI). +

+ +

Writing a new ordering test

+
const opened = await openHarness(page, { mode: "manual", maxPages: 8, lodDisposeTimeoutMs: 200 });
+const h = harness(page);
+await h.addPaged("A", FIXTURES.chunked);
+await h.renderUntilIdle();
+
+// 1. Hold the pipeline at the await you care about
+await h.hooks.hold("lod.beforeCleanup");
+await h.render();
+await h.hooks.waitHeld("lod.beforeCleanup", 1);
+
+// 2. Perform the racing action (scene edit, held fetch release, another render)
+await h.remove("A");
+
+// 3. Resume and let everything settle
+await h.hooks.unhold("lod.beforeCleanup");
+await h.hooks.release("lod.beforeCleanup", 1);
+expect(await h.renderUntilIdle()).toBe(true);
+
+// 4. Assert on tables, not on timing
+await expectInvariants(page);            // pager + Rust cross-check
+expectNoErrors(opened, await h.snapshot());
+
    +
  • Prefer the network seam (holdRequests(page, ..., url => chunkIndexFromUrl(url) !== 0)) when the ordering is reachable by a real application. Use hook points when the race is inside one callback.
    • +
  • New hook points: add a hookPoint(this.hooks, "area.name") line at the await, list the name in the SparkHooks doc comment and in HOOK_NAMES in fuzz.ts so the fuzzer can hold it.
    • +
  • If the fuzzer finds a failing seed, copy the { seed, steps } pair into PINNED in regressions.spec.ts.
    • +
+ +

Commands

+
npm run build:wasm                     # spark-rs (needs Rust + wasm32-unknown-unknown)
+npm run test:fixtures                  # generate test/fixtures/out via build-lod (also done by globalSetup)
+npm run test:browser                   # all browser specs (starts vite on 8080 or reuses a running one)
+npm run test:browser -- churn          # one spec file (config lives at test/playwright.config.ts)
+npm run test:browser -- -g "P1b"       # by test title
+FUZZ_ITERS=10 npm run test:browser -- pager-fuzz
+npx tsc -p test/tsconfig.json          # type-check the test code (Node + DOM types)
+npx playwright show-trace test-results/<run>/trace.zip   # traces are kept on failure
+ +
+ + + + diff --git a/docs/internals/pager/internals.css b/docs/internals/pager/internals.css new file mode 100644 index 00000000..69d7ac04 --- /dev/null +++ b/docs/internals/pager/internals.css @@ -0,0 +1,115 @@ +/* Shared styles for docs/internals/pager/*.html */ +:root { + --bg: #ffffff; + --fg: #1d2330; + --muted: #5b6474; + --line: #e2e6ee; + --soft: #f5f7fb; + --accent: #2457c5; + --ok: #1d8a4a; + --okbg: #e6f6ec; + --bad: #c0392b; + --badbg: #fdecea; + --warn: #a6640a; + --warnbg: #fff4dd; + --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; +} +* { box-sizing: border-box; } +html { scroll-behavior: smooth; } +body { + margin: 0; + font: 15px/1.55 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + color: var(--fg); + background: var(--bg); + display: grid; + grid-template-columns: 260px 1fr; +} +nav { + position: sticky; + top: 0; + height: 100vh; + overflow-y: auto; + border-right: 1px solid var(--line); + padding: 20px 16px; + background: var(--soft); + font-size: 13.5px; +} +nav h1 { font-size: 15px; margin: 0 0 12px; } +nav a { display: block; color: var(--fg); text-decoration: none; padding: 3px 0; } +nav a:hover { color: var(--accent); } +nav .sub a { padding-left: 14px; color: var(--muted); } +nav .group { margin-top: 10px; font-weight: 600; } +nav .see { margin-top: 18px; padding-top: 12px; border-top: 1px solid var(--line); } +nav .see a { color: var(--accent); } +main { padding: 28px 44px 80px; max-width: 1180px; } +h2 { margin-top: 48px; padding-top: 16px; border-top: 1px solid var(--line); font-size: 24px; } +h3 { margin-top: 32px; font-size: 18px; } +h4 { margin-top: 24px; font-size: 15.5px; } +p, li { max-width: 88ch; } +code, pre { font-family: var(--mono); font-size: 0.92em; } +code { background: var(--soft); padding: 1px 5px; border-radius: 4px; } +pre.code { background: var(--soft); padding: 12px 14px; border-radius: 6px; overflow-x: auto; line-height: 1.45; } +pre.code code { background: none; padding: 0; } +table { border-collapse: collapse; width: 100%; margin: 12px 0 20px; font-size: 13.5px; } +th, td { border: 1px solid var(--line); padding: 7px 9px; vertical-align: top; text-align: left; } +th { background: var(--soft); } +td code { white-space: nowrap; } +td.wrap code { white-space: normal; } +.mermaid { background: #fff; border: 1px solid var(--line); border-radius: 6px; padding: 12px; margin: 14px 0 22px; overflow-x: auto; } +.callout { border-left: 4px solid var(--accent); background: var(--soft); padding: 10px 14px; border-radius: 0 6px 6px 0; margin: 14px 0; } +.callout.warn { border-color: var(--warn); background: var(--warnbg); } +.callout.ok { border-color: var(--ok); background: var(--okbg); } +.pill { display: inline-block; font-size: 11.5px; padding: 1px 7px; border-radius: 10px; border: 1px solid var(--line); background: #fff; margin-right: 4px; } +.pill.red { color: var(--bad); border-color: #f1b8b3; background: var(--badbg); } +.pill.green { color: var(--ok); border-color: #b5e0c4; background: var(--okbg); } +.pill.grey { color: var(--muted); } +.file { color: var(--muted); font-family: var(--mono); font-size: 12.5px; } +.two { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; } +@media (max-width: 1000px) { .two { grid-template-columns: 1fr; } body { grid-template-columns: 1fr; } nav { position: static; height: auto; } } + +/* Simulator (used by fixes-2026-09.html) */ +.sim { border: 1px solid var(--line); border-radius: 8px; margin: 16px 0 28px; background: #fff; } +.sim-head { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; padding: 10px 14px; border-bottom: 1px solid var(--line); background: var(--soft); border-radius: 8px 8px 0 0; } +.sim-head .title { font-weight: 600; flex: 1 1 auto; } +.sim button { font: inherit; font-size: 13px; padding: 4px 10px; border: 1px solid #c8cfdb; border-radius: 5px; background: #fff; cursor: pointer; } +.sim button:hover { border-color: var(--accent); } +.sim button:disabled { opacity: 0.45; cursor: default; } +.toggle { display: inline-flex; border: 1px solid #c8cfdb; border-radius: 5px; overflow: hidden; } +.toggle button { border: 0; border-radius: 0; } +.toggle button.on { background: var(--accent); color: #fff; } +.sim-body { display: grid; grid-template-columns: 300px 1fr; gap: 0; } +.sim-steps { border-right: 1px solid var(--line); padding: 8px 0; font-size: 13.5px; } +.sim-steps ol { margin: 0; padding: 0 0 0 30px; } +.sim-steps li { padding: 4px 8px 4px 2px; cursor: pointer; color: var(--muted); } +.sim-steps li.done { color: var(--fg); } +.sim-steps li.cur { color: var(--fg); font-weight: 600; background: #eef3ff; border-radius: 4px; } +.sim-state { padding: 12px 14px; } +.sim-note { min-height: 44px; margin-bottom: 10px; font-size: 14px; } +.sim-note .step-no { color: var(--muted); font-size: 12.5px; } +.panel { margin-bottom: 10px; } +.panel-label { font-size: 12px; color: var(--muted); text-transform: uppercase; letter-spacing: .03em; margin-bottom: 3px; } +.pages { display: flex; gap: 6px; flex-wrap: wrap; } +.page { min-width: 74px; padding: 6px 8px; border: 1px solid var(--line); border-radius: 5px; font-family: var(--mono); font-size: 12.5px; background: var(--soft); text-align: center; } +.page .idx { color: var(--muted); font-size: 11px; display: block; } +.page.mapped { background: #e7efff; border-color: #b9ccf5; } +.page.changed { outline: 2px solid var(--warn); outline-offset: 1px; } +.chips { display: flex; gap: 6px; flex-wrap: wrap; min-height: 26px; } +.chip { font-family: var(--mono); font-size: 12.5px; padding: 3px 8px; border: 1px solid var(--line); border-radius: 12px; background: #fff; } +.chip.new { background: var(--warnbg); border-color: #f0cf8f; } +.chip.insert { border-color: #b5e0c4; } +.chip.evict { border-color: #f1b8b3; } +.empty { color: var(--muted); font-size: 12.5px; font-style: italic; } +.kv { font-family: var(--mono); font-size: 12.5px; } +.kv div { padding: 2px 6px; border-radius: 4px; } +.kv div.new { background: var(--warnbg); } +.flags { display: flex; gap: 6px; flex-wrap: wrap; } +.flag { font-family: var(--mono); font-size: 12.5px; padding: 3px 8px; border-radius: 4px; border: 1px solid var(--line); } +.flag.true { background: #e7efff; border-color: #b9ccf5; } +.flag.new { outline: 2px solid var(--warn); outline-offset: 1px; } +.log { font-family: var(--mono); font-size: 12px; background: var(--soft); padding: 8px 10px; border-radius: 5px; max-height: 160px; overflow-y: auto; } +.log div.new { color: var(--warn); } +.verdict { margin-top: 10px; padding: 8px 12px; border-radius: 5px; font-size: 13.5px; } +.verdict.ok { background: var(--okbg); color: var(--ok); } +.verdict.bad { background: var(--badbg); color: var(--bad); } +.verdict ul { margin: 4px 0 0; padding-left: 18px; } +.verdict code { background: rgba(255,255,255,.6); } diff --git a/examples.html b/examples.html index 6317fe34..9e28e58a 100644 --- a/examples.html +++ b/examples.html @@ -266,7 +266,8 @@ 'nonlod': './nonlod/index.html', 'extsplats': './extsplats/index.html', 'streaming-lod': './streaming-lod/index.html', - 'multi-lod': './multi-lod/index.html' + 'multi-lod': './multi-lod/index.html', + 'on-demand': './on-demand/index.html' }; function getExampleFromHash() { @@ -478,7 +479,8 @@ Simultaneous Non-LoD + LoD Extended Splats encoding Streaming LoDs - Multiple Streaming LoDs + Multiple Streaming LoDs + On-demand Rendering
diff --git a/examples/on-demand/index.html b/examples/on-demand/index.html new file mode 100644 index 00000000..3da52c6d --- /dev/null +++ b/examples/on-demand/index.html @@ -0,0 +1,143 @@ + + + + + + + Spark • On-demand rendering + + + + + +
+

On-demand rendering

+

+ No animation loop: frames are rendered only when the camera moves or when + Spark calls onDirty (sort finished, LoD updated, streamed + chunk landed, mesh loaded). +

+

+
+ + + + + diff --git a/index.html b/index.html index 87e59291..33927855 100644 --- a/index.html +++ b/index.html @@ -164,6 +164,7 @@

Examples

  • Extended Splats encoding
  • Streaming LoDs
  • Multiple Streaming LoDs
    • +
  • On-demand Rendering
    • diff --git a/mkdocs.yml b/mkdocs.yml index 0d5650ab..bef9330d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,7 @@ nav: - Overview: docs/overview.md - System Design: docs/system-design.md - SparkRenderer: docs/spark-renderer.md + - On-demand rendering: docs/on-demand-rendering.md - SplatMesh: docs/splat-mesh.md - PackedSplats: docs/packed-splats.md - ExtSplats: docs/ext-splats.md diff --git a/package-lock.json b/package-lock.json index fa990057..c6da9ead 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,20 @@ { "name": "@sparkjsdev/spark", - "version": "2.1.0", + "version": "2.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@sparkjsdev/spark", - "version": "2.1.0", + "version": "2.2.0", "license": "MIT", "dependencies": { "fflate": "^0.8.2" }, "devDependencies": { "@biomejs/biome": "1.9.4", + "@playwright/test": "^1.63.0", + "@types/node": "^22.20.2", "@types/three": "0.180.0", "fflate": "^0.8.2", "lefthook": "1.11.12", @@ -789,6 +791,22 @@ "resolve": "~1.22.2" } }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@rollup/pluginutils": { "version": "5.1.4", "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.4.tgz", @@ -1223,11 +1241,11 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.15.17", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.17.tgz", - "integrity": "sha512-wIX2aSZL5FE+MR0JlvF87BNVrtFWf6AE6rxSE9X7OwnVvoyCQjpzSRJ+M87se/4QCkCiebQAqrJ0y6fwIyi7nw==", + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", "dev": true, - "peer": true, + "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } @@ -2359,6 +2377,35 @@ "pathe": "^2.0.3" } }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/postcss": { "version": "8.5.3", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", @@ -2749,8 +2796,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "peer": true + "dev": true }, "node_modules/universalify": { "version": "2.0.1", @@ -2942,23 +2988,11 @@ "node": ">=6" } }, - "rust/spark-internal-rs/pkg": { - "name": "spark-internal-rs", - "version": "0.1.0", - "extraneous": true, - "license": "Proprietary" - }, "rust/spark-rs/pkg": { "name": "spark-rs", "version": "0.1.0", "dev": true, "license": "Proprietary" - }, - "rust/spark-worker-rs/pkg": { - "name": "spark-worker-rs", - "version": "0.1.0", - "extraneous": true, - "license": "Proprietary" } } } diff --git a/package.json b/package.json index bb1357fa..0ab78e3a 100644 --- a/package.json +++ b/package.json @@ -38,13 +38,17 @@ "site:deploy": "npm run site:build && node scripts/deploy-site.js", "site:serve": "node scripts/serve-site.js site", "start": "npm run dev", - "test": "node --no-warnings --loader ts-node/esm --test test/**/*.test.ts" + "test": "node --no-warnings --loader ts-node/esm --test test/**/*.test.ts", + "test:browser": "playwright test -c test/playwright.config.ts", + "test:fixtures": "node test/fixtures/gen-fixture.mjs" }, "repository": "sparkjs-dev/spark", "files": ["dist"], "license": "MIT", "devDependencies": { "@biomejs/biome": "1.9.4", + "@playwright/test": "^1.63.0", + "@types/node": "^22.20.2", "@types/three": "0.180.0", "fflate": "^0.8.2", "lefthook": "1.11.12", diff --git a/rust/spark-rs/src/lod_tree.rs b/rust/spark-rs/src/lod_tree.rs index c7121057..e7e73ed1 100644 --- a/rust/spark-rs/src/lod_tree.rs +++ b/rust/spark-rs/src/lod_tree.rs @@ -183,8 +183,12 @@ thread_local! { static STATE: RefCell = RefCell::new(LodState::new()); } -fn set_lod_tree_data(state: &mut LodState, lod_id: u32, page_base: u32, _chunk_base: u32, count: u32, lod_tree_data: &Uint32Array) { - let lod_tree = state.lod_trees.get(&lod_id).unwrap(); +fn unknown_lod_id(lod_id: u32) -> JsValue { + JsValue::from_str(&format!("Unknown lod_id: {}", lod_id)) +} + +fn set_lod_tree_data(state: &mut LodState, lod_id: u32, page_base: u32, _chunk_base: u32, count: u32, lod_tree_data: &Uint32Array) -> Result<(), JsValue> { + let lod_tree = state.lod_trees.get(&lod_id).ok_or_else(|| unknown_lod_id(lod_id))?; let mut splats = lod_tree.splats.borrow_mut(); if state.buffer.is_empty() { @@ -218,6 +222,7 @@ fn set_lod_tree_data(state: &mut LodState, lod_id: u32, page_base: u32, _chunk_b } index += chunk; } + Ok(()) } #[wasm_bindgen] @@ -242,7 +247,7 @@ pub fn new_lod_tree(capacity: u32) -> Result { #[wasm_bindgen] pub fn new_shared_lod_tree(orig_lod_id: u32) -> Result { STATE.with_borrow_mut(|state| { - let lod_tree = state.lod_trees.get(&orig_lod_id).unwrap(); + let lod_tree = state.lod_trees.get(&orig_lod_id).ok_or_else(|| unknown_lod_id(orig_lod_id))?; let splats = lod_tree.splats.clone(); let page_to_chunk = Vec::with_capacity(lod_tree.page_to_chunk.capacity()); let chunk_to_page = Vec::with_capacity(lod_tree.chunk_to_page.capacity()); @@ -269,7 +274,7 @@ pub fn init_lod_tree(num_splats: u32, lod_tree: Uint32Array) -> Resultchunk mapping tables and splat +/// storage size of a LoD tree, or an error if the id is unknown. +#[wasm_bindgen] +pub fn get_lod_tree_info(lod_id: u32) -> Result { + STATE.with_borrow(|state| { + let lod_tree = state.lod_trees.get(&lod_id) + .ok_or_else(|| JsValue::from_str(&format!("Unknown lod_id: {}", lod_id)))?; + let splats = lod_tree.splats.borrow(); + + let page_to_chunk = Uint32Array::new_with_length(lod_tree.page_to_chunk.len() as u32); + page_to_chunk.copy_from(&lod_tree.page_to_chunk); + let chunk_to_page = Uint32Array::new_with_length(lod_tree.chunk_to_page.len() as u32); + chunk_to_page.copy_from(&lod_tree.chunk_to_page); + + let result = Object::new(); + Reflect::set(&result, &JsValue::from_str("lodId"), &JsValue::from(lod_id)).unwrap(); + Reflect::set(&result, &JsValue::from_str("numSplats"), &JsValue::from(splats.len() as u32)).unwrap(); + Reflect::set(&result, &JsValue::from_str("sharedRefs"), &JsValue::from(Rc::strong_count(&lod_tree.splats) as u32)).unwrap(); + Reflect::set(&result, &JsValue::from_str("pageToChunk"), &JsValue::from(page_to_chunk)).unwrap(); + Reflect::set(&result, &JsValue::from_str("chunkToPage"), &JsValue::from(chunk_to_page)).unwrap(); + Ok(result) + }) +} + +/// Debug/test introspection: all live LoD tree ids. +#[wasm_bindgen] +pub fn get_lod_tree_ids() -> Uint32Array { + STATE.with_borrow(|state| { + let mut ids: Vec = state.lod_trees.keys().copied().collect(); + ids.sort_unstable(); + let result = Uint32Array::new_with_length(ids.len() as u32); + result.copy_from(&ids); + result + }) +} + #[wasm_bindgen] pub fn update_lod_trees(lod_ids: &[u32], page_bases: &[u32], chunk_bases: &[u32], counts: &[u32], lod_trees: &Array) -> Result { STATE.with_borrow_mut(|state| { for (&lod_id, &page_base, &chunk_base, &count, lod_tree_data) in izip!(lod_ids, page_bases, chunk_bases, counts, lod_trees.iter()) { - let lod_tree = state.lod_trees.get_mut(&lod_id).unwrap(); + let lod_tree = state.lod_trees.get_mut(&lod_id).ok_or_else(|| unknown_lod_id(lod_id))?; let pages = count.div_ceil(65536); let base_page = page_base >> 16; let base_chunk = chunk_base >> 16; @@ -313,7 +354,7 @@ pub fn update_lod_trees(lod_ids: &[u32], page_bases: &[u32], chunk_bases: &[u32] } let lod_tree_data = Uint32Array::from(lod_tree_data); - set_lod_tree_data(state, lod_id, page_base, chunk_base, count, &lod_tree_data); + set_lod_tree_data(state, lod_id, page_base, chunk_base, count, &lod_tree_data)?; } } @@ -372,7 +413,7 @@ fn is_resident(index: u32, instance: &LodInstance) -> bool { pub fn get_lod_tree_level(lod_id: u32, level: u32) -> anyhow::Result { STATE.with_borrow_mut(|state| { let LodState { lod_trees, .. } = state; - let lod_tree = lod_trees.get(&lod_id).unwrap(); + let lod_tree = lod_trees.get(&lod_id).ok_or_else(|| unknown_lod_id(lod_id))?; let splats = lod_tree.splats.borrow(); let root_size = splats[0].size(); @@ -440,7 +481,7 @@ pub fn traverse_lod_trees( STATE.with_borrow_mut(|state| { let LodState { lod_trees, frontier, output, touched, touched_set, .. } = state; let instances: Vec<_> = lod_ids.iter().enumerate().map(|(index, &lod_id)| { - let lod_tree = lod_trees.get(&lod_id).unwrap(); + let lod_tree = lod_trees.get(&lod_id).ok_or_else(|| unknown_lod_id(lod_id))?; let LodTree { splats, page_to_chunk, chunk_to_page } = &lod_tree; let i16 = index * 16; let forward = Vec3A::from_slice(&view_to_objects[(i16 + 8)..(i16 + 11)]).normalize().map(|x| -x); @@ -451,8 +492,8 @@ pub fn traverse_lod_trees( let cone_dot0 = if cone_fov0s[index] > 0.0 { (0.5 * cone_fov0s[index].clamp(0.0, 180.0)).to_radians().cos() } else { 1.0 }; let cone_dot = if cone_fovs[index] > 0.0 { (0.5 * cone_fovs[index].clamp(0.0, 180.0)).to_radians().cos() } else { 1.0 }; let cone_dot = cone_dot.min(cone_dot0); - (lod_id, splats.borrow(), page_to_chunk, chunk_to_page, origin, forward, lod_scale, behind_foveate, cone_foveate, cone_dot0, cone_dot) - }).collect(); + Ok((lod_id, splats.borrow(), page_to_chunk, chunk_to_page, origin, forward, lod_scale, behind_foveate, cone_foveate, cone_dot0, cone_dot)) + }).collect::, JsValue>>()?; let mut num_splats = 0; frontier.clear(); @@ -466,6 +507,10 @@ pub fn traverse_lod_trees( let root_page = root_pages[inst_index]; let root_page = if root_page == 0xFFFFFFFF { 0 } else { root_page }; let root_index = root_page << 16; + if root_index as usize >= splats.len() { + // Root page has no data (yet); instance contributes no splats + continue; + } let pixel_scale = compute_pixel_scale(&splats[root_index as usize], instance); frontier.push((OrderedFloat(pixel_scale), inst_index as u32, root_index)); num_splats += 1; @@ -664,7 +709,7 @@ pub fn dynamic_traverse_lod_trees( STATE.with_borrow_mut(|state| { let LodState { lod_trees, .. } = state; let instances: Vec<_> = lod_ids.iter().enumerate().map(|(index, &lod_id)| { - let lod_tree = lod_trees.get(&lod_id).unwrap(); + let lod_tree = lod_trees.get(&lod_id).ok_or_else(|| unknown_lod_id(lod_id))?; let LodTree { splats, page_to_chunk, chunk_to_page } = &lod_tree; let i16 = index * 16; let forward = Vec3A::from_slice(&view_to_objects[(i16 + 8)..(i16 + 11)]).normalize().map(|x| -x); @@ -674,8 +719,8 @@ pub fn dynamic_traverse_lod_trees( let cone_foveate = cone_foveates[index]; let cone_dot0 = if cone_fov0s[index] > 0.0 { (0.5 * cone_fov0s[index]).to_radians().cos() } else { 1.0 }; let cone_dot = if cone_fovs[index] > 0.0 { (0.5 * cone_fovs[index]).to_radians().cos() } else { 1.0 }; - (lod_id, splats.borrow(), page_to_chunk, chunk_to_page, origin, forward, lod_scale, behind_foveate, cone_foveate, cone_dot0, cone_dot) - }).collect(); + Ok((lod_id, splats.borrow(), page_to_chunk, chunk_to_page, origin, forward, lod_scale, behind_foveate, cone_foveate, cone_dot0, cone_dot)) + }).collect::, JsValue>>()?; let mut lod_chunk_max: AHashMap> = AHashMap::new(); @@ -685,8 +730,13 @@ pub fn dynamic_traverse_lod_trees( let root_page = root_pages[inst_index]; let root_page = if root_page == 0xFFFFFFFF { 0 } else { root_page }; let root_index = root_page << 16; - let root_scale = compute_pixel_scale(&splats[root_index as usize], instance); - let frontier = vec![(root_index, root_scale)]; + let frontier = if (root_index as usize) < splats.len() { + let root_scale = compute_pixel_scale(&splats[root_index as usize], instance); + vec![(root_index, root_scale)] + } else { + // Root page has no data (yet); instance contributes no splats + Vec::new() + }; let instance_output = Vec::with_capacity(1000); outputs.push((instance_output, frontier)); diff --git a/src/SparkHooks.ts b/src/SparkHooks.ts new file mode 100644 index 00000000..df54aef3 --- /dev/null +++ b/src/SparkHooks.ts @@ -0,0 +1,36 @@ +/** + * Test/debug instrumentation for Spark's asynchronous pipelines. + * + * Spark has several concurrent asynchronous flows (splat sorting, LoD tree + * updates/traversal, paged chunk fetching). To make their interleavings + * deterministic in tests, the code calls `hooks.point(name)` at well-defined + * async boundaries. In production `SparkRenderer.hooks` is undefined and these + * calls compile down to a single undefined check with no extra microtask. + * + * A hook implementation may: + * - return `undefined` to let execution continue immediately, + * - return a `Promise` to hold execution at that point until it resolves, + * - throw (or return a rejected promise) to inject a fault at that point. + * + * Hook point names currently emitted: + * - `sort.start`, `sort.afterReadback`, `sort.afterWorker` + * - `lod.start`, `lod.afterInit`, `lod.afterUpdateTrees`, `lod.afterTraverse`, + * `lod.beforeCleanup` + * - `pager.fetched` (a chunk fetch+decode completed, before it is queued), + * `pager.beforeProcessFetched` (before queued chunks are assigned pages) + */ +export interface SparkHooks { + point(name: string, info?: unknown): undefined | Promise; +} + +/** + * Await a hook point if hooks are installed. Kept as a tiny helper so call + * sites stay one line and production code performs no allocation. + */ +export function hookPoint( + hooks: SparkHooks | undefined, + name: string, + info?: unknown, +): undefined | Promise { + return hooks ? hooks.point(name, info) : undefined; +} diff --git a/src/SparkRenderer.ts b/src/SparkRenderer.ts index 25434513..ec94fc0d 100644 --- a/src/SparkRenderer.ts +++ b/src/SparkRenderer.ts @@ -8,6 +8,7 @@ import { SplatMesh, SplatPager, } from "."; +import { type SparkHooks, hookPoint } from "./SparkHooks"; import { SplatAccumulator } from "./SplatAccumulator"; import { SplatGeometry } from "./SplatGeometry"; import { SplatWorker } from "./SplatWorker"; @@ -222,6 +223,20 @@ export interface SparkRendererOptions { * @default 3 */ numLodFetchers?: number; + /** + * How long (ms) a LoD tree for a SplatMesh that is no longer visible is kept + * alive before it is disposed and its paged splats are released. Trees + * backing meshes that are visible in the current frame are never disposed, + * whatever the value; `0` disposes a tree on the first LoD update after its + * mesh stops being visible. + * @default 3000 + */ + lodDisposeTimeoutMs?: number; + /** + * Optional instrumentation hooks for tests. See `SparkHooks`. + * @default undefined + */ + hooks?: SparkHooks; /** * Full-width angle in degrees of fixed foveation cone along the view direction * with no foveation applied (full resolution, foveate=1.0). Set to 0 to disable. @@ -354,7 +369,7 @@ export class SparkRenderer extends THREE.Mesh { readonly timer: THREE.Timer; private readonly ownsTimer: boolean; lastFrame = -1; - updateTimeoutId = -1; + updateTimeoutId: ReturnType | -1 = -1; onDirty?: () => void; dirty: boolean; @@ -370,10 +385,12 @@ export class SparkRenderer extends THREE.Mesh { sortDirty = false; lastSortTime = 0; sortWorker: SplatWorker | null = null; - sortTimeoutId = -1; + sortTimeoutId: ReturnType | -1 = -1; sortedCenter = new THREE.Vector3().setScalar(Number.NEGATIVE_INFINITY); sortedDir = new THREE.Vector3().setScalar(0); readback32 = new Uint32Array(0); + // Uninitialized SplatMeshes whose completion will call setDirty() + private initWatched: WeakSet = new WeakSet(); enableLod: boolean; enableDriveLod: boolean; @@ -386,6 +403,8 @@ export class SparkRenderer extends THREE.Mesh { pagedExtSplats: boolean; maxPagedSplats: number; numLodFetchers: number; + lodDisposeTimeoutMs: number; + hooks?: SparkHooks; behindFoveate: number; coneFov0: number; coneFov: number; @@ -535,6 +554,8 @@ export class SparkRenderer extends THREE.Mesh { const defaultPages = isMobile() ? (isIos() ? 96 : 128) : 256; this.maxPagedSplats = options.maxPagedSplats ?? defaultPages * 65536; this.numLodFetchers = options.numLodFetchers ?? 3; + this.lodDisposeTimeoutMs = options.lodDisposeTimeoutMs ?? 3000; + this.hooks = options.hooks; this.behindFoveate = options.behindFoveate ?? 0.2; this.coneFov0 = options.coneFov0 ?? 90.0; this.coneFov = options.coneFov ?? 120.0; @@ -944,6 +965,24 @@ export class SparkRenderer extends THREE.Mesh { lodInstances: this.enableLod ? this.lodInstances : undefined, }); + // Generators that are still loading change nothing this frame; make sure + // a render happens once they finish, even without a render loop. + for (const generator of visibleGenerators) { + if ( + generator instanceof SplatMesh && + !generator.isInitialized && + !this.initWatched.has(generator) + ) { + this.initWatched.add(generator); + generator.initialized.then( + () => this.setDirty(), + () => { + // Load failures are reported by the mesh itself + }, + ); + } + } + let doUpdate = true; const needsUpdate = viewChanged || version !== this.current.version; const mappingUpdated = mappingVersion !== this.display.mappingVersion; @@ -1019,6 +1058,25 @@ export class SparkRenderer extends THREE.Mesh { this.sortDirty = false; this.lastSortTime = now; + try { + await this.sortInternal(); + } catch (error) { + // A failed sort must not leave `sorting` stuck (which would block every + // later mapping change). Retry on the next frame. + this.sortDirty = true; + throw error; + } finally { + this.sorting = false; + this.setDirty(); + } + + this.driveSort(); + } + + private async sortInternal() { + const hookStart = hookPoint(this.hooks, "sort.start"); + if (hookStart) await hookStart; + if (this.readPause > 0) { await new Promise((resolve) => setTimeout(resolve, this.readPause)); } @@ -1044,6 +1102,9 @@ export class SparkRenderer extends THREE.Mesh { readback, }); + const hookReadback = hookPoint(this.hooks, "sort.afterReadback"); + if (hookReadback) await hookReadback; + if (this.sortPause > 0) { await new Promise((resolve) => setTimeout(resolve, this.sortPause)); } @@ -1057,6 +1118,9 @@ export class SparkRenderer extends THREE.Mesh { ordering, }); + const hookWorker = hookPoint(this.hooks, "sort.afterWorker"); + if (hookWorker) await hookWorker; + if (this.sortDelay > 0) { await new Promise((resolve) => setTimeout(resolve, this.sortDelay)); } @@ -1107,10 +1171,6 @@ export class SparkRenderer extends THREE.Mesh { this.display = this.current; } } - this.sorting = false; - this.setDirty(); - - this.driveSort(); } private ensureLodWorker() { @@ -1243,103 +1303,167 @@ export class SparkRenderer extends THREE.Mesh { } this.ensureLodWorker().tryExclusive(async (worker) => { - if (hasPaged && !this.pager) { - this.pager = new SplatPager({ - renderer: this.renderer, - extSplats: this.pagedExtSplats, - maxSplats: this.maxPagedSplats, - numFetchers: this.numLodFetchers, - }); - - const { lodId } = await worker.call("newLodTree", { - capacity: this.pager.maxSplats, + try { + await this.lodCallback(worker, { + hasPaged, + lodMeshes, + maxSplats, + viewPos, + viewQuat, + pixelScaleLimit, }); - this.pagerId = lodId; + } finally { + // Frames rendered while this callback ran could not start their own + // callback (tryExclusive), and chunk data may have landed since we + // consumed the pager queues. Both can only be picked up by another + // render, so request one for on-demand applications. + if ( + this.lodDirty || + this.lodInitQueue.length > 0 || + this.pager?.hasPendingUpdates() + ) { + this.setDirty(); + } } + }); + } - // Assign pager to any new meshes that don't have one yet - // (must run every frame, not just when pager is first created) - if (this.pager) { - for (const { mesh } of this.lodMeshes) { - if (mesh.paged && !mesh.paged.pager) { - mesh.paged.pager = this.pager; - } + private async lodCallback( + worker: SplatWorker, + { + hasPaged, + lodMeshes, + maxSplats, + viewPos, + viewQuat, + pixelScaleLimit, + }: { + hasPaged: boolean; + lodMeshes: SplatMesh[]; + maxSplats: number; + viewPos: THREE.Vector3; + viewQuat: THREE.Quaternion; + pixelScaleLimit: number; + }, + ) { + const hookStart = hookPoint(this.hooks, "lod.start"); + if (hookStart) await hookStart; + + if (hasPaged && !this.pager) { + this.pager = new SplatPager({ + renderer: this.renderer, + extSplats: this.pagedExtSplats, + maxSplats: this.maxPagedSplats, + numFetchers: this.numLodFetchers, + }); + this.pager.hooks = this.hooks; + // Chunk data that landed must be consumed by a LoD callback, which + // only runs from render(): ask on-demand applications to render. + this.pager.onUpdate = () => this.setDirty(); + + const { lodId } = await worker.call("newLodTree", { + capacity: this.pager.maxSplats, + }); + this.pagerId = lodId; + } + + // Assign pager to any new meshes that don't have one yet + // (must run every frame, not just when pager is first created) + if (this.pager) { + for (const { mesh } of this.lodMeshes) { + if (mesh.paged && !mesh.paged.pager) { + mesh.paged.pager = this.pager; } } + } - if (this.lodInitQueue.length > 0) { - const lodInitQueue = this.lodInitQueue; - this.lodInitQueue = []; - while (lodInitQueue.length > 0) { - const splats = lodInitQueue.shift(); - if (splats) { - await this.initLodTree(worker, splats); - this.lodDirty = true; - } + if (this.lodInitQueue.length > 0) { + const lodInitQueue = this.lodInitQueue; + this.lodInitQueue = []; + while (lodInitQueue.length > 0) { + const splats = lodInitQueue.shift(); + if (splats) { + await this.initLodTree(worker, splats); + this.lodDirty = true; } } + const hookInit = hookPoint(this.hooks, "lod.afterInit"); + if (hookInit) await hookInit; + } - if (this.pager) { - const updates = this.pager.consumeLodTreeUpdates(); + if (this.pager) { + const updates = this.pager.consumeLodTreeUpdates(); - for (const { splats, page, chunk, numSplats, lodTree } of updates) { - const record = this.lodIds.get(splats); - if (record) { - if (lodTree && chunk === 0) { + for (const { splats, page, chunk, numSplats, lodTree } of updates) { + const record = this.lodIds.get(splats); + if (record) { + if (chunk === 0) { + if (lodTree) { record.rootPage = page; + } else if (record.rootPage === page) { + // Root chunk evicted: the instance must not be traversed + // (it would read foreign data) until the root is re-fetched, + // and its current indices no longer point at resident data. + record.rootPage = undefined; + splats.update(0, EMPTY_INDICES); } - this.lodUpdates.push({ - lodId: record.lodId, - pageBase: page * this.pager.pageSplats, - chunkBase: chunk * this.pager.pageSplats, - count: numSplats, - lodTreeData: lodTree, - }); } + this.lodUpdates.push({ + lodId: record.lodId, + pageBase: page * this.pager.pageSplats, + chunkBase: chunk * this.pager.pageSplats, + count: numSplats, + lodTreeData: lodTree, + }); } } + } - if (this.lodUpdates.length > 0) { - const lodUpdates = this.lodUpdates; - this.lodUpdates = []; - await worker.call("updateLodTrees", { ranges: lodUpdates }); - this.lodDirty = true; + if (this.lodUpdates.length > 0) { + const lodUpdates = this.lodUpdates; + this.lodUpdates = []; + await worker.call("updateLodTrees", { ranges: lodUpdates }); + this.lodDirty = true; + const hookUpdate = hookPoint(this.hooks, "lod.afterUpdateTrees"); + if (hookUpdate) await hookUpdate; + } + + if (this.lodDirty) { + const now = performance.now(); + const deltaPred = new THREE.Vector3(); + if (this.lastLod) { + const deltaTime = Math.max(1, now - this.lastLod.timestamp); + deltaPred + .copy(viewPos) + .sub(this.lastLod.pos) + .multiplyScalar(this.lastTraverseTime / deltaTime); } + this.lastLod = { + pos: viewPos, + quat: viewQuat, + pixelScaleLimit, + maxSplats, + timestamp: now, + }; + this.lodDirty = false; + + await this.updateLodInstances( + worker, + deltaPred, + lodMeshes, + maxSplats, + viewPos, + viewQuat, + pixelScaleLimit, + ); + this.currentLod = this.lastLod; + this.setDirty(); + } - if (this.lodDirty) { - const now = performance.now(); - const deltaPred = new THREE.Vector3(); - if (this.lastLod) { - const deltaTime = Math.max(1, now - this.lastLod.timestamp); - deltaPred - .copy(viewPos) - .sub(this.lastLod.pos) - .multiplyScalar(this.lastTraverseTime / deltaTime); - } - this.lastLod = { - pos: viewPos, - quat: viewQuat, - pixelScaleLimit, - maxSplats, - timestamp: now, - }; - this.lodDirty = false; - - await this.updateLodInstances( - worker, - deltaPred, - lodMeshes, - maxSplats, - viewPos, - viewQuat, - pixelScaleLimit, - ); - this.currentLod = this.lastLod; - this.setDirty(); - } + const hookCleanup = hookPoint(this.hooks, "lod.beforeCleanup"); + if (hookCleanup) await hookCleanup; - await this.cleanupLodTrees(worker); - }); + await this.cleanupLodTrees(worker, lodMeshes); } private async initLodTree( @@ -1360,6 +1484,8 @@ export class SparkRenderer extends THREE.Mesh { }); this.lodIds.set(splats, { lodId, lastTouched: performance.now() }); this.lodIdToSplats.set(lodId, splats); + // Chunk fetches landing from now on belong to this tree + this.pager?.activateSplats(splats); // console.log("*** newSharedLodTree", lodId, this.pagerId, splats); } } @@ -1448,6 +1574,9 @@ export class SparkRenderer extends THREE.Mesh { }); this.lastTraverseTime = performance.now() - traverseStart; + const hookTraverse = hookPoint(this.hooks, "lod.afterTraverse"); + if (hookTraverse) await hookTraverse; + const { keyIndices, chunks, pixelLimit } = result; this.lastPixelLimit = pixelLimit; const totalLodSplats = Object.values(keyIndices).reduce( @@ -1461,6 +1590,16 @@ export class SparkRenderer extends THREE.Mesh { this.updateLodIndices(uuidToMesh, keyIndices); // console.log("chunks.length =", chunks.length); + // Paged meshes that could not be traversed (no tree yet, or root chunk + // not resident) must not keep displaying indices from a previous traverse + // that may now point at evicted pages. + for (const mesh of lodMeshes) { + if (mesh.paged && !instances[mesh.uuid] && mesh.paged.numSplats > 0) { + mesh.paged.update(0, EMPTY_INDICES); + mesh.updateMappingVersion(); + } + } + if (this.pager) { this.pager.processUploads(); @@ -1534,12 +1673,36 @@ export class SparkRenderer extends THREE.Mesh { } } - private async cleanupLodTrees(worker: SplatWorker) { - const DISPOSE_TIMEOUT_MS = 3000; + private async cleanupLodTrees(worker: SplatWorker, lodMeshes: SplatMesh[]) { + const DISPOSE_TIMEOUT_MS = this.lodDisposeTimeoutMs; const now = performance.now(); + // Splats backing visible meshes are never eligible: their lastTouched is + // stamped in updateLod before this (possibly long) async callback runs, + // so a short timeout could otherwise see them as stale. Check both the + // meshes this callback was started with and this.lodMeshes, which frames + // rendered while the callback ran have kept up to date (a mesh re-added + // during the callback has a fresh stamp but is not in `lodMeshes`). + const visible = new Set(); + const addVisible = (mesh: SplatMesh) => { + const splats = + mesh.packedSplats?.lodSplats ?? mesh.extSplats?.lodSplats ?? mesh.paged; + if (splats) { + visible.add(splats); + } + }; + for (const mesh of lodMeshes) { + addVisible(mesh); + } + for (const { mesh } of this.lodMeshes) { + addVisible(mesh); + } + let oldest = null; for (const [splats, record] of this.lodIds.entries()) { + if (visible.has(splats)) { + continue; + } if (oldest == null || record.lastTouched < oldest.lastTouched) { oldest = { splats, @@ -1564,6 +1727,10 @@ export class SparkRenderer extends THREE.Mesh { if (oldest.splats instanceof PagedSplats) { this.pager?.removeSplats(oldest.splats); + // Its pages are released; drop indices that point into them + if (oldest.splats.pager) { + oldest.splats.update(0, EMPTY_INDICES); + } } await worker.call("disposeLodTree", { lodId: oldest.lodId }); @@ -2076,3 +2243,5 @@ export class SparkRenderer extends THREE.Mesh { function checkIsXRRenderTarget(renderTarget: THREE.RenderTarget | null) { return (renderTarget as unknown as Record)?.isXRRenderTarget; } + +const EMPTY_INDICES = new Uint32Array(0); diff --git a/src/SplatLoader.ts b/src/SplatLoader.ts index 38a95460..0cee0d9b 100644 --- a/src/SplatLoader.ts +++ b/src/SplatLoader.ts @@ -88,10 +88,11 @@ export class SplatLoader extends Loader { lodAbove?: number; lodBase?: number; }) { - if (fileBytes instanceof ArrayBuffer) { - fileBytes = new Uint8Array(fileBytes); - } - const resolvedURL = fileBytes + // A const keeps the narrowed type inside the worker closure below; + // reassigning the parameter would lose it there. + const bytes: Uint8Array | undefined = + fileBytes instanceof ArrayBuffer ? new Uint8Array(fileBytes) : fileBytes; + const resolvedURL = bytes ? undefined : this.manager.resolveURL((this.path ?? "") + (url ?? "")); @@ -151,7 +152,7 @@ export class SplatLoader extends Loader { url: basedUrl, requestHeader: this.requestHeader, withCredentials: this.withCredentials, - fileBytes: fileBytes?.slice(), + fileBytes: bytes?.slice(), fileType, pathName: resolvedURL || fileName, chunked: stream !== undefined, diff --git a/src/SplatPager.ts b/src/SplatPager.ts index baa2869e..3e5039b0 100644 --- a/src/SplatPager.ts +++ b/src/SplatPager.ts @@ -4,6 +4,7 @@ import { decode_rad_header } from "spark-rs"; import { LN_SCALE_MAX, LN_SCALE_MIN, dyno } from "."; import { evaluateExtSH } from "./ExtSplats"; import { evaluatePackedSH } from "./PackedSplats"; +import { type SparkHooks, hookPoint } from "./SparkHooks"; import { getSplatFileType, getSplatFileTypeFromPath } from "./SplatLoader"; import type { SplatSource } from "./SplatMesh"; import { workerPool } from "./SplatWorker"; @@ -334,9 +335,21 @@ export class PagedSplats implements SplatSource { throw new Error("PagedSplats.pager not set"); } + if (this.abortController.signal.aborted) { + // Disposed while a traverse was in flight: keep the indices texture + // released and render nothing. + this.numSplats = 0; + this.dynoNumSplats.value = 0; + return; + } + const renderer = this.pager.renderer; this.numSplats = numSplats; this.dynoNumSplats.value = this.numSplats; + if (numSplats === 0) { + // Nothing is read from the indices texture when numSplats is 0 + return; + } const rows = Math.ceil(numSplats / 16384); let indicesTexture = @@ -525,6 +538,26 @@ interface PageUpload { shArrays: Array; } +/** Snapshot of SplatPager internal tables, see SplatPager.debugState() */ +export interface SplatPagerDebugState { + maxPages: number; + freelist: number[]; + freeable: number[]; + lruSize: number; + mapped: { page: number; splats: PagedSplats; chunk: number }[]; + fetchers: { splats: PagedSplats; chunk: number }[]; + fetched: { splats: PagedSplats; chunk: number }[]; + lodTreeUpdates: { + splats: PagedSplats; + chunk: number; + page: number; + insert: boolean; + }[]; + newUploads: number[]; + readyUploads: number[]; + fetchPriority: { splats: PagedSplats; chunk: number }[]; +} + export class SplatPager { readonly renderer: THREE.WebGLRenderer; @@ -540,6 +573,14 @@ export class SplatPager { numFetchers: number; fetchPause = 0; + /** + * Called whenever new chunk data has been assigned a page and is waiting to + * be consumed (via consumeLodTreeUpdates/processUploads) by the LoD driver. + */ + onUpdate?: () => void; + /** Optional test instrumentation, see SparkHooks. */ + hooks?: SparkHooks; + splatsChunkToPage: Map< PagedSplats, ({ page: number; lru: number } | undefined)[] @@ -568,6 +609,14 @@ export class SplatPager { data: PackedResult | ExtResult; }[]; fetchPriority: { splats: PagedSplats; chunk: number }[]; + /** + * PagedSplats whose pages were released via removeSplats() and that have + * not been re-activated since. Chunk fetches that were already in flight + * when the splats were removed land here and must not be mapped: their LoD + * tree is gone (or about to be re-created empty), so a page mapped now + * would never be reflected in the tree and would never be fetched again. + */ + private readonly retiredSplats: WeakSet = new WeakSet(); packedTexture: dyno.DynoUsampler2DArray< "packedTexture", @@ -958,28 +1007,79 @@ export class SplatPager { } } + /** + * Mark splats as active again after removeSplats(). Called when a LoD tree + * is (re-)created for them; from then on landing chunk fetches are mapped. + */ + activateSplats(splats: PagedSplats) { + this.retiredSplats.delete(splats); + } + + isRetired(splats: PagedSplats): boolean { + return this.retiredSplats.has(splats); + } + + /** + * Release every page owned by splats and drop all queued work that refers + * to them. Until activateSplats() is called again, chunk fetches that land + * for these splats are discarded. + */ removeSplats(splats: PagedSplats) { - const chunks = this.splatsChunkToPage.get(splats); - if (!chunks) { - return; + this.retiredSplats.add(splats); + + // Fetched-but-not-yet-mapped chunks would otherwise be mapped by the next + // processFetched() for a tree that no longer exists. + for (let i = this.fetched.length - 1; i >= 0; i--) { + if (this.fetched[i].splats === splats) { + this.fetched.splice(i, 1); + } } + const chunks = this.splatsChunkToPage.get(splats); const freedPages = new Set(); - while (chunks.length > 0) { - const chunk = chunks.pop(); - if (chunk) { - const { page } = chunk; - this.pageToSplatsChunk[page] = undefined; - freedPages.add(page); - this.pageFreelist.push(page); - this.pageLru.delete(chunk); + if (chunks) { + while (chunks.length > 0) { + const chunk = chunks.pop(); + if (chunk) { + const { page } = chunk; + this.pageToSplatsChunk[page] = undefined; + freedPages.add(page); + this.pageFreelist.push(page); + this.pageLru.delete(chunk); + } + } + this.splatsChunkToPage.delete(splats); + while ( + this.pageToSplatsChunk.length > 0 && + this.pageToSplatsChunk[this.pageToSplatsChunk.length - 1] === undefined + ) { + this.pageToSplatsChunk.pop(); + } + this.freeablePages = this.freeablePages.filter( + (page) => !freedPages.has(page), + ); + } + + // Pending tree updates for these splats refer to pages that are now free + // (and may be handed to another chunk before they are consumed). Their + // tree is disposed, so nothing needs to be applied for them. + if (this.lodTreeUpdates.some((update) => update.splats === splats)) { + this.lodTreeUpdates = this.lodTreeUpdates.filter( + (update) => update.splats !== splats, + ); + } + // Texture uploads for freed pages are dead data. + if (freedPages.size > 0) { + this.newUploads = this.newUploads.filter( + ({ page }) => !freedPages.has(page), + ); + for (let i = this.readyUploads.length - 1; i >= 0; i--) { + if (freedPages.has(this.readyUploads[i].page)) { + this.readyUploads.splice(i, 1); + } } } - this.splatsChunkToPage.delete(splats); - this.freeablePages = this.freeablePages.filter( - (page) => !freedPages.has(page), - ); } private uploadPage( @@ -1076,8 +1176,18 @@ export class SplatPager { .fetchDecodeChunk(chunk) .then( async (data) => { - // Make sure the originating PagedSplat hasn't been disposed in the meantime - if (splats.abortController.signal.aborted) { + const hook = hookPoint(this.hooks, "pager.fetched", { + splats, + chunk, + }); + if (hook) await hook; + + // Make sure the originating PagedSplat hasn't been disposed or + // removed (LoD tree cleaned up) in the meantime + if ( + splats.abortController.signal.aborted || + this.retiredSplats.has(splats) + ) { return; } @@ -1100,12 +1210,18 @@ export class SplatPager { await new Promise((resolve) => setTimeout(resolve, backoff)); }, ) - .finally(() => { + .finally(async () => { // Remove this fetcher from active fetchers list const fetchIndex = this.fetchers.indexOf(fetcher); this.fetchers[fetchIndex] = this.fetchers[this.fetchers.length - 1]; this.fetchers.length--; + const hook = hookPoint(this.hooks, "pager.beforeProcessFetched", { + splats, + chunk, + }); + if (hook) await hook; + this.processFetched(); }); @@ -1163,22 +1279,31 @@ export class SplatPager { private processFetched() { const now = performance.now(); + let updated = false; while (true) { const fetched = this.fetched.shift(); if (!fetched) { break; } const { splats, chunk, data } = fetched; + if ( + splats.abortController.signal.aborted || + this.retiredSplats.has(splats) + ) { + // Owner went away between landing and processing + continue; + } let page = this.allocatePage(); if (page === undefined) { page = this.allocateFreeable(); if (page === undefined) { // No pages available, stop for now - return; + break; } } + updated = true; this.insertSplatsChunkPage(splats, chunk, page, now); const { numSplats, extra } = data; this.lodTreeUpdates.push({ @@ -1229,6 +1354,10 @@ export class SplatPager { }); } } + + if (updated) { + this.onUpdate?.(); + } } processUploads() { @@ -1242,6 +1371,11 @@ export class SplatPager { } } + /** Chunk data waiting to be consumed by the next LoD callback. */ + hasPendingUpdates(): boolean { + return this.lodTreeUpdates.length > 0 || this.newUploads.length > 0; + } + consumeLodTreeUpdates() { const updates = this.lodTreeUpdates; this.lodTreeUpdates = []; @@ -1251,6 +1385,217 @@ export class SplatPager { return updates; } + /** + * Snapshot of the pager's internal tables for debugging and tests. + */ + debugState(): SplatPagerDebugState { + const mapped: SplatPagerDebugState["mapped"] = []; + for (let page = 0; page < this.pageToSplatsChunk.length; page++) { + const entry = this.pageToSplatsChunk[page]; + if (entry) { + mapped.push({ page, splats: entry.splats, chunk: entry.chunk }); + } + } + return { + maxPages: this.maxPages, + freelist: this.pageFreelist.slice(), + freeable: this.freeablePages.slice(), + lruSize: this.pageLru.size, + mapped, + fetchers: this.fetchers.map(({ splats, chunk }) => ({ splats, chunk })), + fetched: this.fetched.map(({ splats, chunk }) => ({ splats, chunk })), + lodTreeUpdates: this.lodTreeUpdates.map( + ({ splats, chunk, page, lodTree }) => ({ + splats, + chunk, + page, + insert: !!lodTree, + }), + ), + newUploads: this.newUploads.map(({ page }) => page), + readyUploads: this.readyUploads.map(({ page }) => page), + fetchPriority: this.fetchPriority.map(({ splats, chunk }) => ({ + splats, + chunk, + })), + }; + } + + /** + * Verify internal consistency of the page tables. Returns a list of + * human-readable violations (empty when everything is consistent). + * + * @param liveSplats If provided, every PagedSplats that owns mapped pages + * must be in this set (i.e. the SparkRenderer still tracks it in lodIds). + */ + checkInvariants(liveSplats?: Set): string[] { + const errors: string[] = []; + const { maxPages } = this; + + // 1. Freelist: unique, in range + const freeSet = new Set(); + for (const page of this.pageFreelist) { + if (page < 0 || page >= maxPages || !Number.isInteger(page)) { + errors.push(`freelist page out of range: ${page}`); + } + if (freeSet.has(page)) { + errors.push(`freelist has duplicate page: ${page}`); + } + freeSet.add(page); + } + + // 2. Mapped pages disjoint from freelist, union covers all pages + const mappedSet = new Set(); + for (let page = 0; page < this.pageToSplatsChunk.length; page++) { + const entry = this.pageToSplatsChunk[page]; + if (!entry) continue; + if (page >= maxPages) { + errors.push(`mapped page out of range: ${page}`); + } + mappedSet.add(page); + if (freeSet.has(page)) { + errors.push(`page ${page} is both mapped and in freelist`); + } + } + if (this.pageToSplatsChunk.length > maxPages) { + errors.push( + `pageToSplatsChunk.length ${this.pageToSplatsChunk.length} > maxPages ${maxPages}`, + ); + } + if (freeSet.size + mappedSet.size !== maxPages) { + errors.push( + `freelist (${freeSet.size}) + mapped (${mappedSet.size}) != maxPages (${maxPages})`, + ); + } + + // 3. Bidirectional mapping consistency and 4. pageLru equals mapping entries + let numEntries = 0; + const lruEntries = new Set(this.pageLru); + for (const [splats, chunks] of this.splatsChunkToPage.entries()) { + if (chunks.length === 0) { + errors.push("splatsChunkToPage has an entry with no chunks"); + } + if (chunks.length > 0 && chunks[chunks.length - 1] === undefined) { + errors.push("splatsChunkToPage chunk array not trimmed"); + } + for (let chunk = 0; chunk < chunks.length; chunk++) { + const entry = chunks[chunk]; + if (!entry) continue; + numEntries += 1; + const back = this.pageToSplatsChunk[entry.page]; + if (!back) { + errors.push( + `chunk ${chunk} -> page ${entry.page} but page is unmapped`, + ); + } else if (back.splats !== splats || back.chunk !== chunk) { + errors.push( + `chunk ${chunk} -> page ${entry.page} but page maps back to chunk ${back.chunk} of ${back.splats === splats ? "same" : "different"} splats`, + ); + } + if (!lruEntries.delete(entry)) { + errors.push( + `mapping entry for chunk ${chunk} (page ${entry.page}) missing from pageLru`, + ); + } + } + if (liveSplats && !liveSplats.has(splats)) { + errors.push( + `mapped pages for splats not in lodIds (${chunks.filter((c) => c).length} pages)`, + ); + } + } + if (numEntries !== mappedSet.size) { + errors.push( + `splatsChunkToPage entries (${numEntries}) != mapped pages (${mappedSet.size})`, + ); + } + if (lruEntries.size > 0) { + errors.push(`pageLru has ${lruEntries.size} orphaned entries`); + } + + // 5. freeablePages subset of mapped, unique + const freeableSet = new Set(); + for (const page of this.freeablePages) { + if (freeableSet.has(page)) { + errors.push(`freeablePages has duplicate page: ${page}`); + } + freeableSet.add(page); + if (!mappedSet.has(page)) { + errors.push(`freeable page ${page} is not mapped`); + } + } + + // 6. No duplicate (splats, chunk) across fetchers / fetched / mapping + const seen = new Map>(); + const mark = (splats: PagedSplats, chunk: number, where: string) => { + let set = seen.get(splats); + if (!set) { + set = new Set(); + seen.set(splats, set); + } + if (set.has(chunk)) { + errors.push(`duplicate (splats, chunk ${chunk}) in ${where}`); + } + set.add(chunk); + }; + for (const [splats, chunks] of this.splatsChunkToPage.entries()) { + chunks.forEach((entry, chunk) => { + if (entry) mark(splats, chunk, "mapping"); + }); + } + for (const { splats, chunk } of this.fetched) { + mark(splats, chunk, "fetched"); + } + for (const { splats, chunk } of this.fetchers) { + mark(splats, chunk, "fetchers"); + } + + // 7. Pending uploads refer to mapped pages + for (const { page } of this.newUploads) { + if (!mappedSet.has(page)) { + errors.push(`newUploads page ${page} is not mapped`); + } + } + for (const { page } of this.readyUploads) { + if (!mappedSet.has(page)) { + errors.push(`readyUploads page ${page} is not mapped`); + } + } + // Replaying pending lodTreeUpdates in order must end at the current mapping + // for every page they touch. + const replay = new Map(); + const touched = new Set(); + for (const { splats, chunk, page, lodTree } of this.lodTreeUpdates) { + touched.add(page); + if (lodTree) { + replay.set(page, { splats, chunk }); + } else { + replay.delete(page); + } + } + for (const page of touched) { + const expected = replay.get(page); + const actual = this.pageToSplatsChunk[page]; + if (expected) { + if ( + !actual || + actual.splats !== expected.splats || + actual.chunk !== expected.chunk + ) { + errors.push( + `pending lodTreeUpdates end with chunk ${expected.chunk} on page ${page} but mapping has ${actual ? `chunk ${actual.chunk}` : "nothing"}`, + ); + } + } else if (actual) { + errors.push( + `pending lodTreeUpdates end with page ${page} evicted but mapping has chunk ${actual.chunk}`, + ); + } + } + + return errors; + } + static emptyUint32x4 = (() => { const { width, height, depth, maxSplats } = getTextureSize(1); const emptyArray = new Uint32Array(maxSplats * 4); diff --git a/src/index.ts b/src/index.ts index 2ce39cc9..2bbbfe85 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,7 @@ export { transcodeSpz, writeSpz } from "./spz"; export { PackedSplats, type PackedSplatsOptions } from "./PackedSplats"; export { ExtSplats, type ExtSplatsOptions } from "./ExtSplats"; export * from "./SplatPager"; +export { type SparkHooks, hookPoint } from "./SparkHooks"; export { SplatGenerator, type GsplatGenerator, diff --git a/src/worker.ts b/src/worker.ts index 3687a25c..2249b2e9 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -18,6 +18,8 @@ import init_wasm, { tiny_lod_extsplats, bhatt_lod_extsplats, get_lod_tree_level, + get_lod_tree_info, + get_lod_tree_ids, } from "spark-rs"; import type { ExtResult, PackedResult, SplatEncoding } from "./defines"; @@ -37,6 +39,8 @@ const rpcHandlers = { updateLodTrees, traverseLodTrees, getLodTreeLevel, + getLodTreeInfo, + getLodTreeIds, nextChunk, }; export type rpcHandlers = typeof rpcHandlers; @@ -820,6 +824,24 @@ function getLodTreeLevel({ return get_lod_tree_level(lodId, level) as { indices: Uint32Array }; } +export type LodTreeInfo = { + lodId: number; + numSplats: number; + sharedRefs: number; + pageToChunk: Uint32Array; + chunkToPage: Uint32Array; +}; + +// Debug/test introspection of a LoD tree's page<->chunk tables +function getLodTreeInfo({ lodId }: { lodId: number }) { + return get_lod_tree_info(lodId) as LodTreeInfo; +} + +// Debug/test introspection: all live LoD tree ids in this worker +function getLodTreeIds(_args: Record | undefined) { + return { lodIds: Array.from(get_lod_tree_ids()) }; +} + let nextChunkWaiter = (_chunk: Uint8Array) => {}; async function nextChunk({ chunk }: { chunk: Uint8Array }) { diff --git a/test/browser/churn.spec.ts b/test/browser/churn.spec.ts new file mode 100644 index 00000000..121ff235 --- /dev/null +++ b/test/browser/churn.spec.ts @@ -0,0 +1,239 @@ +import { expect, test } from "@playwright/test"; +import { + FIXTURES, + chunkIndexFromUrl, + expectInvariants, + expectNoErrors, + harness, + holdRequests, + openHarness, + waitForSnapshot, +} from "./helpers"; + +// Pager table invariants under add/remove churn (P1a, P1b natural variant, +// P2, P5). All tests use the network seam (held chunk requests) rather than +// in-source hooks, so they reflect what real applications can hit. + +/** + * Reach the P1a state: mesh A has its root resident, a child chunk fetch is + * held in flight, A is removed and its LoD tree disposed. Returns the gate. + */ +async function removedWhileChildFetchInFlight( + page: import("@playwright/test").Page, + name = "A", +) { + const h = harness(page); + const gate = await holdRequests( + page, + undefined, + (url) => chunkIndexFromUrl(url) !== 0, + ); + await h.addPaged(name, FIXTURES.chunked); + + // Root resident, LoD traversed, a child chunk requested (held by the gate) + await waitForSnapshot( + page, + (s) => + s.pager.mapped.some((m) => m.name === name && m.chunk === 0) && + s.pager.fetchers.some((f) => f.name === name && f.chunk !== 0), + { + label: `${name}: root resident + child fetch in flight`, + timeoutMs: 20_000, + }, + ); + await gate.waitHeld(1); + + // Remove (not dispose) and wait for the LoD tree to be cleaned up + await h.remove(name); + await waitForSnapshot( + page, + (s) => + !s.lodIds.some((l) => l.name === name) && + !s.pager.mapped.some((m) => m.name === name), + { label: `${name}: LoD tree disposed after remove`, timeoutMs: 10_000 }, + ); + // The held fetch is still in flight for the now-untracked PagedSplats + const s = await h.snapshot(); + expect(s.pager.fetchers.some((f) => f.name === name)).toBe(true); + return gate; +} + +test("P1a: a chunk that lands for a removed mesh does not stay mapped", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "loop", + maxPages: 8, + lodSplatCount: 40000, + numLodFetchers: 1, + lodDisposeTimeoutMs: 200, + }); + const h = harness(page); + const gate = await removedWhileChildFetchInFlight(page, "A"); + + // Let the stale fetch land and be seen by at least one LoD callback + await gate.releaseAll(); + gate.passthrough(); + await waitForSnapshot( + page, + (s) => + s.pager.fetchers.length === 0 && + s.pager.fetched === 0 && + s.pager.lodTreeUpdates === 0, + { label: "stale fetch completed and consumed", timeoutMs: 10_000 }, + ); + await page.waitForTimeout(500); + + // Desired: nothing owned by A remains in the page tables, pool fully free + const s = await h.snapshot(); + expect(s.pager.mapped.filter((m) => m.name === "A")).toEqual([]); + expect(s.pager.freelist.length).toBe(s.pager.maxPages); + await expectInvariants(page); + await gate.dispose(); + expectNoErrors(opened, s); +}); + +test("P1b (natural): re-adding a mesh after a stale chunk landed keeps pager and Rust tree consistent", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "loop", + maxPages: 8, + lodSplatCount: 40000, + numLodFetchers: 1, + lodDisposeTimeoutMs: 200, + }); + const h = harness(page); + const gate = await removedWhileChildFetchInFlight(page, "A"); + await gate.releaseAll(); + gate.passthrough(); + await waitForSnapshot(page, (s) => s.pager.fetchers.length === 0, { + label: "stale fetch completed", + timeoutMs: 10_000, + }); + + // Re-add the same mesh (same PagedSplats). A new LoD tree is created; every + // chunk the pager believes resident must also be resident in that tree. + await h.readd("A"); + await waitForSnapshot( + page, + (s) => + s.lodIds.some((l) => l.name === "A" && l.rootPage !== undefined) && + (s.meshes.A.pagedNumSplats ?? 0) > 0, + { label: "A re-added and displayed", timeoutMs: 15_000 }, + ); + expect(await h.waitIdle({ timeoutMs: 15_000 })).toBe(true); + const result = await expectInvariants(page); + expect(result.skipped).toEqual([]); + + // And it must be able to become fully resident again (no chunk stuck as + // "resident in pager, missing in Rust" which would never be re-fetched) + const resident = await h.residentChunks("A"); + expect(Object.keys(resident).length).toBeGreaterThanOrEqual(2); + await gate.dispose(); + expectNoErrors(opened, await h.snapshot()); +}); + +test("P2: evicting a mesh's root chunk does not leave a stale rootPage", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "loop", + maxPages: 2, + lodSplatCount: 40000, + numLodFetchers: 1, + lodDisposeTimeoutMs: 30_000, + }); + const h = harness(page); + + await h.addPaged("M1", FIXTURES.chunked); + await waitForSnapshot( + page, + (s) => + s.pager.mapped.filter((m) => m.name === "M1").length === 2 && + s.pager.fetchers.length === 0 && + s.pager.lodTreeUpdates === 0, + { label: "M1 fills the 2-page pool", timeoutMs: 20_000 }, + ); + expect(await h.waitIdle({ timeoutMs: 20_000 })).toBe(true); + + // Hide M1 (stays within the dispose timeout) and add M2, whose chunks must + // evict all of M1's pages including its root. + await h.setVisible("M1", false); + await h.addPaged("M2", FIXTURES.chunked); + await waitForSnapshot( + page, + (s) => + s.pager.mapped.filter((m) => m.name === "M2").length === 2 && + s.pager.mapped.filter((m) => m.name === "M1").length === 0 && + s.pager.lodTreeUpdates === 0, + { label: "M2 evicted all of M1's pages", timeoutMs: 20_000 }, + ); + expect(await h.waitIdle({ timeoutMs: 20_000 })).toBe(true); + + // Desired: M1's record no longer claims a root page (cross-checked against + // the Rust tree's chunk_to_page[0] by checkInvariants). + const s1 = await h.snapshot(); + const m1 = s1.lodIds.find((l) => l.name === "M1"); + expect(m1).toBeDefined(); + expect(m1?.rootPage).toBeUndefined(); + await expectInvariants(page); + + // Make M1 visible again: it must not render with a stale root page (foreign + // data) while it has no resident pages. + await h.setVisible("M1", true); + let sawForeignRoot = false; + const start = Date.now(); + while (Date.now() - start < 6_000) { + const s = await h.snapshot(); + const resident = s.pager.mapped.filter((m) => m.name === "M1").length; + if ((s.meshes.M1.pagedNumSplats ?? 0) > 0 && resident === 0) { + sawForeignRoot = true; + break; + } + if (resident > 0 && (s.meshes.M1.pagedNumSplats ?? 0) > 0) break; + await page.waitForTimeout(50); + } + expect(sawForeignRoot, "M1 displayed splats with no resident pages").toBe( + false, + ); + await expectInvariants(page); + expectNoErrors(opened, await h.snapshot()); +}); + +test("P5: dispose() then re-adding the same mesh does not spin on aborted fetches", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "loop", + maxPages: 8, + lodSplatCount: 30000, + numLodFetchers: 1, + }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + await waitForSnapshot(page, (s) => (s.meshes.A.pagedNumSplats ?? 0) > 0, { + label: "A displayed", + timeoutMs: 20_000, + }); + expect(await h.waitIdle({ timeoutMs: 10_000 })).toBe(true); + + // Application mistake / edge case: dispose then put the same object back + await h.dispose("A"); + await h.readd("A"); + await page.waitForTimeout(1500); + + // Desired: the pager does not keep retrying fetches for a disposed + // PagedSplats (each retry fails immediately with AbortError + backoff). + let spinning = 0; + for (let i = 0; i < 6; i++) { + const s = await h.snapshot(); + if (s.pager.fetchers.some((f) => f.name === "A")) spinning++; + await page.waitForTimeout(300); + } + const s = await h.snapshot(); + expect(spinning, "fetcher for disposed A observed in pager.fetchers").toBe(0); + expect(s.meshes.A.pagedAborted).toBe(true); + await expectInvariants(page); + expectNoErrors(opened, s); +}); diff --git a/test/browser/fuzz-shared.ts b/test/browser/fuzz-shared.ts new file mode 100644 index 00000000..dda21bbe --- /dev/null +++ b/test/browser/fuzz-shared.ts @@ -0,0 +1,30 @@ +// Shared configuration between pager-fuzz.spec.ts and regressions.spec.ts so +// that a pinned seed replays exactly the run that found it. + +export const FUZZ_HARNESS = { + mode: "manual" as const, + maxPages: 4, + lodSplatCount: 40000, + numLodFetchers: 2, + lodDisposeTimeoutMs: 150, +}; + +export function formatFuzzFailure(result: { + seed: number; + violations: string[]; + errors: string[]; + trace: string[]; + skippedChecks: string[]; +}) { + return [ + `seed ${result.seed}`, + "violations:", + ...result.violations.map((v) => ` ${v}`), + "errors:", + ...result.errors.map((e) => ` ${e}`), + "skipped checks:", + ...result.skippedChecks.map((s) => ` ${s}`), + "trace:", + ...result.trace.map((t) => ` ${t}`), + ].join("\n"); +} diff --git a/test/browser/global-setup.ts b/test/browser/global-setup.ts new file mode 100644 index 00000000..3f42cb00 --- /dev/null +++ b/test/browser/global-setup.ts @@ -0,0 +1,19 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +// Generates the synthetic LoD fixtures (needs a Rust toolchain for build-lod) +// if they are not already present under test/fixtures/out/. +export default async function globalSetup() { + const here = path.dirname(fileURLToPath(import.meta.url)); + const generator = path.resolve(here, "../fixtures/gen-fixture.mjs"); + const mod = (await import(generator)) as { + generateFixtures: (opts?: { force?: boolean }) => Record; + fixturesExist: () => boolean; + }; + if (!mod.fixturesExist()) { + console.log( + "[spark test] generating LoD fixtures with build-lod (requires Rust)...", + ); + } + mod.generateFixtures(); +} diff --git a/test/browser/helpers.ts b/test/browser/helpers.ts new file mode 100644 index 00000000..80e3cef0 --- /dev/null +++ b/test/browser/helpers.ts @@ -0,0 +1,293 @@ +import { type Page, expect } from "@playwright/test"; +import type { Harness, HarnessInitOptions } from "./pages/harness"; + +export type Snapshot = ReturnType; + +export const FIXTURES = { + /** Chunked LoD RAD: fixture-lod.rad + fixture-lod-.radc (7 chunks) */ + chunked: "/test/fixtures/out/chunked/fixture-lod.rad", + /** Single-file LoD RAD */ + lodRad: "/test/fixtures/out/fixture-lod.rad", + /** Plain (non-LoD) PLY, 300K splats (slow to rasterize on SwiftShader) */ + ply: "/test/fixtures/out/fixture.ply", + /** Plain (non-LoD) PLY, 20K splats */ + smallPly: "/test/fixtures/out/fixture-small.ply", +}; + +export const CHUNK_URL_GLOB = "**/fixture-lod-*.radc"; + +export interface OpenedHarness { + page: Page; + pageErrors: string[]; + consoleErrors: string[]; +} + +/** Navigate to the harness page and initialize it. */ +export async function openHarness( + page: Page, + options: HarnessInitOptions = {}, +): Promise { + const pageErrors: string[] = []; + const consoleErrors: string[] = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + page.on("console", (message) => { + if (message.type() === "error") consoleErrors.push(message.text()); + }); + await page.goto("/test/browser/pages/harness.html"); + await page.waitForFunction(() => window.harnessReady, null, { + timeout: 30_000, + }); + await page.evaluate((o) => window.harness.init(o), options); + return { page, pageErrors, consoleErrors }; +} + +/** Thin typed wrappers around page.evaluate(window.harness.*) */ +export function harness(page: Page) { + return { + snapshot: () => page.evaluate(() => window.harness.snapshot()), + render: () => page.evaluate(() => window.harness.render()), + renderN: (n: number) => + page.evaluate(async (count) => { + for (let i = 0; i < count; i++) { + window.harness.render(); + await new Promise((r) => requestAnimationFrame(r)); + } + }, n), + setMode: (mode: "loop" | "ondemand" | "manual") => + page.evaluate((m) => window.harness.setMode(m), mode), + addPaged: ( + name: string, + url: string = FIXTURES.chunked, + opts: { position?: [number, number, number]; lodScale?: number } = {}, + ) => + page.evaluate(([n, u, o]) => window.harness.addPaged(n, u, o), [ + name, + url, + opts, + ] as const), + addMesh: ( + name: string, + url: string, + opts: { position?: [number, number, number]; lod?: boolean } = {}, + ) => + page.evaluate(([n, u, o]) => window.harness.addMesh(n, u, o), [ + name, + url, + opts, + ] as const), + remove: (name: string) => + page.evaluate((n) => window.harness.remove(n), name), + readd: (name: string) => + page.evaluate((n) => window.harness.readd(n), name), + dispose: (name: string) => + page.evaluate((n) => window.harness.dispose(n), name), + destroy: (name: string) => + page.evaluate((n) => window.harness.destroy(n), name), + setVisible: (name: string, visible: boolean) => + page.evaluate(([n, v]) => window.harness.setVisible(n, v), [ + name, + visible, + ] as const), + setPosition: (name: string, position: [number, number, number]) => + page.evaluate(([n, p]) => window.harness.setPosition(n, p), [ + name, + position, + ] as const), + moveCamera: ( + position: [number, number, number], + lookAt?: [number, number, number], + ) => + page.evaluate(([p, l]) => window.harness.moveCamera(p, l), [ + position, + lookAt, + ] as const), + awaitInitialized: (name: string) => + page.evaluate((n) => window.harness.awaitInitialized(n), name), + countLitPixels: () => page.evaluate(() => window.harness.countLitPixels()), + residentChunks: (name: string) => + page.evaluate((n) => window.harness.residentChunks(n), name), + waitIdle: (opts: { timeoutMs?: number; settleMs?: number } = {}) => + page.evaluate((o) => window.harness.waitIdle(o), opts), + renderUntilIdle: (opts: { timeoutMs?: number; idleFrames?: number } = {}) => + page.evaluate((o) => window.harness.renderUntilIdle(o), opts), + isBusy: () => page.evaluate(() => window.harness.isBusy()), + waitQuiet: ( + opts: { + settleMs?: number; + timeoutMs?: number; + ignoreLoop?: boolean; + } = {}, + ) => page.evaluate((o) => window.harness.waitQuiet(o), opts), + sleep: (ms: number) => page.evaluate((t) => window.harness.sleep(t), ms), + checkInvariants: () => + page.evaluate(() => window.harness.checkInvariants()), + hooks: { + hold: (name: string) => + page.evaluate((n) => window.harness.hooks.hold(n), name), + unhold: (name: string) => + page.evaluate((n) => window.harness.hooks.unhold(n), name), + unholdAll: () => page.evaluate(() => window.harness.hooks.unholdAll()), + release: (name?: string, count = 1) => + page.evaluate(([n, c]) => window.harness.hooks.release(n, c), [ + name, + count, + ] as const), + releaseAll: () => page.evaluate(() => window.harness.hooks.releaseAll()), + heldCount: (name?: string) => + page.evaluate((n) => window.harness.hooks.heldCount(n), name), + heldNames: () => page.evaluate(() => window.harness.hooks.heldNames()), + failNext: (name: string, message?: string, count = 1) => + page.evaluate(([n, m, c]) => window.harness.hooks.failNext(n, m, c), [ + name, + message, + count, + ] as const), + rejectHeld: (name: string, message?: string) => + page.evaluate(([n, m]) => window.harness.hooks.rejectHeld(n, m), [ + name, + message, + ] as const), + /** Wait until at least `count` executions are parked at `name` */ + waitHeld: (name: string, count = 1, timeout = 15_000) => + page.waitForFunction( + ([n, c]) => window.harness.hooks.heldCount(n) >= c, + [name, count] as const, + { timeout, polling: 10 }, + ), + log: () => page.evaluate(() => window.harness.hooks.log), + }, + }; +} + +/** Poll a snapshot-derived predicate until true or timeout. */ +export async function waitForSnapshot( + page: Page, + predicate: (s: Snapshot) => boolean, + { timeoutMs = 15_000, intervalMs = 50, label = "condition" } = {}, +): Promise { + const start = Date.now(); + let last: Snapshot | undefined; + while (Date.now() - start < timeoutMs) { + last = await page.evaluate(() => window.harness.snapshot()); + if (predicate(last)) return last; + await page.waitForTimeout(intervalMs); + } + throw new Error( + `Timed out waiting for ${label}. Last snapshot: ${JSON.stringify(last, null, 1)}`, + ); +} + +/** Assert no page errors / console errors / harness-captured errors. */ +export function expectNoErrors(opened: OpenedHarness, snapshot: Snapshot) { + expect(opened.pageErrors, "pageerror").toEqual([]); + expect( + snapshot.errors.filter((e) => !isBenignError(e)), + "harness errors", + ).toEqual([]); +} + +function isBenignError(message: string) { + // Chrome/SwiftShader occasionally logs GL performance warnings as errors + return /GL_|performance warning|GPU stall/i.test(message); +} + +export async function expectInvariants(page: Page) { + const result = await page.evaluate(() => window.harness.checkInvariants()); + expect(result.errors, "invariants").toEqual([]); + return result; +} + +/** + * Hold chunk requests matching `glob`. Returns a controller to release them + * (individually, in order, or all at once), or fail them. + */ +export async function holdRequests( + page: Page, + glob = CHUNK_URL_GLOB, + filter: (url: string) => boolean = () => true, +) { + type Held = { + url: string; + go: () => Promise; + fail: () => Promise; + }; + const held: Held[] = []; + let holding = true; + const waiters: (() => void)[] = []; + + await page.route(glob, async (route) => { + const url = route.request().url(); + if (!holding || !filter(url)) { + await route.continue(); + return; + } + await new Promise((resolve, reject) => { + held.push({ + url, + go: async () => { + await route.continue(); + resolve(); + }, + fail: async () => { + await route.abort(); + reject(new Error("aborted by test")); + }, + }); + for (const w of waiters.splice(0)) w(); + }).catch(() => {}); + }); + + const controller = { + held, + heldUrls: () => held.map((h) => h.url), + /** wait until at least `count` requests are held */ + async waitHeld(count = 1, timeoutMs = 15_000) { + const start = Date.now(); + while (held.length < count) { + if (Date.now() - start > timeoutMs) { + throw new Error( + `Timed out waiting for ${count} held requests (have ${held.length}: ${held.map((h) => h.url).join(", ")})`, + ); + } + await new Promise((r) => { + waiters.push(r); + setTimeout(r, 50); + }); + } + }, + async releaseOne(match?: (url: string) => boolean) { + const index = match ? held.findIndex((h) => match(h.url)) : 0; + if (index < 0 || index >= held.length) return undefined; + const [h] = held.splice(index, 1); + await h.go(); + return h.url; + }, + async releaseAll() { + const all = held.splice(0); + for (const h of all) await h.go(); + return all.map((h) => h.url); + }, + async failAll() { + const all = held.splice(0); + for (const h of all) await h.fail(); + }, + /** stop holding new requests (already-held ones stay held) */ + passthrough() { + holding = false; + }, + resume() { + holding = true; + }, + async dispose() { + holding = false; + await controller.releaseAll(); + await page.unroute(glob); + }, + }; + return controller; +} + +export function chunkIndexFromUrl(url: string): number { + const match = /fixture-lod-(\d+)\.radc/.exec(url); + return match ? Number(match[1]) : -1; +} diff --git a/test/browser/on-demand.spec.ts b/test/browser/on-demand.spec.ts new file mode 100644 index 00000000..3b0e4727 --- /dev/null +++ b/test/browser/on-demand.spec.ts @@ -0,0 +1,285 @@ +import { expect, test } from "@playwright/test"; +import { + FIXTURES, + expectNoErrors, + harness, + holdRequests, + openHarness, + waitForSnapshot, +} from "./helpers"; + +// Dirty-flag / on-demand rendering matrix (D1, D2, D3, D5). +// +// "on-demand mode" = no animation loop; SparkRenderer.onDirty schedules exactly +// one render() via requestAnimationFrame. Each test is written as the desired +// behavior: the scene must converge without the application calling render() +// on its own. + +test("D1: paged chunk landing triggers a render in on-demand mode", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "ondemand", + maxPages: 8, + lodSplatCount: 30000, + }); + const h = harness(page); + + await h.addPaged("A", FIXTURES.chunked); + // The application renders exactly once; everything else must come from onDirty + await h.render(); + + const converged = await waitForSnapshot( + page, + (s) => + s.activeSplats > 0 && + s.pager.lodTreeUpdates === 0 && + s.pager.mapped.length >= 1 && + (s.meshes.A.pagedNumSplats ?? 0) > 0, + { + label: "paged mesh displayed via onDirty-driven renders", + timeoutMs: 15_000, + }, + ).catch(async (error) => { + const s = await h.snapshot(); + throw new Error( + `${error.message}\nrenders=${s.renders} dirtyEvents=${s.dirtyEvents} pagerUpdates=${s.pagerUpdates} lodTreeUpdates=${s.pager.lodTreeUpdates} mapped=${JSON.stringify(s.pager.mapped)}`, + ); + }); + + expect(converged.renders).toBeGreaterThan(1); + await expect + .poll(() => h.countLitPixels(), { timeout: 10_000 }) + .toBeGreaterThan(50); + expectNoErrors(opened, await h.snapshot()); +}); + +test("D1b: on-demand pipeline keeps converging after camera moves", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "ondemand", + maxPages: 8, + lodSplatCount: 40000, + }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + await h.render(); + await waitForSnapshot(page, (s) => s.activeSplats > 0, { + label: "initial display", + timeoutMs: 15_000, + }); + + // Move the camera close to one blob; new chunks become relevant and must be + // fetched + displayed without further application renders (only one here). + await h.moveCamera([1.5, 1.0, -2.5], [1.5, 1.0, -4]); + const before = await h.snapshot(); + await h.render(); + + const after = await waitForSnapshot( + page, + (s) => + s.pager.lodTreeUpdates === 0 && + s.pager.fetched === 0 && + s.pager.fetchers.length === 0 && + s.renders > before.renders + 1 && + !s.lodDirty, + { label: "converged after camera move", timeoutMs: 15_000 }, + ); + expect(after.activeSplats).toBeGreaterThan(0); + // The pipeline drains on its own (only onDirty-driven renders happen here): + // nothing stays queued that would need an application render to be consumed. + expect(await h.waitIdle({ timeoutMs: 10_000 })).toBe(true); + expect(await h.isBusy()).toBe(false); + expectNoErrors(opened, await h.snapshot()); +}); + +test("D2: SplatMesh async initialization triggers a render in on-demand mode", async ({ + page, +}) => { + const opened = await openHarness(page, { mode: "ondemand" }); + const h = harness(page); + + // Hold the PLY so that initialization completes only after the pipeline has + // gone quiet (initial render + sort-completion render are done). + const gate = await holdRequests(page, "**/fixture-small.ply"); + await h.addMesh("P", FIXTURES.smallPly, { position: [0, 0, -2] }); + await h.render(); + await gate.waitHeld(1); + const quiet = await h.waitQuiet({ settleMs: 300, timeoutMs: 10_000 }); + expect(quiet.filter((r) => !r.startsWith("init:"))).toEqual([]); + const before = await h.snapshot(); + expect(before.meshes.P.initialized).toBe(false); + + await gate.releaseAll(); + await h.awaitInitialized("P"); + + const after = await waitForSnapshot( + page, + (s) => s.activeSplats > 0 && s.renders > before.renders, + { label: "mesh displayed after initialization", timeoutMs: 5_000 }, + ).catch(async (error) => { + const s = await h.snapshot(); + throw new Error( + `${error.message}\nrenders before=${before.renders} after=${s.renders}, activeSplats=${s.activeSplats}, initialized=${s.meshes.P.initialized}`, + ); + }); + expect(after.meshes.P.numSplats).toBeGreaterThan(0); + await gate.dispose(); + expectNoErrors(opened, await h.snapshot()); +}); + +test("D2b: LoD (non-paged) SplatMesh initialization triggers a render in on-demand mode", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "ondemand", + lodSplatCount: 30000, + }); + const h = harness(page); + + const gate = await holdRequests(page, "**/out/fixture-lod.rad"); + await h.addMesh("L", FIXTURES.lodRad, { lod: true }); + await h.render(); + await gate.waitHeld(1); + const quiet = await h.waitQuiet({ settleMs: 300, timeoutMs: 10_000 }); + expect(quiet.filter((r) => !r.startsWith("init:"))).toEqual([]); + const before = await h.snapshot(); + + await gate.releaseAll(); + await h.awaitInitialized("L"); + + const after = await waitForSnapshot( + page, + (s) => + s.activeSplats > 0 && + s.renders > before.renders && + s.lodIds.some((l) => l.name === "L"), + { label: "LoD mesh displayed after initialization", timeoutMs: 8_000 }, + ).catch(async (error) => { + const s = await h.snapshot(); + throw new Error( + `${error.message}\nrenders before=${before.renders} after=${s.renders}, activeSplats=${s.activeSplats}`, + ); + }); + expect(after.meshes.L.numSplats).toBeGreaterThan(0); + await gate.dispose(); + expectNoErrors(opened, await h.snapshot()); +}); + +test("D3: LoD callback that exits with lodDirty pending requests a render", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "manual", + maxPages: 8, + lodSplatCount: 30000, + }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + expect(await h.renderUntilIdle({ timeoutMs: 30_000 })).toBe(true); + + // Switch to on-demand: from here on only onDirty may cause renders + await h.setMode("ondemand"); + await h.hooks.hold("lod.beforeCleanup"); + + // Camera move #1 -> render -> LoD callback traverses and parks at beforeCleanup + await h.moveCamera([0.5, 0.2, -1.0], [0.5, 0.2, -4]); + await h.render(); + await h.hooks.waitHeld("lod.beforeCleanup", 1); + // Let the sort for camera #1 finish (otherwise updateInternal skips the + // accumulator update because the mapping changed while sorting). + await waitForSnapshot( + page, + (s) => + !s.sorting && + !s.sortDirty && + !s.inFlight.includes("renderScheduled") && + s.held.includes("lod.beforeCleanup"), + { label: "sort settled while LoD callback parked", timeoutMs: 10_000 }, + ); + + // Change a LoD-only parameter while the callback is parked and render once + // (e.g. the app changed the splat budget). This does not move the camera, so + // no sort is triggered that could mask the problem. The sync part of driveLod + // sets lodDirty; tryExclusive is skipped because the worker is busy. + await page.evaluate(() => { + window.harness.spark.lodSplatCount = 12000; + }); + await h.render(); + const parked = await h.snapshot(); + expect(parked.lodDirty).toBe(true); + expect(parked.held).toEqual(["lod.beforeCleanup"]); + expect(parked.sorting).toBe(false); + const dirtyEventsBefore = parked.dirtyEvents; + const activeBefore = parked.activeSplats; + + // Let the callback finish. Desired: it notices pending LoD work and calls + // setDirty so the on-demand app re-renders and the LoD converges to the + // new budget. + await h.hooks.unhold("lod.beforeCleanup"); + await h.hooks.release("lod.beforeCleanup", 1); + + const after = await waitForSnapshot( + page, + (s) => + !s.lodDirty && + s.dirtyEvents > dirtyEventsBefore && + s.activeSplats < activeBefore && + s.activeSplats > 0, + { label: "LoD re-traversed with the new splat budget", timeoutMs: 5_000 }, + ).catch(async (error) => { + const s = await h.snapshot(); + throw new Error( + `${error.message}\nlodDirty=${s.lodDirty} dirtyEvents before=${dirtyEventsBefore} after=${s.dirtyEvents} renders=${s.renders} activeSplats before=${activeBefore} after=${s.activeSplats} held=${s.held}`, + ); + }); + expect(await h.waitIdle({ timeoutMs: 5_000 })).toBe(true); + expectNoErrors(opened, after); +}); + +test("D5: a failed sort does not leave `sorting` stuck", async ({ page }) => { + const opened = await openHarness(page, { + mode: "manual", + lodSplatCount: 30000, + }); + const h = harness(page); + await h.addMesh("P", FIXTURES.smallPly, { position: [0, 0, -2] }); + expect(await h.renderUntilIdle({ timeoutMs: 30_000 })).toBe(true); + + // Next sort fails after the worker returns + await h.hooks.failNext("sort.afterWorker", "injected sort failure"); + await h.moveCamera([0.3, 0.1, 0.2], [0, 0, -4]); + await h.render(); + // Wait for the failure to be recorded + await waitForSnapshot( + page, + (s) => s.errors.some((e) => e.includes("injected sort failure")), + { label: "sort failure surfaced", timeoutMs: 5_000 }, + ); + + // Desired: sorting flag recovers and later frames sort again + const recovered = await waitForSnapshot(page, (s) => !s.sorting, { + label: "sorting flag cleared after failure", + timeoutMs: 3_000, + }).catch(async (error) => { + const s = await h.snapshot(); + throw new Error(`${error.message}\nsorting=${s.sorting}`); + }); + expect(recovered.sorting).toBe(false); + + // A mapping change (second mesh) must still be displayable + await h.addMesh("Q", FIXTURES.smallPly, { position: [1, 0, -2] }); + await h.awaitInitialized("Q"); + const before = await h.snapshot(); + expect(await h.renderUntilIdle({ timeoutMs: 15_000 })).toBe(true); + const after = await h.snapshot(); + expect(after.activeSplats).toBeGreaterThan(before.activeSplats); + expect(after.sorting).toBe(false); + + // Only the injected failure may have been reported + expect( + after.errors.filter((e) => !e.includes("injected sort failure")), + ).toEqual([]); + expect(opened.pageErrors.filter((e) => !e.includes("injected"))).toEqual([]); +}); diff --git a/test/browser/orderings.spec.ts b/test/browser/orderings.spec.ts new file mode 100644 index 00000000..b978b445 --- /dev/null +++ b/test/browser/orderings.spec.ts @@ -0,0 +1,193 @@ +import { expect, test } from "@playwright/test"; +import { + FIXTURES, + chunkIndexFromUrl, + expectInvariants, + expectNoErrors, + harness, + holdRequests, + openHarness, + waitForSnapshot, +} from "./helpers"; + +// Targeted interleavings using in-source hook points (P1b via hooks, P6) and +// pool-pressure / worker hardening checks (P3, R1). + +test("P1b (hooks): a chunk landing between consume and cleanup of the callback that disposes its tree is not applied to a re-created tree", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "manual", + maxPages: 8, + lodSplatCount: 40000, + numLodFetchers: 1, + lodDisposeTimeoutMs: 200, + }); + const h = harness(page); + const gate = await holdRequests( + page, + undefined, + (url) => chunkIndexFromUrl(url) !== 0, + ); + + // Render until the root is resident and a child fetch is held + await h.addPaged("A", FIXTURES.chunked); + const start = Date.now(); + while (Date.now() - start < 20_000) { + await h.render(); + await page.waitForTimeout(60); + const s = await h.snapshot(); + if ( + s.pager.mapped.some((m) => m.name === "A" && m.chunk === 0) && + s.pager.fetchers.some((f) => f.name === "A" && f.chunk !== 0) + ) { + break; + } + } + await gate.waitHeld(1); + + // Remove A, render once so it is no longer touched, wait past the timeout + await h.remove("A"); + await h.render(); + await page.waitForTimeout(400); + + // Park the callback that will dispose A right before cleanup + await h.hooks.hold("lod.beforeCleanup"); + await h.render(); + await h.hooks.waitHeld("lod.beforeCleanup", 1); + + // While parked (after consumeLodTreeUpdates ran), let the child chunk land: + // it gets a page and queues an insert update for A. + await gate.releaseAll(); + gate.passthrough(); + await waitForSnapshot( + page, + (s) => + s.pager.fetchers.length === 0 && + s.pager.lodTreeUpdates >= 1 && + s.pager.mapped.some((m) => m.name === "A" && m.chunk !== 0), + { label: "stale chunk landed while callback parked", timeoutMs: 10_000 }, + ); + + // Resume: cleanup disposes A's tree and frees its pages + await h.hooks.unhold("lod.beforeCleanup"); + await h.hooks.release("lod.beforeCleanup", 1); + await waitForSnapshot( + page, + (s) => !s.lodIds.some((l) => l.name === "A") && s.inFlight.length === 0, + { label: "A's LoD tree disposed", timeoutMs: 5_000 }, + ); + + // Pending pager updates must not reference pages that are no longer mapped + const afterDispose = await h.snapshot(); + expect(afterDispose.pager.mapped.filter((m) => m.name === "A")).toEqual([]); + await expectInvariants(page); + + // Re-add A: the new tree must only learn about chunks the pager has mapped + await h.readd("A"); + expect(await h.renderUntilIdle({ timeoutMs: 30_000 })).toBe(true); + const result = await expectInvariants(page); + expect(result.skipped).toEqual([]); + await gate.dispose(); + expectNoErrors(opened, await h.snapshot()); +}); + +test("P3: fetched chunks are never dropped when the pool is full", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "loop", + maxPages: 3, + lodSplatCount: 250000, + numLodFetchers: 3, + }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + await waitForSnapshot( + page, + (s) => s.pager.mapped.length === s.pager.maxPages, + { label: "pool full", timeoutMs: 30_000 }, + ); + // Sustained pressure: the traverse wants more chunks than there are pages + await page.waitForTimeout(6_000); + const s = await h.snapshot(); + expect(s.pagerDrops, "fetched chunks dropped for lack of pages").toBe(0); + expect(s.pager.mapped.length).toBeLessThanOrEqual(s.pager.maxPages); + await expectInvariants(page); + expectNoErrors(opened, s); +}); + +test("P6: disposing a paged mesh during a traverse does not recreate its indices texture", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "loop", + maxPages: 8, + lodSplatCount: 30000, + }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + await waitForSnapshot(page, (s) => (s.meshes.A.pagedNumSplats ?? 0) > 0, { + label: "A displayed", + timeoutMs: 20_000, + }); + + await h.hooks.hold("lod.afterTraverse"); + // Loop mode: the next callback with lodDirty parks after the traverse. + // Force one by nudging the camera. + await h.moveCamera([0.2, 0.1, 0.0]); + await h.hooks.waitHeld("lod.afterTraverse", 1); + + await h.dispose("A"); + const disposed = await h.snapshot(); + expect(disposed.meshes.A.pagedHasIndicesTexture).toBe(false); + expect(disposed.meshes.A.pagedAborted).toBe(true); + + await h.hooks.unhold("lod.afterTraverse"); + await h.hooks.release("lod.afterTraverse", 1); + await page.waitForTimeout(500); + + const after = await h.snapshot(); + expect(after.meshes.A.pagedHasIndicesTexture).toBe(false); + expectNoErrors(opened, after); +}); + +test("R1 (gated hardening): unknown lodId in a worker call rejects without poisoning the worker", async ({ + page, +}) => { + await openHarness(page, { mode: "manual" }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + await h.render(); + await waitForSnapshot(page, (s) => s.hasPager, { + label: "pager created", + timeoutMs: 10_000, + }); + + const outcome = await page.evaluate(async () => { + const results: { first: string; second: string } = { + first: "", + second: "", + }; + try { + await window.harness.lodWorkerCall("updateLodTrees", { + ranges: [{ lodId: 999999, pageBase: 0, chunkBase: 0, count: 65536 }], + }); + results.first = "resolved"; + } catch (error) { + results.first = `rejected: ${String((error as Error)?.message ?? error)}`; + } + try { + const ids = await window.harness.lodWorkerCall<{ lodIds: number[] }>( + "getLodTreeIds", + {}, + ); + results.second = `resolved: ${ids.lodIds.length} trees`; + } catch (error) { + results.second = `rejected: ${String((error as Error)?.message ?? error)}`; + } + return results; + }); + expect(outcome.first).toMatch(/^rejected/); + expect(outcome.second).toMatch(/^resolved/); +}); diff --git a/test/browser/pager-fuzz.spec.ts b/test/browser/pager-fuzz.spec.ts new file mode 100644 index 00000000..257f7ba5 --- /dev/null +++ b/test/browser/pager-fuzz.spec.ts @@ -0,0 +1,39 @@ +import { expect, test } from "@playwright/test"; +import { FUZZ_HARNESS, formatFuzzFailure } from "./fuzz-shared"; +import { FIXTURES, openHarness } from "./helpers"; + +// Seeded random interleavings of {add, remove, re-add, dispose, toggle +// visibility, move camera, render, release held chunk fetch, hold/release +// hook points} with pager invariants checked after every step and a full +// pager <-> Rust cross-check at each drain. Failing seeds are reported with +// their action trace; pin them in regressions.spec.ts. +// +// FUZZ_ITERS=20 FUZZ_SEED=1 npm run test:browser -- pager-fuzz + +const iters = Number(process.env.FUZZ_ITERS ?? 3); +const baseSeed = Number(process.env.FUZZ_SEED ?? 1); +const steps = Number(process.env.FUZZ_STEPS ?? 60); + +for (let i = 0; i < iters; i++) { + const seed = baseSeed + i; + test(`fuzz seed ${seed}`, async ({ page }) => { + test.setTimeout(180_000); + const opened = await openHarness(page, FUZZ_HARNESS); + const result = await page.evaluate( + ([s, n, url]) => + window.harness.fuzz({ seed: s, steps: n, url, budgetMs: 120_000 }), + [seed, steps, FIXTURES.chunked] as const, + ); + const benign = (e: string) => /GL_|performance warning/i.test(e); + expect( + result.violations, + `${formatFuzzFailure(result)}\npageErrors: ${opened.pageErrors.join("; ")}`, + ).toEqual([]); + expect( + result.errors.filter((e) => !benign(e)), + formatFuzzFailure(result), + ).toEqual([]); + expect(opened.pageErrors).toEqual([]); + expect(result.finalCleanupOk).toBe(true); + }); +} diff --git a/test/browser/pages/fuzz.ts b/test/browser/pages/fuzz.ts new file mode 100644 index 00000000..2727055c --- /dev/null +++ b/test/browser/pages/fuzz.ts @@ -0,0 +1,324 @@ +// Seeded in-page fuzzer for the paged LoD pipeline. Runs inside the harness +// page (see harness.ts) so that every action, hook release and chunk-fetch +// release is chosen by one PRNG; the resulting trace is returned to the test. + +import { PagedSplats } from "../../../src/index"; +import type { Harness } from "./harness"; + +export interface FuzzOptions { + seed: number; + steps: number; + /** Check invariants + drain every N steps */ + drainEvery?: number; + maxMeshes?: number; + /** Probability that a chunk fetch is held until explicitly released */ + holdFetchProbability?: number; + /** Fixture URL for paged meshes */ + url: string; + /** Wall-clock budget for the whole run (ms) */ + budgetMs?: number; + /** Stop at the first violation (default true) */ + stopOnViolation?: boolean; +} + +export interface FuzzResult { + seed: number; + stepsRun: number; + trace: string[]; + violations: string[]; + errors: string[]; + drains: number; + skippedChecks: string[]; + finalCleanupOk: boolean; + durationMs: number; +} + +function mulberry32(seed: number) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const HOOK_NAMES = [ + "lod.afterInit", + "lod.afterUpdateTrees", + "lod.afterTraverse", + "lod.beforeCleanup", + "pager.fetched", + "pager.beforeProcessFetched", + "sort.afterReadback", +]; + +const CAMERAS: [[number, number, number], [number, number, number]][] = [ + [ + [0, 0, 0], + [0, 0, -4], + ], + [ + [1.5, 1.0, -2.5], + [1.5, 1.0, -4], + ], + [ + [-1.5, -1.0, -2.0], + [-1.5, -1.0, -4], + ], + [ + [0, 3, 0], + [0, 0, -4], + ], + [ + [0, 0, 4], + [0, 0, -4], + ], +]; + +const POSITIONS: [number, number, number][] = [ + [0, 0, 0], + [0.5, 0.2, -0.5], + [-0.5, -0.3, 0.5], +]; + +/** Intercepts chunk fetches so they can be held/released deterministically. */ +export class FetchGate { + held: { url: string; release: () => void }[] = []; + active = false; + shouldHold: () => boolean = () => false; + private original?: typeof window.fetch; + private pattern = /fixture-lod-\d+\.radc/; + + install() { + if (this.original) return; + this.original = window.fetch.bind(window); + const original = this.original; + window.fetch = async (input, init) => { + const url = + typeof input === "string" + ? input + : input instanceof Request + ? input.url + : input.toString(); + if (this.active && this.pattern.test(url) && this.shouldHold()) { + const signal = + init?.signal ?? (input instanceof Request ? input.signal : null); + await new Promise((resolve, reject) => { + const entry = { + url, + release: () => { + const index = this.held.indexOf(entry); + if (index >= 0) this.held.splice(index, 1); + resolve(); + }, + }; + this.held.push(entry); + signal?.addEventListener("abort", () => { + const index = this.held.indexOf(entry); + if (index >= 0) this.held.splice(index, 1); + reject(new DOMException("Aborted", "AbortError")); + }); + }); + } + return original(input, init); + }; + } + + releaseAll() { + const all = this.held.slice(); + for (const h of all) h.release(); + return all.length; + } +} + +export async function runFuzz( + harness: Harness, + options: FuzzOptions, +): Promise { + const start = performance.now(); + const rand = mulberry32(options.seed); + const pick = (list: T[]): T => list[Math.floor(rand() * list.length)]; + const chance = (p: number) => rand() < p; + const drainEvery = options.drainEvery ?? 12; + const maxMeshes = options.maxMeshes ?? 4; + const holdP = options.holdFetchProbability ?? 0.5; + const budgetMs = options.budgetMs ?? 60_000; + const stopOnViolation = options.stopOnViolation ?? true; + + const gate = new FetchGate(); + gate.install(); + gate.active = true; + gate.shouldHold = () => chance(holdP); + + const trace: string[] = []; + const violations: string[] = []; + const skippedChecks: string[] = []; + let drains = 0; + let meshCounter = 0; + const disposed = new Set(); + const errorsBefore = harness.errors.length; + + const names = () => Array.from(harness.meshes.keys()); + const inScene = () => names().filter((n) => harness.inScene(n)); + const removed = () => + names().filter((n) => !harness.inScene(n) && !disposed.has(n)); + + const log = (step: number, message: string) => { + trace.push(`${step}: ${message}`); + }; + + const quickCheck = (step: number, label: string) => { + const pager = harness.spark.pager; + if (!pager) return; + const live = new Set(); + for (const splats of harness.spark.lodIds.keys()) { + if (splats instanceof PagedSplats) live.add(splats); + } + const errors = pager.checkInvariants(live); + for (const error of errors) { + violations.push(`step ${step} (${label}): ${error}`); + } + }; + + const drain = async (step: number) => { + drains += 1; + log(step, "drain: release hooks + fetches, render until idle, full check"); + harness.hooks.unholdAll(); + harness.hooks.releaseAll(); + gate.active = false; + gate.releaseAll(); + const idle = await harness.renderUntilIdle({ timeoutMs: 15_000 }); + if (!idle) { + skippedChecks.push(`step ${step}: did not reach idle within 15s`); + } + const { errors, skipped } = await harness.checkInvariants(); + for (const error of errors) + violations.push(`step ${step} (drain): ${error}`); + for (const s of skipped) skippedChecks.push(`step ${step}: ${s}`); + gate.active = true; + }; + + let step = 0; + for (; step < options.steps; step++) { + if (performance.now() - start > budgetMs) { + log(step, "budget exhausted"); + break; + } + if (stopOnViolation && violations.length > 0) break; + + if (step > 0 && step % drainEvery === 0) { + await drain(step); + continue; + } + + const r = rand(); + if (r < 0.1 && names().length - disposed.size < maxMeshes) { + const name = `F${meshCounter++}`; + const position = pick(POSITIONS); + harness.addPaged(name, options.url, { position }); + log(step, `add ${name} at ${position.join(",")}`); + } else if (r < 0.18 && inScene().length > 0) { + const name = pick(inScene()); + harness.remove(name); + log(step, `remove ${name}`); + } else if (r < 0.26 && removed().length > 0) { + const name = pick(removed()); + harness.readd(name); + log(step, `readd ${name}`); + } else if (r < 0.3 && names().length - disposed.size > 0) { + const name = pick(names().filter((n) => !disposed.has(n))); + harness.dispose(name); + disposed.add(name); + log(step, `dispose ${name}`); + } else if (r < 0.36 && inScene().length > 0) { + const name = pick(inScene()); + const visible = !harness.meshes.get(name)?.mesh.visible; + harness.setVisible(name, visible); + log(step, `visible ${name} ${visible}`); + } else if (r < 0.44) { + const [pos, look] = pick(CAMERAS); + harness.moveCamera(pos, look); + log(step, `camera ${pos.join(",")}`); + } else if (r < 0.62) { + const frames = 1 + Math.floor(rand() * 3); + for (let i = 0; i < frames; i++) { + harness.render(); + await new Promise((resolve) => requestAnimationFrame(resolve)); + } + log(step, `render x${frames}`); + } else if (r < 0.72 && gate.held.length > 0) { + const entry = pick(gate.held); + entry.release(); + log(step, `release fetch ${entry.url.replace(/^.*\//, "")}`); + } else if (r < 0.8 && harness.hooks.heldCount() > 0) { + const name = pick(harness.hooks.heldNames()); + harness.hooks.release(name, 1); + log(step, `release hook ${name}`); + } else if (r < 0.86) { + const name = pick(HOOK_NAMES); + harness.hooks.hold(name); + log(step, `hold ${name}`); + } else if (r < 0.9) { + const name = pick(HOOK_NAMES); + harness.hooks.unhold(name); + harness.hooks.release(name, 100); + log(step, `unhold ${name}`); + } else { + const ms = Math.floor(rand() * 80); + await harness.sleep(ms); + log(step, `sleep ${ms}`); + } + + // Give queued microtasks/timers a chance to run before checking + await harness.sleep(0); + quickCheck(step, trace[trace.length - 1]); + } + + // Final drain and full cleanup: remove everything, wait past the dispose + // timeout, and require the pool to be fully released. + await drain(step); + let finalCleanupOk = true; + if (violations.length === 0 || !stopOnViolation) { + for (const name of inScene()) harness.remove(name); + log(step, "final: remove all meshes"); + harness.render(); + await harness.sleep(harness.spark.lodDisposeTimeoutMs + 50); + const idle = await harness.renderUntilIdle({ timeoutMs: 15_000 }); + const s = harness.snapshot(); + const pager = harness.spark.pager; + if (pager) { + const state = pager.debugState(); + if (state.freelist.length !== state.maxPages || state.mapped.length > 0) { + finalCleanupOk = false; + violations.push( + `final: pool not released (free ${state.freelist.length}/${state.maxPages}, mapped ${state.mapped.length})`, + ); + } + } + if (s.lodIds.length > 0) { + finalCleanupOk = false; + violations.push(`final: lodIds not empty (${s.lodIds.length})`); + } + if (!idle) skippedChecks.push("final: did not reach idle"); + const { errors, skipped } = await harness.checkInvariants(); + for (const error of errors) violations.push(`final: ${error}`); + for (const sk of skipped) skippedChecks.push(`final: ${sk}`); + } + + gate.active = false; + gate.releaseAll(); + + return { + seed: options.seed, + stepsRun: step, + trace, + violations, + errors: harness.errors.slice(errorsBefore), + drains, + skippedChecks, + finalCleanupOk, + durationMs: performance.now() - start, + }; +} diff --git a/test/browser/pages/harness.html b/test/browser/pages/harness.html new file mode 100644 index 00000000..0cea8109 --- /dev/null +++ b/test/browser/pages/harness.html @@ -0,0 +1,19 @@ + + + + + Spark browser test harness + + + + + + diff --git a/test/browser/pages/harness.ts b/test/browser/pages/harness.ts new file mode 100644 index 00000000..f49fdec8 --- /dev/null +++ b/test/browser/pages/harness.ts @@ -0,0 +1,752 @@ +// Browser-side test harness for Spark's paged LoD / on-demand rendering tests. +// Loaded by harness.html through the Vite dev server and driven from Playwright +// via `window.harness`. + +import * as THREE from "three"; +import { + PagedSplats, + type SparkHooks, + SparkRenderer, + SplatMesh, + SplatPager, +} from "../../../src/index"; +import type { LodTreeInfo } from "../../../src/worker"; +import { type FuzzOptions, type FuzzResult, runFuzz } from "./fuzz"; + +export type RenderMode = "loop" | "ondemand" | "manual"; + +export interface HarnessInitOptions { + mode?: RenderMode; + width?: number; + height?: number; + /** Page pool size in pages of 65536 splats */ + maxPages?: number; + lodSplatCount?: number; + numLodFetchers?: number; + lodDisposeTimeoutMs?: number; + lodRaycast?: number; + fetchPause?: number; + /** + * Extra scalar SparkRenderer options. Only JSON-serializable values can + * cross page.evaluate, and a narrow type keeps Playwright's argument + * serialization types from recursing into SparkRendererOptions. + */ + spark?: Record; +} + +interface HeldPoint { + name: string; + info: unknown; + seq: number; + resolve: () => void; + reject: (error: Error) => void; +} + +/** + * Hook controller: lets a test hold execution at named async points, release + * them in a chosen order, or inject faults. + */ +class HookController implements SparkHooks { + private holds = new Set(); + private held: HeldPoint[] = []; + private faults = new Map(); + private seq = 0; + log: { seq: number; name: string; t: number; held: boolean }[] = []; + /** Names to record in the log (all when undefined) */ + logFilter?: Set; + + point(name: string, info?: unknown): undefined | Promise { + const seq = ++this.seq; + const fault = this.faults.get(name); + if (fault && fault.length > 0) { + const message = fault.shift() as string; + this.record(seq, name, false); + throw new Error(message); + } + if (!this.holds.has(name)) { + this.record(seq, name, false); + return undefined; + } + this.record(seq, name, true); + return new Promise((resolve, reject) => { + this.held.push({ name, info, seq, resolve, reject }); + }); + } + + private record(seq: number, name: string, held: boolean) { + if (!this.logFilter || this.logFilter.has(name)) { + this.log.push({ seq, name, t: performance.now(), held }); + } + } + + hold(name: string) { + this.holds.add(name); + } + + unhold(name: string) { + this.holds.delete(name); + } + + unholdAll() { + this.holds.clear(); + } + + /** Number of executions currently parked at `name` (all names if omitted) */ + heldCount(name?: string) { + return this.held.filter((h) => !name || h.name === name).length; + } + + heldNames() { + return this.held.map((h) => h.name); + } + + /** Release up to `count` parked executions at `name` (FIFO). Returns released count. */ + release(name?: string, count = 1) { + let released = 0; + for (let i = 0; i < this.held.length && released < count; ) { + const h = this.held[i]; + if (!name || h.name === name) { + this.held.splice(i, 1); + h.resolve(); + released++; + } else { + i++; + } + } + return released; + } + + releaseAll() { + const held = this.held; + this.held = []; + for (const h of held) h.resolve(); + return held.length; + } + + /** Make the next `count` executions of `name` throw `message`. */ + failNext(name: string, message = `Injected fault at ${name}`, count = 1) { + const list = this.faults.get(name) ?? []; + for (let i = 0; i < count; i++) list.push(message); + this.faults.set(name, list); + } + + /** Reject a parked execution (fault injection at a held point). */ + rejectHeld(name: string, message = `Injected fault at ${name}`) { + const index = this.held.findIndex((h) => h.name === name); + if (index < 0) return false; + const [h] = this.held.splice(index, 1); + h.reject(new Error(message)); + return true; + } + + reset() { + this.releaseAll(); + this.holds.clear(); + this.faults.clear(); + this.log = []; + } +} + +interface MeshRecord { + name: string; + mesh: SplatMesh; + url: string; +} + +export class Harness { + THREE = THREE; + renderer!: THREE.WebGLRenderer; + scene!: THREE.Scene; + camera!: THREE.PerspectiveCamera; + spark!: SparkRenderer; + hooks = new HookController(); + mode: RenderMode = "manual"; + + meshes = new Map(); + /** Stable names for PagedSplats instances (survive remove/re-add) */ + private splatsNames = new Map(); + + renders = 0; + dirtyEvents: number[] = []; + errors: string[] = []; + private scheduled = false; + private loopHandle = false; + pagerUpdates = 0; + initialized = false; + + constructor() { + window.addEventListener("error", (event) => { + this.errors.push(`error: ${event.message}`); + }); + window.addEventListener("unhandledrejection", (event) => { + const reason = event.reason as { message?: string } | string; + const message = + typeof reason === "string" + ? reason + : (reason?.message ?? String(reason)); + this.errors.push(`unhandledrejection: ${message}`); + }); + const origError = console.error.bind(console); + console.error = (...args: unknown[]) => { + this.errors.push(`console.error: ${args.map(String).join(" ")}`); + origError(...args); + }; + const origWarn = console.warn.bind(console); + console.warn = (...args: unknown[]) => { + this.warnings.push(`console.warn: ${args.map(String).join(" ")}`); + origWarn(...args); + }; + } + warnings: string[] = []; + + init(options: HarnessInitOptions = {}) { + if (this.initialized) { + throw new Error("Harness already initialized; reload the page"); + } + this.initialized = true; + const width = options.width ?? 256; + const height = options.height ?? 256; + this.mode = options.mode ?? "manual"; + + this.renderer = new THREE.WebGLRenderer({ + antialias: false, + preserveDrawingBuffer: true, + }); + this.renderer.setPixelRatio(1); + this.renderer.setSize(width, height); + document.body.appendChild(this.renderer.domElement); + + this.scene = new THREE.Scene(); + this.camera = new THREE.PerspectiveCamera(60, width / height, 0.05, 100); + this.camera.position.set(0, 0, 0); + this.camera.lookAt(0, 0, -4); + this.camera.updateMatrixWorld(); + + const maxPages = options.maxPages ?? 4; + this.spark = new SparkRenderer({ + renderer: this.renderer, + maxPagedSplats: maxPages * 65536, + lodSplatCount: options.lodSplatCount ?? 30000, + numLodFetchers: options.numLodFetchers ?? 1, + lodDisposeTimeoutMs: options.lodDisposeTimeoutMs ?? 3000, + // Disable raycast traverses unless a test asks for them; they add a + // second traverse per LoD callback that only makes traces noisier. + lodRaycast: options.lodRaycast ?? 0, + hooks: this.hooks, + onDirty: () => this.onDirty(), + ...(options.spark ?? {}), + }); + this.scene.add(this.spark); + + if (options.fetchPause !== undefined) { + // Applied when the pager is created (see pagerHook) + this.fetchPause = options.fetchPause; + } + + if (this.mode === "loop") { + this.loopHandle = true; + this.renderer.setAnimationLoop(() => this.render()); + } + } + private fetchPause?: number; + + private onDirty() { + this.dirtyEvents.push(performance.now()); + if (this.mode === "ondemand" && !this.scheduled) { + this.scheduled = true; + requestAnimationFrame(() => { + this.scheduled = false; + this.render(); + }); + } + } + + setMode(mode: RenderMode) { + if (this.mode === "loop" && mode !== "loop") { + this.renderer.setAnimationLoop(null); + this.loopHandle = false; + } + this.mode = mode; + if (mode === "loop" && !this.loopHandle) { + this.loopHandle = true; + this.renderer.setAnimationLoop(() => this.render()); + } + } + + /** Render one frame now (works in every mode). */ + render() { + this.renders += 1; + this.renderer.render(this.scene, this.camera); + // Pick up the pager once it exists so the tests can observe its counters + const pager = this.spark.pager; + if (pager && !this.pagerHooked) { + this.pagerHooked = true; + if (this.fetchPause !== undefined) pager.fetchPause = this.fetchPause; + const prev = pager.onUpdate; + pager.onUpdate = () => { + this.pagerUpdates += 1; + prev?.(); + }; + // Count fetched chunks dropped because no page could be allocated + // (processFetched's "no pages available" branch). + const privatePager = pager as unknown as { + allocateFreeable(): number | undefined; + }; + const origAllocateFreeable = privatePager.allocateFreeable.bind(pager); + privatePager.allocateFreeable = () => { + const page = origAllocateFreeable(); + if (page === undefined) this.pagerDrops += 1; + return page; + }; + } + } + private pagerHooked = false; + pagerDrops = 0; + + get pager(): SplatPager | undefined { + return this.spark.pager; + } + + private nameFor(splats: object | undefined): string { + if (!splats) return "?"; + return this.splatsNames.get(splats) ?? ""; + } + + /** Add a paged SplatMesh loading a chunked -lod.rad URL. */ + addPaged( + name: string, + url: string, + opts: { position?: [number, number, number]; lodScale?: number } = {}, + ) { + if (this.meshes.has(name)) { + throw new Error(`mesh ${name} already exists`); + } + const mesh = new SplatMesh({ url, paged: true, lodScale: opts.lodScale }); + if (opts.position) mesh.position.set(...opts.position); + if (mesh.paged) this.splatsNames.set(mesh.paged, name); + this.meshes.set(name, { name, mesh, url }); + this.scene.add(mesh); + return name; + } + + /** Add a regular (non-paged) SplatMesh; `lod` enables LoD for -lod.rad files */ + addMesh( + name: string, + url: string, + opts: { position?: [number, number, number]; lod?: boolean } = {}, + ) { + if (this.meshes.has(name)) { + throw new Error(`mesh ${name} already exists`); + } + const mesh = new SplatMesh({ url, lod: opts.lod }); + if (opts.position) mesh.position.set(...opts.position); + this.meshes.set(name, { name, mesh, url }); + this.scene.add(mesh); + mesh.initialized.then(() => { + const splats = mesh.packedSplats?.lodSplats ?? mesh.extSplats?.lodSplats; + if (splats) this.splatsNames.set(splats, name); + }); + return name; + } + + private rec(name: string) { + const rec = this.meshes.get(name); + if (!rec) throw new Error(`unknown mesh ${name}`); + return rec; + } + + remove(name: string) { + this.scene.remove(this.rec(name).mesh); + } + + /** Re-add a previously removed (not disposed) mesh */ + readd(name: string) { + this.scene.add(this.rec(name).mesh); + } + + dispose(name: string) { + const rec = this.rec(name); + this.scene.remove(rec.mesh); + rec.mesh.dispose(); + } + + /** Remove + dispose + forget the name so it can be reused */ + destroy(name: string) { + this.dispose(name); + this.meshes.delete(name); + } + + setVisible(name: string, visible: boolean) { + this.rec(name).mesh.visible = visible; + } + + setPosition(name: string, position: [number, number, number]) { + this.rec(name).mesh.position.set(...position); + } + + inScene(name: string) { + return this.rec(name).mesh.parent === this.scene; + } + + async awaitInitialized(name: string) { + await this.rec(name).mesh.initialized; + } + + moveCamera( + position: [number, number, number], + lookAt: [number, number, number] = [0, 0, -4], + ) { + this.camera.position.set(...position); + this.camera.lookAt(...lookAt); + this.camera.updateMatrixWorld(); + } + + /** Count pixels with any visible color in the last rendered frame. */ + countLitPixels(threshold = 24) { + const gl = this.renderer.getContext() as WebGL2RenderingContext; + const { width, height } = gl.canvas; + const pixels = new Uint8Array(width * height * 4); + gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels); + let lit = 0; + for (let i = 0; i < pixels.length; i += 4) { + if (pixels[i] + pixels[i + 1] + pixels[i + 2] > threshold) lit++; + } + return lit; + } + + /** Is there any asynchronous work in flight? */ + inFlight() { + const spark = this.spark; + const pagerState = spark.pager?.debugState(); + const lodWorker = spark.lodWorker as { queue: unknown[] | null } | null; + const reasons: string[] = []; + if (spark.sorting) reasons.push("sorting"); + if (spark.sortTimeoutId !== -1) reasons.push("sortTimeout"); + if (lodWorker && lodWorker.queue != null) reasons.push("lodCallback"); + if (pagerState && pagerState.fetchers.length > 0) reasons.push("fetchers"); + if (this.scheduled) reasons.push("renderScheduled"); + if (this.mode === "loop") reasons.push("loop"); + for (const [name, rec] of this.meshes) { + if (!rec.mesh.isInitialized && rec.mesh.parent === this.scene) { + // Only count if it can still complete + reasons.push(`init:${name}`); + } + } + return reasons; + } + + /** + * Wait until nothing asynchronous is in flight for `settleMs`, or until + * `timeoutMs` passes. Returns the in-flight reasons at exit (empty = quiet). + * Loop mode never counts as quiet unless `ignoreLoop` is true. + */ + async waitQuiet({ + settleMs = 150, + timeoutMs = 10000, + ignoreLoop = false, + } = {}): Promise { + const start = performance.now(); + let quietSince: number | null = null; + while (performance.now() - start < timeoutMs) { + const reasons = this.inFlight().filter( + (r) => !(ignoreLoop && r === "loop"), + ); + if (reasons.length === 0) { + if (quietSince === null) quietSince = performance.now(); + if (performance.now() - quietSince >= settleMs) return []; + } else { + quietSince = null; + } + await new Promise((r) => setTimeout(r, 20)); + } + return this.inFlight(); + } + + /** + * Wait until the pipeline is fully drained: quiet, and no pending pager + * queues. In loop/ondemand mode this is the "converged" state. + */ + async waitIdle({ timeoutMs = 15000, settleMs = 200 } = {}) { + const start = performance.now(); + let idleSince: number | null = null; + while (performance.now() - start < timeoutMs) { + if (!this.isBusy()) { + if (idleSince === null) idleSince = performance.now(); + if (performance.now() - idleSince >= settleMs) return true; + } else { + idleSince = null; + } + await new Promise((r) => setTimeout(r, 25)); + } + return false; + } + + /** Any in-flight work or un-consumed queued work (ignoring the render loop) */ + isBusy() { + const s = this.snapshot(); + return ( + s.inFlight.filter((r) => r !== "loop").length > 0 || + s.pager.fetched > 0 || + s.pager.lodTreeUpdates > 0 || + s.pager.newUploads > 0 || + s.pager.readyUploads > 0 || + s.lodUpdates > 0 || + s.sortDirty || + s.lodInitQueue > 0 + ); + } + + /** + * Manual-mode convergence: keep rendering frames until nothing is busy for + * `idleFrames` consecutive frames. Returns true if converged. + */ + async renderUntilIdle({ timeoutMs = 30000, idleFrames = 3 } = {}) { + const start = performance.now(); + let idle = 0; + while (performance.now() - start < timeoutMs) { + this.render(); + await new Promise((r) => requestAnimationFrame(r)); + await this.sleep(30); + if (!this.isBusy() && !this.spark.dirty) { + idle += 1; + if (idle >= idleFrames) return true; + } else { + idle = 0; + } + } + return false; + } + + sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); + } + + /** JSON-serializable view of renderer + pager state */ + snapshot() { + const spark = this.spark; + const pager = spark.pager; + const state = pager?.debugState(); + const lodIds: { + name: string; + lodId: number; + rootPage?: number; + paged: boolean; + }[] = []; + for (const [splats, record] of spark.lodIds.entries()) { + lodIds.push({ + name: this.nameFor(splats), + lodId: record.lodId, + rootPage: record.rootPage, + paged: splats instanceof PagedSplats, + }); + } + const meshes: Record< + string, + { + inScene: boolean; + visible: boolean; + initialized: boolean; + numSplats: number; + pagedNumSplats?: number; + pagedAborted?: boolean; + pagedHasIndicesTexture?: boolean; + } + > = {}; + for (const [name, rec] of this.meshes) { + const paged = rec.mesh.paged; + meshes[name] = { + inScene: rec.mesh.parent === this.scene, + visible: rec.mesh.visible, + initialized: rec.mesh.isInitialized, + numSplats: rec.mesh.numSplats, + pagedNumSplats: paged?.numSplats, + pagedAborted: paged?.abortController.signal.aborted, + pagedHasIndicesTexture: paged + ? paged.dynoIndices.value !== SplatPager.emptyIndicesTexture + : undefined, + }; + } + return { + renders: this.renders, + dirtyEvents: this.dirtyEvents.length, + pagerUpdates: this.pagerUpdates, + pagerDrops: this.pagerDrops, + activeSplats: spark.activeSplats, + sorting: spark.sorting, + sortDirty: spark.sortDirty, + lodDirty: spark.lodDirty, + lodUpdates: spark.lodUpdates.length, + lodInitQueue: spark.lodInitQueue.length, + lodIds, + pagerId: spark.pagerId, + hasPager: !!pager, + inFlight: this.inFlight(), + accumulatorsFree: spark.accumulators.length, + meshes, + pager: { + maxPages: state?.maxPages ?? 0, + freelist: state?.freelist ?? [], + freeable: state?.freeable ?? [], + mapped: (state?.mapped ?? []).map(({ page, splats, chunk }) => ({ + page, + name: this.nameFor(splats), + chunk, + })), + fetchers: (state?.fetchers ?? []).map(({ splats, chunk }) => ({ + name: this.nameFor(splats), + chunk, + })), + fetched: state?.fetched.length ?? 0, + lodTreeUpdates: state?.lodTreeUpdates.length ?? 0, + newUploads: state?.newUploads.length ?? 0, + readyUploads: state?.readyUploads.length ?? 0, + fetchPriority: (state?.fetchPriority ?? []).map( + ({ splats, chunk }) => ({ name: this.nameFor(splats), chunk }), + ), + }, + held: this.hooks.heldNames(), + errors: this.errors.slice(), + warnings: this.warnings.slice(), + }; + } + + /** Pages currently mapped for a named paged mesh, keyed by chunk */ + residentChunks(name: string): Record { + const rec = this.rec(name); + const pager = this.spark.pager; + if (!pager || !rec.mesh.paged) return {}; + const chunks = pager.splatsChunkToPage.get(rec.mesh.paged); + const out: Record = {}; + chunks?.forEach((entry, chunk) => { + if (entry) out[chunk] = entry.page; + }); + return out; + } + + /** Seeded random interleaving of actions, see fuzz.ts */ + fuzz(options: FuzzOptions): Promise { + return runFuzz(this, options); + } + + /** Direct call into the LoD worker (test introspection) */ + async lodWorkerCall(name: string, args: unknown): Promise { + const worker = ( + this.spark as unknown as { ensureLodWorker(): unknown } + ).ensureLodWorker() as { + call(name: string, args: unknown): Promise; + }; + return worker.call(name, args); + } + + /** + * Pager table invariants plus cross-check against the Rust LoD trees. + * The Rust cross-check is only meaningful once pending updates have been + * drained; when they have not, it is skipped (reported in `skipped`). + */ + async checkInvariants(): Promise<{ errors: string[]; skipped: string[] }> { + const spark = this.spark; + const errors: string[] = []; + const skipped: string[] = []; + const pager = spark.pager; + if (!pager) return { errors, skipped: ["no pager"] }; + + const live = new Set(); + for (const splats of spark.lodIds.keys()) { + if (splats instanceof PagedSplats) live.add(splats); + } + errors.push(...pager.checkInvariants(live)); + + // accumulators freelist sanity: 2 pool + display/current juggling + const free = spark.accumulators.length; + if (free < 1 || free > 2) { + errors.push(`accumulators freelist size ${free} (expected 1..2)`); + } + + const state = pager.debugState(); + const lodCallbackActive = + (spark.lodWorker as { queue: unknown[] | null } | null)?.queue != null; + const drained = + state.lodTreeUpdates.length === 0 && + spark.lodUpdates.length === 0 && + !lodCallbackActive; + if (!drained) { + skipped.push("rust cross-check (pending updates or LoD callback active)"); + return { errors, skipped }; + } + + // Live tree ids in Rust must be exactly the pager tree + tracked lodIds + const { lodIds: rustIds } = await this.lodWorkerCall<{ + lodIds: number[]; + }>("getLodTreeIds", {}); + const expectedIds = new Set(); + if (spark.pagerId) expectedIds.add(spark.pagerId); + for (const record of spark.lodIds.values()) expectedIds.add(record.lodId); + // Re-check drained-ness: if a callback started meanwhile, results are moot + if ( + (spark.lodWorker as { queue: unknown[] | null } | null)?.queue != null + ) { + skipped.push("rust cross-check (LoD callback started during check)"); + return { errors, skipped }; + } + const rustSet = new Set(rustIds); + for (const id of rustSet) { + if (!expectedIds.has(id)) errors.push(`rust has leaked lod tree ${id}`); + } + for (const id of expectedIds) { + if (!rustSet.has(id)) errors.push(`rust missing lod tree ${id}`); + } + + for (const [splats, record] of spark.lodIds.entries()) { + if (!(splats instanceof PagedSplats)) continue; + if (!rustSet.has(record.lodId)) continue; + const info = await this.lodWorkerCall("getLodTreeInfo", { + lodId: record.lodId, + }); + const name = this.nameFor(splats); + const chunks = pager.splatsChunkToPage.get(splats) ?? []; + const NONE = 0xffffffff; + const maxChunks = Math.max(chunks.length, info.chunkToPage.length); + for (let chunk = 0; chunk < maxChunks; chunk++) { + const jsPage = chunks[chunk]?.page; + const rustPage = + chunk < info.chunkToPage.length ? info.chunkToPage[chunk] : NONE; + const rustPageOrUndef = rustPage === NONE ? undefined : rustPage; + if (jsPage !== rustPageOrUndef) { + errors.push( + `${name} chunk ${chunk}: pager page ${jsPage} != rust chunk_to_page ${rustPageOrUndef}`, + ); + } + } + for (let page = 0; page < info.pageToChunk.length; page++) { + const rustChunk = info.pageToChunk[page]; + if (rustChunk === NONE) continue; + const owner = pager.pageToSplatsChunk[page]; + if (!owner || owner.splats !== splats || owner.chunk !== rustChunk) { + errors.push( + `${name} rust page_to_chunk[${page}]=${rustChunk} but pager page owner is ${owner ? `${this.nameFor(owner.splats)}:${owner.chunk}` : "none"}`, + ); + } + } + const rustRoot = + info.chunkToPage.length > 0 && info.chunkToPage[0] !== NONE + ? info.chunkToPage[0] + : undefined; + if (record.rootPage !== rustRoot) { + errors.push( + `${name} rootPage ${record.rootPage} != rust chunk_to_page[0] ${rustRoot}`, + ); + } + } + return { errors, skipped }; + } +} + +declare global { + interface Window { + harness: Harness; + harnessReady: boolean; + } +} + +window.harness = new Harness(); +window.harnessReady = true; diff --git a/test/browser/regressions.spec.ts b/test/browser/regressions.spec.ts new file mode 100644 index 00000000..7da1e03d --- /dev/null +++ b/test/browser/regressions.spec.ts @@ -0,0 +1,142 @@ +import { expect, test } from "@playwright/test"; +import { FUZZ_HARNESS, formatFuzzFailure } from "./fuzz-shared"; +import { + FIXTURES, + expectInvariants, + expectNoErrors, + harness, + openHarness, +} from "./helpers"; + +// Fuzz seeds that produced invariant violations against the pre-fix pager. +// Each entry pins the exact harness configuration and step count of the run +// that found it so the same interleaving is replayed. +// +// seed 12 / 17 (120 steps): pages stayed mapped for PagedSplats whose LoD +// tree had been disposed (a chunk fetch landed after cleanupLodTrees -> +// pager.removeSplats), and the run never reached idle afterwards. +const PINNED: { seed: number; steps: number }[] = [ + { seed: 12, steps: 120 }, + { seed: 17, steps: 120 }, +]; + +for (const { seed, steps } of PINNED) { + test(`fuzz regression seed ${seed} (${steps} steps)`, async ({ page }) => { + test.setTimeout(180_000); + const opened = await openHarness(page, FUZZ_HARNESS); + const result = await page.evaluate( + ([s, n, url]) => + window.harness.fuzz({ seed: s, steps: n, url, budgetMs: 120_000 }), + [seed, steps, FIXTURES.chunked] as const, + ); + const benign = (e: string) => /GL_|performance warning/i.test(e); + expect( + result.violations, + `${formatFuzzFailure(result)}\npageErrors: ${opened.pageErrors.join("; ")}`, + ).toEqual([]); + expect( + result.errors.filter((e) => !benign(e)), + formatFuzzFailure(result), + ).toEqual([]); + expect(opened.pageErrors).toEqual([]); + expect(result.finalCleanupOk).toBe(true); + }); +} + +// cleanupLodTrees compared lastTouched (stamped at the start of updateLod) +// against performance.now() at the end of the async LoD callback. With a +// short lodDisposeTimeoutMs the tree of a mesh that was visible this very +// frame looked stale and was disposed, then re-created next frame (flicker). +test("lodDisposeTimeoutMs 0 never disposes the LoD tree of a visible mesh", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "manual", + maxPages: 8, + lodSplatCount: 40000, + numLodFetchers: 1, + lodDisposeTimeoutMs: 0, + }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + expect(await h.renderUntilIdle({ timeoutMs: 30_000 })).toBe(true); + + const s0 = await h.snapshot(); + const initial = s0.lodIds.find((l) => l.name === "A"); + expect(initial, "A has a LoD tree after converging").toBeDefined(); + expect(s0.meshes.A.pagedNumSplats ?? 0).toBeGreaterThan(0); + + // Keep rendering with A visible: its record must survive every callback + // with the same lodId (no dispose / re-init churn). + for (let i = 0; i < 20; i++) { + await h.render(); + await page.waitForTimeout(40); + const s = await h.snapshot(); + const record = s.lodIds.find((l) => l.name === "A"); + expect(record?.lodId, `frame ${i}: A's tree`).toBe(initial?.lodId); + } + expect(await h.renderUntilIdle({ timeoutMs: 15_000 })).toBe(true); + const s1 = await h.snapshot(); + expect(s1.lodIds.find((l) => l.name === "A")?.lodId).toBe(initial?.lodId); + expect(s1.meshes.A.pagedNumSplats ?? 0).toBeGreaterThan(0); + await expectInvariants(page); + + // Once removed, a zero timeout disposes the tree on the next callback and + // returns every page to the pool. + await h.remove("A"); + expect(await h.renderUntilIdle({ timeoutMs: 15_000 })).toBe(true); + const s2 = await h.snapshot(); + expect(s2.lodIds).toEqual([]); + expect(s2.pager.mapped.filter((m) => m.name === "A")).toEqual([]); + expect(s2.pager.freelist.length).toBe(s2.pager.maxPages); + await expectInvariants(page); + expectNoErrors(opened, s2); +}); + +// Overlapping frames: a mesh removed in frame N (whose callback is still +// running) and re-added in frame N+1 gets a fresh lastTouched from N+1's +// updateLod, but callback N's cleanup only knew frame N's lodMeshes. It must +// consult the latest visibility rather than dispose the now-visible tree. +test("cleanup does not dispose a tree made visible again while the callback ran", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "manual", + maxPages: 8, + lodSplatCount: 40000, + numLodFetchers: 1, + lodDisposeTimeoutMs: 0, + }); + const h = harness(page); + await h.addPaged("A", FIXTURES.chunked); + expect(await h.renderUntilIdle({ timeoutMs: 30_000 })).toBe(true); + const initial = (await h.snapshot()).lodIds.find((l) => l.name === "A"); + expect(initial).toBeDefined(); + + // Frame N: A removed; its callback parks right before cleanup. + await h.hooks.hold("lod.beforeCleanup"); + await h.remove("A"); + await h.render(); + await h.hooks.waitHeld("lod.beforeCleanup", 1); + + // Frame N+1 (callback skipped, worker busy): A is visible again and its + // record is re-stamped. + await h.readd("A"); + await h.render(); + await page.waitForTimeout(50); + + // Resume callback N's cleanup with the zero timeout. + await h.hooks.unhold("lod.beforeCleanup"); + await h.hooks.release("lod.beforeCleanup", 1); + await page.waitForTimeout(200); + + const after = (await h.snapshot()).lodIds.find((l) => l.name === "A"); + expect(after?.lodId, "A's tree survived cleanup").toBe(initial?.lodId); + + expect(await h.renderUntilIdle({ timeoutMs: 15_000 })).toBe(true); + const s = await h.snapshot(); + expect(s.lodIds.find((l) => l.name === "A")?.lodId).toBe(initial?.lodId); + expect(s.meshes.A.pagedNumSplats ?? 0).toBeGreaterThan(0); + await expectInvariants(page); + expectNoErrors(opened, s); +}); diff --git a/test/browser/sanity.spec.ts b/test/browser/sanity.spec.ts new file mode 100644 index 00000000..3d36969f --- /dev/null +++ b/test/browser/sanity.spec.ts @@ -0,0 +1,91 @@ +import { expect, test } from "@playwright/test"; +import { + FIXTURES, + expectInvariants, + expectNoErrors, + harness, + openHarness, + waitForSnapshot, +} from "./helpers"; + +// Baseline: continuous rendering with a paged mesh must load, render pixels, +// keep the page tables consistent, and fully clean up after removal. +// This must be green before and after any fix. + +test("WebGL2 + workers + WASM: paged mesh loads and renders in loop mode", async ({ + page, +}) => { + const opened = await openHarness(page, { + mode: "loop", + maxPages: 8, + lodSplatCount: 60000, + lodDisposeTimeoutMs: 300, + }); + const h = harness(page); + + await h.addPaged("A", FIXTURES.chunked); + + const loaded = await waitForSnapshot( + page, + (s) => s.activeSplats > 0 && s.pager.mapped.length >= 2, + { label: "paged mesh resident and active", timeoutMs: 30_000 }, + ); + expect(loaded.hasPager).toBe(true); + expect(loaded.lodIds.find((l) => l.name === "A")?.rootPage).toBe( + loaded.pager.mapped.find((m) => m.name === "A" && m.chunk === 0)?.page, + ); + + // Let fetching settle, then check pixels + invariants + expect(await h.waitIdle({ timeoutMs: 30_000 })).toBe(true); + const lit = await h.countLitPixels(); + expect(lit).toBeGreaterThan(100); + + const s1 = await h.snapshot(); + expect(s1.pager.mapped.length).toBeLessThanOrEqual(s1.pager.maxPages); + expect(s1.pager.mapped.length).toBeGreaterThanOrEqual(2); + await expectInvariants(page); + expectNoErrors(opened, s1); + + // Remove the mesh; after the dispose timeout everything must be released + await h.remove("A"); + const cleaned = await waitForSnapshot( + page, + (s) => + s.lodIds.length === 0 && + s.pager.freelist.length === s.pager.maxPages && + s.pager.mapped.length === 0, + { label: "pager fully released after remove", timeoutMs: 15_000 }, + ); + expect(cleaned.activeSplats).toBe(0); + const inv = await expectInvariants(page); + expect(inv.skipped).toEqual([]); + expectNoErrors(opened, cleaned); +}); + +test("non-paged LoD mesh and plain PLY load in loop mode", async ({ page }) => { + const opened = await openHarness(page, { + mode: "loop", + lodSplatCount: 30000, + }); + const h = harness(page); + await h.addMesh("L", FIXTURES.lodRad, { lod: true }); + await h.addMesh("P", FIXTURES.smallPly, { position: [0, 0, -2] }); + + const s = await waitForSnapshot( + page, + (s) => + s.meshes.L.initialized && + s.meshes.P.initialized && + s.activeSplats > 0 && + s.lodIds.some((l) => l.name === "L" && !l.paged), + { + label: "both meshes initialized and LoD tree created", + timeoutMs: 30_000, + }, + ); + expect(s.meshes.P.numSplats).toBeGreaterThan(0); + await expect + .poll(() => h.countLitPixels(), { timeout: 15_000 }) + .toBeGreaterThan(100); + expectNoErrors(opened, await h.snapshot()); +}); diff --git a/test/fixtures/gen-fixture.mjs b/test/fixtures/gen-fixture.mjs new file mode 100644 index 00000000..5180bae5 --- /dev/null +++ b/test/fixtures/gen-fixture.mjs @@ -0,0 +1,200 @@ +// Generates deterministic synthetic splat fixtures for the browser tests. +// +// test/fixtures/out/fixture.ply plain 3DGS PLY (non-LoD) +// test/fixtures/out/fixture-lod.rad single-file LoD RAD (built by build-lod) +// test/fixtures/out/chunked/fixture-lod.rad + fixture-lod-.radc +// chunked LoD RAD for paged loading +// +// Requires a Rust toolchain: the LoD files are produced by rust/build-lod. +// Usage: node test/fixtures/gen-fixture.mjs [--force] + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, "../.."); +const outDir = path.join(here, "out"); +const chunkedDir = path.join(outDir, "chunked"); + +// ~300K splats -> ~5-6 chunks of 65536 after LoD nodes are added. +export const FIXTURE_SPLATS = 300_000; + +// Small non-LoD PLY for tests that only need a regular mesh (SwiftShader is +// slow at rasterizing hundreds of thousands of splats). +export const FIXTURE_SMALL_SPLATS = 20_000; + +export const FIXTURE_FILES = { + ply: path.join(outDir, "fixture.ply"), + smallPly: path.join(outDir, "fixture-small.ply"), + lodRad: path.join(outDir, "fixture-lod.rad"), + chunkedRad: path.join(chunkedDir, "fixture-lod.rad"), +}; + +// Small deterministic PRNG (mulberry32) +function rng(seed) { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export function writeSyntheticPly(filename, numSplats, seed = 1234) { + const rand = rng(seed); + const props = [ + "x", + "y", + "z", + "f_dc_0", + "f_dc_1", + "f_dc_2", + "opacity", + "scale_0", + "scale_1", + "scale_2", + "rot_0", + "rot_1", + "rot_2", + "rot_3", + ]; + const header = [ + "ply", + "format binary_little_endian 1.0", + `element vertex ${numSplats}`, + ...props.map((p) => `property float ${p}`), + "end_header", + "", + ].join("\n"); + + const stride = props.length * 4; + const body = Buffer.alloc(numSplats * stride); + const view = new DataView(body.buffer, body.byteOffset, body.byteLength); + const SH_C0 = 0.28209479177387814; + + for (let i = 0; i < numSplats; i++) { + // Points spread over a few blobs so the LoD tree has real structure and + // the camera can see a subset of chunks at a time. + const blob = i % 4; + const bx = (blob & 1 ? 1 : -1) * 1.5; + const by = (blob & 2 ? 1 : -1) * 1.0; + const r = Math.sqrt(-2 * Math.log(1 - rand())) * 0.6; + const theta = rand() * Math.PI * 2; + const phi = Math.acos(2 * rand() - 1); + const x = bx + r * Math.sin(phi) * Math.cos(theta); + const y = by + r * Math.sin(phi) * Math.sin(theta); + const z = -4 + r * Math.cos(phi); + + // Bright, saturated colors per blob so pixels are clearly non-black + const cr = blob === 0 || blob === 3 ? 1.0 : 0.2; + const cg = blob === 1 || blob === 3 ? 1.0 : 0.2; + const cb = blob === 2 ? 1.0 : 0.2; + + const lnScale = Math.log(0.01 + rand() * 0.02); + // Random unit quaternion + const u1 = rand(); + const u2 = rand() * Math.PI * 2; + const u3 = rand() * Math.PI * 2; + const qw = Math.sqrt(1 - u1) * Math.sin(u2); + const qx = Math.sqrt(1 - u1) * Math.cos(u2); + const qy = Math.sqrt(u1) * Math.sin(u3); + const qz = Math.sqrt(u1) * Math.cos(u3); + + const o = i * stride; + const vals = [ + x, + y, + z, + (cr - 0.5) / SH_C0, + (cg - 0.5) / SH_C0, + (cb - 0.5) / SH_C0, + // logit(opacity) with opacity ~ 0.8 + Math.log(0.8 / 0.2), + lnScale, + lnScale, + lnScale, + qw, + qx, + qy, + qz, + ]; + for (let j = 0; j < vals.length; j++) { + view.setFloat32(o + j * 4, vals[j], true); + } + } + + fs.mkdirSync(path.dirname(filename), { recursive: true }); + fs.writeFileSync( + filename, + Buffer.concat([Buffer.from(header, "ascii"), body]), + ); +} + +function runBuildLod(args, cwd) { + const manifest = path.join(repoRoot, "rust/build-lod/Cargo.toml"); + execFileSync( + "cargo", + [ + "run", + "--quiet", + "--manifest-path", + manifest, + "--release", + "--no-default-features", + "--", + ...args, + ], + { cwd, stdio: process.env.SPARK_FIXTURE_VERBOSE ? "inherit" : "pipe" }, + ); +} + +export function fixturesExist() { + return Object.values(FIXTURE_FILES).every((f) => fs.existsSync(f)); +} + +export function generateFixtures({ force = false } = {}) { + if (!force && fixturesExist()) { + return FIXTURE_FILES; + } + fs.rmSync(outDir, { recursive: true, force: true }); + fs.mkdirSync(chunkedDir, { recursive: true }); + + writeSyntheticPly(FIXTURE_FILES.ply, FIXTURE_SPLATS); + writeSyntheticPly(FIXTURE_FILES.smallPly, FIXTURE_SMALL_SPLATS, 4321); + + // Single-file LoD RAD next to the PLY: fixture.ply -> fixture-lod.rad + runBuildLod(["--quick", "--rad", "--max-sh=0", FIXTURE_FILES.ply], outDir); + + // Chunked RAD in its own directory (build-lod writes chunks next to the input) + const chunkedPly = path.join(chunkedDir, "fixture.ply"); + fs.copyFileSync(FIXTURE_FILES.ply, chunkedPly); + runBuildLod( + ["--quick", "--rad-chunked", "--max-sh=0", chunkedPly], + chunkedDir, + ); + fs.rmSync(chunkedPly); + + if (!fixturesExist()) { + throw new Error("Fixture generation did not produce the expected files"); + } + return FIXTURE_FILES; +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + const force = process.argv.includes("--force"); + const files = generateFixtures({ force }); + for (const [key, file] of Object.entries(files)) { + console.log( + `${key}: ${path.relative(repoRoot, file)} (${fs.statSync(file).size} bytes)`, + ); + } + const chunks = fs.readdirSync(chunkedDir).filter((f) => f.endsWith(".radc")); + console.log(`chunks: ${chunks.length}`); +} diff --git a/test/playwright.config.ts b/test/playwright.config.ts new file mode 100644 index 00000000..66dc3a83 --- /dev/null +++ b/test/playwright.config.ts @@ -0,0 +1,60 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { defineConfig, devices } from "@playwright/test"; + +// Browser tests run against the Vite dev server so that src/*.ts, GLSL and the +// inline worker are transformed on the fly, with real WebGL2 (SwiftShader), +// Web Workers and the spark-rs WASM module. +// +// This file lives under test/ so it is covered by test/tsconfig.json (Node +// types); `npm run test:browser` points Playwright at it. +const PORT = Number(process.env.SPARK_TEST_PORT ?? 8080); +const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", +); + +export default defineConfig({ + testDir: "browser", + testMatch: /.*\.spec\.ts/, + globalSetup: "./browser/global-setup.ts", + outputDir: path.join(repoRoot, "test-results"), + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 60_000, + expect: { timeout: 15_000 }, + reporter: process.env.CI ? [["list"], ["github"]] : [["list"]], + use: { + baseURL: `http://localhost:${PORT}`, + trace: "retain-on-failure", + viewport: { width: 320, height: 320 }, + }, + projects: [ + { + name: "chromium", + use: { + ...devices["Desktop Chrome"], + viewport: { width: 320, height: 320 }, + launchOptions: { + args: [ + "--use-angle=swiftshader", + "--ignore-gpu-blocklist", + "--enable-unsafe-swiftshader", + "--enable-webgl", + "--use-gl=angle", + ], + }, + }, + }, + ], + webServer: { + command: `npx vite --port ${PORT} --strictPort`, + cwd: repoRoot, + url: `http://localhost:${PORT}/test/browser/pages/harness.html`, + reuseExistingServer: !process.env.CI, + timeout: 120_000, + stdout: "ignore", + stderr: "pipe", + }, +}); diff --git a/test/tsconfig.json b/test/tsconfig.json new file mode 100644 index 00000000..d6a110bd --- /dev/null +++ b/test/tsconfig.json @@ -0,0 +1,22 @@ +{ + // Editor / type-check project for everything under test/ (Playwright config, + // specs, the in-page harness). These files run under Node (config, specs) or + // in the browser (browser/pages/*), so they need both the Node globals and the + // same Vite ambient types as src/. Nothing is emitted. + // Check with: npx tsc -p test/tsconfig.json + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "emitDeclarationOnly": false, + "declaration": false, + "sourceMap": false, + "rootDir": "..", + "types": [ + "node", + "vite/client", + "vite-plugin-glsl/ext", + "vite-plugin-arraybuffer/types" + ] + }, + "include": ["**/*.ts"] +}