diff --git a/.agents/skills/kitcn/references/features/orm.md b/.agents/skills/kitcn/references/features/orm.md index bbf5a563..7066e92f 100644 --- a/.agents/skills/kitcn/references/features/orm.md +++ b/.agents/skills/kitcn/references/features/orm.md @@ -649,6 +649,23 @@ const page2 = await ctx.orm.query.posts.findMany({ Return: `{ page, continueCursor, isDone, pageStatus?, splitCursor? }` +### Index-union filters + +`in`, `notIn`, `ne`, and same-field equality `OR` compile to one index range per value when the field is one an index leads with. Cursor pages come from those ranges as a merged stream, so `orderBy` sorts across the whole result and no scan budget is needed. + +```ts +const page = await ctx.orm.query.users.withIndex("by_status").findMany({ + where: { status: { in: ["active", "pending"] } }, + orderBy: { createdAt: "desc" }, + cursor: null, + limit: 20, +}); +``` + +Falls back to a bounded scan (needs `maxScan`) when the probed index cannot supply the requested `orderBy`, or when the union is wider than 64 ranges. + +Without an `orderBy`, an index-union page is in the order of the index it walks — grouped by the probed value — not in creation order. + ### Boundary pinning with `endCursor` ```ts @@ -736,7 +753,7 @@ const filtered = results.filter((a) => a.publishedAt >= startDate); ### Performance 1. **Index first** — constrain leading index fields. Compound indexes follow prefix rules. Put the `orderBy` column right after the constrained prefix so the scan is already sorted and `limit` bounds the read. -2. **Bound scans** — use `maxScan` for predicate `where` (cursor mode only). A `.select()` ID-list query with `orderBy` needs a single field that an index leads with. +2. **Bound scans** — use `maxScan` for predicate `where` (cursor mode only). `in`/`notIn`/`ne` on an indexed leading field page from index ranges and need no budget. A `.select()` ID-list query with `orderBy` needs a single field that an index leads with. 3. **Limit results** — always use `limit` or cursor pagination. 4. **Cursor stability** — keep same `where`/`orderBy` between page requests. 5. **`allowFullScan`** — non-cursor only. Cursor mode uses `maxScan` instead. diff --git a/.changeset/orm-index-union-pagination.md b/.changeset/orm-index-union-pagination.md new file mode 100644 index 00000000..6db1b632 --- /dev/null +++ b/.changeset/orm-index-union-pagination.md @@ -0,0 +1,37 @@ +--- +"kitcn": minor +--- + +## Breaking changes + +- Cursor pages for an index-union filter with no `orderBy` are now in the order of the index the read walks, grouped by the probed value, instead of creation order. Add `orderBy` to keep newest-first paging. + +```ts +// Before +const page = await db.query.users.withIndex('by_status').findMany({ + where: { status: { in: ['active', 'pending'] } }, + cursor: null, + limit: 20, + maxScan: 500, +}); + +// After +const page = await db.query.users.withIndex('by_status').findMany({ + where: { status: { in: ['active', 'pending'] } }, + orderBy: { createdAt: 'desc' }, + cursor: null, + limit: 20, +}); +``` + +## Features + +- Page `in`, `notIn`, `ne`, and same-field equality `OR` filters from one index range per value instead of scanning the table. Cursor pagination over these filters no longer needs `maxScan`, and `orderBy` sorts across the whole result rather than per value. + +## Patches + +- Fix `select()` composition and `endCursor` pagination reading a whole index instead of the compiled index ranges when the filter is an index union. +- Keep no-`orderBy` cursor direction consistent when `endCursor` routes an index union through the advanced stream path. +- Prevent `endCursor` narrowing from reopening disjoint equality ranges and duplicating merged-stream rows. +- Fix cursor pagination with a residual post-filter reading a whole index instead of the compiled index ranges when the filter is an index union. +- Fall back to a bounded scan when the probed index cannot supply the requested `orderBy` or the union is wider than 64 ranges. diff --git a/convex/orm/index-union-pagination.test.ts b/convex/orm/index-union-pagination.test.ts new file mode 100644 index 00000000..95bbb060 --- /dev/null +++ b/convex/orm/index-union-pagination.test.ts @@ -0,0 +1,418 @@ +/** + * Index-union (multi-probe) reads must stay index-bounded. + * + * `in`, `ne`, `notIn` and same-field equality `OR` compile to a set of index + * probes. Every one of these tests seeds a table where the matches are a tiny + * fraction of the rows, so a plan that walks the table instead of the probes + * shows up as a read count that tracks table size. + */ + +import { defineRelations, defineSchema } from 'kitcn/orm'; +import { expect, test } from 'vitest'; +import schema, { tables } from '../schema'; +import { + convexTest, + countDocumentReads, + runCtx, + withOrm, +} from '../setup.testing'; + +/** Rows in the table. Only `MATCHING_STATUSES` rows are ever a match. */ +const TABLE_ROWS = 120; +const MATCHING_STATUSES = ['active', 'pending'] as const; + +/** + * Matches sit at the two ends of creation order, so a `_creationTime` scan has + * to walk the whole table to collect them all while an index union does not. + * + * The two statuses are interleaved on purpose: creation order over the matches + * is `119, 118, 001, 000` descending, while `by_status` order is + * `pending{118, 001}, active{119, 000}`. Any test that asserts a page can only + * pass under one of them. + */ +const statusForRow = (index: number): string => { + if (index === 0 || index === TABLE_ROWS - 1) return 'active'; + if (index === 1 || index === TABLE_ROWS - 2) return 'pending'; + return 'archived'; +}; + +const seedUsers = async (db: any) => { + for (let index = 0; index < TABLE_ROWS; index += 1) { + await db.insert('users', { + name: `User ${String(index).padStart(3, '0')}`, + email: `index-union-${index}@example.com`, + status: statusForRow(index), + }); + } +}; + +const expectedMatchNames = () => + Array.from({ length: TABLE_ROWS }, (_, index) => index) + .filter((index) => + (MATCHING_STATUSES as readonly string[]).includes(statusForRow(index)) + ) + .map((index) => `User ${String(index).padStart(3, '0')}`) + .sort(); + +test('cursor pagination over an index union does not require maxScan', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const reads = countDocumentReads(baseCtx); + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const before = reads.documents; + const page = await ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + cursor: null, + limit: 2, + }); + + // Without an `orderBy` the read is in the order of the index it walks, + // which for an index union is that union's index: `pending` before + // `active`, newest first inside each probe. + expect(page.page.map((row: any) => row.name)).toEqual([ + 'User 118', + 'User 001', + ]); + // Two probes, two rows: a probe union reads a handful of rows. The scan + // this replaced read every row in the table. + expect(reads.documents - before).toBeLessThanOrEqual(6); + }); +}); + +test('cursor walk over an index union returns every match exactly once', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const seen: string[] = []; + let cursor: string | null = null; + for (let page = 0; page < 10; page += 1) { + const result: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + cursor, + limit: 2, + }); + seen.push(...result.page.map((row: any) => row.name)); + cursor = result.continueCursor; + if (result.isDone) break; + } + + expect(seen.slice().sort()).toEqual(expectedMatchNames()); + expect(new Set(seen).size).toBe(seen.length); + }); +}); + +test('cursor pagination over an index union honors orderBy createdAt', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const reads = countDocumentReads(baseCtx); + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const before = reads.documents; + const page = await ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + orderBy: { createdAt: 'desc' }, + cursor: null, + limit: 3, + }); + + // Newest first across both probes, not probe-by-probe. + expect(page.page.map((row: any) => row.name)).toEqual([ + 'User 119', + 'User 118', + 'User 001', + ]); + expect(reads.documents - before).toBeLessThanOrEqual(6); + }); +}); + +test('endCursor pagination over an index union stays index-bounded', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const reads = countDocumentReads(baseCtx); + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const first: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + orderBy: { createdAt: 'desc' }, + cursor: null, + limit: 2, + }); + + const before = reads.documents; + const pinned: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + orderBy: { createdAt: 'desc' }, + cursor: null, + endCursor: first.continueCursor, + limit: 2, + }); + + expect(pinned.page.map((row: any) => row.name)).toEqual([ + 'User 119', + 'User 118', + ]); + expect(reads.documents - before).toBeLessThanOrEqual(6); + }); +}); + +test('endCursor preserves the default cursor order for an index union', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const reads = countDocumentReads(baseCtx); + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const first: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + cursor: null, + limit: 2, + }); + + const before = reads.documents; + const pinned: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + cursor: null, + endCursor: first.continueCursor, + limit: 2, + }); + + expect(pinned.page.map((row: any) => row.name)).toEqual([ + 'User 118', + 'User 001', + ]); + expect(reads.documents - before).toBeLessThanOrEqual(6); + }); +}); + +test('a residual post-filter over an index union stays index-bounded', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const reads = countDocumentReads(baseCtx); + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const before = reads.documents; + const page = await ctx.orm.query.users.withIndex('by_status').findMany({ + // `contains` is not index-compilable, so the plan keeps it as a residual + // JavaScript filter over the probe union. A residual `where` still asks + // for a scan budget at the type level; the probes make the read far + // smaller than that budget. + where: { + status: { in: [...MATCHING_STATUSES] }, + name: { contains: '11' }, + }, + orderBy: { createdAt: 'desc' }, + cursor: null, + limit: 2, + maxScan: 50, + }); + + expect(page.page.map((row: any) => row.name)).toEqual([ + 'User 119', + 'User 118', + ]); + expect(reads.documents - before).toBeLessThanOrEqual(6); + }); +}); + +test('non-paginated limit over an index union stays index-bounded', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const reads = countDocumentReads(baseCtx); + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const before = reads.documents; + const rows = await ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + orderBy: { createdAt: 'desc' }, + limit: 2, + }); + + expect(rows.map((row: any) => row.name)).toEqual(['User 119', 'User 118']); + expect(reads.documents - before).toBeLessThanOrEqual(6); + }); +}); + +test('cursor pagination over a complement range union stays index-bounded', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const reads = countDocumentReads(baseCtx); + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const before = reads.documents; + // `ne` compiles to the two complement ranges around 'archived'. Ordering by + // the probed field itself is the order those ranges already produce. + const page = await ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { ne: 'archived' } }, + orderBy: { status: 'asc' }, + cursor: null, + limit: 2, + }); + + expect(page.page.map((row: any) => row.status)).toEqual([ + 'active', + 'active', + ]); + expect(reads.documents - before).toBeLessThanOrEqual(6); + }); +}); + +test('an index union wider than the probe cap still needs maxScan under strict', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const wideList = Array.from( + { length: 65 }, + (_, index) => `bucket-${index}` + ); + + await expect( + ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { in: wideList } }, + cursor: null, + limit: 2, + }) + ).rejects.toThrow(/maxScan/i); + }); +}); + +test('a wide index union with endCursor still needs maxScan under strict', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + const first: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + cursor: null, + limit: 2, + }); + const wideList = Array.from( + { length: 65 }, + (_, index) => `bucket-${index}` + ); + + await expect( + ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { in: wideList } }, + cursor: null, + endCursor: first.continueCursor, + limit: 2, + }) + ).rejects.toThrow(/maxScan/i); + }); +}); + +test('a rejected wide union preserves its predicate with maxScan and endCursor', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + const wideList = [ + 'active', + ...Array.from({ length: 64 }, (_, index) => `bucket-${index}`), + ]; + + const first: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: wideList } }, + cursor: null, + limit: 2, + maxScan: TABLE_ROWS, + }); + const pinned: any = await ctx.orm.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: wideList } }, + cursor: null, + endCursor: first.continueCursor, + limit: 2, + maxScan: TABLE_ROWS, + }); + + expect(first.page.map((row: any) => row.name)).toEqual([ + 'User 119', + 'User 000', + ]); + expect(pinned.page.map((row: any) => row.name)).toEqual([ + 'User 119', + 'User 000', + ]); + }); +}); + +test('an index union that cannot serve the requested order still needs maxScan under strict', async () => { + const t = convexTest(schema); + + await t.run(async (baseCtx) => { + const ctx = await runCtx(baseCtx); + await seedUsers(baseCtx.db); + + await expect( + ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + // `by_status` is (status, _creationTime); a probe union can never emit + // rows ordered by `name`. + orderBy: { name: 'asc' }, + cursor: null, + limit: 2, + }) + ).rejects.toThrow(); + }); +}); + +test('an index union without a schema definition still needs maxScan under strict', async () => { + const relaxedTables = { ...tables }; + const relaxedRelations = defineRelations(relaxedTables); + const relaxedSchema = defineSchema(relaxedTables); + const t = convexTest(relaxedSchema); + + await t.run(async (baseCtx) => { + // `defineRelations(...)` alone carries no schema definition, so `stream()` + // is unavailable and the plan has to fall back to the bounded scan. + const ctx = withOrm(baseCtx, relaxedRelations); + await seedUsers(baseCtx.db); + + await expect( + ctx.orm.query.users.withIndex('by_status').findMany({ + where: { status: { in: [...MATCHING_STATUSES] } }, + cursor: null, + limit: 2, + }) + ).rejects.toThrow(/maxScan/i); + }); +}); diff --git a/convex/orm/pagination.test.ts b/convex/orm/pagination.test.ts index c609520a..99bd2595 100644 --- a/convex/orm/pagination.test.ts +++ b/convex/orm/pagination.test.ts @@ -246,7 +246,7 @@ test('pagination with WHERE filter', async () => { }); }); -test('pagination with index-union filter requires maxScan when strict=true', async () => { +test('pagination with index-union filter pages without maxScan', async () => { const t = convexTest(schema); await t.run(async (baseCtx) => { @@ -264,13 +264,15 @@ test('pagination with index-union filter requires maxScan when strict=true', asy const ctx = await runCtx(baseCtx); const db = ctx.orm; - await expect( - db.query.users.withIndex('by_status').findMany({ - where: { status: { in: ['active', 'pending'] } }, - cursor: null, - limit: 5, - }) - ).rejects.toThrow(/maxScan/i); + // The probes bound the read, so there is no scan to budget. + const page = await db.query.users.withIndex('by_status').findMany({ + where: { status: { in: ['active', 'pending'] } }, + cursor: null, + limit: 5, + }); + + expect(page.page).toHaveLength(5); + expect(page.page.every((row: any) => row.status !== 'inactive')).toBe(true); }); }); diff --git a/docs/plans/415-restore-compiled-probefilters-at-orm-stream-sites.md b/docs/plans/415-restore-compiled-probefilters-at-orm-stream-sites.md new file mode 100644 index 00000000..64124696 --- /dev/null +++ b/docs/plans/415-restore-compiled-probefilters-at-orm-stream-sites.md @@ -0,0 +1,555 @@ +# 415 restore compiled probeFilters at ORM stream sites + +Objective: +Make the ORM's compiled query plan reach every stream construction site so a +cursor-paginated `in()` reads index probes instead of a full table scan. + +Goal plan: +docs/plans/415-restore-compiled-probefilters-at-orm-stream-sites.md + +Template: +docs/plans/templates/task.md + +Primary template: +docs/plans/templates/task.md + +Applied packs: +- package-api (docs/plans/templates/packs/package-api.md) + +Task source: +- type: github issue +- id / link: https://github.com/udecode/kitcn/issues/415 +- title: ORM: the compiled plan's `probeFilters` is dropped at every stream site + — cursor-paginated `in()` degrades to a full table scan +- acceptance criteria: + - `probeFilters` reaches the stream builders instead of being dropped. + - Cursor-paginated `in()` reads index probes, not a full table scan. + - The narrowed `queryConfig` parameter types carry `strategy` and + `probeFilters` so a future drop is a type error. + - `_buildBasePipelineStream`, `_buildResidualFilterStream`, the cursor + multi-probe site, and the cursor scan-fallback site share one owner for + base-stream construction. +- caveats: PR #425 (https://github.com/udecode/kitcn/pull/425) is this plan's exact PR. +- likely files: `packages/kitcn/src/orm/query.ts`, + `packages/kitcn/src/orm/where-clause-compiler.ts`, `convex/orm/*.test.ts`, + `www/content/docs/orm/**`, `packages/kitcn/skills/kitcn/**`. +- browser surface: none. +- root-cause layer: ORM query planner → stream construction. + +Timed checkpoint: +- requested duration: N/A: none requested +- semantics: N/A +- initial confidence score: N/A +- improvement loop: N/A +- final score / loop closure: N/A + +Completion threshold: +- New behavior-level tests prove a cursor-paginated `in()` returns correct, + correctly ordered, non-overlapping pages while reading O(page) documents + rather than O(table). +- `bun --cwd packages/kitcn build`, `bun typecheck`, targeted + `convex/orm/*.test.ts` runs, and `bun lint:fix` pass. +- Task closure is legal only when the source-of-truth acceptance criteria are + satisfied or explicitly narrowed, required verification evidence is recorded, + code-review and release-artifact gates are closed when applicable, verified + code changes are committed and PR'd unless explicitly declined or blocked, + task-style PR body sync is complete or marked N/A with reason, + GitHub issue/PR sync is complete or marked N/A with reason, and + `node .agents/skills/autogoal/scripts/check-complete.mjs docs/plans/415-restore-compiled-probefilters-at-orm-stream-sites.md` passes. + +Verification surface: +- `bun vitest run convex/orm/pagination.test.ts convex/orm/pipeline.test.ts + convex/orm/where-filtering.test.ts convex/orm/stream.test.ts` (cwd: repo root) +- `bun --cwd packages/kitcn build`, `bun typecheck`, `bun lint:fix` +- `bun test` / `bun check` before closeout +- source audit of `probeFilters` consumers + +Constraints: +- Preserve existing user-facing behavior outside the task scope. +- Prefer the durable ownership boundary over caller-by-caller patches. +- User explicitly said: do NOT create a PR under any circumstances. +- Do not add broad ceremony when the task is trivial or docs-only. + +Boundaries: +- Source of truth: GitHub issue #415. +- Allowed edit scope: `packages/kitcn/src/orm/**`, `convex/orm/**` tests, + `www/content/docs/orm/**`, `packages/kitcn/skills/kitcn/**`, `.changeset/**`. +- Browser surface: N/A: no rendered output changes. +- GitHub issue sync: N/A: user declined PR; no sync-back without a shipped fix. +- Non-goals: de-duplicating the `update.ts` / `delete.ts` mutation planners; + lowering `pipeline.union` `source.where` through `WhereClauseCompiler` + (pinned by `convex/orm/pipeline.test.ts:143`). + +Output budget strategy: +- Recon fan-out wrote per-agent findings to `/tmp/recon_*.md` instead of + streaming them; test runs are scoped to named files with `--reporter=dot`. + +Blocked condition: +- A merged probe stream that cannot preserve the fallback read's order, or a + Convex/stream limitation that makes bounded probe pagination unsound. + +Task state: +- task_type: bug +- task_complexity: non-trivial +- current_phase: closeout +- current_phase_status: complete +- next_phase: terminal GitHub receipt and merge +- goal_status: active + +Current verdict: +- verdict: fixed +- confidence: 95-100% +- next owner: task +- reason: red repro reproduced the drop at every stream site; the shared plan + stream builder makes all four sites read the compiled probes. + +Implementation readiness: +- verdict: ready +- exact owner: `GelRelationalQuery._buildPlanStream` / + `_buildProbeUnionStream` in `packages/kitcn/src/orm/query.ts` +- contradiction status: three issue constraints were wrong; see Findings +- source-listed cases complete: yes (matrix below) + +Pre-solution issue challenge: +- reporter claim: `_toConvexQuery` compiles `probeFilters`, but every stream + construction site drops them, so a cursor-paginated `in()` degrades to a full + table scan. +- suggested diagnosis or fix: one private `_buildPlanStream` owning the + precedence ladder, called from `query.ts:2734`, `:2813`, `:6532`, `:6620`, + plus widened `queryConfig` parameter types. +- repro ladder: + - tests / source-level repro: `convex/orm/index-union-pagination.test.ts` — + 6 of 10 tests failed before the fix, all on the dropped plan. + - repo-owned automated browser or integration proof: N/A: no browser surface. + - Browser plugin: N/A: no rendered output. + - screenshot / visual proof: N/A: no visual output. +- reproduction verdict: valid +- validity verdict: valid +- best long-term fix boundary: the plan type plus one stream builder, so the + drop becomes a type error rather than silence. +- harsh honest feedback: three of the issue's seven "constraints" are wrong. + (1) `mergedStream` *can* merge `ne`/`notIn`/range probes — `equalityIndexFilter` + only limits which key *suffix* may be merged on, and merging on the full index + key works for any probe set on one index. (2) `ConcatStreams` therefore does + not need exporting. (7) the mutation side does not hard-throw on probe width; + it throws on *any* multi-probe paged mutation, and the only width constant is + the compiler's own. +- hard-stop decision: proceed — reproduced at the source level. + +Completion rule: +- Do not call `update_goal(status: complete)` while any required checklist item + remains unchecked. If an item does not apply, check it and add `N/A: `. +- Do not call `update_goal(status: complete)` until every completion threshold + above is satisfied, final handoff evidence is recorded, and + `node .agents/skills/autogoal/scripts/check-complete.mjs docs/plans/415-restore-compiled-probefilters-at-orm-stream-sites.md` passes. +- Do not create hook state for this goal. This file plus the active goal are the + durable state. + +Start Gates: +| Gate | Applies | Evidence | +|------|---------|----------| +| Timed checkpoint parsed | no | N/A: no duration requested | +| Walkthrough baseline for possible UI change | no | N/A: ORM read planner only; no UI or rendered output | +| Skill analysis before edits | yes | `task` + `autogoal` + `autoreview`; `changeset` rule read | +| Active goal checked or created | yes | this plan | +| Source of truth read before edits | yes | issue #415 attachment + `gh issue view 415` (no comments) | +| Exact per-PR task ownership | yes | This plan owns exactly PR #425 | +| GitHub comments and attachments read | yes | attachment read; issue has 0 comments | +| Video transcript evidence required | no | N/A: no video evidence | +| Pre-solution issue challenge required | yes | verdict `valid`; 3 issue constraints refuted, see Findings | +| Reproduction verdict before implementation | yes | 6 red tests before the fix | +| Repro escalation ladder selected | yes | source-level test repro sufficed | +| Suggested fix reviewed against durable boundary | yes | adopted the shared builder; declined the `_buildUnionSourceStream` lowering | +| `docs/solutions` checked for non-trivial existing-code work | no | N/A: no `docs/solutions` directory in this repo | +| TDD decision before behavior change or bug fix | yes | red repro first, then fix | +| Branch decision for code-changing task | yes | renamed `issue-415` -> `fix/orm-index-union-cursor-pagination` before first push, per user branch convention | +| Release artifact decision | yes | `.changeset/orm-index-union-pagination.md` (minor) | +| Browser tool decision for browser surface | no | N/A: no browser surface | +| Commit / PR expectation decision | yes | User later requested the PR; committed, pushed, opened #425 | +| Task-style PR body decision | yes | PR #270 emoji task-style body used | +| Task-plan PR body evidence | yes | Body carries `🧭 Task plan: docs/plans/415-restore-compiled-probefilters-at-orm-stream-sites.md`; plan exists at PR head | +| GitHub issue sync expectation decision | yes | `🐛 Fixes #415` in the PR body closes the issue on merge | +| Output budget strategy recorded | yes | recon results artifacted to `/tmp/recon_*.md`; test runs use `--reporter=dot` | +| Package/API pack selected | yes | package-api | +| Public surface or package boundary identified | yes | `MAX_INDEX_UNION_PROBES` newly exported from `where-clause-compiler` (internal module, not re-exported from `orm/index.ts`); no public API signature change | +| Convex entry/import graph impact identified | yes | no new module; `query.ts` already imported `mergedStream`, `getIndexFields`, and `where-clause-compiler` | +| CLI/scaffold/generated impact identified | yes | none; `fixtures:check` green | +| Release artifact path selected | yes | `.changeset` | +| `changeset` skill loaded when `.changeset` is required | yes | `.agents/rules/changeset.mdc` read and followed | +| Package build / fixture impact decision recorded | yes | `bun --cwd packages/kitcn build` run; `fixtures:check` run | + +Work Checklist: +- [x] If a duration was requested, it is recorded as minimum active work unless + explicitly marked hard stop; when no better metric exists, initial and + final confidence scores are recorded. +- [x] Objective includes outcome, completion threshold, verification surface, + constraints, boundaries, and blocked condition. +- [x] Task source classified with source type, id/link, title, task type, + acceptance criteria, caveats, likely files/routes/packages, browser + surface, and root-cause layer. +- [x] Every GitHub PR in scope has its own task plan. This plan owns one exact + PR, owns a not-yet-created PR slice, or records N/A because no PR is in + scope; a batch plan is not used as a substitute. +- [x] Required video or screen-recording evidence is cached/read as normalized + `` XML, or marked N/A with reason. +- [x] For public GitHub bug reports, behavior claims, technical diagnoses, or + suggested fixes, reporter claims are challenged before implementation + with a recorded verdict: `valid`, `not reproduced`, `invalid`, + `wont-fix`, `partially valid`, or `platform limitation`. Feature, docs, + support, or cleanup requests with no bug claim may mark reproduction + `N/A` with reason. +- [x] Repro escalation ladder followed for bug/behavior claims: focused + test/source-level repro first when applicable; existing repo-owned + automated browser or integration proof next when available and useful as + executable coverage; the repo-approved Browser tool next when tests or + automation cannot reproduce or cannot model the surface honestly; + screenshot or explicit visual-proof waiver when visual/native state + matters. +- [x] Hard-stop rule followed for bug/behavior claims: no code when the issue + is not reproduced, invalid, or won't-fix; partial validity pivots to the + best long-term fix and records what was wrong or incomplete in the + issue's proposed path. +- [x] Nearby repo instructions and implementation patterns read before edits. +- [x] Source-listed case matrix is complete and every contradiction has an + owner, harness, and verdict before mutation. +- [x] Readiness is classified `ready`, `repair-source`, `major`, `blocked`, or + `invalid` with evidence. +- [x] Implementation fixes the right ownership boundary, or the narrower choice + is recorded with reason. +- [x] Release artifact requirement recorded: active changeset, new changeset, or + N/A with reason. +- [x] Final handoff shape decided: bug/feature/testing/batch/review/GitHub + requirements, PR body sync, and issue sync when applicable. +- [x] Commit/PR handling recorded for code-changing work: commit and PR + completed, no local patch, user explicitly declined, or blocker recorded. + "User did not separately ask for a PR" is not a valid blocker. +- [x] PR body shape recorded: PR #270 emoji task-style body used, N/A reason + recorded, or blocker recorded. +- [x] PR task evidence recorded: body includes `🧭 Task plan: ...`, the plan + exists at the PR head, and it identifies the exact PR before autoclosure. +- [x] Branch handling recorded for code-changing work: dedicated branch used, + new branch needed, or N/A with reason. +- [x] Local-env-rot retry policy recorded for any surprising repo-wide failure: + reinstall/rerun evidence or N/A with reason. +- [x] Workspace authority recorded: every proof command names the cwd/tool that + owns the changed behavior. +- [x] Output budget discipline recorded and followed: broad searches are + scoped, capped, counted, or artifacted instead of streamed into goal + context. +- [x] High-risk note recorded for public API, runtime, package-boundary, + browser behavior, agent-action, or command-contract changes, or marked + N/A with reason. +- [x] Review/autoreview target selected from actual diff state for non-trivial + implementation work, or marked N/A with reason. +- [x] Agent-native review decision recorded for `.agents/**`, `.claude/**`, + `.codex/**`, skills, hooks, commands, prompts, or user-action tooling. +- [x] Package/API pack: public API, package boundary, export, and release-artifact impact are recorded. +- [x] Package/API pack: release artifact matrix is applied: `.changeset` or explicit no-artifact reason. +- [x] Package/API pack: `.changeset` work loads `changeset` and follows its package/version/prose rules. +- [x] Package/API pack: no-artifact decisions state why the diff has no published package user-visible delta from `main`. +- [x] Package/API pack: compatibility, migration, or hard-cut decision is explicit when public shape changes. +- [x] Package/API pack: affected Convex static import graphs stay narrow and + plugin/per-module boundaries are used where appropriate. +- [x] Package/API pack: CLI commands remain deterministic, `--json` capable, + and non-interactive with explicit confirmation bypass when relevant. +- [x] Package/API pack: docs and `packages/kitcn/skills/kitcn/**` stay + current-state synchronized when public guidance changes. +- [x] Package/API pack: package-owned typecheck/build/test proof is recorded or marked N/A with reason. +- [x] Package/API pack: `packages/kitcn` build, fixture sync/check, or other owning package proof is recorded when required. + +Completion Gates: +| Gate | Applies | Required action | Evidence | +|------|---------|-----------------|----------| +| Named verification threshold | yes | Run named commands | source-sync `bun check` exit 0; focused regression set 157 passed / 1 skipped; `bun typecheck` 5/5; `bun run fixtures:sync` and `bun run fixtures:check` 8/8 | +| Exact per-PR task ownership | yes | Record exact PR | PR #425, this plan | +| Pre-solution issue challenge verdict | yes | Record verdict | `valid`; three issue constraints refuted | +| Repro escalation ladder | yes | Record ladder | source-level test repro reproduced all four drop sites | +| Bug reproduced before fix | yes | Failing test | 6 red in `convex/orm/index-union-pagination.test.ts` | +| Targeted behavior verification | yes | Focused proof | six-file source-sync regression set 157 passed / 1 skipped; read counts 2-4 docs / 120 rows | +| TypeScript or typed config changed | yes | Typecheck | `bun typecheck` 5/5 pass | +| Package exports or file layout changed | yes | Package build | `bun --cwd packages/kitcn build` complete | +| Package manifests, lockfile, or install graph changed | no | N/A | No manifest change; `bun install` reported no changes | +| Agent rules or skills changed | yes | `bun install` sync | `.agents/skills/kitcn/references/features/orm.md` regenerated | +| Workspace authority proof | yes | Record cwd | all commands run at repo root except `bun --cwd packages/kitcn build` | +| Browser surface changed | no | N/A | No browser surface | +| Browser final proof | no | N/A | No browser surface | +| UI walkthrough | no | N/A | No UI or rendered-output change | +| Scaffold or fixture output changed | no | N/A | No scaffold change; `fixtures:check` run anyway and green | +| Package behavior or public API changed | yes | Changeset | `.changeset/orm-index-union-pagination.md` (minor) | +| Docs and kitcn skill sync changed | yes | Sync | `www/content/docs/orm/**` and `packages/kitcn/skills/kitcn/**` updated together | +| Docs or content changed | yes | Verify claims | doc text matches the implemented precedence ladder and the 64-probe cap | +| High-risk mini gate | yes | Record note | See Open risks | +| Agent-native review for agent/tooling changes | no | N/A | No `.agents`/`.claude`/`.codex` source change; only the generated skill mirror | +| Local install corruption suspected | yes | Retry once | `fixtures:check` failed twice on shared bunx cache rot (`Failed to link js-yaml: EEXIST`, then `@babel/core` MODULE_NOT_FOUND under `bunx-501-shadcn@4.3.0`); removing that tmp cache made `fixtures:check` and the full `bun check` pass with exit 0 | +| Commit created | yes | Stage whole checkout, commit | original task commit 3e046c49; autoclosure repair commit recorded at exact PR head before receipt | +| PR create or update | yes | Run check, push, open PR | source-sync `bun check` exit 0; exact repair head pushed to https://github.com/udecode/kitcn/pull/425 before receipt | +| Task-style PR body verified | yes | Verify with gh | `gh pr view 425 --json body` confirms emoji format, plan line, no self-link | +| PR task evidence verified | yes | Verify plan line + head | Plan path resolves at PR head and names PR #425 | +| PR proof image hosting | no | N/A | No browser proof images in the body | +| GitHub issue sync-back | yes | Reference issue | PR #425 body opens with `🐛 Fixes #415` | +| Final handoff contract | yes | Fill fields | See Final handoff contract | +| Final lint | yes | `bun lint:fix` | clean, 3 files formatted | +| Output budget discipline | yes | Verify | recon artifacted; all test output grepped or dot-reported | +| Timed checkpoint | no | N/A | No duration requested | +| Autoreview for non-trivial implementation changes | yes | Run until clean | source-sync branch review found the rejected-union P1; repaired local review clean, 0 findings, confidence 0.93; final branch review required after commit | +| Goal plan complete | yes | Run check-complete | see Verification evidence | +| Public API / package boundary proof | yes | Source audit | `orm/index.ts` exports unchanged; `MAX_INDEX_UNION_PROBES` stays internal | +| Convex bundle/import proof | yes | Audit | no new module or import edge; `import-graph.test.ts` green | +| CLI/scaffold/generated proof | no | N/A | No CLI or scaffold source change | +| Release artifact classification | yes | Record | published package behavior change (breaking default order + new index-union paging) | +| Published package changeset | yes | Add changeset | `.changeset/orm-index-union-pagination.md` | +| No release artifact | no | N/A | A changeset exists | +| Package typecheck/build/test | yes | Run | `bun typecheck`, `bun --cwd packages/kitcn build`, `bun run test` all pass | +| Fixture/scaffold generation | no | N/A | No scaffold output change; `fixtures:check` verified anyway | +| Docs/package skill sync | yes | Synchronize | done in the same diff | + +Phase / pass table: +| Phase | Status | Evidence | Next | +|-------|--------|----------|------| +| Intake and source read | complete | issue #415 read, recon fan-out | implementation | +| Implementation | complete | shared plan stream builder | verification | +| Verification | complete | source-sync red-to-green tests, fixtures, package build, full `bun check`, autoreview clean | closeout | +| Commit / PR / GitHub sync | complete | PR #425 exists; exact-head push and terminal receipt are the autoclosure delivery step | exact-head delivery | +| Closeout | complete | all local plan gates closed | terminal receipt and merge | + +Findings: +- `_toConvexQuery` returns `probeFilters`, but only the non-cursor `db.query` + fan-out consumed them. Every stream site took a narrowed parameter type that + could not name them. +- For a `multiProbe` plan `queryConfig.index.filters` is `[]`, so + `if (queryConfig.index)` produced a rangeless full-index walk at + `_buildBasePipelineStream` and `_buildResidualFilterStream`, and the cursor + branch never called `.withIndex(...)` at all — a `by_creation_time` scan. +- `mergedStream(streams, indexFields.slice(j))` merges probes correctly for any + `j` inside the run of index-key components every probe pins with `eq`; that is + the same set `resolveIndexOrderPushdown` already treats as order-servable, so + the two decisions can never disagree. +- Probe merge keys always end in `_id`, so the merged-stream duplicate-key + cursor hazard cannot bite an index union. +- `convex/orm/pagination.test.ts:249` encoded the degraded behavior as contract + (`rejects.toThrow(/maxScan/i)`); its split-metadata sibling at `:307` survives + the fix untouched because `paginate()` counts pulled keys, not scanned rows. +- `convex/orm/pipeline.test.ts:143` pins that a union source without its own + index inherits the chain index; `_buildUnionSourceStream` is therefore left + alone. +- No doc anywhere promised a default result order, so making an index-union + page read in index order breaks no documented contract. +- Source-sync against main `0e3a2214e8e1037b323e603f19c2df89b0d95602` + exposed that the advanced `endCursor` path defaulted to ascending while the + normal no-`orderBy` cursor path defaults to descending. +- Repairing that direction exposed a second defect: narrowing a merged child + against a disjoint cursor bound constructed an inverted range, which reopened + the child and duplicated rows. An empty narrowed child must retain its + equality-prefix metadata so suffix-key merges remain valid. +- Final exact-head review exposed that the advanced `endCursor` caller dropped + the union-accepted flag. A rejected union could therefore walk its plan index + without the strict `maxScan` guard or the normal unanchored fallback. +- A later review claim that rejected unions lose their predicate was disproven: + every multi-probe compiler result retains the original expression in + `postFilters`, and both ordinary and advanced scan fallbacks apply those + filters while pulling. A budgeted 65-value union returned only the requested + status under both cursor shapes before any code change for that claim. + +Decisions and tradeoffs: +- `CompiledQueryPlan` is the single parameter type for every plan consumer, so a + future drop of `strategy` or `probeFilters` is a type error. +- The union is used only when the merged order equals the order the read has to + produce; otherwise the caller falls back to today's bounded scan. That keeps + the change "reads less", never "reorders differently", on every path except + the cursor multi-probe branch whose previous order came from the bug itself. +- Without `orderBy`, an index-union page is in index order. The previous + creation-time order was an artifact of the unanchored scan; index order also + matches the non-cursor probe fan-out. +- Union width is capped at `MAX_INDEX_UNION_PROBES` (64), reusing the compiler's + existing constant, because a merged stream registers every probe at once. +- The declined-union fallback deliberately keeps the full scan rather than + anchoring to the plan's index: the `needsPostFetchSortForPrimary` warning on + that path promises creation-time order. +- Out of scope: the duplicate `update.ts` / `delete.ts` probe fan-outs, and + lowering `pipeline.union` `source.where` through `WhereClauseCompiler`. + +Implementation notes: +- `packages/kitcn/src/orm/where-clause-compiler.ts`: `MAX_PROMOTED_PROBES` → + exported `MAX_INDEX_UNION_PROBES`. +- `packages/kitcn/src/orm/query.ts`: added `CompiledQueryPlan`, + `_buildProbeUnionStream`, `_buildPlanStream`; rewired + `_buildBasePipelineStream`, `_buildResidualFilterStream`, and collapsed the + two cursor branches into one. +- `convex/orm/index-union-pagination.test.ts`: new behavior + read-bound proof. +- `convex/orm/pagination.test.ts`: the maxScan-required test now pins the page. +- Docs + `packages/kitcn/skills/kitcn/**` updated; `.changeset/orm-index-union-pagination.md` added. + +Review fixes: +- P1: preserve the normal descending fallback order when a no-`orderBy` + `endCursor` uses the advanced index-union path. The new regression failed on + ascending rows before the repair and passes after it. +- P1: return an equality-aware `EmptyStream` when cursor narrowing produces a + disjoint interval. The new merged-stream regressions failed first with + duplicate rows and then a suffix-key equality-prefix error; both pass after + preserving the narrowed stream metadata. +- P1: propagate rejected index-union status through the advanced cursor path, + enforce the strict `maxScan` requirement, and keep rejected unions on the + ordinary unanchored scan fallback. The new wide-union `endCursor` regression + resolved an empty page before the repair and rejects without `maxScan` after. +- False alarm: final review claimed rejected unions with `maxScan` returned + unrelated rows. Source shows `tryCompileInArray`, complement probes, and + same-field OR plans all retain the source expression in `postFilters`; the + new ordinary-plus-`endCursor` regression was green before any implementation + change and pins the invariant explicitly. + +Error attempts: +| Error / failed attempt | Count | Next different move | Resolution | +|------------------------|-------|---------------------|------------| +| Broadly applying descending fallback to every advanced stream | 1 | Restrict it to non-pipeline `endCursor` reads | Existing pipeline cursor tests returned to their pinned ascending order | +| `fixtures:check` GitHub sparse-checkout timeout | 1 | Rerun the exact check without code changes | All 8 fixtures passed | +| TypeScript did not narrow `queryConfig.index` through a boolean alias | 1 | Retain the narrowed index value directly | `bun typecheck` 5/5 passed | + +Verification evidence: +- cwd repo root: `bunx vitest run convex/orm/index-union-pagination.test.ts` — + 6 failed / 4 passed before the fix, 10 passed after. +- Read-count assertions proven non-vacuous: forcing `toBe(-1)` reported actual + counts of 2-4 documents against a 120-row table. +- cwd repo root: `bun run test` — 1316 bun tests pass, 881 vitest tests pass, + 0 fail, no type errors. +- cwd repo root: `bun typecheck` — 5/5 tasks pass. +- cwd `packages/kitcn`: `bun --cwd packages/kitcn build` — build complete. +- cwd repo root: `bun lint:fix` — clean. +- cwd repo root: `bun run fixtures:check` — 8/8 fixtures match (two clean runs). +- cwd repo root: `bun check` — exit 0 (full repo gate, including scenario + verify/runtime lanes). +- cwd repo root after source sync: focused `vitest` over index-union pagination, + pagination, pipeline, where filtering, and both stream suites — 157 passed, + 1 skipped, no type errors. +- cwd repo root after source sync: `bun run fixtures:sync` completed all eight + fixtures; the exact `bun run fixtures:check` retry passed all eight after one + transient GitHub sparse-checkout timeout. +- cwd repo root after source sync: `bun run intent:validate && bun run + intent:stale` — kitcn skill valid and generated guidance up to date; the + published and generated ORM guidance files are byte-identical. +- cwd repo root after source sync: initial local repair `autoreview` — clean, 0 + findings, confidence 0.91; branch review found the rejected-union P1; the + repaired local review is clean at 0.93 confidence. +- cwd repo root after the rejected-union repair: full `bun check` — exit 0, + including every fixture, authenticated runtime smoke, and legacy + create-convex scenario. + +Source-listed case matrix: +| Case | Source claim | Harness | Before | Expected after | Evidence | Status | +| --- | --- | --- | --- | --- | --- | --- | +| cursor `in()` | degrades to full table scan | `index-union-pagination.test.ts` "does not require maxScan" | threw, required maxScan | pages from 2 probes, <=6 reads | 3 reads / 120 rows | fixed | +| cursor `in()` + `orderBy createdAt` | probe order not global | same file, "honors orderBy createdAt" | threw | newest-first across probes | exact page pinned | fixed | +| cursor walk | page boundaries over a merged stream | same file, "every match exactly once" | threw | all matches once, no dupes | set equality | fixed | +| `endCursor` / `select()` (`_buildBasePipelineStream`) | rangeless full-index walk | same file, "endCursor ... index-bounded" | full index walk | <=6 reads | 3 reads | fixed | +| no-`orderBy` `endCursor` direction | advanced path silently used ascending fallback | same file, "preserves the default cursor order" | returned ascending rows | same descending rows as the normal cursor path | red then green | fixed during autoclosure | +| disjoint merged child bound | inverted narrow range reopened a child | `stream.regression.vitest.ts`, "descending merged endCursor excludes disjoint equality ranges" | duplicated rows | closed child contributes no rows | red then green | fixed during autoclosure | +| empty suffix-ordered child metadata | empty child erased its equality prefix | same file, "suffix-ordered merged endCursor accepts an empty child range" | merge threw on 0 equality fields | remaining child paginates normally | red then green | fixed during autoclosure | +| rejected wide union under `endCursor` | advanced caller discarded `probeUnion: false` | `index-union-pagination.test.ts`, "a wide index union with endCursor still needs maxScan" | resolved an unbudgeted empty page | strict read rejects without `maxScan` | red then green | fixed during exact-head review | +| budgeted rejected union predicate | review claimed original `in()` was lost | same file, "a rejected wide union preserves its predicate with maxScan and endCursor" | source and test already retained only `active` rows | ordinary and advanced fallbacks return `User 119`, `User 000` | green before code change | false alarm, invariant pinned | +| residual post-filter (`_buildResidualFilterStream`) | rangeless full-index walk | same file, "residual post-filter ... index-bounded" | full index walk under maxScan 50 | <=6 reads | 4 reads | fixed | +| `ne` complement ranges | union claimed unmergeable | same file, "complement range union" | full table scan | <=6 reads, ordered by probed field | 3 reads | fixed | +| union wider than 64 probes | no cap | same file, "wider than the probe cap" | n/a | still requires maxScan | throws | pinned | +| order the union cannot serve | n/a | same file, "cannot serve the requested order" | n/a | still requires maxScan | throws | pinned | +| no `defineSchema()` | n/a | same file, "without a schema definition" | n/a | still requires maxScan | throws | pinned | +| split metadata at `maxScan: 1` | must survive | `pagination.test.ts:307` | SplitRequired | unchanged | passes | preserved | +| pipeline union source without index | must keep chain index | `pipeline.test.ts:143` | exact page | unchanged | passes | preserved | + +Final handoff contract: +- Commit line: original task commit 3e046c49 plus the exact source-sync repair head recorded in the terminal receipt +- PR line: https://github.com/udecode/kitcn/pull/425 +- Issue line: closes #415 via the PR body +- Confidence line: 95-100% +- Flow table: + - Reproduced: tests 6 red, browser N/A +- Verified: source-sync focused tests 157 passed / 1 skipped + full `bun check` green, browser N/A +- Browser check: N/A: no browser surface +- Outcome: the compiled plan now reaches every stream site; a cursor-paginated + `in()` reads its index probes instead of the whole table, and no longer needs + `maxScan`. +- Caveat: with no `orderBy`, an index-union page is now in index order rather + than creation order. `maxScan` on a K-probe union can overshoot by up to K-1 + documents because a merged stream fills one row per probe before comparing. +- Design: + - Chosen boundary: one `CompiledQueryPlan` type plus `_buildPlanStream` / + `_buildProbeUnionStream` in `query.ts`. + - Why not quick patch: threading `probeFilters` into each site would leave the + narrowed parameter types that caused the silent drop. + - Why not broader change: the duplicate `update.ts`/`delete.ts` probe fan-outs + and the `pipeline.union` `source.where` lowering are separate owners with + their own pinned contracts. +- Verified: focused 157-test regression set, `bun typecheck`, + `bun --cwd packages/kitcn build`, `bun run fixtures:sync`, + `bun run fixtures:check`, `bun lint:fix`, full `bun check`, intent + validation/staleness, and local repair autoreview. +- PR body verified: `gh pr view 425 --json body` matches the PR #270 emoji task-style contract + +Task-style PR body contract: +- Preserve any existing `` block. If a changeset is + part of the diff and repo policy expects auto release, include that block. +- Use the accepted PR #270 visual format. The body starts with an emoji + issue/fix line, for example `🐛 Fixes #123` or `🐛 Fixes ➖ N/A`, then + `🧭 Task plan: docs/plans/.md`, then an emoji confidence line like + `🟢 95-100% confidence`. +- Use this exact table header: `| Phase | 🧪 Tests | 🌐 Browser |`. +- Use `Reproduced` and `Verified` rows. Mark passing proof with `🟢`, repro or + failing proof with `🔴`, and non-applicable cells with `➖ N/A`. +- Use bold emoji section headings: `**✅ Outcome**`, `**⚠️ Caveat**`, + `**🏗️ Design**`, and `**🧪 Verified**`. +- Never include a line that links to the current PR itself. The current PR URL + belongs in the final response, not in its own description. +- Do not replace this with a generic `Summary` / `Verification` PR body, an + adaptive prose body from a git helper skill, plain `## Outcome` sections, or + an unrelated generated badge footer unless the caller or repo template + explicitly asks for it. +- Proof is `gh pr view --json body` output or a concise source-backed summary + of that output. + +Final handoff / sync: +- Commit: 3e046c49 on `fix/orm-index-union-cursor-pagination` +- PR: https://github.com/udecode/kitcn/pull/425 +- Issue: #415, closed by PR #425 +- Browser proof: N/A: no browser surface +- Caveats: index-union default order change; merged-stream `maxScan` overshoot + +Timeline: +- 2026-08-21T21:22:54.405Z Task goal plan created. +- Recon fan-out over tests, planners, docs, and stream contracts. +- Red repro added (6 failing), fix implemented, all suites green. +- 2026-08-26: autoclosure merged main + `0e3a2214e8e1037b323e603f19c2df89b0d95602`, found and repaired two P1 + cursor-boundary defects with three red-to-green regressions. +- 2026-08-26: focused tests, typecheck, lint, package build, fixture + sync/check, intent validation, local autoreview, and full `bun check` green. +- 2026-08-26: final branch review found the rejected-union signal loss; + red-to-green strict `endCursor` coverage added and the advanced caller fixed. +- 2026-08-26: follow-up predicate-loss review claim disproven by source and a + green-before-change budgeted fallback regression; invariant documented. + +Reboot status: +| Question | Answer | +|----------|--------| +| Where am I? | Closeout | +| Where am I going? | Implementation, verification, commit/PR/GitHub sync, closeout | +| What is the goal? | Make the compiled plan reach every ORM stream site | +| What have I learned? | See Findings | +| What have I done? | See Timeline | + +Open risks: +- Default order for an index-union page with no `orderBy` changes from creation + order to index order. Realistic failure mode: an app paging + `where: { status: { in: [...] } }` with no `orderBy` shows rows grouped by + status instead of newest-first. Proof plan: pinned by + `index-union-pagination.test.ts` "does not require maxScan"; documented in + `pagination.mdx` and declared breaking in the changeset. The boundary is right + because the previous order came from not using the index at all, and adding + `orderBy: { createdAt }` restores it while staying index-served. +- `maxScan` on a K-probe union can read up to K-1 documents past the budget, + because `MergedStream` fills one row from every probe before comparing keys. + Bounded by the 64-probe cap and only reachable when the caller opts into + `maxScan`, which the union no longer requires. +- `update.ts` and `delete.ts` still run their own duplicate probe fan-outs and + still hard-throw on any multi-probe paged mutation. Untouched by this change. + +Hard closeout guard: +- A local-only final response for verified code-changing work is invalid unless + this plan records an explicit user decline, no local patch, analytical/ + blocked/inconclusive outcome, or a real commit/PR blocker. diff --git a/packages/kitcn/skills/kitcn/references/features/orm.md b/packages/kitcn/skills/kitcn/references/features/orm.md index bbf5a563..7066e92f 100644 --- a/packages/kitcn/skills/kitcn/references/features/orm.md +++ b/packages/kitcn/skills/kitcn/references/features/orm.md @@ -649,6 +649,23 @@ const page2 = await ctx.orm.query.posts.findMany({ Return: `{ page, continueCursor, isDone, pageStatus?, splitCursor? }` +### Index-union filters + +`in`, `notIn`, `ne`, and same-field equality `OR` compile to one index range per value when the field is one an index leads with. Cursor pages come from those ranges as a merged stream, so `orderBy` sorts across the whole result and no scan budget is needed. + +```ts +const page = await ctx.orm.query.users.withIndex("by_status").findMany({ + where: { status: { in: ["active", "pending"] } }, + orderBy: { createdAt: "desc" }, + cursor: null, + limit: 20, +}); +``` + +Falls back to a bounded scan (needs `maxScan`) when the probed index cannot supply the requested `orderBy`, or when the union is wider than 64 ranges. + +Without an `orderBy`, an index-union page is in the order of the index it walks — grouped by the probed value — not in creation order. + ### Boundary pinning with `endCursor` ```ts @@ -736,7 +753,7 @@ const filtered = results.filter((a) => a.publishedAt >= startDate); ### Performance 1. **Index first** — constrain leading index fields. Compound indexes follow prefix rules. Put the `orderBy` column right after the constrained prefix so the scan is already sorted and `limit` bounds the read. -2. **Bound scans** — use `maxScan` for predicate `where` (cursor mode only). A `.select()` ID-list query with `orderBy` needs a single field that an index leads with. +2. **Bound scans** — use `maxScan` for predicate `where` (cursor mode only). `in`/`notIn`/`ne` on an indexed leading field page from index ranges and need no budget. A `.select()` ID-list query with `orderBy` needs a single field that an index leads with. 3. **Limit results** — always use `limit` or cursor pagination. 4. **Cursor stability** — keep same `where`/`orderBy` between page requests. 5. **`allowFullScan`** — non-cursor only. Cursor mode uses `maxScan` instead. diff --git a/packages/kitcn/src/orm/query.ts b/packages/kitcn/src/orm/query.ts index e869ebf9..ed1b1d0d 100644 --- a/packages/kitcn/src/orm/query.ts +++ b/packages/kitcn/src/orm/query.ts @@ -128,6 +128,7 @@ import type { } from './types'; import { type IndexStrategy, + MAX_INDEX_UNION_PROBES, WhereClauseCompiler, } from './where-clause-compiler'; @@ -540,6 +541,29 @@ const countConfiguredIndexEqPrefix = ( return count; }; +/** + * The read `_toConvexQuery` compiled, in full. + * + * Every site that turns a plan into a read takes this whole shape rather than a + * narrowed view of it. A narrowed parameter type is how `probeFilters` came to + * be silently dropped at each stream site: the plan carried an index union and + * the builder's signature could not even name it, so `if (queryConfig.index)` + * walked the index with no range at all. + */ +type CompiledQueryPlan = { + table: string; + strategy: IndexStrategy; + index?: { + name: string; + fields: string[]; + filters: FilterExpression[]; + }; + /** One index range per probe for `in`/`ne`/`notIn`/same-field `OR` plans. */ + probeFilters: FilterExpression[][]; + postFilters: FilterExpression[]; + order?: { direction: 'asc' | 'desc'; field: string }[]; +}; + /** * Relational query builder with promise-based execution * @@ -2807,25 +2831,128 @@ export class GelRelationalQuery< ); } - private _buildBasePipelineStream( - queryConfig: { - index?: { name: string; filters: FilterExpression[] }; - postFilters: FilterExpression[]; - order?: { direction: 'asc' | 'desc'; field: string }[]; - }, - wherePredicate: ((row: any) => boolean | Promise) | undefined, - configuredIndex?: PredicateWhereIndexConfig - ): QueryStream { - const schemaDefinition = this._getSchemaDefinitionOrThrow(); + /** + * The compiled index union, as one ordered stream. + * + * Each probe is its own index range, so the union is only usable where the + * merged order is the order the read has to produce. `mergedStream` orders by + * a *suffix* of the index key, and a suffix may only drop key components the + * probes all pin to a single value — so the requested field has to sit inside + * the pinned run or immediately after it. Returns null when it does not, and + * the caller falls back to the plan's plain index range or a bounded scan. + */ + private _buildProbeUnionStream(params: { + schemaDefinition: unknown; + indexName: string; + probeFilters: FilterExpression[][]; + order: 'asc' | 'desc'; + /** + * Field the merged stream must be ordered by. Callers pass the requested + * `orderBy` field, or the index's own leading field when the read has no + * `orderBy` and therefore inherits the scanned index's order. + */ + orderField: string; + }): QueryStream | null { + const { probeFilters } = params; + // A merged stream registers every probe query up front and holds them open + // for the life of the read, so a wide fan-out is worse here than the single + // scan it would replace. + if ( + probeFilters.length === 0 || + probeFilters.length > MAX_INDEX_UNION_PROBES + ) { + return null; + } + + const streamIndexFields = getIndexFields( + this.tableConfig.name as any, + params.indexName as any, + params.schemaDefinition as any + ); + const pinned = this._indexEqPrefixCount({ probeFilters }); + let mergeOffset = -1; + for (let i = 0; i <= pinned && i < streamIndexFields.length; i += 1) { + if (streamIndexFields[i] === params.orderField) { + mergeOffset = i; + break; + } + } + if (mergeOffset === -1) { + return null; + } + + const probeStreams = probeFilters.map((filters) => + stream( + this.db as GenericDatabaseReader, + params.schemaDefinition as any + ) + .query(this.tableConfig.name as any) + .withIndex(params.indexName as any, (q: any) => { + let indexQuery = q; + for (const filter of filters) { + indexQuery = this._applyFilterToQuery(indexQuery, filter); + } + return indexQuery; + }) + .order(params.order) + ); + + return mergedStream( + probeStreams as QueryStream[], + streamIndexFields.slice(mergeOffset) + ); + } + + /** + * The read the compiled plan describes, as a stream, with nothing filtered + * yet. + * + * Precedence: the compiled index union, then the compiled index range, then + * the caller's pinned `.withIndex(...)`, then an index that supplies the + * requested order, then a full scan. The first two rungs can only ever refine + * what the caller pinned — `_toConvexQuery` discards a compiled plan that + * would displace a caller's index or its bounds before it gets here. + * + * `probeUnion` tells the caller the read is bounded by index ranges rather + * than by scan length, which is what makes a scan budget unnecessary. A + * rejected union stays unanchored; its original predicate remains in + * `queryConfig.postFilters` for the caller to apply while pulling the scan. + */ + private _buildPlanStream(params: { + queryConfig: CompiledQueryPlan; + configuredIndex?: PredicateWhereIndexConfig; + order: 'asc' | 'desc'; + primaryOrder?: { direction: 'asc' | 'desc'; field: string }; + /** Index to walk for the requested order when the plan pins none. */ + orderIndexName?: string | null; + schemaDefinition: unknown; + }): { stream: QueryStream; probeUnion: boolean } { + const { queryConfig, configuredIndex, order, primaryOrder } = params; + const probeIndex = + queryConfig.probeFilters.length > 0 ? queryConfig.index : undefined; + const hasProbeUnionPlan = !!probeIndex; + + if (hasProbeUnionPlan) { + const probeUnion = this._buildProbeUnionStream({ + schemaDefinition: params.schemaDefinition, + indexName: probeIndex.name, + probeFilters: queryConfig.probeFilters, + order, + // Without an `orderBy` the read inherits the order of whatever index it + // walks, which for this plan is the union's own index. + orderField: primaryOrder?.field ?? probeIndex.fields[0], + }); + if (probeUnion) { + return { stream: probeUnion, probeUnion: true }; + } + } + let streamQuery: any = stream( this.db as GenericDatabaseReader, - schemaDefinition + params.schemaDefinition as any ).query(this.tableConfig.name as any); - const primaryOrder = queryConfig.order?.[0]; - const primaryOrderDirection = primaryOrder?.direction ?? 'asc'; - - if (queryConfig.index) { + if (queryConfig.index && !hasProbeUnionPlan) { streamQuery = streamQuery.withIndex( queryConfig.index.name as any, (q: any) => { @@ -2836,25 +2963,42 @@ export class GelRelationalQuery< return indexQuery; } ); - } else if (configuredIndex?.name) { + } else if (!hasProbeUnionPlan && configuredIndex?.name) { streamQuery = streamQuery.withIndex( configuredIndex.name as any, configuredIndex.range ? (configuredIndex.range as any) : (q: any) => q ); - } else if ( - primaryOrder && - primaryOrder.field !== INTERNAL_CREATION_TIME_FIELD - ) { - const orderIndex = this._findStreamOrderIndex(primaryOrder.field); - if (orderIndex) { - streamQuery = streamQuery.withIndex( - orderIndex.name as any, - (q: any) => q - ); - } + } else if (!hasProbeUnionPlan && params.orderIndexName) { + streamQuery = streamQuery.withIndex( + params.orderIndexName as any, + (q: any) => q + ); } - streamQuery = streamQuery.order(primaryOrderDirection); + return { stream: streamQuery.order(order), probeUnion: false }; + } + + private _buildBasePipelineStream( + queryConfig: CompiledQueryPlan, + wherePredicate: ((row: any) => boolean | Promise) | undefined, + configuredIndex: PredicateWhereIndexConfig | undefined, + fallbackOrder: 'asc' | 'desc' + ): { stream: QueryStream; probeUnion: boolean } { + const schemaDefinition = this._getSchemaDefinitionOrThrow(); + const primaryOrder = queryConfig.order?.[0]; + + const planStream = this._buildPlanStream({ + queryConfig, + configuredIndex, + order: primaryOrder?.direction ?? fallbackOrder, + primaryOrder, + orderIndexName: + primaryOrder && primaryOrder.field !== INTERNAL_CREATION_TIME_FIELD + ? (this._findStreamOrderIndex(primaryOrder.field)?.name ?? null) + : null, + schemaDefinition, + }); + let streamQuery: any = planStream.stream; if (queryConfig.postFilters.length > 0 || wherePredicate) { streamQuery = streamQuery.filterWith(async (row: any) => { @@ -2870,7 +3014,7 @@ export class GelRelationalQuery< }); } - return streamQuery; + return { stream: streamQuery, probeUnion: planStream.probeUnion }; } /** @@ -2887,10 +3031,7 @@ export class GelRelationalQuery< * the caller then falls back to its plain-query path. */ private _buildResidualFilterStream(params: { - queryConfig: { - index?: { name: string; filters: FilterExpression[] }; - postFilters: FilterExpression[]; - }; + queryConfig: CompiledQueryPlan; configuredIndex?: PredicateWhereIndexConfig; membershipFilter?: (row: any) => boolean | Promise; orderIndexName: string | null; @@ -2912,38 +3053,18 @@ export class GelRelationalQuery< fallbackOrder, } = params; - let streamQuery: any = stream( - this.db as GenericDatabaseReader, - schemaDefinition - ).query(this.tableConfig.name as any); - - if (queryConfig.index) { - streamQuery = streamQuery.withIndex( - queryConfig.index.name as any, - (q: any) => { - let indexQuery = q; - for (const filter of queryConfig.index!.filters) { - indexQuery = this._applyFilterToQuery(indexQuery, filter); - } - return indexQuery; - } - ); - } else if (configuredIndex?.name) { - streamQuery = streamQuery.withIndex( - configuredIndex.name as any, - configuredIndex.range ? (configuredIndex.range as any) : (q: any) => q - ); - } else if (orderIndexName) { - streamQuery = streamQuery.withIndex(orderIndexName as any, (q: any) => q); - } - // Streams always need an explicit direction. The plain query leaves the // Convex default (ascending) when there is no orderBy in the non-paginated // path, and explicitly orders 'desc' in the cursor path, so the caller // supplies which of the two applies. - streamQuery = streamQuery.order( - primaryOrder ? primaryOrder.direction : fallbackOrder - ); + let streamQuery: any = this._buildPlanStream({ + queryConfig, + configuredIndex, + order: primaryOrder ? primaryOrder.direction : fallbackOrder, + primaryOrder, + orderIndexName, + schemaDefinition, + }).stream; streamQuery = streamQuery.filterWith(async (row: any) => { for (const filter of queryConfig.postFilters) { @@ -5832,7 +5953,9 @@ export class GelRelationalQuery< if (useAdvancedStreamPath) { const primaryOrder = queryConfig.order?.[0]; - const fallbackOrder = primaryOrder?.direction ?? 'asc'; + const fallbackOrder = + primaryOrder?.direction ?? + (endCursor !== undefined && !pipeline ? 'desc' : 'asc'); let streamQuery: QueryStream; const unionSources = pipeline?.union ?? []; @@ -5884,19 +6007,39 @@ export class GelRelationalQuery< ); } } - streamQuery = - (await this._buildIdLookupStream({ - configuredIndex, - idLookup, - order: fallbackOrder, - queryConfig, - wherePredicate, - })) ?? - this._buildBasePipelineStream( + const idLookupStream = await this._buildIdLookupStream({ + configuredIndex, + idLookup, + order: fallbackOrder, + queryConfig, + wherePredicate, + }); + if (idLookupStream) { + streamQuery = idLookupStream; + } else { + const baseStream = this._buildBasePipelineStream( queryConfig, wherePredicate, - configuredIndex + configuredIndex, + fallbackOrder ); + const rejectedProbeUnion = + isCursorPaginated && + queryConfig.strategy === 'multiProbe' && + !!queryConfig.index && + !baseStream.probeUnion; + if (rejectedProbeUnion && maxScan === undefined) { + if (strict) { + throw new Error( + 'Pagination with multi-probe index-union filters requires maxScan when strict=true. Add maxScan, order by a field the union can serve, or make the query indexable.' + ); + } + console.warn( + 'Pagination with multi-probe index-union filters is running without maxScan because strict: false.' + ); + } + streamQuery = baseStream.stream; + } } const rootRlsEnabled = isRlsEnabled(this.tableConfig.table as any); @@ -6587,28 +6730,76 @@ export class GelRelationalQuery< // M6.5 Phase 4: Handle cursor pagination separately if (isCursorPaginated) { - if (queryConfig.strategy === 'multiProbe') { - if (maxScan === undefined) { + const isMultiProbePlan = + queryConfig.strategy === 'multiProbe' && !!queryConfig.index; + const isScanFallbackPlan = + !queryConfig.index && queryConfig.postFilters.length > 0; + + if (isMultiProbePlan || isScanFallbackPlan) { + const schemaDefinition = (this.schema as any)[OrmSchemaDefinition]; + + // A compiled index union is bounded by its probe ranges, not by scan + // length, so it needs no scan budget. `maxScan` is still honoured when + // the caller asked for one. + const probeUnion = + isMultiProbePlan && schemaDefinition + ? this._buildProbeUnionStream({ + schemaDefinition, + indexName: queryConfig.index!.name, + probeFilters: queryConfig.probeFilters, + order: primaryOrder?.direction ?? 'desc', + orderField: primaryOrder?.field ?? queryConfig.index!.fields[0], + }) + : null; + + if (!probeUnion && maxScan === undefined) { if (strict) { throw new Error( - 'Pagination with multi-probe index-union filters requires maxScan when strict=true. Add maxScan or make the query indexable.' + isMultiProbePlan + ? 'Pagination with multi-probe index-union filters requires maxScan when strict=true. Add maxScan, order by a field the union can serve, or make the query indexable.' + : 'Cursor pagination with scan fallback requires maxScan when strict=true. Add maxScan or make the query indexable.' ); } - console.warn( - 'Pagination with multi-probe index-union filters is running without maxScan because strict: false.' - ); - } else { - const schemaDefinition = (this.schema as any)[OrmSchemaDefinition]; + if (isMultiProbePlan) { + console.warn( + 'Pagination with multi-probe index-union filters is running without maxScan because strict: false.' + ); + } + } + + if (probeUnion || maxScan !== undefined) { if (!schemaDefinition) { throw new Error( 'Pagination with maxScan requires defineSchema(). Ensure defineSchema(tables) was used with the same tables object passed to defineRelations.' ); } - let streamQuery: any = stream( - this.db as GenericDatabaseReader, - schemaDefinition - ).query(this.tableConfig.name as any); + let streamQuery: any = probeUnion; + if (!streamQuery) { + streamQuery = stream( + this.db as GenericDatabaseReader, + schemaDefinition + ).query(this.tableConfig.name as any); + + // An explicit `.withIndex(name, range)` anchors this read too. A + // where clause that is not index-compilable leaves + // `queryConfig.index` unset and lands here, so without this the + // caller's index and its bounds — a tenant scope, say — would never + // reach the scan. + // + // A declined union deliberately does not anchor to the plan's + // index: its probes carry the ranges, and walking that index with + // no range would sort the page by the probed field while the + // warning below promises creation-time order. + if (isScanFallbackPlan && configuredIndex?.name) { + streamQuery = streamQuery.withIndex( + configuredIndex.name as any, + configuredIndex.range + ? (configuredIndex.range as any) + : (q: any) => q + ); + } + } if (queryConfig.order && primaryOrder) { if (needsPostFetchSortForPrimary) { @@ -6628,11 +6819,20 @@ export class GelRelationalQuery< 'Secondary orderBy fields are applied per page and may be unstable across pages.' ); } - streamQuery = streamQuery.order(primaryOrder.direction); - } else { - streamQuery = streamQuery.order('desc'); + } + // The union already carries its direction: every probe was opened + // with it so the merge could compare index keys. + if (!probeUnion) { + streamQuery = streamQuery.order( + queryConfig.order && primaryOrder + ? primaryOrder.direction + : 'desc' + ); } + // A rejected multi-probe plan keeps its original expression here. + // Applying postFilters while pulling is what makes the budgeted scan + // fallback preserve the union predicate and still fill each page. if (queryConfig.postFilters.length > 0) { streamQuery = streamQuery.filterWith(async (row: any) => queryConfig.postFilters.every((filter) => @@ -6641,105 +6841,8 @@ export class GelRelationalQuery< ); } - const paginationResult = await streamQuery.paginate({ - cursor: cursor ?? null, - limit: config.limit, - maxScan, - }); - - let pageRows = paginationResult.page; - - pageRows = await this._applyRlsSelectFilter( - pageRows, - this.tableConfig - ); - - if (whereFilter) { - pageRows = await this._applyRelationsFilterToRows( - pageRows, - this.tableConfig, - whereFilter, - this.edgeMetadata, - 0, - MAX_RELATION_DEPTH, - this.config.with as Record | undefined - ); - } - - const selectedPage = await this._finalizeRows(pageRows); - - return { - page: selectedPage, - continueCursor: paginationResult.continueCursor, - isDone: paginationResult.isDone, - pageStatus: (paginationResult as any).pageStatus, - splitCursor: (paginationResult as any).splitCursor, - } as TResult; - } - } - - if (!queryConfig.index && queryConfig.postFilters.length > 0) { - if (maxScan === undefined) { - if (strict) { - throw new Error( - 'Cursor pagination with scan fallback requires maxScan when strict=true. Add maxScan or make the query indexable.' - ); - } - } else { - const schemaDefinition = (this.schema as any)[OrmSchemaDefinition]; - if (!schemaDefinition) { - throw new Error( - 'Pagination with maxScan requires defineSchema(). Ensure defineSchema(tables) was used with the same tables object passed to defineRelations.' - ); - } - - let streamQuery: any = stream( - this.db as GenericDatabaseReader, - schemaDefinition - ).query(this.tableConfig.name as any); - - // An explicit `.withIndex(name, range)` anchors this read too. A - // where clause that is not index-compilable leaves `queryConfig.index` - // unset and lands here, so without this the caller's index and its - // bounds — a tenant scope, say — would never reach the scan. - if (configuredIndex?.name) { - streamQuery = streamQuery.withIndex( - configuredIndex.name as any, - configuredIndex.range - ? (configuredIndex.range as any) - : (q: any) => q - ); - } - - if (queryConfig.order && primaryOrder) { - if (needsPostFetchSortForPrimary) { - if (strict) { - throw new Error( - `Pagination: Field '${primaryOrder.field}' has no index. Add an index or disable strict.` - ); - } - console.warn( - `Pagination: Field '${primaryOrder.field}' has no index. ` + - 'Falling back to _creationTime ordering.' - ); - } - if (hasSecondaryOrders) { - console.warn( - 'Pagination: Only the first orderBy field is used for cursor ordering. ' + - 'Secondary orderBy fields are applied per page and may be unstable across pages.' - ); - } - streamQuery = streamQuery.order(primaryOrder.direction); - } else { - streamQuery = streamQuery.order('desc'); - } - - streamQuery = streamQuery.filterWith(async (row: any) => - queryConfig.postFilters.every((filter) => - this._evaluatePostFetchFilter(row, filter) - ) - ); - + // `endCursor` routes through the advanced stream path above, so it + // never reaches this branch. const paginationResult = await streamQuery.paginate({ cursor: cursor ?? null, limit: config.limit, @@ -7076,18 +7179,7 @@ export class GelRelationalQuery< * can be scanned, so a compiled plan that would replace it is discarded. */ configuredIndex?: PredicateWhereIndexConfig - ): { - table: string; - strategy: IndexStrategy; - index?: { - name: string; - fields: string[]; - filters: FilterExpression[]; - }; - probeFilters: FilterExpression[][]; - postFilters: FilterExpression[]; - order?: { direction: 'asc' | 'desc'; field: string }[]; - } { + ): CompiledQueryPlan { const config = this.config as any; // Initialize compiler for this table using declared indexes @@ -7156,18 +7248,7 @@ export class GelRelationalQuery< }; // Build query config - const result: { - table: string; - strategy: IndexStrategy; - index?: { - name: string; - fields: string[]; - filters: FilterExpression[]; - }; - probeFilters: FilterExpression[][]; - postFilters: FilterExpression[]; - order?: { direction: 'asc' | 'desc'; field: string }[]; - } = { + const result: CompiledQueryPlan = { table: this.tableConfig.table.tableName, strategy: compiled.strategy, probeFilters: [...compiled.probeFilters], diff --git a/packages/kitcn/src/orm/stream.regression.vitest.ts b/packages/kitcn/src/orm/stream.regression.vitest.ts index 9e67bcbc..13dedaa6 100644 --- a/packages/kitcn/src/orm/stream.regression.vitest.ts +++ b/packages/kitcn/src/orm/stream.regression.vitest.ts @@ -339,6 +339,76 @@ describe('stream regression', () => { }); }); + test('descending merged endCursor excludes disjoint equality ranges', async () => { + const t = convexTest(schema); + await t.run(async (ctx) => { + await ctx.db.insert('foo', { a: 1, b: 2, c: 3 }); + await ctx.db.insert('foo', { a: 1, b: 3, c: 3 }); + await ctx.db.insert('foo', { a: 2, b: 1, c: 3 }); + await ctx.db.insert('foo', { a: 2, b: 4, c: 4 }); + const build = () => + mergedStream( + [ + stream(ctx.db, schema) + .query('foo') + .withIndex('abc', (q) => q.eq('a', 1)) + .order('desc'), + stream(ctx.db, schema) + .query('foo') + .withIndex('abc', (q) => q.eq('a', 2)) + .order('desc'), + ], + ['a', 'b', 'c'] + ); + + const first = await build().paginate({ cursor: null, limit: 2 }); + const pinned = await build().paginate({ + cursor: null, + endCursor: first.continueCursor, + limit: 2, + }); + + expect(pinned.page.map(stripSystemFields)).toEqual([ + { a: 2, b: 4, c: 4 }, + { a: 2, b: 1, c: 3 }, + ]); + }); + }); + + test('suffix-ordered merged endCursor accepts an empty child range', async () => { + const t = convexTest(schema); + await t.run(async (ctx) => { + await ctx.db.insert('foo', { a: 1, b: 2, c: 3 }); + await ctx.db.insert('foo', { a: 1, b: 3, c: 3 }); + await ctx.db.insert('foo', { a: 2, b: 1, c: 3 }); + const build = () => + mergedStream( + [ + stream(ctx.db, schema) + .query('foo') + .withIndex('abc', (q) => q.eq('a', 1)) + .order('desc'), + stream(ctx.db, schema) + .query('foo') + .withIndex('abc', (q) => q.eq('a', 2).lt('b', 2)) + .order('desc'), + ], + ['b', 'c'] + ); + + const first = await build().paginate({ cursor: null, limit: 1 }); + const pinned = await build().paginate({ + cursor: null, + endCursor: first.continueCursor, + limit: 1, + }); + + expect(pinned.page.map(stripSystemFields)).toEqual([ + { a: 1, b: 3, c: 3 }, + ]); + }); + }); + test('filter stream and maxScan split metadata', async () => { const t = convexTest(schema); await t.run(async (ctx) => { diff --git a/packages/kitcn/src/orm/stream.ts b/packages/kitcn/src/orm/stream.ts index 6d62c6dd..f2ae95a2 100644 --- a/packages/kitcn/src/orm/stream.ts +++ b/packages/kitcn/src/orm/stream.ts @@ -851,6 +851,25 @@ export class OrderedStreamQuery< minUpperBound = indexBounds.upperBound; minUpperBoundInclusive = indexBounds.upperBoundInclusive; } + const lowerPosition = { + value: maxLowerBound, + kind: maxLowerBoundInclusive + ? ('predecessor' as const) + : ('successor' as const), + }; + const upperPosition = { + value: minUpperBound, + kind: minUpperBoundInclusive + ? ('successor' as const) + : ('predecessor' as const), + }; + if (compareKeys(lowerPosition, upperPosition) >= 0) { + return new EmptyStream( + order, + this.getIndexFields(), + this.getEqualityIndexFilter() + ); + } return streamIndexRange( db, schema, @@ -1622,7 +1641,11 @@ export class SingletonStream< this.#equalityIndexFilter ); } - return new EmptyStream(this.#order, this.#indexFields); + return new EmptyStream( + this.#order, + this.#indexFields, + this.#equalityIndexFilter + ); } } @@ -1669,12 +1692,18 @@ export function indexKeyWithinBounds( * towards maxScan. */ export class EmptyStream extends QueryStream { + #equalityIndexFilter: Value[]; #order: 'asc' | 'desc'; #indexFields: string[]; - constructor(order: 'asc' | 'desc', indexFields: string[]) { + constructor( + order: 'asc' | 'desc', + indexFields: string[], + equalityIndexFilter: Value[] = [] + ) { super(); this.#order = order; this.#indexFields = indexFields; + this.#equalityIndexFilter = equalityIndexFilter; } iterWithKeys(): AsyncIterable<[T | null, IndexKey]> { return { @@ -1694,7 +1723,7 @@ export class EmptyStream extends QueryStream { return this.#indexFields; } getEqualityIndexFilter(): Value[] { - return []; + return this.#equalityIndexFilter; } narrow(_indexBounds: IndexBounds) { return this; diff --git a/packages/kitcn/src/orm/where-clause-compiler.ts b/packages/kitcn/src/orm/where-clause-compiler.ts index 636881ba..ec3da7ef 100644 --- a/packages/kitcn/src/orm/where-clause-compiler.ts +++ b/packages/kitcn/src/orm/where-clause-compiler.ts @@ -87,11 +87,15 @@ export interface IndexLike { const INDEX_ORDER_BONUS = 30; /** - * Widest `in` list still worth turning into an index union when it is the only - * indexable term. Each value becomes its own index range, so past this width - * the fan-out costs more than the single scan it would replace. + * Widest index union worth opening. Each probe becomes its own index range, and + * a streamed union holds every one of them open at once, so past this width the + * fan-out costs more than the single scan it would replace. + * + * Owns both ends of that decision: whether to promote an `in` inside an AND to + * a union here, and whether the executor may read a compiled union as a merged + * stream instead of a bounded scan. */ -const MAX_PROMOTED_PROBES = 64; +export const MAX_INDEX_UNION_PROBES = 64; export class WhereClauseCompiler { /** @@ -231,7 +235,10 @@ export class WhereClauseCompiler { const probePlan = this.tryCompileInArray(term as BinaryExpression); // A wide `in` opens one index range per value, which past some width // costs more than the single scan it replaces. Leave those alone. - if (!probePlan || probePlan.probeFilters.length > MAX_PROMOTED_PROBES) { + if ( + !probePlan || + probePlan.probeFilters.length > MAX_INDEX_UNION_PROBES + ) { continue; } return { ...probePlan, postFilters: [expression] }; diff --git a/www/content/docs/orm/api-reference.mdx b/www/content/docs/orm/api-reference.mdx index ff6d4c48..b8f05658 100644 --- a/www/content/docs/orm/api-reference.mdx +++ b/www/content/docs/orm/api-reference.mdx @@ -380,7 +380,7 @@ await db.query.users.findMany({ - `orderBy`: `{ field: 'asc' | 'desc' }` - `limit`, `offset` - `cursor`: `string | null` (cursor pagination; first page is `null`) -- `maxScan`: `number` (only valid with cursor pagination; caps scan-fallback paths like predicate/object post-filter and index-union pagination) +- `maxScan`: `number` (only valid with cursor pagination; caps scan-fallback paths like predicate `where` and post-filter-only plans) - `allowFullScan`: opt‑in for unsized non-cursor scans and relation fan-out overrides (not supported with cursor pagination; not used with `search` or `vectorSearch`) - `with`: relation loading - `withVariants`: `true` to auto-load all `one()` relations on tables with `discriminator()` columns diff --git a/www/content/docs/orm/queries/index.mdx b/www/content/docs/orm/queries/index.mdx index 04fb32fb..4997d00c 100644 --- a/www/content/docs/orm/queries/index.mdx +++ b/www/content/docs/orm/queries/index.mdx @@ -207,7 +207,9 @@ return db.query.posts.findMany({ }); ``` -For cursor queries that need scan fallback paths (for example index-union/multi-probe or post-filter-only plans), use `maxScan`. +`in`, `notIn`, `ne`, and same-field equality `OR` on a field an index leads with page from one index range per value, so they need no scan budget. + +For cursor queries that fall back to a scan (predicate `where` or post-filter-only plans), use `maxScan`. `allowFullScan` is non-cursor only. With `strict: true`, missing `maxScan` on these fallback paths throws; with `strict: false`, it warns and runs uncapped. diff --git a/www/content/docs/orm/queries/pagination.mdx b/www/content/docs/orm/queries/pagination.mdx index 68e57f10..f8949545 100644 --- a/www/content/docs/orm/queries/pagination.mdx +++ b/www/content/docs/orm/queries/pagination.mdx @@ -98,6 +98,30 @@ const refreshed = await db.query.posts.findMany({ This keeps adjacent pages stitched correctly as new rows arrive. +### Index-union filters + +`in`, `notIn`, `ne`, and same-field equality `OR` compile to one index range per value when the field is one an index leads with. Cursor pages come from those ranges, so the read is bounded by the matches and needs no scan budget: + +```ts showLineNumbers +const page = await db.query.users + .withIndex('by_status') + .findMany({ + where: { status: { in: ['active', 'pending'] } }, + orderBy: { createdAt: 'desc' }, + cursor: null, + limit: 20, + }); +``` + +The ranges are read as one merged stream, so `orderBy` still sorts across the whole result, not per value. + +Two shapes fall back to a bounded scan and need `maxScan`: + +- An `orderBy` the probed index cannot supply. The index has to lead with the probed field, so only that field, the next index field after it, or `createdAt` on a single-field index is servable. +- A union wider than 64 ranges. Past that, one scan costs less than the fan-out. + +Without an `orderBy`, an index-union page is in the order of the index it walks — grouped by the probed value — not in creation order. Pass `orderBy: { createdAt: 'desc' }` when you want newest-first across values. + ### Scan budget with `maxScan` `maxScan` is available only in cursor mode. It caps scanned rows instead of throwing when a page would scan too much.