test(orm): count documents scanned behind .filter()

convex/orm/example-comment-tree-reads.test.ts

@@ -88,7 +88,7 @@ type ExampleCtx = { orm: OrmWriter<typeof schema>; db: any };
  * silently reports zero for every read the ORM issues.
  */
 const withCountedExampleOrm = async (
-  run: (ctx: ExampleCtx, reads: { documents: number }) => Promise<void>
+  run: (ctx: ExampleCtx, reads: { scanned: number }) => Promise<void>
 ) => {
   await withExampleEnv(async () => {
     const t = convexTest(schema);
@@ -160,13 +160,13 @@ const seedThreads = async (
   }
 };
 
-/** The exact read `todoComments.getTodoComments` issues, counting documents. */
+/** The exact read `todoComments.getTodoComments` issues, counting scans. */
 const readCommentTree = async (
   ctx: ExampleCtx,
-  reads: { documents: number },
+  reads: { scanned: number },
   todoId: string
 ) => {
-  const before = reads.documents;
+  const before = reads.scanned;
 
   const results = await ctx.orm.query.todoComments.findMany({
     where: { todoId, parentId: { isNull: true } },
@@ -180,9 +180,9 @@ const readCommentTree = async (
     },
   });
 
-  const documents = reads.documents - before;
+  const scanned = reads.scanned - before;
   const rows = CommentRowWithRepliesSchema.array().parse(results.page);
-  return { documents, page: rows.map(toReply) };
+  return { scanned, page: rows.map(toReply) };
 };
 
 const countNodes = (replies: { replies: any[] }[]): number =>
@@ -194,11 +194,18 @@ const countNodes = (replies: { replies: any[] }[]): number =>
  * second count per node, and buys nothing: the relation loader resolves `_count`
  * at every level it returns, including the deepest.
  *
- * Measured marginal cost over 30 extra nodes: 100 documents for the second pass
- * versus 50 for one pass. Fixed setup cost (the author, the root index scan)
- * does not scale, so the bound is on the marginal document cost per node.
+ * The bound is on marginal cost per node, not total: fixed setup (the author,
+ * the root index scan) does not scale with the thread.
+ *
+ * Measured marginal cost over 30 extra nodes is 80 scanned documents, against
+ * 60 returned. The gap is the root page: `todoComments` indexes `todoId` and
+ * `parentId` separately, so `where: { todoId, parentId: { isNull: true } }`
+ * rides `todoId` and leaves `parentId` to a Convex `.filter()` — every reply on
+ * the todo is read and rejected to find the roots. A compound `(todoId,
+ * parentId)` index would erase it. Counting `documents` hid this entirely,
+ * which is the whole point of asserting on `scanned`.
  */
-const MAX_DOCUMENTS_PER_COMMENT = 2;
+const MAX_DOCUMENTS_PER_COMMENT = 3;
 
 test('comment tree cost does not track the number of comments returned', async () => {
   await withCountedExampleOrm(async (ctx, reads) => {
@@ -219,7 +226,7 @@ test('comment tree cost does not track the number of comments returned', async (
     expect(smallNodes).toBe(MAX_REPLY_DEPTH + 1);
     expect(largeNodes).toBe(6 * (MAX_REPLY_DEPTH + 1));
 
-    expect(large.documents - small.documents).toBeLessThanOrEqual(
+    expect(large.scanned - small.scanned).toBeLessThanOrEqual(
       MAX_DOCUMENTS_PER_COMMENT * (largeNodes - smallNodes)
     );
   });

convex/orm/example-invite-member-reads.test.ts

@@ -203,7 +203,7 @@ test('organization seat counting does not read one row per member', async () =>
       // Aggregate bucket reads only: 4 observed, flat in organization size.
       // Collecting the rows reported 43 reads here
       // (DECOY_MEMBERS + PENDING_INVITATIONS).
-      expect(reads.documents).toBeLessThanOrEqual(6);
+      expect(reads.scanned).toBeLessThanOrEqual(6);
     });
   });
 });

convex/orm/example-project-existence-reads.test.ts

@@ -91,7 +91,7 @@ test('project existence check does not scan projects owned by other users', asyn
       await expect(hasAnyProject(ctx, viewer.id)).resolves.toBe(false);
       // Both legs hit an index range that is empty for this user. The
       // unindexed read model reported OTHER_USER_PROJECTS reads here.
-      expect(reads.documents).toBeLessThanOrEqual(2);
+      expect(reads.scanned).toBeLessThanOrEqual(2);
     });
   });
 });
@@ -150,7 +150,7 @@ test('project existence check reads only the viewer own and member rows', async
       await expect(hasAnyProject(ctx, viewer.id)).resolves.toBe(true);
       // One membership and one related project row. Nothing proportional to
       // OTHER_USER_PROJECTS.
-      expect(reads.documents).toBeLessThanOrEqual(3);
+      expect(reads.scanned).toBeLessThanOrEqual(3);
     });
   });
 });

convex/orm/example-tag-merge-reads.test.ts

@@ -115,7 +115,7 @@ test('merge cost does not track the target tag size', async () => {
         sourceTagId,
         targetTagId,
       });
-      const mergeReads = reads.documents;
+      const mergeReads = reads.scanned;
 
       const [sourceTag, targetJoins] = await Promise.all([
         ctx.orm.query.tags.findFirst({ where: { id: sourceTagId } }),
@@ -181,7 +181,7 @@ test('merge probe finds a target row the source todo already carries', async ()
       expect(targetJoins).toHaveLength(1);
       // The probe is index-backed: the merge must not scan the target's other
       // rows or insert a duplicate for the shared todo.
-      expect(reads.documents).toBeLessThanOrEqual(20);
+      expect(reads.scanned).toBeLessThanOrEqual(20);
     });
   });
 });

convex/orm/ordering.test.ts

@@ -654,7 +654,7 @@ describe('M5: OrderBy - Index Selection', () => {
 
     expect(users.map((user) => user.name)).toEqual(['u59', 'u58']);
     // One bounded read per probe, not the whole matching set.
-    expect(reads.documents).toBeLessThanOrEqual(8);
+    expect(reads.scanned).toBeLessThanOrEqual(8);
   });
 
   test('orderBy an index suffix field reads only the requested window', async ({
@@ -678,7 +678,7 @@ describe('M5: OrderBy - Index Selection', () => {
     });
 
     expect(posts.map((post) => post.numLikes)).toEqual([59, 58]);
-    expect(reads.documents).toBeLessThanOrEqual(4);
+    expect(reads.scanned).toBeLessThanOrEqual(4);
   });
 
   test('orderBy createdAt under an unpinned index suffix sorts by createdAt', async ({

convex/orm/pipeline.test.ts

@@ -62,7 +62,7 @@ test('union sources anchor their own index range', async () => {
     expect(result.page.map((p) => p.numLikes)).toEqual([1, 2, 3, 4, 5, 6]);
     // 6 anchored rows plus each source's one-row lookahead past its range.
     // The 20 'author-z' rows sit outside both ranges and are never touched.
-    expect(reads.documents).toBeLessThanOrEqual(8);
+    expect(reads.scanned).toBeLessThanOrEqual(8);
   });
 });
 
@@ -753,7 +753,7 @@ test('flatMap limit reads each child once and stays under maxScan', async () =>
       .paginate({ cursor: null, limit: 5, maxScan: 6 });
 
     // 1 parent + at most 6 child reads for maxScan: 6.
-    expect(reads.documents).toBeLessThanOrEqual(7);
+    expect(reads.scanned).toBeLessThanOrEqual(7);
     // And that walk has to be visible to maxScan, not happen behind it.
     expect(result.pageStatus).toBe('SplitRequired');
   });
@@ -789,7 +789,7 @@ test('flatMap limit stops on the limit-th child, not one past it', async () => {
     expect(rows.map((row) => row.text)).toEqual(['p0', 'p1']);
     // 1 parent + exactly 2 children. A third child read would be a document
     // nothing can emit and maxScan never sees, since only replayed rows count.
-    expect(reads.documents).toBe(3);
+    expect(reads.scanned).toBe(3);
   });
 });
 
@@ -900,7 +900,7 @@ test('select() pipeline reads an id-only where by key, not by scan', async () =>
       .limit(10);
 
     expect(mapped).toEqual([{ onlyTitle: 'title-39' }]);
-    expect(reads.documents).toBe(1);
+    expect(reads.scanned).toBe(1);
   });
 });
 
@@ -936,7 +936,7 @@ test('select() pipeline reads an id `in` where by key, in list order', async ()
       { onlyTitle: 'title-30' },
       { onlyTitle: 'title-5' },
     ]);
-    expect(reads.documents).toBe(2);
+    expect(reads.scanned).toBe(2);
   });
 });
 
@@ -1009,7 +1009,7 @@ test('select() pages an id list without re-reading the whole list', async () =>
         .where({ id: { in: ids } })
         .map((row) => ({ onlyTitle: row.title }))
         .paginate({ cursor, limit: 10 });
-      perPageReads.push(reads.documents);
+      perPageReads.push(reads.scanned);
       walked.push(...result.page.map((row) => row.onlyTitle as string));
       cursor = result.continueCursor;
       if (result.isDone) {
@@ -1054,7 +1054,7 @@ test('select() honors maxScan on id-list pagination', async () => {
 
     expect(result.page).toHaveLength(1);
     expect(result.isDone).toBe(false);
-    expect(reads.documents).toBe(1);
+    expect(reads.scanned).toBe(1);
   });
 });
 
@@ -1091,7 +1091,7 @@ test('select() honors maxScan on an id list ordered by an index', async () => {
         .orderBy({ title: 'asc' })
         .map((row) => ({ onlyTitle: row.title }))
         .paginate({ cursor, limit: 2, maxScan: 2 });
-      perPageReads.push(reads.documents);
+      perPageReads.push(reads.scanned);
       walked.push(...result.page.map((row) => row.onlyTitle as string));
       cursor = result.continueCursor;
       if (result.isDone) {
@@ -1203,7 +1203,7 @@ test('select() flatMap runs off an id-only where without scanning', async () =>
 
     expect(rows.map((row) => row.text)).toEqual(['Bo0', 'Bo1', 'Bo2']);
     // 1 parent read by key + its 3 children. Ada is never touched.
-    expect(reads.documents).toBe(4);
+    expect(reads.scanned).toBe(4);
   });
 });