From 9a2eaaad42223ecd824f5e0df7e33b35121c5e62 Mon Sep 17 00:00:00 2001
From: adrinr <adria@budibase.com>
Date: Thu, 16 Mar 2023 13:37:17 +0100
Subject: [PATCH] Test deep pagination

---
 packages/backend-core/src/db/lucene.ts        | 19 ++++++++++---------
 .../backend-core/src/db/tests/lucene.spec.ts  | 16 +++++++++++++++-
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts
index d8ea223402..b9463b12fb 100644
--- a/packages/backend-core/src/db/lucene.ts
+++ b/packages/backend-core/src/db/lucene.ts
@@ -58,6 +58,8 @@ export class QueryBuilder<T> {
   #noEscaping = false
   #skip?: number
 
+  static readonly maxLimit = 200
+
   constructor(dbName: string, index: string, base?: SearchFilters) {
     this.#dbName = dbName
     this.#index = index
@@ -459,7 +461,7 @@ export class QueryBuilder<T> {
   buildSearchBody() {
     let body: any = {
       q: this.buildSearchQuery(),
-      limit: Math.min(this.#limit, 200),
+      limit: Math.min(this.#limit, QueryBuilder.maxLimit),
       include_docs: this.#includeDocs,
     }
     if (this.#bookmark) {
@@ -492,14 +494,13 @@ export class QueryBuilder<T> {
     const prevLimit = this.#limit
 
     this.excludeDocs()
-    const maxPageSize = 1000
     let skipRemaining = skip
     do {
-      const toSkip = Math.min(maxPageSize, skipRemaining)
+      const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining)
       this.setLimit(toSkip)
-      const { bookmark } = await this.#execute()
+      const { bookmark, rows } = await this.#execute()
       this.setBookmark(bookmark)
-      skipRemaining -= toSkip
+      skipRemaining -= rows.length
     } while (skipRemaining > 0)
 
     this.#includeDocs = prevIncludeDocs
@@ -596,8 +597,8 @@ async function recursiveSearch<T>(
   if (rows.length >= params.limit) {
     return rows
   }
-  let pageSize = 200
-  if (rows.length > params.limit - 200) {
+  let pageSize = QueryBuilder.maxLimit
+  if (rows.length > params.limit - QueryBuilder.maxLimit) {
     pageSize = params.limit - rows.length
   }
   const page = await new QueryBuilder<T>(dbName, index, query)
@@ -612,7 +613,7 @@ async function recursiveSearch<T>(
   if (!page.rows.length) {
     return rows
   }
-  if (page.rows.length < 200) {
+  if (page.rows.length < QueryBuilder.maxLimit) {
     return [...rows, ...page.rows]
   }
   const newParams = {
@@ -650,7 +651,7 @@ export async function paginatedSearch<T>(
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
   }
-  limit = Math.min(limit, 200)
+  limit = Math.min(limit, QueryBuilder.maxLimit)
   const search = new QueryBuilder<T>(dbName, index, query)
   if (params.version) {
     search.setVersion(params.version)
diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts
index 3ac9b5febe..2e2fa9f2e7 100644
--- a/packages/backend-core/src/db/tests/lucene.spec.ts
+++ b/packages/backend-core/src/db/tests/lucene.spec.ts
@@ -148,7 +148,7 @@ describe("lucene", () => {
       beforeAll(async () => {
         const db = getDB(skipDbName)
 
-        docs = Array(1500)
+        docs = Array(QueryBuilder.maxLimit * 2.5)
           .fill(0)
           .map((_, i) => ({
             _id: i.toString().padStart(4, "0"),
@@ -196,6 +196,20 @@ describe("lucene", () => {
           docs.slice(50, 60).map(expect.objectContaining)
         )
       })
+
+      it("should be able to skip searching through multiple responses", async () => {
+        const builder = new QueryBuilder(skipDbName, INDEX_NAME)
+        // Skipping 2 max limits plus a little bit more
+        const skip = QueryBuilder.maxLimit * 2 + 37
+        builder.setSkip(skip)
+        builder.setSort("_id")
+        const resp = await builder.run()
+
+        expect(resp.rows.length).toBe(50)
+        expect(resp.rows).toEqual(
+          docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining)
+        )
+      })
     })
   })