CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-08-03 20:31:50 +12:00
Code Issues Projects Releases Wiki Activity

Test deep pagination

Browse source
This commit is contained in:
adrinr 2023-03-16 13:37:17 +01:00
parent 30c66748af
commit 9a2eaaad42
2 changed files with 25 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
packages/backend-core/src/db/lucene.ts
View file

@ -58,6 +58,8 @@ export class QueryBuilder<T> {
#noEscaping = false
#skip?: number
static readonly maxLimit = 200
constructor(dbName: string, index: string, base?: SearchFilters) {
this.#dbName = dbName
this.#index = index
@ -459,7 +461,7 @@ export class QueryBuilder<T> {
buildSearchBody() {
let body: any = {
q: this.buildSearchQuery(),
limit: Math.min(this.#limit, 200),
limit: Math.min(this.#limit, QueryBuilder.maxLimit),
include_docs: this.#includeDocs,
}
if (this.#bookmark) {
@ -492,14 +494,13 @@ export class QueryBuilder<T> {
const prevLimit = this.#limit
this.excludeDocs()
const maxPageSize = 1000
let skipRemaining = skip
do {
const toSkip = Math.min(maxPageSize, skipRemaining)
const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining)
this.setLimit(toSkip)
const { bookmark } = await this.#execute()
const { bookmark, rows } = await this.#execute()
this.setBookmark(bookmark)
skipRemaining -= toSkip
skipRemaining -= rows.length
} while (skipRemaining > 0)
this.#includeDocs = prevIncludeDocs
@ -596,8 +597,8 @@ async function recursiveSearch<T>(
if (rows.length >= params.limit) {
return rows
}
let pageSize = 200
if (rows.length > params.limit - 200) {
let pageSize = QueryBuilder.maxLimit
if (rows.length > params.limit - QueryBuilder.maxLimit) {
pageSize = params.limit - rows.length
}
const page = await new QueryBuilder<T>(dbName, index, query)
@ -612,7 +613,7 @@ async function recursiveSearch<T>(
if (!page.rows.length) {
return rows
}
if (page.rows.length < 200) {
if (page.rows.length < QueryBuilder.maxLimit) {
return [...rows, ...page.rows]
}
const newParams = {
@ -650,7 +651,7 @@ export async function paginatedSearch<T>(
if (limit == null || isNaN(limit) || limit < 0) {
limit = 50
}
limit = Math.min(limit, 200)
limit = Math.min(limit, QueryBuilder.maxLimit)
const search = new QueryBuilder<T>(dbName, index, query)
if (params.version) {
search.setVersion(params.version)

16
packages/backend-core/src/db/tests/lucene.spec.ts
View file

@ -148,7 +148,7 @@ describe("lucene", () => {
beforeAll(async () => {
const db = getDB(skipDbName)
docs = Array(1500)
docs = Array(QueryBuilder.maxLimit * 2.5)
.fill(0)
.map((_, i) => ({
_id: i.toString().padStart(4, "0"),
@ -196,6 +196,20 @@ describe("lucene", () => {
docs.slice(50, 60).map(expect.objectContaining)
)
})
it("should be able to skip searching through multiple responses", async () => {
const builder = new QueryBuilder(skipDbName, INDEX_NAME)
// Skipping 2 max limits plus a little bit more
const skip = QueryBuilder.maxLimit * 2 + 37
builder.setSkip(skip)
builder.setSort("_id")
const resp = await builder.run()
expect(resp.rows.length).toBe(50)
expect(resp.rows).toEqual(
docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining)
)
})
})
})