From 212b93cbe9080f5af22ec5f55a66385cf5edab6a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 Mar 2023 19:00:22 +0000 Subject: [PATCH 1/2] Fix for not contains with all or - variety of changes needed to achieve the actual expected functionality. --- packages/backend-core/src/db/lucene.ts | 61 +++++++++++++++++++++----- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index cba2f0138a..a177d97bbe 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -236,6 +236,36 @@ export class QueryBuilder { return value } + isMultiCondition() { + let count = 0 + for (let filters of Object.values(this.query)) { + // not contains is one massive filter in allOr mode + if (typeof filters === "object") { + count += Object.keys(filters).length + } + } + return count > 1 + } + + compressFilters(filters: Record) { + const compressed: typeof filters = {} + for (let key of Object.keys(filters)) { + const finalKey = removeKeyNumbering(key) + if (compressed[finalKey]) { + compressed[finalKey] = compressed[finalKey].concat(filters[key]) + } else { + compressed[finalKey] = filters[key] + } + } + // add prefixes back + const final: typeof filters = {} + let count = 1 + for (let [key, value] of Object.entries(compressed)) { + final[`${count++}:${key}`] = value + } + return final + } + buildSearchQuery() { const builder = this let allOr = this.query && this.query.allOr @@ -272,9 +302,9 @@ export class QueryBuilder { } const notContains = (key: string, value: any) => { - // @ts-ignore - const allPrefix = allOr === "" ? "*:* AND" : "" - return allPrefix + "NOT " + contains(key, value) + const allPrefix = allOr ? "*:* AND " : "" + const mode = allOr ? "AND" : undefined + return allPrefix + "NOT " + contains(key, value, mode) } const containsAny = (key: string, value: any) => { @@ -299,21 +329,32 @@ export class QueryBuilder { return `${key}:(${orStatement})` } - function build(structure: any, queryFn: any) { + function build( + structure: any, + queryFn: (key: string, value: any) => string | null, + opts?: { returnBuilt?: boolean; mode?: string } + ) { + let built = "" for (let [key, value] of Object.entries(structure)) { // check for new format - remove numbering if needed key = removeKeyNumbering(key) key = builder.preprocess(builder.handleSpaces(key), { escape: true, }) - const expression = queryFn(key, value) + let expression = queryFn(key, value) if (expression == null) { continue } - if (query.length > 0) { - query += ` ${allOr ? "OR" : "AND"} ` + if (built.length > 0 || query.length > 0) { + const mode = opts?.mode ? opts.mode : allOr ? "OR" : "AND" + built += ` ${mode} ` } - query += expression + built += expression + } + if (opts?.returnBuilt) { + return built + } else { + query += built } } @@ -384,14 +425,14 @@ export class QueryBuilder { build(this.query.contains, contains) } if (this.query.notContains) { - build(this.query.notContains, notContains) + build(this.compressFilters(this.query.notContains), notContains) } if (this.query.containsAny) { build(this.query.containsAny, containsAny) } // make sure table ID is always added as an AND if (tableId) { - query = `(${query})` + query = this.isMultiCondition() ? `(${query})` : query allOr = false build({ tableId }, equal) } From bff6a51af28c6b782d5ed497768c94a12b751789 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 Mar 2023 19:15:13 +0000 Subject: [PATCH 2/2] Adding test case for scenario. --- packages/backend-core/src/db/lucene.ts | 4 +++ .../backend-core/src/db/tests/lucene.spec.ts | 28 +++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index a177d97bbe..71ce4ba9ac 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -199,6 +199,10 @@ export class QueryBuilder { return this } + setAllOr() { + this.query.allOr = true + } + handleSpaces(input: string) { if (this.noEscaping) { return input diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index 23b01e18df..52017cc94c 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -6,9 +6,13 @@ import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene" const INDEX_NAME = "main" const index = `function(doc) { - let props = ["property", "number"] + let props = ["property", "number", "array"] for (let key of props) { - if (doc[key]) { + if (Array.isArray(doc[key])) { + for (let val of doc[key]) { + index(key, val) + } + } else if (doc[key]) { index(key, doc[key]) } } @@ -21,9 +25,14 @@ describe("lucene", () => { dbName = `db-${newid()}` // create the DB for testing db = getDB(dbName) - await db.put({ _id: newid(), property: "word" }) - await db.put({ _id: newid(), property: "word2" }) - await db.put({ _id: newid(), property: "word3", number: 1 }) + await db.put({ _id: newid(), property: "word", array: ["1", "4"] }) + await db.put({ _id: newid(), property: "word2", array: ["3", "1"] }) + await db.put({ + _id: newid(), + property: "word3", + number: 1, + array: ["1", "2"], + }) }) it("should be able to create a lucene index", async () => { @@ -118,6 +127,15 @@ describe("lucene", () => { const resp = await builder.run() expect(resp.rows.length).toBe(2) }) + + it("should be able to perform an or not contains search", async () => { + const builder = new QueryBuilder(dbName, INDEX_NAME) + builder.addNotContains("array", ["1"]) + builder.addNotContains("array", ["2"]) + builder.setAllOr() + const resp = await builder.run() + expect(resp.rows.length).toBe(2) + }) }) describe("paginated search", () => {