Merge pull request #9998 from Budibase/fix/budi-6720-dev

Match any filter + not contains fix

packages/backend-core/src/db/lucene.ts

@@ -199,6 +199,10 @@ export class QueryBuilder<T> {
     return this
   }
 
+  setAllOr() {
+    this.query.allOr = true
+  }
+
   handleSpaces(input: string) {
     if (this.noEscaping) {
       return input
@@ -236,6 +240,36 @@ export class QueryBuilder<T> {
     return value
   }
 
+  isMultiCondition() {
+    let count = 0
+    for (let filters of Object.values(this.query)) {
+      // not contains is one massive filter in allOr mode
+      if (typeof filters === "object") {
+        count += Object.keys(filters).length
+      }
+    }
+    return count > 1
+  }
+
+  compressFilters(filters: Record<string, string[]>) {
+    const compressed: typeof filters = {}
+    for (let key of Object.keys(filters)) {
+      const finalKey = removeKeyNumbering(key)
+      if (compressed[finalKey]) {
+        compressed[finalKey] = compressed[finalKey].concat(filters[key])
+      } else {
+        compressed[finalKey] = filters[key]
+      }
+    }
+    // add prefixes back
+    const final: typeof filters = {}
+    let count = 1
+    for (let [key, value] of Object.entries(compressed)) {
+      final[`${count++}:${key}`] = value
+    }
+    return final
+  }
+
   buildSearchQuery() {
     const builder = this
     let allOr = this.query && this.query.allOr
@@ -272,9 +306,9 @@ export class QueryBuilder<T> {
     }
 
     const notContains = (key: string, value: any) => {
-      // @ts-ignore
+      const allPrefix = allOr ? "*:* AND " : ""
-      const allPrefix = allOr === "" ? "*:* AND" : ""
+      const mode = allOr ? "AND" : undefined
-      return allPrefix + "NOT " + contains(key, value)
+      return allPrefix + "NOT " + contains(key, value, mode)
     }
 
     const containsAny = (key: string, value: any) => {
@@ -299,21 +333,32 @@ export class QueryBuilder<T> {
       return `${key}:(${orStatement})`
     }
 
-    function build(structure: any, queryFn: any) {
+    function build(
+      structure: any,
+      queryFn: (key: string, value: any) => string | null,
+      opts?: { returnBuilt?: boolean; mode?: string }
+    ) {
+      let built = ""
       for (let [key, value] of Object.entries(structure)) {
         // check for new format - remove numbering if needed
         key = removeKeyNumbering(key)
         key = builder.preprocess(builder.handleSpaces(key), {
           escape: true,
         })
-        const expression = queryFn(key, value)
+        let expression = queryFn(key, value)
         if (expression == null) {
           continue
         }
-        if (query.length > 0) {
+        if (built.length > 0 || query.length > 0) {
-          query += ` ${allOr ? "OR" : "AND"} `
+          const mode = opts?.mode ? opts.mode : allOr ? "OR" : "AND"
+          built += ` ${mode} `
         }
-        query += expression
+        built += expression
+      }
+      if (opts?.returnBuilt) {
+        return built
+      } else {
+        query += built
       }
     }
 
@@ -384,14 +429,14 @@ export class QueryBuilder<T> {
       build(this.query.contains, contains)
     }
     if (this.query.notContains) {
-      build(this.query.notContains, notContains)
+      build(this.compressFilters(this.query.notContains), notContains)
     }
     if (this.query.containsAny) {
       build(this.query.containsAny, containsAny)
     }
     // make sure table ID is always added as an AND
     if (tableId) {
-      query = `(${query})`
+      query = this.isMultiCondition() ? `(${query})` : query
       allOr = false
       build({ tableId }, equal)
     }

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -6,9 +6,13 @@ import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
 const INDEX_NAME = "main"
 
 const index = `function(doc) {
-  let props = ["property", "number"]
+  let props = ["property", "number", "array"]
   for (let key of props) {
-    if (doc[key]) {
+    if (Array.isArray(doc[key])) {
+      for (let val of doc[key]) {
+        index(key, val)
+      }
+    } else if (doc[key]) {
       index(key, doc[key])
     }
   }
@@ -21,9 +25,14 @@ describe("lucene", () => {
     dbName = `db-${newid()}`
     // create the DB for testing
     db = getDB(dbName)
-    await db.put({ _id: newid(), property: "word" })
+    await db.put({ _id: newid(), property: "word", array: ["1", "4"] })
-    await db.put({ _id: newid(), property: "word2" })
+    await db.put({ _id: newid(), property: "word2", array: ["3", "1"] })
-    await db.put({ _id: newid(), property: "word3", number: 1 })
+    await db.put({
+      _id: newid(),
+      property: "word3",
+      number: 1,
+      array: ["1", "2"],
+    })
   })
 
   it("should be able to create a lucene index", async () => {
@@ -118,6 +127,15 @@ describe("lucene", () => {
       const resp = await builder.run()
       expect(resp.rows.length).toBe(2)
     })
+
+    it("should be able to perform an or not contains search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotContains("array", ["1"])
+      builder.addNotContains("array", ["2"])
+      builder.setAllOr()
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
   })
 
   describe("paginated search", () => {