Merge pull request #2064 from Budibase/fix/lucene-filtering

Fix lucene filtering

packages/auth/constants.js

@@ -0,0 +1 @@
+module.exports = require("./src/constants")

packages/auth/src/constants.js

@@ -8,6 +8,13 @@ exports.Cookies = {
   Auth: "budibase:auth",
 }
 
+exports.Headers = {
+  API_KEY: "x-budibase-api-key",
+  API_VER: "x-budibase-api-version",
+  APP_ID: "x-budibase-app-id",
+  TYPE: "x-budibase-type",
+}
+
 exports.GlobalRoles = {
   OWNER: "owner",
   ADMIN: "admin",

packages/auth/src/middleware/authenticated.js

@@ -1,4 +1,4 @@
-const { Cookies } = require("../constants")
+const { Cookies, Headers } = require("../constants")
 const database = require("../db")
 const { getCookie, clearCookie } = require("../utils")
 const { StaticDatabases } = require("../db/utils")
@@ -22,15 +22,17 @@ function buildNoAuthRegex(patterns) {
   })
 }
 
-function finalise(ctx, { authenticated, user, internal } = {}) {
+function finalise(ctx, { authenticated, user, internal, version } = {}) {
   ctx.isAuthenticated = authenticated || false
   ctx.user = user
   ctx.internal = internal || false
+  ctx.version = version
 }
 
 module.exports = (noAuthPatterns = [], opts) => {
   const noAuthOptions = noAuthPatterns ? buildNoAuthRegex(noAuthPatterns) : []
   return async (ctx, next) => {
+    const version = ctx.request.headers[Headers.API_VER]
     // the path is not authenticated
     const found = noAuthOptions.find(({ regex, method }) => {
       return (
@@ -58,7 +60,7 @@ module.exports = (noAuthPatterns = [], opts) => {
           clearCookie(ctx, Cookies.Auth)
         }
       }
-      const apiKey = ctx.request.headers["x-budibase-api-key"]
+      const apiKey = ctx.request.headers[Headers.API_KEY]
       // this is an internal request, no user made it
       if (!authenticated && apiKey && apiKey === env.INTERNAL_API_KEY) {
         authenticated = true
@@ -69,12 +71,12 @@ module.exports = (noAuthPatterns = [], opts) => {
         authenticated = false
       }
       // isAuthenticated is a function, so use a variable to be able to check authed state
-      finalise(ctx, { authenticated, user, internal })
+      finalise(ctx, { authenticated, user, internal, version })
       return next()
     } catch (err) {
       // allow configuring for public access
       if (opts && opts.publicAllowed) {
-        finalise(ctx, { authenticated: false })
+        finalise(ctx, { authenticated: false, version })
       } else {
         ctx.throw(err.status || 403, err)
       }

packages/auth/src/utils.js

@@ -8,6 +8,7 @@ const jwt = require("jsonwebtoken")
 const { options } = require("./middleware/passport/jwt")
 const { createUserEmailView } = require("./db/views")
 const { getDB } = require("./db")
+const { Headers } = require("./constants")
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -23,7 +24,7 @@ function confirmAppId(possibleAppId) {
  * @returns {string|undefined} If an appId was found it will be returned.
  */
 exports.getAppId = ctx => {
-  const options = [ctx.headers["x-budibase-app-id"], ctx.params.appId]
+  const options = [ctx.headers[Headers.APP_ID], ctx.params.appId]
   if (ctx.subdomains) {
     options.push(ctx.subdomains[1])
   }
@@ -102,7 +103,7 @@ exports.clearCookie = (ctx, name) => {
  * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder).
  */
 exports.isClient = ctx => {
-  return ctx.headers["x-budibase-type"] === "client"
+  return ctx.headers[Headers.TYPE] === "client"
 }
 
 /**

packages/client/src/api/api.js

@@ -1,7 +1,9 @@
+import { notificationStore } from "../store"
+import { ApiVersion } from "../constants"
+
 /**
  * API cache for cached request responses.
  */
-import { notificationStore } from "../store"
 let cache = {}
 
 /**
@@ -22,6 +24,7 @@ const makeApiCall = async ({ method, url, body, json = true }) => {
     const headers = {
       Accept: "application/json",
       "x-budibase-app-id": window["##BUDIBASE_APP_ID##"],
+      "x-budibase-api-version": ApiVersion,
       ...(json && { "Content-Type": "application/json" }),
       ...(!inBuilder && { "x-budibase-type": "client" }),
     }

packages/client/src/constants.js

@@ -7,3 +7,11 @@ export const ActionTypes = {
   RefreshDatasource: "RefreshDatasource",
   SetDataProviderQuery: "SetDataProviderQuery",
 }
+
+export const ApiVersion = "1"
+
+/**
+ * API Version Changelog
+ * v1:
+ *   - Coerce types for search endpoint
+ */

packages/server/src/api/controllers/row/internal.js

@@ -278,6 +278,7 @@ exports.search = async ctx => {
   const { tableId } = ctx.params
   const db = new CouchDB(appId)
   const { paginate, query, ...params } = ctx.request.body
+  params.version = ctx.version
   params.tableId = tableId
 
   let response

packages/server/src/api/controllers/row/internalSearch.js

@@ -2,16 +2,6 @@ const { SearchIndexes } = require("../../../db/utils")
 const env = require("../../../environment")
 const fetch = require("node-fetch")
 
-/**
- * Escapes any characters in a string which lucene searches require to be
- * escaped.
- * @param value The value to escape
- * @returns {string}
- */
-const luceneEscape = value => {
-  return `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
-}
-
 /**
  * Class to build lucene query URLs.
  * Optionally takes a base lucene query object.
@@ -33,6 +23,12 @@ class QueryBuilder {
     this.sortOrder = "ascending"
     this.sortType = "string"
     this.includeDocs = true
+    this.version = null
+  }
+
+  setVersion(version) {
+    this.version = version
+    return this
   }
 
   setTable(tableId) {
@@ -108,12 +104,43 @@ class QueryBuilder {
     return this
   }
 
+  /**
+   * Preprocesses a value before going into a lucene search.
+   * Transforms strings to lowercase and wraps strings and bools in quotes.
+   * @param value The value to process
+   * @param options The preprocess options
+   * @returns {string|*}
+   */
+  preprocess(value, { escape, lowercase, wrap } = {}) {
+    const hasVersion = !!this.version
+    // Determine if type needs wrapped
+    const originalType = typeof value
+    // Convert to lowercase
+    if (value && lowercase) {
+      value = value.toLowerCase ? value.toLowerCase() : value
+    }
+    // Escape characters
+    if (escape && originalType === "string") {
+      value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
+    }
+    // Wrap in quotes
+    if (hasVersion && wrap) {
+      value = originalType === "number" ? value : `"${value}"`
+    }
+    return value
+  }
+
   buildSearchQuery() {
+    const builder = this
     let query = "*:*"
+    const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
 
     function build(structure, queryFn) {
       for (let [key, value] of Object.entries(structure)) {
-        const expression = queryFn(luceneEscape(key.replace(/ /, "_")), value)
+        key = builder.preprocess(key.replace(/ /, "_"), {
+          escape: true,
+        })
+        const expression = queryFn(key, value)
         if (expression == null) {
           continue
         }
@@ -124,7 +151,14 @@ class QueryBuilder {
     // Construct the actual lucene search query string from JSON structure
     if (this.query.string) {
       build(this.query.string, (key, value) => {
-        return value ? `${key}:${luceneEscape(value.toLowerCase())}*` : null
+        if (!value) {
+          return null
+        }
+        value = builder.preprocess(value, {
+          escape: true,
+          lowercase: true,
+        })
+        return `${key}:${value}*`
       })
     }
     if (this.query.range) {
@@ -138,30 +172,37 @@ class QueryBuilder {
         if (value.high == null || value.high === "") {
           return null
         }
-        return `${key}:[${value.low} TO ${value.high}]`
+        const low = builder.preprocess(value.low, allPreProcessingOpts)
+        const high = builder.preprocess(value.high, allPreProcessingOpts)
+        return `${key}:[${low} TO ${high}]`
       })
     }
     if (this.query.fuzzy) {
       build(this.query.fuzzy, (key, value) => {
-        return value ? `${key}:${luceneEscape(value.toLowerCase())}~` : null
+        if (!value) {
+          return null
+        }
+        value = builder.preprocess(value, {
+          escape: true,
+          lowercase: true,
+        })
+        return `${key}:${value}~`
       })
     }
     if (this.query.equal) {
       build(this.query.equal, (key, value) => {
-        const escapedValue = luceneEscape(value.toLowerCase())
-        // have to do the or to manage straight values, or strings
-        return value
-          ? `(${key}:${escapedValue} OR ${key}:"${escapedValue}")`
-          : null
+        if (!value) {
+          return null
+        }
+        return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
       })
     }
     if (this.query.notEqual) {
       build(this.query.notEqual, (key, value) => {
-        const escapedValue = luceneEscape(value.toLowerCase())
-        // have to do the or to manage straight values, or strings
-        return value
-          ? `(!${key}:${escapedValue} OR !${key}:"${escapedValue}")`
-          : null
+        if (!value) {
+          return null
+        }
+        return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}`
       })
     }
     if (this.query.empty) {
@@ -250,6 +291,7 @@ const recursiveSearch = async (appId, query, params) => {
     pageSize = params.limit - rows.length
   }
   const page = await new QueryBuilder(appId, query)
+    .setVersion(params.version)
     .setTable(params.tableId)
     .setBookmark(bookmark)
     .setLimit(pageSize)
@@ -294,6 +336,7 @@ exports.paginatedSearch = async (appId, query, params) => {
   }
   limit = Math.min(limit, 200)
   const search = new QueryBuilder(appId, query)
+    .setVersion(params.version)
     .setTable(params.tableId)
     .setSort(params.sort)
     .setSortOrder(params.sortOrder)

packages/server/src/middleware/currentapp.js

@@ -12,7 +12,12 @@ module.exports = async (ctx, next) => {
   // try to get the appID from the request
   const requestAppId = getAppId(ctx)
   // get app cookie if it exists
-  const appCookie = getCookie(ctx, Cookies.CurrentApp)
+  let appCookie = null
+  try {
+    appCookie = getCookie(ctx, Cookies.CurrentApp)
+  } catch (err) {
+    clearCookie(ctx, Cookies.CurrentApp)
+  }
   if (!appCookie && !requestAppId) {
     return next()
   }

packages/server/src/tests/utilities/TestConfiguration.js

@@ -14,7 +14,7 @@ const {
 const controllers = require("./controllers")
 const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
-const { Cookies } = require("@budibase/auth").constants
+const { Cookies, Headers } = require("@budibase/auth").constants
 const { jwt } = require("@budibase/auth").auth
 const { StaticDatabases } = require("@budibase/auth/db")
 const CouchDB = require("../../db")
@@ -118,7 +118,7 @@ class TestConfiguration {
       ],
     }
     if (this.appId) {
-      headers["x-budibase-app-id"] = this.appId
+      headers[Headers.APP_ID] = this.appId
     }
     return headers
   }
@@ -128,7 +128,7 @@ class TestConfiguration {
       Accept: "application/json",
     }
     if (this.appId) {
-      headers["x-budibase-app-id"] = this.appId
+      headers[Headers.APP_ID] = this.appId
     }
     return headers
   }
@@ -349,7 +349,7 @@ class TestConfiguration {
         `${Cookies.Auth}=${authToken}`,
         `${Cookies.CurrentApp}=${appToken}`,
       ],
-      "x-budibase-app-id": this.appId,
+      [Headers.APP_ID]: this.appId,
     }
   }
 }

packages/server/src/utilities/workerRequests.js

@@ -3,13 +3,14 @@ const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
 const { getDeployedAppID } = require("@budibase/auth/db")
 const { updateAppRole, getGlobalUser } = require("./global")
+const { Headers } = require("@budibase/auth/constants")
 
 function request(ctx, request, noApiKey) {
   if (!request.headers) {
     request.headers = {}
   }
   if (!noApiKey) {
-    request.headers["x-budibase-api-key"] = env.INTERNAL_API_KEY
+    request.headers[Headers.API_KEY] = env.INTERNAL_API_KEY
   }
   if (request.body && Object.keys(request.body).length > 0) {
     request.headers["Content-Type"] = "application/json"

packages/standard-components/src/lucene.js

@@ -15,10 +15,16 @@ export const buildLuceneQuery = filter => {
   if (Array.isArray(filter)) {
     filter.forEach(expression => {
       let { operator, field, type, value } = expression
-      // Ensure date fields are transformed into ISO strings
+      // Parse all values into correct types
       if (type === "datetime" && value) {
         value = new Date(value).toISOString()
       }
+      if (type === "number") {
+        value = parseFloat(value)
+      }
+      if (type === "boolean") {
+        value = value?.toLowerCase() === "true"
+      }
       if (operator.startsWith("range")) {
         if (!query.range[field]) {
           query.range[field] = {
@@ -42,10 +48,10 @@ export const buildLuceneQuery = filter => {
           // Transform boolean filters to cope with null.
           // "equals false" needs to be "not equals true"
           // "not equals false" needs to be "equals true"
-          if (operator === "equal" && value === "false") {
-            query.notEqual[field] = "true"
-          } else if (operator === "notEqual" && value === "false") {
-            query.equal[field] = "true"
+          if (operator === "equal" && value === false) {
+            query.notEqual[field] = true
+          } else if (operator === "notEqual" && value === false) {
+            query.equal[field] = true
           } else {
             query[operator][field] = value
           }