From 3cea3dd510d3e0cf4b1b2004308b4e2a0f02a0d1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 6 Oct 2021 17:38:32 +0100
Subject: [PATCH] Fixing issue #2788 - making client side queries and SQL
 queries consistent with internal table ones.

---
 packages/builder/src/constants/lucene.js      |  9 +++++++
 packages/builder/src/helpers/lucene.js        | 25 +++++++++++++++++++
 .../src/api/controllers/row/external.js       | 14 +++++++++++
 packages/server/src/constants/index.js        | 23 +++++++++++++++++
 packages/server/src/integrations/base/sql.ts  |  6 +++++
 5 files changed, 77 insertions(+)

diff --git a/packages/builder/src/constants/lucene.js b/packages/builder/src/constants/lucene.js
index 00da0c29bc..132790739c 100644
--- a/packages/builder/src/constants/lucene.js
+++ b/packages/builder/src/constants/lucene.js
@@ -44,6 +44,15 @@ export const OperatorOptions = {
   },
 }
 
+export const NoEmptyFilterStrings = [
+  OperatorOptions.StartsWith.value,
+  OperatorOptions.Like.value,
+  OperatorOptions.Equals.value,
+  OperatorOptions.NotEquals.value,
+  OperatorOptions.Contains.value,
+  OperatorOptions.NotContains.value,
+]
+
 /**
  * Returns the valid operator options for a certain data type
  * @param type the data type
diff --git a/packages/builder/src/helpers/lucene.js b/packages/builder/src/helpers/lucene.js
index 03baa751cc..d344b462d1 100644
--- a/packages/builder/src/helpers/lucene.js
+++ b/packages/builder/src/helpers/lucene.js
@@ -1,3 +1,26 @@
+import { NoEmptyFilterStrings } from "../constants/lucene"
+
+/**
+ * Removes any fields that contain empty strings that would cause inconsistent
+ * behaviour with how backend tables are filtered (no value means no filter).
+ */
+function cleanupQuery(query) {
+  if (!query) {
+    return query
+  }
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!query[filterField]) {
+      continue
+    }
+    for (let [key, value] of Object.entries(query[filterField])) {
+      if (!value || value === "") {
+        delete query[filterField][key]
+      }
+    }
+  }
+  return query
+}
+
 /**
  * Builds a lucene JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
@@ -76,6 +99,8 @@ export const luceneQuery = (docs, query) => {
   if (!query) {
     return docs
   }
+  // make query consistent first
+  query = cleanupQuery(query)
 
   // Iterates over a set of filters and evaluates a fail function against a doc
   const match = (type, failFn) => doc => {
diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js
index 3a96064a9f..534f2207b0 100644
--- a/packages/server/src/api/controllers/row/external.js
+++ b/packages/server/src/api/controllers/row/external.js
@@ -2,6 +2,7 @@ const {
   DataSourceOperation,
   SortDirection,
   FieldTypes,
+  NoEmptyFilterStrings,
 } = require("../../../constants")
 const {
   breakExternalTableId,
@@ -11,6 +12,19 @@ const ExternalRequest = require("./ExternalRequest")
 const CouchDB = require("../../../db")
 
 async function handleRequest(appId, operation, tableId, opts = {}) {
+  // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
+  if (opts && opts.filters) {
+    for (let filterField of NoEmptyFilterStrings) {
+      if (!opts.filters[filterField]) {
+        continue
+      }
+      for (let [key, value] of Object.entries(opts.filters[filterField])) {
+        if (!value || value === "") {
+          delete opts.filters[filterField][key]
+        }
+      }
+    }
+  }
   return new ExternalRequest(appId, operation, tableId, opts.datasource).run(
     opts
   )
diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js
index bea58fd260..d19f9ff313 100644
--- a/packages/server/src/constants/index.js
+++ b/packages/server/src/constants/index.js
@@ -6,6 +6,29 @@ exports.JobQueues = {
   AUTOMATIONS: "automationQueue",
 }
 
+const FilterTypes = {
+  STRING: "string",
+  FUZZY: "fuzzy",
+  RANGE: "range",
+  EQUAL: "equal",
+  NOT_EQUAL: "notEqual",
+  EMPTY: "empty",
+  NOT_EMPTY: "notEmpty",
+  CONTAINS: "contains",
+  NOT_CONTAINS: "notContains",
+  ONE_OF: "oneOf",
+}
+
+exports.FilterTypes = FilterTypes
+exports.NoEmptyFilterStrings = [
+  FilterTypes.STRING,
+  FilterTypes.FUZZY,
+  FilterTypes.EQUAL,
+  FilterTypes.NOT_EQUAL,
+  FilterTypes.CONTAINS,
+  FilterTypes.NOT_CONTAINS,
+]
+
 exports.FieldTypes = {
   STRING: "string",
   LONGFORM: "longform",
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 6bd2137f99..a12d8c3c08 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -55,6 +55,12 @@ function addFilters(
       query = query[fnc](key, "ilike", `${value}%`)
     })
   }
+  if (filters.fuzzy) {
+    iterate(filters.fuzzy, (key, value) => {
+      const fnc = allOr ? "orWhere" : "where"
+      query = query[fnc](key, "ilike", `%${value}%`)
+    })
+  }
   if (filters.range) {
     iterate(filters.range, (key, value) => {
       if (!value.high || !value.low) {