Move when filter empty option into filter drawer (#11262)

* Add when filter empty dropdown

* Add unit tests

* "fix" typescript issue

* Add empty filter check for external

* Add unit test

* Hide empty filter option for custom query

* Make onEmptyFilter optional

* Fix unit tests

* Remove onEmptyFilter automation input

* Remove unused var

* Refactor

* Fix path

* Fix type issue

* Fix types

packages/backend-core/src/db/lucene.ts

@@ -1,7 +1,6 @@
 import fetch from "node-fetch"
 import { getCouchInfo } from "./couch"
-import { SearchFilters, Row } from "@budibase/types"
+import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
-import { createUserIndex } from "./searchIndexes/searchIndexes"
 
 const QUERY_START_REGEX = /\d[0-9]*:/g
 
@@ -65,6 +64,7 @@ export class QueryBuilder<T> {
     this.#index = index
     this.#query = {
       allOr: false,
+      onEmptyFilter: EmptyFilterOption.RETURN_ALL,
       string: {},
       fuzzy: {},
       range: {},
@@ -218,6 +218,10 @@ export class QueryBuilder<T> {
     this.#query.allOr = true
   }
 
+  setOnEmptyFilter(value: EmptyFilterOption) {
+    this.#query.onEmptyFilter = value
+  }
+
   handleSpaces(input: string) {
     if (this.#noEscaping) {
       return input
@@ -289,8 +293,9 @@ export class QueryBuilder<T> {
     const builder = this
     let allOr = this.#query && this.#query.allOr
     let query = allOr ? "" : "*:*"
+    let allFiltersEmpty = true
     const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
-    let tableId
+    let tableId: string = ""
     if (this.#query.equal!.tableId) {
       tableId = this.#query.equal!.tableId
       delete this.#query.equal!.tableId
@@ -305,7 +310,7 @@ export class QueryBuilder<T> {
     }
 
     const contains = (key: string, value: any, mode = "AND") => {
-      if (Array.isArray(value) && value.length === 0) {
+      if (!value || (Array.isArray(value) && value.length === 0)) {
         return null
       }
       if (!Array.isArray(value)) {
@@ -384,6 +389,12 @@ export class QueryBuilder<T> {
           built += ` ${mode} `
         }
         built += expression
+        if (
+          (typeof value !== "string" && value != null) ||
+          (typeof value === "string" && value !== tableId && value !== "")
+        ) {
+          allFiltersEmpty = false
+        }
       }
       if (opts?.returnBuilt) {
         return built
@@ -463,6 +474,13 @@ export class QueryBuilder<T> {
       allOr = false
       build({ tableId }, equal)
     }
+    if (allFiltersEmpty) {
+      if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) {
+        return ""
+      } else if (this.#query?.allOr) {
+        return query.replace("()", "(*:*)")
+      }
+    }
     return query
   }
 

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -1,6 +1,6 @@
 import { newid } from "../../docIds/newid"
 import { getDB } from "../db"
-import { Database } from "@budibase/types"
+import { Database, EmptyFilterOption } from "@budibase/types"
 import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
 
 const INDEX_NAME = "main"
@@ -156,6 +156,76 @@ describe("lucene", () => {
       expect(resp.rows.length).toBe(2)
     })
 
+    describe("empty filters behaviour", () => {
+      it("should return all rows by default", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(3)
+      })
+
+      it("should return all rows when onEmptyFilter is ALL", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL)
+        builder.setAllOr()
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(3)
+      })
+
+      it("should return no rows when onEmptyFilter is NONE", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
+        builder.addEqual("property", "")
+        builder.addEqual("number", null)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(0)
+      })
+
+      it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => {
+        const builder = new QueryBuilder(dbName, INDEX_NAME)
+        builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
+        builder.addEqual("property", "")
+        builder.addEqual("number", 1)
+        builder.addString("property", "")
+        builder.addFuzzy("property", "")
+        builder.addNotEqual("number", undefined)
+        builder.addOneOf("number", null)
+        builder.addContains("array", undefined)
+        builder.addNotContains("array", null)
+        builder.addContainsAny("array", null)
+
+        const resp = await builder.run()
+        expect(resp.rows.length).toBe(1)
+      })
+    })
+
     describe("skip", () => {
       const skipDbName = `db-${newid()}`
       let docs: {

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -35,22 +35,28 @@
     { value: "and", label: "Match all filters" },
     { value: "or", label: "Match any filter" },
   ]
+  const onEmptyOptions = [
+    { value: "all", label: "Return all table rows" },
+    { value: "none", label: "Return no rows" },
+  ]
 
   let rawFilters
   let matchAny = false
+  let onEmptyFilter = "all"
 
   $: parseFilters(filters)
-  $: dispatch("change", enrichFilters(rawFilters, matchAny))
+  $: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter))
   $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
   $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
   $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
-  // Remove field key prefixes and determine whether to use the "match all"
+  // Remove field key prefixes and determine which behaviours to use
-  // or "match any" behaviour
   const parseFilters = filters => {
     matchAny = filters?.find(filter => filter.operator === "allOr") != null
+    onEmptyFilter =
+      filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
     rawFilters = (filters || [])
-      .filter(filter => filter.operator !== "allOr")
+      .filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter)
       .map(filter => {
         const { field } = filter
         let newFilter = { ...filter }
@@ -74,8 +80,8 @@
   })
 
   // Add field key prefixes and a special metadata filter object to indicate
-  // whether to use the "match all" or "match any" behaviour
+  // how to handle filter behaviour
-  const enrichFilters = (rawFilters, matchAny) => {
+  const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => {
     let count = 1
     return rawFilters
       .filter(filter => filter.field)
@@ -84,6 +90,7 @@
         field: `${count++}:${filter.field}`,
       }))
       .concat(matchAny ? [{ operator: "allOr" }] : [])
+      .concat([{ onEmptyFilter }])
   }
 
   const addFilter = () => {
@@ -195,6 +202,17 @@
             on:change={e => (matchAny = e.detail === "or")}
             placeholder={null}
           />
+          {#if datasource?.type === "table"}
+            <Select
+              label="When filter empty"
+              value={onEmptyFilter}
+              options={onEmptyOptions}
+              getOptionLabel={opt => opt.label}
+              getOptionValue={opt => opt.value}
+              on:change={e => (onEmptyFilter = e.detail)}
+              placeholder={null}
+            />
+          {/if}
         </div>
         <div>
           <div class="filter-label">

packages/server/jest.config.ts

@@ -12,6 +12,7 @@ const baseConfig: Config.InitialProjectOptions = {
   },
   moduleNameMapper: {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
+    "@budibase/shared-core/(.*)": "<rootDir>/../shared-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/shared-core": "<rootDir>/../shared-core/src",
     "@budibase/types": "<rootDir>/../types/src",

packages/server/src/api/controllers/row/external.ts

@@ -13,8 +13,10 @@ import {
   Row,
   Table,
   UserCtx,
+  EmptyFilterOption,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import { hasFilters } from "@budibase/shared-core/src/filters"
 import * as utils from "./utils"
 
 export async function handleRequest(
@@ -38,6 +40,13 @@ export async function handleRequest(
     }
   }
 
+  if (
+    !hasFilters(opts?.filters) &&
+    opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
+  ) {
+    return []
+  }
+
   return new ExternalRequest(operation, tableId, opts?.datasource).run(
     opts || {}
   )

packages/server/src/automations/steps/queryRows.ts

@@ -11,6 +11,7 @@ import {
   AutomationStepInput,
   AutomationStepSchema,
   AutomationStepType,
+  EmptyFilterOption,
   SearchFilters,
   Table,
 } from "@budibase/types"
@@ -26,16 +27,6 @@ const SortOrderPretty = {
   [SortOrder.DESCENDING]: "Descending",
 }
 
-enum EmptyFilterOption {
-  RETURN_ALL = "all",
-  RETURN_NONE = "none",
-}
-
-const EmptyFilterOptionPretty = {
-  [EmptyFilterOption.RETURN_ALL]: "Return all table rows",
-  [EmptyFilterOption.RETURN_NONE]: "Return no rows",
-}
-
 export const definition: AutomationStepSchema = {
   description: "Query rows from the database",
   icon: "Search",
@@ -77,12 +68,6 @@ export const definition: AutomationStepSchema = {
           title: "Limit",
           customType: AutomationCustomIOType.QUERY_LIMIT,
         },
-        onEmptyFilter: {
-          pretty: Object.values(EmptyFilterOptionPretty),
-          enum: Object.values(EmptyFilterOption),
-          type: AutomationIOType.STRING,
-          title: "When Filter Empty",
-        },
       },
       required: ["tableId"],
     },

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -2,6 +2,7 @@ import { GenericContainer } from "testcontainers"
 
 import {
   Datasource,
+  EmptyFilterOption,
   FieldType,
   Row,
   SourceName,
@@ -123,6 +124,22 @@ describe.skip("external", () => {
       })
     })
 
+    it("empty filters search returns no data", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tableId = config.table!._id!
+
+        const searchParams: SearchParams = {
+          tableId,
+          query: {
+            onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+          },
+        }
+        const result = await search(searchParams)
+
+        expect(result.rows).toHaveLength(0)
+      })
+    })
+
     it("querying by fields will always return data attribute columns", async () => {
       await config.doInContext(config.appId, async () => {
         const tableId = config.table!._id!

packages/shared-core/src/filters.ts

@@ -1,11 +1,11 @@
 import {
   Datasource,
   FieldType,
+  SortDirection,
+  SortType,
   SearchFilter,
   SearchQuery,
   SearchQueryFields,
-  SortDirection,
-  SortType,
 } from "@budibase/types"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet } from "./helpers"
@@ -138,7 +138,8 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
   }
   if (Array.isArray(filter)) {
     filter.forEach(expression => {
-      let { operator, field, type, value, externalType } = expression
+      let { operator, field, type, value, externalType, onEmptyFilter } =
+        expression
       const isHbs =
         typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
       // Parse all values into correct types
@@ -146,6 +147,10 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
         query.allOr = true
         return
       }
+      if (onEmptyFilter) {
+        query.onEmptyFilter = onEmptyFilter
+        return
+      }
       if (
         type === "datetime" &&
         !isHbs &&
@@ -203,7 +208,7 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
         ) {
           query.range[field].high = value
         }
-      } else if (query[operator]) {
+      } else if (query[operator] && operator !== "onEmptyFilter") {
         if (type === "boolean") {
           // Transform boolean filters to cope with null.
           // "equals false" needs to be "not equals true"
@@ -418,7 +423,7 @@ export const hasFilters = (query?: SearchQuery) => {
   if (!query) {
     return false
   }
-  const skipped = ["allOr"]
+  const skipped = ["allOr", "onEmptyFilter"]
   for (let [key, value] of Object.entries(query)) {
     if (skipped.includes(key) || typeof value !== "object") {
       continue

packages/types/src/api/web/searchFilter.ts

@@ -1,7 +1,9 @@
 import { FieldType } from "../../documents"
+import { EmptyFilterOption } from "../../sdk"
 
 export type SearchFilter = {
   operator: keyof SearchQuery
+  onEmptyFilter?: EmptyFilterOption
   field: string
   type?: FieldType
   value: any
@@ -10,6 +12,7 @@ export type SearchFilter = {
 
 export type SearchQuery = {
   allOr?: boolean
+  onEmptyFilter?: EmptyFilterOption
   string?: {
     [key: string]: string
   }
@@ -48,4 +51,4 @@ export type SearchQuery = {
   }
 }
 
-export type SearchQueryFields = Omit<SearchQuery, "allOr">
+export type SearchQueryFields = Omit<SearchQuery, "allOr" | "onEmptyFilter">

packages/types/src/sdk/search.ts

@@ -4,6 +4,7 @@ import { SortType } from "../api"
 
 export interface SearchFilters {
   allOr?: boolean
+  onEmptyFilter?: EmptyFilterOption
   string?: {
     [key: string]: string
   }
@@ -99,3 +100,8 @@ export interface SqlQuery {
   sql: string
   bindings?: string[]
 }
+
+export enum EmptyFilterOption {
+  RETURN_ALL = "all",
+  RETURN_NONE = "none",
+}