Fix filter drawer and filtering in general across data, design and automate and update to support keyed fields and 'match any'

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -38,13 +38,15 @@
   export let testData
   export let schemaProperties
   export let isTestModal = false
+
   let webhookModal
   let drawer
-  let tempFilters = lookForFilters(schemaProperties) || []
   let fillWidth = true
   let codeBindingOpen = false
   let inputData
 
+  $: filters = lookForFilters(schemaProperties) || []
+  $: tempFilters = filters
   $: stepId = block.stepId
   $: bindings = getAvailableBindings(
     block || $automationStore.selectedBlock,
@@ -222,16 +224,17 @@
       {:else if value.customType === "filters"}
         <ActionButton on:click={drawer.show}>Define filters</ActionButton>
         <Drawer bind:this={drawer} {fillWidth} title="Filtering">
-          <Button cta slot="buttons" on:click={() => saveFilters(key)}
-            >Save</Button
-          >
+          <Button cta slot="buttons" on:click={() => saveFilters(key)}>
+            Save
+          </Button>
           <FilterDrawer
             slot="body"
-            bind:filters={tempFilters}
+            {filters}
             {bindings}
             {schemaFields}
             panel={AutomationBindingPanel}
             fillWidth
+            on:change={e => (tempFilters = e.detail)}
           />
         </Drawer>
       {:else if value.customType === "password"}

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -25,6 +25,7 @@
   import { API } from "api"
 
   let hideAutocolumns = true
+  let filters
 
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
   $: type = $tables.selected?.type
@@ -102,8 +103,9 @@
 
   // Fetch data whenever filters change
   const onFilter = e => {
+    filters = e.detail
     fetch.update({
-      filter: e.detail,
+      filter: filters,
     })
   }
 
@@ -178,6 +180,7 @@
         {#key id}
           <TableFilterButton
             {schema}
+            {filters}
             on:change={onFilter}
             disabled={!hasCols}
           />

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -8,9 +8,10 @@
   export let disabled = false
 
   const dispatch = createEventDispatcher()
-  let modal
-  let tempValue = filters || []
 
+  let modal
+
+  $: tempValue = filters || []
   $: schemaFields = Object.values(schema || {})
 </script>
 
@@ -34,8 +35,9 @@
     <div class="wrapper">
       <FilterDrawer
         allowBindings={false}
-        bind:filters={tempValue}
+        {filters}
         {schemaFields}
+        on:change={e => (tempValue = e.detail)}
       />
     </div>
   </ModalContent>

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -17,22 +17,30 @@
   import { generate } from "shortid"
   import { LuceneUtils, Constants } from "@budibase/frontend-core"
   import { getFields } from "helpers/searchFields"
-  import { createEventDispatcher, onMount } from "svelte"
-
-  const dispatch = createEventDispatcher()
-  const { OperatorOptions } = Constants
-  const { getValidOperatorsForType } = LuceneUtils
+  import { createEventDispatcher } from "svelte"
 
   export let schemaFields
   export let filters = []
   export let bindings = []
   export let panel = ClientBindingPanel
   export let allowBindings = true
-  export let allOr = false
   export let fillWidth = false
   export let tableId
 
-  $: dispatch("change", filters)
+  const dispatch = createEventDispatcher()
+  const { OperatorOptions } = Constants
+  const { getValidOperatorsForType } = LuceneUtils
+  const KeyedFieldRegex = /\d[0-9]*:/g
+  const behaviourOptions = [
+    { value: "and", label: "Match all of the following filters" },
+    { value: "or", label: "Match any of the following filters" },
+  ]
+
+  let rawFilters
+  let matchAny = false
+
+  $: parseFilters(filters)
+  $: dispatch("change", enrichFilters(rawFilters, matchAny))
   $: enrichedSchemaFields = getFields(
     schemaFields || [],
     { allowLinks: true },
@@ -41,14 +49,41 @@
   $: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
   $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
-  let behaviourValue
-  const behaviourOptions = [
-    { value: "and", label: "Match all of the following filters" },
-    { value: "or", label: "Match any of the following filters" },
-  ]
+  // Remove field key prefixes and determine whether to use the "match all"
+  // or "match any" behaviour
+  const parseFilters = filters => {
+    matchAny = filters?.find(filter => filter.operator === "allOr") != null
+    rawFilters = (filters || [])
+      .filter(filter => filter.operator !== "allOr")
+      .map(filter => {
+        const { field } = filter
+        let newFilter = { ...filter }
+        delete newFilter.allOr
+        if (typeof field === "string" && field.match(KeyedFieldRegex) != null) {
+          const parts = field.split(":")
+          parts.shift()
+          newFilter.field = parts.join(":")
+        }
+        return newFilter
+      })
+  }
+
+  // Add field key prefixes and a special metadata filter object to indicate
+  // whether to use the "match all" or "match any" behaviour
+  const enrichFilters = (rawFilters, matchAny) => {
+    let count = 1
+    return rawFilters
+      .filter(filter => filter.field)
+      .map(filter => ({
+        ...filter,
+        field: `${count++}:${filter.field}`,
+      }))
+      .concat(matchAny ? [{ operator: "allOr" }] : [])
+  }
+
   const addFilter = () => {
-    filters = [
-      ...filters,
+    rawFilters = [
+      ...rawFilters,
       {
         id: generate(),
         field: null,
@@ -60,13 +95,13 @@
   }
 
   const removeFilter = id => {
-    filters = filters.filter(field => field.id !== id)
+    rawFilters = rawFilters.filter(field => field.id !== id)
   }
 
   const duplicateFilter = id => {
-    const existingFilter = filters.find(filter => filter.id === id)
+    const existingFilter = rawFilters.find(filter => filter.id === id)
     const duplicate = { ...existingFilter, id: generate() }
-    filters = [...filters, duplicate]
+    rawFilters = [...rawFilters, duplicate]
   }
 
   const getSchema = filter => {
@@ -133,32 +168,28 @@
     const schema = enrichedSchemaFields.find(x => x.name === field)
     return schema?.constraints?.inclusion || []
   }
-
-  onMount(() => {
-    behaviourValue = allOr ? "or" : "and"
-  })
 </script>
 
 <DrawerContent>
   <div class="container">
     <Layout noPadding>
       <Body size="S">
-        {#if !filters?.length}
+        {#if !rawFilters?.length}
           Add your first filter expression.
         {:else}
           Results are filtered to only those which match all of the following
           constraints.
         {/if}
       </Body>
-      {#if filters?.length}
+      {#if rawFilters?.length}
         <div class="fields">
           <Select
             label="Behaviour"
-            value={behaviourValue}
+            value={matchAny ? "or" : "and"}
             options={behaviourOptions}
             getOptionLabel={opt => opt.label}
             getOptionValue={opt => opt.value}
-            on:change={e => (allOr = e.detail === "or")}
+            on:change={e => (matchAny = e.detail === "or")}
             placeholder={null}
           />
         </div>
@@ -167,7 +198,7 @@
             <Label>Filters</Label>
           </div>
           <div class="fields">
-            {#each filters as filter, idx}
+            {#each rawFilters as filter, idx}
               <Select
                 bind:value={filter.field}
                 options={fieldOptions}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte

@@ -8,74 +8,25 @@
   import FilterDrawer from "./FilterDrawer.svelte"
   import { currentAsset } from "builderStore"
 
-  const QUERY_START_REGEX = /\d[0-9]*:/g
   const dispatch = createEventDispatcher()
 
   export let value = []
   export let componentInstance
   export let bindings = []
 
-  let drawer,
-    toSaveFilters = null,
-    allOr,
-    initialAllOr
+  let drawer
 
-  $: initialFilters = correctFilters(value || [])
+  $: tempValue = value
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
   $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
   $: schemaFields = Object.values(schema || {})
 
-  function addNumbering(filters) {
-    let count = 1
-    for (let value of filters) {
-      if (value.field && value.field?.match(QUERY_START_REGEX) == null) {
-        value.field = `${count++}:${value.field}`
-      }
-    }
-    return filters
-  }
-
-  function correctFilters(filters) {
-    const corrected = []
-    for (let filter of filters) {
-      let field = filter.field
-      if (filter.operator === "allOr") {
-        initialAllOr = allOr = true
-        continue
-      }
-      if (
-        typeof filter.field === "string" &&
-        filter.field.match(QUERY_START_REGEX) != null
-      ) {
-        const parts = field.split(":")
-        const number = parts[0]
-        // it's the new format, remove number
-        if (!isNaN(parseInt(number))) {
-          parts.shift()
-          field = parts.join(":")
-        }
-      }
-      corrected.push({
-        ...filter,
-        field,
-      })
-    }
-    return corrected
-  }
-
   async function saveFilter() {
-    if (!toSaveFilters && allOr !== initialAllOr) {
-      toSaveFilters = initialFilters
+    if (tempValue !== value) {
+      console.log("saving", tempValue)
+      dispatch("change", tempValue)
     }
-    const filters = toSaveFilters?.filter(filter => filter.operator !== "allOr")
-    if (allOr && filters) {
-      filters.push({ operator: "allOr" })
-    }
-    // only save if anything was updated
-    if (filters) {
-      dispatch("change", addNumbering(filters))
-    }
-    notifications.success("Filters saved.")
+    notifications.success("Filters saved")
     drawer.hide()
   }
 </script>
@@ -85,13 +36,10 @@
   <Button cta slot="buttons" on:click={saveFilter}>Save</Button>
   <FilterDrawer
     slot="body"
-    filters={initialFilters}
+    filters={value}
     {bindings}
     {schemaFields}
     tableId={dataSource.tableId}
-    bind:allOr
-    on:change={event => {
-      toSaveFilters = event.detail
-    }}
+    on:change={e => (tempValue = e.detail)}
   />
 </Drawer>