Allow data providers to inherit each other and add full client side lucene implementation

packages/builder/src/builderStore/dataBinding.js

@@ -136,7 +136,7 @@ const getContextBindings = (asset, componentId) => {
       if (!datasource) {
         return
       }
-      const info = getSchemaForDatasource(datasource)
+      const info = getSchemaForDatasource(asset, datasource)
       schema = info.schema
       readablePrefix = info.table?.name
     }
@@ -191,7 +191,7 @@ const getContextBindings = (asset, componentId) => {
  */
 const getUserBindings = () => {
   let bindings = []
-  const { schema } = getSchemaForDatasource({
+  const { schema } = getSchemaForDatasource(null, {
     type: "table",
     tableId: TableNames.USERS,
   })
@@ -244,11 +244,15 @@ const getUrlBindings = asset => {
 /**
  * Gets a schema for a datasource object.
  */
-export const getSchemaForDatasource = (datasource, isForm = false) => {
+export const getSchemaForDatasource = (asset, datasource, isForm = false) => {
   let schema, table
   if (datasource) {
     const { type } = datasource
-    if (type === "query") {
+    if (type === "provider") {
+      const component = findComponent(asset.props, datasource.providerId)
+      const source = getDatasourceForProvider(asset, component)
+      return getSchemaForDatasource(asset, source, isForm)
+    } else if (type === "query") {
       const queries = get(queriesStores).list
       table = queries.find(query => query._id === datasource._id)
     } else {

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -174,7 +174,7 @@ const fieldTypeToComponentMap = {
 }
 
 export function makeDatasourceFormComponents(datasource) {
-  const { schema } = getSchemaForDatasource(datasource, true)
+  const { schema } = getSchemaForDatasource(null, datasource, true)
   let components = []
   let fields = Object.keys(schema || {})
   fields.forEach(field => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte

@@ -1,5 +1,8 @@
 <script>
-  import { getBindableProperties } from "builderStore/dataBinding"
+  import {
+    getBindableProperties,
+    getDataProviderComponents,
+  } from "builderStore/dataBinding"
   import {
     Button,
     Popover,
@@ -61,6 +64,17 @@
     $currentAsset,
     $store.selectedComponentId
   )
+  $: dataProviders = getDataProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId
+  ).map(provider => ({
+    label: provider._instanceName,
+    name: provider._instanceName,
+    providerId: provider._id,
+    value: `{{ literal [${provider._id}] }}`,
+    type: "provider",
+    schema: provider.schema,
+  }))
   $: queryBindableProperties = bindableProperties.map(property => ({
     ...property,
     category: property.type === "instance" ? "Component" : "Table",
@@ -182,7 +196,20 @@
         </li>
       {/each}
     </ul>
-
+    <Divider size="S" />
+    <div class="title">
+      <Heading size="XS">Data Providers</Heading>
+    </div>
+    <ul>
+      {#each dataProviders as provider}
+        <li
+          class:selected={value === provider}
+          on:click={() => handleSelected(provider)}
+        >
+          {provider.label}
+        </li>
+      {/each}
+    </ul>
     {#if otherSources?.length}
       <Divider size="S" />
       <div class="title">

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveRow.svelte

@@ -14,11 +14,11 @@
     $currentAsset,
     $store.selectedComponentId
   )
-  $: schemaFields = getSchemaFields(parameters?.tableId)
+  $: schemaFields = getSchemaFields($currentAsset, parameters?.tableId)
   $: tableOptions = $tables.list || []
 
-  const getSchemaFields = tableId => {
+  const getSchemaFields = (asset, tableId) => {
-    const { schema } = getSchemaForDatasource({ type: "table", tableId })
+    const { schema } = getSchemaForDatasource(asset, { type: "table", tableId })
     return Object.values(schema || {})
   }
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FieldSelect.svelte

@@ -14,7 +14,7 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
   $: options = Object.keys(schema || {})
   $: boundValue = getValidValue(value, options)
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -27,19 +27,16 @@
     ? tempValue.length
     : Object.keys(tempValue || {}).length
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(dataSource)?.schema
+  $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
   $: schemaFields = Object.values(schema || {})
   $: internalTable = dataSource?.type === "table"
 
   // Reset value if value is wrong type for the datasource.
   // Lucene editor needs an array, and simple editor needs an object.
   $: {
-    if (internalTable && !Array.isArray(value)) {
+    if (!Array.isArray(value)) {
       tempValue = []
       dispatch("change", [])
-    } else if (!internalTable && Array.isArray(value)) {
-      tempValue = {}
-      dispatch("change", {})
     }
   }
 
@@ -63,28 +60,7 @@
           constaints.
         {/if}
       </Body>
-      {#if internalTable}
+      <LuceneFilterBuilder bind:value={tempValue} {schemaFields} />
-        <LuceneFilterBuilder bind:value={tempValue} {schemaFields} />
-      {:else}
-        <div class="fields">
-          <SaveFields
-            parameterFields={Array.isArray(value) ? {} : value}
-            {schemaFields}
-            valueLabel="Equals"
-            on:change={e => (tempValue = e.detail)}
-          />
-        </div>
-      {/if}
     </Layout>
   </DrawerContent>
 </Drawer>
-
-<style>
-  .fields {
-    display: grid;
-    column-gap: var(--spacing-l);
-    row-gap: var(--spacing-s);
-    align-items: center;
-    grid-template-columns: auto 1fr auto 1fr auto;
-  }
-</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FormFieldSelect.svelte

@@ -18,7 +18,7 @@
     component => component._component === "@budibase/standard-components/form"
   )
   $: datasource = getDatasourceForProvider($currentAsset, form)
-  $: schema = getSchemaForDatasource(datasource, true).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource, true).schema
   $: options = getOptions(schema, type)
 
   const getOptions = (schema, fieldType) => {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/MultiFieldSelect.svelte

@@ -14,7 +14,7 @@
 
   const dispatch = createEventDispatcher()
   $: datasource = getDatasourceForProvider($currentAsset, componentInstance)
-  $: schema = getSchemaForDatasource(datasource).schema
+  $: schema = getSchemaForDatasource($currentAsset, datasource).schema
   $: options = Object.keys(schema || {})
   $: boundValue = getValidOptions(value, options)
 

packages/standard-components/src/DataProvider.svelte

@@ -1,6 +1,12 @@
 <script>
   import { getContext } from "svelte"
   import { ProgressCircle, Pagination } from "@budibase/bbui"
+  import {
+    buildLuceneQuery,
+    luceneQuery,
+    luceneSort,
+    luceneLimit,
+  } from "./lucene"
 
   export let dataSource
   export let filter
@@ -27,7 +33,7 @@
   let pageNumber = 0
 
   $: internalTable = dataSource?.type === "table"
-  $: query = internalTable ? buildLuceneQuery(filter) : null
+  $: query = buildLuceneQuery(filter)
   $: hasNextPage = bookmarks[pageNumber + 1] != null
   $: hasPrevPage = pageNumber > 0
   $: getSchema(dataSource)
@@ -48,18 +54,21 @@
     }
   }
   $: {
-    // Sort and limit rows in memory when we aren't searching internal tables
     if (internalTable) {
+      // Internal tables are already processed server-side
       rows = allRows
     } else {
-      const sortedRows = sortRows(allRows, sortColumn, sortOrder)
+      // For anything else we use client-side implementations to filter, sort
-      rows = limitRows(sortedRows, limit)
+      // and limit
+      const filtered = luceneQuery(allRows, query)
+      const sorted = luceneSort(filtered, sortColumn, sortOrder, sortType)
+      rows = luceneLimit(sorted, limit)
     }
   }
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
-      callback: () => fetchData(dataSource),
+      callback: () => refresh(),
       metadata: { dataSource },
     },
   ]
@@ -73,36 +82,18 @@
     return type === "number" ? "number" : "string"
   }
 
-  const buildLuceneQuery = filter => {
+  const refresh = async () => {
-    let query = {
+    if (schemaLoaded) {
-      string: {},
+      fetchData(
-      fuzzy: {},
+        dataSource,
-      range: {},
+        query,
-      equal: {},
+        limit,
-      notEqual: {},
+        sortColumn,
-      empty: {},
+        sortOrder,
-      notEmpty: {},
+        sortType,
+        paginate
+      )
     }
-    if (Array.isArray(filter)) {
-      filter.forEach(({ operator, field, type, value }) => {
-        if (operator.startsWith("range")) {
-          if (!query.range[field]) {
-            query.range[field] = {
-              low: type === "number" ? Number.MIN_SAFE_INTEGER : "0000",
-              high: type === "number" ? Number.MAX_SAFE_INTEGER : "9999",
-            }
-          }
-          if (operator === "rangeLow") {
-            query.range[field].low = value
-          } else if (operator === "rangeHigh") {
-            query.range[field].high = value
-          }
-        } else if (query[operator]) {
-          query[operator][field] = value
-        }
-      })
-    }
-    return query
   }
 
   const fetchData = async (
@@ -116,6 +107,7 @@
   ) => {
     loading = true
     if (dataSource?.type === "table") {
+      // For internal tables we use server-side processing
       const res = await API.searchTable({
         tableId: dataSource.tableId,
         query,
@@ -132,55 +124,27 @@
       } else {
         bookmarks = [null]
       }
+    } else if (dataSource?.type === "provider") {
+      // For providers referencing another provider, just use the rows it
+      // provides
+      allRows = dataSource?.value?.rows ?? []
     } else {
-      const rows = await API.fetchDatasource(dataSource)
+      // For other data sources like queries or views, fetch all rows from the
-      allRows = inMemoryFilterRows(rows, filter)
+      // server
+      allRows = await API.fetchDatasource(dataSource)
     }
     loading = false
     loaded = true
   }
 
-  const inMemoryFilterRows = (rows, filter) => {
-    let filteredData = [...rows]
-    Object.entries(filter || {}).forEach(([field, value]) => {
-      if (value != null && value !== "") {
-        filteredData = filteredData.filter(row => {
-          return row[field] === value
-        })
-      }
-    })
-    return filteredData
-  }
-
-  const sortRows = (rows, sortColumn, sortOrder) => {
-    if (!sortColumn || !sortOrder) {
-      return rows
-    }
-    return rows.slice().sort((a, b) => {
-      const colA = a[sortColumn]
-      const colB = b[sortColumn]
-      if (sortOrder === "Descending") {
-        return colA > colB ? -1 : 1
-      } else {
-        return colA > colB ? 1 : -1
-      }
-    })
-  }
-
-  const limitRows = (rows, limit) => {
-    const numLimit = parseFloat(limit)
-    if (isNaN(numLimit)) {
-      return rows
-    }
-    return rows.slice(0, numLimit)
-  }
-
   const getSchema = async dataSource => {
     if (dataSource?.schema) {
       schema = dataSource.schema
     } else if (dataSource?.tableId) {
       const definition = await API.fetchTableDefinition(dataSource.tableId)
       schema = definition?.schema ?? {}
+    } else if (dataSource?.type === "provider") {
+      schema = dataSource.value?.schema ?? {}
     } else {
       schema = {}
     }

packages/standard-components/src/lucene.js

@@ -0,0 +1,140 @@
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = filter => {
+  let query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(({ operator, field, type, value }) => {
+      if (operator.startsWith("range")) {
+        if (!query.range[field]) {
+          query.range[field] = {
+            low: type === "number" ? Number.MIN_SAFE_INTEGER : "0000",
+            high: type === "number" ? Number.MAX_SAFE_INTEGER : "9999",
+          }
+        }
+        if (operator === "rangeLow") {
+          query.range[field].low = value
+        } else if (operator === "rangeHigh") {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        query[operator][field] = value
+      }
+    })
+  }
+  return query
+}
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const luceneQuery = (docs, query) => {
+  if (!query) {
+    return docs
+  }
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match = (type, failFn) => doc => {
+    const filters = Object.entries(query[type] || {})
+    for (let i = 0; i < filters.length; i++) {
+      if (failFn(filters[i][0], filters[i][1], doc)) {
+        return false
+      }
+    }
+    return true
+  }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (key, value, doc) => {
+    return !doc[key] || !doc[key].startsWith(value)
+  })
+
+  // Process a range match
+  const rangeMatch = match("range", (key, value, doc) => {
+    return !doc[key] || doc[key] < value.low || doc[key] > value.high
+  })
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match("equal", (key, value, doc) => {
+    return doc[key] !== value
+  })
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match("notEqual", (key, value, doc) => {
+    return doc[key] === value
+  })
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (key, value, doc) => {
+    return doc[key] != null && doc[key] !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (key, value, doc) => {
+    return doc[key] == null || doc[key] === ""
+  })
+
+  // Match a document against all criteria
+  const docMatch = doc => {
+    return (
+      stringMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
+  return docs.slice().sort((a, b) => {
+    const colA = parse(a[sort])
+    const colB = parse(b[sort])
+    if (sortOrder === "Descending") {
+      return colA > colB ? -1 : 1
+    } else {
+      return colA > colB ? 1 : -1
+    }
+  })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs, limit) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
+}