From 230dc169fa86bdaa63976b03407e1d3ab166db4b Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 26 Oct 2023 10:11:34 +0100
Subject: [PATCH 1/4] Revert previous fix for relationship column dragging

---
 .../grid/cells/RelationshipCell.svelte        | 50 +++++++++----------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
index e6d83e0bea..925c840478 100644
--- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
@@ -260,31 +260,29 @@
       class:wrap={editable || contentLines > 1}
       on:wheel={e => (focused ? e.stopPropagation() : null)}
     >
-      {#if Array.isArray(value) && value.length}
-        {#each value as relationship}
-          {#if relationship[primaryDisplay] || relationship.primaryDisplay}
-            <div class="badge">
-              <span
-                on:click={editable
-                  ? () => showRelationship(relationship._id)
-                  : null}
-              >
-                {readable(
-                  relationship[primaryDisplay] || relationship.primaryDisplay
-                )}
-              </span>
-              {#if editable}
-                <Icon
-                  name="Close"
-                  size="XS"
-                  hoverable
-                  on:click={() => toggleRow(relationship)}
-                />
-              {/if}
-            </div>
-          {/if}
-        {/each}
-      {/if}
+      {#each value || [] as relationship}
+        {#if relationship[primaryDisplay] || relationship.primaryDisplay}
+          <div class="badge">
+            <span
+              on:click={editable
+                ? () => showRelationship(relationship._id)
+                : null}
+            >
+              {readable(
+                relationship[primaryDisplay] || relationship.primaryDisplay
+              )}
+            </span>
+            {#if editable}
+              <Icon
+                name="Close"
+                size="XS"
+                hoverable
+                on:click={() => toggleRow(relationship)}
+              />
+            {/if}
+          </div>
+        {/if}
+      {/each}
       {#if editable}
         <div class="add" on:click={open}>
           <Icon name="Add" size="S" />
@@ -320,7 +318,7 @@
         <div class="searching">
           <ProgressCircle size="S" />
         </div>
-      {:else if Array.isArray(searchResults) && searchResults.length}
+      {:else if searchResults?.length}
         <div class="results">
           {#each searchResults as row, idx}
             <div

From 23cdac5906605a34af5aadf6f4bb9ab07998c8fb Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 26 Oct 2023 17:37:59 +0100
Subject: [PATCH 2/4] Update grids to support pagination with REST queries and
 to simplify some logic

---
 .../src/components/grid/stores/datasource.js  |  6 ------
 .../grid/stores/datasources/nonPlus.js        |  6 ------
 .../grid/stores/datasources/table.js          |  6 ------
 .../grid/stores/datasources/viewV2.js         |  6 ------
 .../src/components/grid/stores/rows.js        | 10 ++++-----
 packages/frontend-core/src/fetch/DataFetch.js | 21 +++++++++++++++----
 6 files changed, 22 insertions(+), 33 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 958f4541bd..7ee3a19b8a 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -160,11 +160,6 @@ export const createActions = context => {
     return getAPI()?.actions.canUseColumn(name)
   }
 
-  // Gets the default number of rows for a single page
-  const getFeatures = () => {
-    return getAPI()?.actions.getFeatures()
-  }
-
   return {
     datasource: {
       ...datasource,
@@ -177,7 +172,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
index 017c16a03c..acdf509278 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
@@ -35,11 +35,6 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
-  const getFeatures = () => {
-    // We don't support any features
-    return {}
-  }
-
   return {
     nonPlus: {
       actions: {
@@ -50,7 +45,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/table.js b/packages/frontend-core/src/components/grid/stores/datasources/table.js
index 2f49ab1d38..847dfd2c6b 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/table.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/table.js
@@ -1,5 +1,4 @@
 import { get } from "svelte/store"
-import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
@@ -46,10 +45,6 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
-  const getFeatures = () => {
-    return new TableFetch({ API }).determineFeatureFlags()
-  }
-
   return {
     table: {
       actions: {
@@ -60,7 +55,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 35f57a5fc4..ed31d0ae44 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -1,5 +1,4 @@
 import { get } from "svelte/store"
-import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
@@ -46,10 +45,6 @@ export const createActions = context => {
     )
   }
 
-  const getFeatures = () => {
-    return new ViewV2Fetch({ API }).determineFeatureFlags()
-  }
-
   return {
     viewV2: {
       actions: {
@@ -60,7 +55,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 51c46f8263..82185d6b91 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -114,10 +114,6 @@ export const createActions = context => {
     const $allFilters = get(allFilters)
     const $sort = get(sort)
 
-    // Determine how many rows to fetch per page
-    const features = datasource.actions.getFeatures()
-    const limit = features?.supportsPagination ? RowPageSize : null
-
     // Create new fetch model
     const newFetch = fetchData({
       API,
@@ -126,8 +122,12 @@ export const createActions = context => {
         filter: $allFilters,
         sortColumn: $sort.column,
         sortOrder: $sort.order,
-        limit,
+        limit: RowPageSize,
         paginate: true,
+
+        // Disable client side limiting, so that for queries and custom data
+        // sources we don't impose fake row limits. We want all the data.
+        clientSideLimiting: false,
       },
     })
 
diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js
index 857072601e..92115efef0 100644
--- a/packages/frontend-core/src/fetch/DataFetch.js
+++ b/packages/frontend-core/src/fetch/DataFetch.js
@@ -43,6 +43,11 @@ export default class DataFetch {
 
       // Pagination config
       paginate: true,
+
+      // Client side feature customisation
+      clientSideSearching: true,
+      clientSideSorting: true,
+      clientSideLimiting: true,
     }
 
     // State of the fetch
@@ -208,24 +213,32 @@ export default class DataFetch {
    * Fetches some filtered, sorted and paginated data
    */
   async getPage() {
-    const { sortColumn, sortOrder, sortType, limit } = this.options
+    const {
+      sortColumn,
+      sortOrder,
+      sortType,
+      limit,
+      clientSideSearching,
+      clientSideSorting,
+      clientSideLimiting,
+    } = this.options
     const { query } = get(this.store)
 
     // Get the actual data
     let { rows, info, hasNextPage, cursor, error } = await this.getData()
 
     // If we don't support searching, do a client search
-    if (!this.features.supportsSearch) {
+    if (!this.features.supportsSearch && clientSideSearching) {
       rows = runLuceneQuery(rows, query)
     }
 
     // If we don't support sorting, do a client-side sort
-    if (!this.features.supportsSort) {
+    if (!this.features.supportsSort && clientSideSorting) {
       rows = luceneSort(rows, sortColumn, sortOrder, sortType)
     }
 
     // If we don't support pagination, do a client-side limit
-    if (!this.features.supportsPagination) {
+    if (!this.features.supportsPagination && clientSideLimiting) {
       rows = luceneLimit(rows, limit)
     }
 

From fd15f771ef8266f2f8852cd1a585106add862e40 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 26 Oct 2023 19:03:04 +0100
Subject: [PATCH 3/4] Refactor how relationship cells fetch and cache primary
 display columns to fix issues with store stale data

---
 .../grid/cells/RelationshipCell.svelte        | 25 ++--------
 .../src/components/grid/stores/cache.js       | 49 +++++++++++++++++++
 .../src/components/grid/stores/index.js       |  2 +
 3 files changed, 56 insertions(+), 20 deletions(-)
 create mode 100644 packages/frontend-core/src/components/grid/stores/cache.js

diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
index 925c840478..0db022777f 100644
--- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
@@ -1,27 +1,10 @@
-<script context="module">
-  // We can create a module level cache for all relationship cells to avoid
-  // having to fetch the table definition one time for each cell
-  let primaryDisplayCache = {}
-
-  const getPrimaryDisplayForTableId = async (API, tableId) => {
-    if (primaryDisplayCache[tableId]) {
-      return primaryDisplayCache[tableId]
-    }
-    const definition = await API.fetchTableDefinition(tableId)
-    const primaryDisplay =
-      definition?.primaryDisplay || definition?.schema?.[0]?.name
-    primaryDisplayCache[tableId] = primaryDisplay
-    return primaryDisplay
-  }
-</script>
-
 <script>
   import { getColor } from "../lib/utils"
   import { onMount, getContext } from "svelte"
   import { Icon, Input, ProgressCircle, clickOutside } from "@budibase/bbui"
   import { debounce } from "../../../utils/utils"
 
-  const { API, dispatch } = getContext("grid")
+  const { API, dispatch, cache } = getContext("grid")
 
   export let value
   export let api
@@ -147,7 +130,9 @@
     // Find the primary display for the related table
     if (!primaryDisplay) {
       searching = true
-      primaryDisplay = await getPrimaryDisplayForTableId(API, schema.tableId)
+      primaryDisplay = await cache.actions.getPrimaryDisplayForTableId(
+        schema.tableId
+      )
     }
 
     // Show initial list of results
@@ -195,7 +180,7 @@
   const toggleRow = async row => {
     if (value?.some(x => x._id === row._id)) {
       // If the row is already included, remove it and update the candidate
-      // row to be the the same position if possible
+      // row to be the same position if possible
       if (oneRowOnly) {
         await onChange([])
       } else {
diff --git a/packages/frontend-core/src/components/grid/stores/cache.js b/packages/frontend-core/src/components/grid/stores/cache.js
new file mode 100644
index 0000000000..7d48627314
--- /dev/null
+++ b/packages/frontend-core/src/components/grid/stores/cache.js
@@ -0,0 +1,49 @@
+export const createActions = context => {
+  const { API } = context
+
+  // Cache for the primary display columns of different tables.
+  // If we ever need to cache table definitions for other purposes then we can
+  // expand this to be a more generic cache.
+  let primaryDisplayCache = {}
+
+  const resetPrimaryDisplayCache = () => {
+    primaryDisplayCache = {}
+  }
+
+  const getPrimaryDisplayForTableId = async tableId => {
+    // If we've never encountered this tableId before then store a promise that
+    // resolves to the primary display so that subsequent invocations before the
+    // promise completes can reuse this promise
+    if (!primaryDisplayCache[tableId]) {
+      primaryDisplayCache[tableId] = new Promise(resolve => {
+        API.fetchTableDefinition(tableId).then(def => {
+          const display = def?.primaryDisplay || def?.schema?.[0]?.name
+          primaryDisplayCache[tableId] = display
+          resolve(display)
+        })
+      })
+    }
+
+    // We await the result so that we account for both promises and primitives
+    return await primaryDisplayCache[tableId]
+  }
+
+  return {
+    cache: {
+      actions: {
+        getPrimaryDisplayForTableId,
+        resetPrimaryDisplayCache,
+      },
+    },
+  }
+}
+
+export const initialise = context => {
+  const { datasource, cache } = context
+
+  // Wipe the caches whenever the datasource changes to ensure we aren't
+  // storing any stale information
+  datasource.subscribe(() => {
+    cache.actions.resetPrimaryDisplayCache()
+  })
+}
diff --git a/packages/frontend-core/src/components/grid/stores/index.js b/packages/frontend-core/src/components/grid/stores/index.js
index 10fe932aab..6dfff6531b 100644
--- a/packages/frontend-core/src/components/grid/stores/index.js
+++ b/packages/frontend-core/src/components/grid/stores/index.js
@@ -19,6 +19,7 @@ import * as Datasource from "./datasource"
 import * as Table from "./datasources/table"
 import * as ViewV2 from "./datasources/viewV2"
 import * as NonPlus from "./datasources/nonPlus"
+import * as Cache from "./cache"
 
 const DependencyOrderedStores = [
   Sort,
@@ -42,6 +43,7 @@ const DependencyOrderedStores = [
   Clipboard,
   Config,
   Notifications,
+  Cache,
 ]
 
 export const attachStores = context => {

From 69cffd0f091c63c27268f387cebf567a71a7fbf9 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Thu, 26 Oct 2023 19:14:36 +0100
Subject: [PATCH 4/4] Clean up

---
 packages/frontend-core/src/components/grid/stores/cache.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/cache.js b/packages/frontend-core/src/components/grid/stores/cache.js
index 7d48627314..7eab6795e4 100644
--- a/packages/frontend-core/src/components/grid/stores/cache.js
+++ b/packages/frontend-core/src/components/grid/stores/cache.js
@@ -43,7 +43,5 @@ export const initialise = context => {
 
   // Wipe the caches whenever the datasource changes to ensure we aren't
   // storing any stale information
-  datasource.subscribe(() => {
-    cache.actions.resetPrimaryDisplayCache()
-  })
+  datasource.subscribe(cache.actions.resetPrimaryDisplayCache)
 }