Optimise sheet data loading and add sort button

packages/frontend-core/src/components/sheet/Sheet.svelte

@@ -86,10 +86,6 @@
   // Expose ability to retrieve context externally to allow sheet control
   export const getContext = () => context
 
-  // Local flag for if the sheet has ever had data
-  let initialised = false
-  loaded.subscribe(state => (initialised = initialised || state))
-
   // Initialise websocket for multi-user
   onMount(() => createWebsocket(context))
 </script>
@@ -112,7 +108,7 @@
       <UserAvatars />
     </div>
   </div>
-  {#if initialised}
+  {#if $loaded}
     <div class="sheet-data">
       <StickyColumn />
       <div class="sheet-main">

packages/frontend-core/src/components/sheet/SheetControls.svelte

@@ -1,9 +1,10 @@
 <script>
   import { getContext } from "svelte"
   import { ActionButton } from "@budibase/bbui"
+  import SortButton from "./controls/SortButton.svelte"
 
   const { rows } = getContext("sheet")
 </script>
 
-<ActionButton icon="SortOrderDown" quiet size="M">Sort</ActionButton>
+<SortButton />
 <ActionButton icon="VisibilityOff" quiet size="M">Hide fields</ActionButton>

packages/frontend-core/src/components/sheet/controls/SortButton.svelte

@@ -0,0 +1,52 @@
+<script>
+  import { getContext } from "svelte"
+  import { ActionButton, Popover, Select } from "@budibase/bbui"
+
+  const { sort, columns, stickyColumn } = getContext("sheet")
+  const orderOptions = [
+    { label: "Ascending", value: "ascending" },
+    { label: "Descending", value: "descending" },
+  ]
+
+  let open = false
+  let anchor
+
+  $: columnOptions = getColumnOptions($stickyColumn, $columns)
+  $: console.log($sort)
+
+  const getColumnOptions = (stickyColumn, columns) => {
+    let options = []
+    if (stickyColumn) {
+      options.push(stickyColumn.name)
+    }
+    return [...options, ...columns.map(col => col.name)]
+  }
+</script>
+
+<div bind:this={anchor}>
+  <ActionButton
+    icon="SortOrderDown"
+    quiet
+    size="M"
+    on:click={() => (open = !open)}
+    selected={!!$sort.order}
+  >
+    Sort
+  </ActionButton>
+</div>
+
+<Popover bind:open {anchor} align="left">
+  <div class="content">
+    <Select bind:value={$sort.column} options={columnOptions} autoWidth />
+    <Select bind:value={$sort.order} options={orderOptions} autoWidth />
+  </div>
+</Popover>
+
+<style>
+  .content {
+    padding: 12px 12px;
+    display: flex;
+    align-items: center;
+    gap: 8px;
+  }
+</style>

packages/frontend-core/src/components/sheet/stores/rows.js

@@ -1,5 +1,4 @@
 import { writable, derived, get } from "svelte/store"
-import { LuceneUtils } from "../../../index"
 import { fetchData } from "../../../fetch/fetchData"
 import { notifications } from "@budibase/bbui"
 
@@ -11,20 +10,25 @@ export const createRowsStore = context => {
   const table = writable(null)
   const filter = writable([])
   const loaded = writable(false)
+  const fetch = writable(null)
   const sort = writable({
     column: null,
     order: null,
   })
+
+  // Enrich rows with an index property
   const enrichedRows = derived(rows, $rows => {
     return $rows.map((row, idx) => ({
       ...row,
       __idx: idx,
     }))
   })
+
+  // Generate a lookup map to quick find a row by ID
   const rowLookupMap = derived(enrichedRows, $rows => {
     let map = {}
-    for (let i = 0; i < $rows.length; i++) {
-      map[$rows[i]._id] = i
+    for (let row of $rows) {
+      map[row._id] = row.__idx
     }
     return map
   })
@@ -33,69 +37,84 @@ export const createRowsStore = context => {
   let rowCacheMap = {}
 
   // Reset everything when table ID changes
-  tableId.subscribe(() => {
-    filter.set([])
+  let unsubscribe = null
+  tableId.subscribe($tableId => {
+    // Unsub from previous fetch if one exists
+    unsubscribe?.()
+    fetch.set(null)
+
+    // Reset state
     sort.set({
       column: null,
       order: null,
     })
-  })
+    filter.set([])
 
-  // Local stores for managing fetching data
-  const query = derived(filter, $filter =>
-    LuceneUtils.buildLuceneQuery($filter)
-  )
-  const fetch = derived([tableId, query, sort], ([$tableId, $query, $sort]) => {
-    if (!$tableId) {
-      return null
-    }
-    // Wipe state and fully hydrate next time our fetch returns data
-    loaded.set(false)
-
-    // Create fetch and load initial data
-    return fetchData({
+    // Create new fetch model
+    const newFetch = fetchData({
       API,
       datasource: {
         type: "table",
         tableId: $tableId,
       },
       options: {
-        sortColumn: $sort.column,
-        sortOrder: $sort.order,
-        query: $query,
+        filter: [],
+        sortColumn: null,
+        sortOrder: null,
         limit: 100,
         paginate: true,
       },
     })
-  })
 
-  // Observe each data fetch and extract some data
-  fetch.subscribe($fetch => {
-    if (!$fetch) {
-      return
-    }
-    $fetch.subscribe($$fetch => {
-      if ($$fetch.loaded) {
-        if (!get(loaded)) {
+    // Subscribe to changes of this fetch model
+    unsubscribe = newFetch.subscribe($fetch => {
+      if ($fetch.loaded && !$fetch.loading) {
+        if ($fetch.pageNumber === 0) {
           // Hydrate initial data
-          loaded.set(true)
           rowCacheMap = {}
           rows.set([])
+
+          // Update sorting from fetch if required
+          const $sort = get(sort)
+          if (!$sort.column) {
+            sort.set({
+              column: $fetch.sortColumn,
+              order: $fetch.sortOrder,
+            })
+          }
         }
 
         // Update schema and enrich primary display into schema
-        let newSchema = $$fetch.schema
-        const primaryDisplay = $$fetch.definition?.primaryDisplay
+        let newSchema = $fetch.schema
+        const primaryDisplay = $fetch.definition?.primaryDisplay
         if (primaryDisplay && newSchema[primaryDisplay]) {
           newSchema[primaryDisplay].primaryDisplay = true
         }
         schema.set(newSchema)
-        table.set($$fetch.definition)
+        table.set($fetch.definition)
 
         // Process new rows
-        handleNewRows($$fetch.rows)
+        handleNewRows($fetch.rows)
+
+        // Notify that we're loaded
+        loaded.set(true)
       }
     })
+
+    fetch.set(newFetch)
+  })
+
+  // Update fetch when filter or sort config changes
+  filter.subscribe($filter => {
+    get(fetch)?.update({
+      filter: $filter,
+    })
+  })
+  sort.subscribe($sort => {
+    get(fetch)?.update({
+      sortOrder: $sort.order,
+      sortColumn: $sort.column,
+    })
   })
 
   // Adds a new empty row

packages/frontend-core/src/fetch/DataFetch.js

@@ -21,11 +21,11 @@ export default class DataFetch {
     this.API = null
 
     // Feature flags
-    this.featureStore = writable({
+    this.features = {
       supportsSearch: false,
       supportsSort: false,
       supportsPagination: false,
-    })
+    }
 
     // Config
     this.options = {
@@ -81,17 +81,14 @@ export default class DataFetch {
     this.prevPage = this.prevPage.bind(this)
 
     // Derive certain properties to return
-    this.derivedStore = derived(
-      [this.store, this.featureStore],
-      ([$store, $featureStore]) => {
-        return {
-          ...$store,
-          ...$featureStore,
-          hasNextPage: this.hasNextPage($store),
-          hasPrevPage: this.hasPrevPage($store),
-        }
+    this.derivedStore = derived(this.store, $store => {
+      return {
+        ...$store,
+        ...this.features,
+        hasNextPage: this.hasNextPage($store),
+        hasPrevPage: this.hasPrevPage($store),
       }
-    )
+    })
 
     // Mark as loaded if we have no datasource
     if (!this.options.datasource) {
@@ -120,11 +117,11 @@ export default class DataFetch {
     // Fetch datasource definition and determine feature flags
     const definition = await this.getDefinition(datasource)
     const features = this.determineFeatureFlags(definition)
-    this.featureStore.set({
+    this.features = {
       supportsSearch: !!features?.supportsSearch,
       supportsSort: !!features?.supportsSort,
       supportsPagination: paginate && !!features?.supportsPagination,
-    })
+    }
 
     // Fetch and enrich schema
     let schema = this.getSchema(datasource, definition)
@@ -138,11 +135,17 @@ export default class DataFetch {
       this.options.sortOrder = "ascending"
     }
 
-    // If no sort column, use the first field in the schema
+    // If no sort column, use the primary display and fallback to first column
     if (!this.options.sortColumn) {
-      this.options.sortColumn = Object.keys(schema)[0]
+      let newSortColumn
+      if (definition?.primaryDisplay && schema[definition.primaryDisplay]) {
+        newSortColumn = definition.primaryDisplay
+      } else {
+        newSortColumn = Object.keys(schema)[0]
+      }
+      this.options.sortColumn = newSortColumn
     }
-    const { sortColumn } = this.options
+    const { sortOrder, sortColumn } = this.options
 
     // Determine what sort type to use
     let sortType = "string"
@@ -167,6 +170,8 @@ export default class DataFetch {
       loading: true,
       cursors: [],
       cursor: null,
+      sortOrder,
+      sortColumn,
     }))
 
     // Actually fetch data
@@ -189,23 +194,23 @@ export default class DataFetch {
   async getPage() {
     const { sortColumn, sortOrder, sortType, limit } = this.options
     const { query } = get(this.store)
-    const features = get(this.featureStore)
 
     // Get the actual data
+    console.log("===== FETCH =====")
     let { rows, info, hasNextPage, cursor, error } = await this.getData()
 
     // If we don't support searching, do a client search
-    if (!features.supportsSearch) {
+    if (!this.features.supportsSearch) {
       rows = runLuceneQuery(rows, query)
     }
 
     // If we don't support sorting, do a client-side sort
-    if (!features.supportsSort) {
+    if (!this.features.supportsSort) {
       rows = luceneSort(rows, sortColumn, sortOrder, sortType)
     }
 
     // If we don't support pagination, do a client-side limit
-    if (!features.supportsPagination) {
+    if (!this.features.supportsPagination) {
       rows = luceneLimit(rows, limit)
     }
 

packages/frontend-core/src/fetch/QueryFetch.js

@@ -31,7 +31,7 @@ export default class QueryFetch extends DataFetch {
 
   async getData() {
     const { datasource, limit, paginate } = this.options
-    const { supportsPagination } = get(this.featureStore)
+    const { supportsPagination } = this.features
     const { cursor, definition } = get(this.store)
     const type = definition?.fields?.pagination?.type