Merge pull request #11856 from Budibase/chore/field_with_large_relationships

Fix display of field with large relationships

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,12 +14,12 @@
   export let autocomplete = false
   export let sort = false
   export let autoWidth = false
-  export let fetchTerm = null
-  export let useFetch = false
+  export let searchTerm = null
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let open = false
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -82,6 +82,7 @@
 </script>
 
 <Picker
+  on:loadMore
   {id}
   {error}
   {disabled}
@@ -90,9 +91,8 @@
   {options}
   isPlaceholder={!arrayValue.length}
   {autocomplete}
-  bind:fetchTerm
+  bind:searchTerm
   bind:open
-  {useFetch}
   {isOptionSelected}
   {getOptionLabel}
   {getOptionValue}
@@ -102,4 +102,5 @@
   {customPopoverHeight}
   {customPopoverOffsetBelow}
   {customPopoverMaxHeight}
+  {loading}
 />

packages/bbui/src/Form/Core/Picker.svelte

@@ -2,7 +2,7 @@
   import "@spectrum-css/picker/dist/index-vars.css"
   import "@spectrum-css/popover/dist/index-vars.css"
   import "@spectrum-css/menu/dist/index-vars.css"
-  import { createEventDispatcher } from "svelte"
+  import { createEventDispatcher, onDestroy } from "svelte"
   import clickOutside from "../../Actions/click_outside"
   import Search from "./Search.svelte"
   import Icon from "../../Icon/Icon.svelte"
@@ -10,6 +10,7 @@
   import Popover from "../../Popover/Popover.svelte"
   import Tags from "../../Tags/Tags.svelte"
   import Tag from "../../Tags/Tag.svelte"
+  import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
 
   export let id = null
   export let disabled = false
@@ -35,19 +36,20 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
-  export let fetchTerm = null
-  export let useFetch = false
+  export let searchTerm = null
   export let customPopoverHeight
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
   export let align = "left"
   export let footer = null
   export let customAnchor = null
+  export let loading
+
   const dispatch = createEventDispatcher()
 
-  let searchTerm = null
   let button
   let popover
+  let component
 
   $: sortedOptions = getSortedOptions(options, getOptionLabel, sort)
   $: filteredOptions = getFilteredOptions(
@@ -82,7 +84,7 @@
   }
 
   const getFilteredOptions = (options, term, getLabel) => {
-    if (autocomplete && term && !fetchTerm) {
+    if (autocomplete && term) {
       const lowerCaseTerm = term.toLowerCase()
       return options.filter(option => {
         return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm)
@@ -90,6 +92,20 @@
     }
     return options
   }
+
+  const onScroll = e => {
+    const scrollPxThreshold = 100
+    const scrollPositionFromBottom =
+      e.target.scrollHeight - e.target.clientHeight - e.target.scrollTop
+    if (scrollPositionFromBottom < scrollPxThreshold) {
+      dispatch("loadMore")
+    }
+  }
+
+  $: component?.addEventListener("scroll", onScroll)
+  onDestroy(() => {
+    component?.removeEventListener("scroll", null)
+  })
 </script>
 
 <button
@@ -163,14 +179,13 @@
   >
     {#if autocomplete}
       <Search
-        value={useFetch ? fetchTerm : searchTerm}
-        on:change={event =>
-          useFetch ? (fetchTerm = event.detail) : (searchTerm = event.detail)}
+        value={searchTerm}
+        on:change={event => (searchTerm = event.detail)}
         {disabled}
         placeholder="Search"
       />
     {/if}
-    <ul class="spectrum-Menu" role="listbox">
+    <ul class="spectrum-Menu" role="listbox" bind:this={component}>
       {#if placeholderOption}
         <li
           class="spectrum-Menu-item placeholder"
@@ -248,6 +263,12 @@
       {/if}
     </ul>
 
+    {#if loading}
+      <div class="loading" class:loading--withAutocomplete={autocomplete}>
+        <ProgressCircle size="S" />
+      </div>
+    {/if}
+
     {#if footer}
       <div class="footer">
         {footer}
@@ -323,18 +344,19 @@
   /* Search styles inside popover */
   .popover-content :global(.spectrum-Search) {
     margin-top: -1px;
-    margin-left: -1px;
-    width: calc(100% + 2px);
+    width: 100%;
   }
   .popover-content :global(.spectrum-Search input) {
     height: auto;
     border-bottom-left-radius: 0;
     border-bottom-right-radius: 0;
+    border-left: 0;
+    border-right: 0;
     padding-top: var(--spectrum-global-dimension-size-100);
     padding-bottom: var(--spectrum-global-dimension-size-100);
   }
   .popover-content :global(.spectrum-Search .spectrum-ClearButton) {
-    right: 1px;
+    right: 2px;
     top: 2px;
   }
   .popover-content :global(.spectrum-Search .spectrum-Textfield-icon) {
@@ -359,4 +381,14 @@
   .option-tag :global(.spectrum-Tags-item > .spectrum-Icon) {
     margin-top: 2px;
   }
+
+  .loading {
+    position: fixed;
+    justify-content: center;
+    right: var(--spacing-s);
+    top: var(--spacing-s);
+  }
+  .loading--withAutocomplete {
+    top: calc(34px + var(--spacing-m));
+  }
 </style>

packages/bbui/src/Form/Core/Select.svelte

@@ -25,6 +25,8 @@
   export let tag = null
   export let customPopoverOffsetBelow
   export let customPopoverMaxHeight
+  export let searchTerm = null
+  export let loading
 
   const dispatch = createEventDispatcher()
 
@@ -65,6 +67,8 @@
 <Picker
   on:click
   bind:open
+  bind:searchTerm
+  on:loadMore
   {quiet}
   {id}
   {error}
@@ -92,4 +96,5 @@
   placeholderOption={placeholder === false ? null : placeholder}
   isOptionSelected={option => option === value}
   onSelectOption={selectOption}
+  {loading}
 />

packages/bbui/src/Form/Multiselect.svelte

@@ -16,8 +16,7 @@
   export let sort = false
   export let autoWidth = false
   export let autocomplete = false
-  export let fetchTerm = null
-  export let useFetch = false
+  export let searchTerm = null
   export let customPopoverHeight
 
   const dispatch = createEventDispatcher()
@@ -41,8 +40,7 @@
     {autoWidth}
     {autocomplete}
     {customPopoverHeight}
-    bind:fetchTerm
-    {useFetch}
+    bind:searchTerm
     on:change={onChange}
     on:click
   />

packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte

@@ -86,8 +86,16 @@
   $: userPage = $userPageInfo.page
   $: logsPage = $logsPageInfo.page
 
+  let usersObj = {}
+  $: usersObj = {
+    ...usersObj,
+    ...$users.data?.reduce((accumulator, user) => {
+      accumulator[user._id] = user
+      return accumulator
+    }, {}),
+  }
   $: sortedUsers = sort(
-    enrich($users.data || [], selectedUsers, "_id"),
+    enrich(Object.values(usersObj), selectedUsers, "_id"),
     "email"
   )
   $: sortedEvents = sort(
@@ -256,8 +264,7 @@
   <div class="controls">
     <div class="select">
       <Multiselect
-        bind:fetchTerm={userSearchTerm}
-        useFetch
+        bind:searchTerm={userSearchTerm}
         placeholder="All users"
         label="Users"
         autocomplete

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,11 +1,6 @@
 <script>
-  import {
-    CoreSelect,
-    CoreMultiselect,
-    Input,
-    ProgressCircle,
-  } from "@budibase/bbui"
-  import { fetchData, Utils } from "@budibase/frontend-core"
+  import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
+  import { fetchData } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
   import { FieldTypes } from "../../../constants"
@@ -26,16 +21,8 @@
   let fieldApi
   let fieldSchema
   let tableDefinition
-  let primaryDisplay
-  let options
-  let selectedOptions = []
-  let isOpen = false
-  let hasFilter
-
-  let searchResults
-  let searchString
-  let searching = false
-  let lastSearchId
+  let searchTerm
+  let open
 
   $: multiselect = fieldSchema?.relationshipType !== "one-to-many"
   $: linkedTableId = fieldSchema?.tableId
@@ -50,54 +37,74 @@
       limit: 100,
     },
   })
-  $: hasFilter = !!filter?.filter(f => !!f.field)?.length
-  $: fetch.update({ filter })
-  $: {
-    options = searchResults ? searchResults : $fetch.rows
-    const nonMatchingOptions = selectedOptions.filter(
-      option => !options.map(opt => opt._id).includes(option._id)
-    )
-    // Append initially selected options if there is no filter
-    // and hasn't already been appended
-    if (!hasFilter) {
-      options = [...options, ...nonMatchingOptions]
-    }
-  }
+
   $: tableDefinition = $fetch.definition
-  $: primaryDisplay = tableDefinition?.primaryDisplay || "_id"
-  $: singleValue = flatten(fieldState?.value)?.[0]
-  $: multiValue = flatten(fieldState?.value) ?? []
+  $: selectedValue = multiselect
+    ? flatten(fieldState?.value) ?? []
+    : flatten(fieldState?.value)?.[0]
   $: component = multiselect ? CoreMultiselect : CoreSelect
   $: expandedDefaultValue = expand(defaultValue)
-  $: debouncedSearch(searchString)
+  $: primaryDisplay = tableDefinition?.primaryDisplay
+
+  let optionsObj = {}
+  let initialValuesProcessed
+
   $: {
-    if (searching) {
-      isOpen = true
+    if (!initialValuesProcessed && primaryDisplay) {
+      // Persist the initial values as options, allowing them to be present in the dropdown,
+      // even if they are not in the inital fetch results
+      initialValuesProcessed = true
+      optionsObj = fieldState?.value?.reduce((accumulator, value) => {
+        accumulator[value._id] = {
+          _id: value._id,
+          [primaryDisplay]: value.primaryDisplay,
+        }
+        return accumulator
+      }, optionsObj)
     }
   }
 
-  // Fetch the initially selected values
-  // as they may not be within the first 100 records
+  $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
+  const enrichOptions = (optionsObj, fetchResults) => {
+    const result = (fetchResults || [])?.reduce((accumulator, row) => {
+      if (!accumulator[row._id]) {
+        accumulator[row._id] = row
+      }
+      return accumulator
+    }, optionsObj)
+
+    return Object.values(result)
+  }
   $: {
-    if (
-      primaryDisplay !== "_id" &&
-      fieldState?.value?.length &&
-      !selectedOptions?.length
-    ) {
-      API.searchTable({
-        paginate: false,
-        tableId: linkedTableId,
-        limit: 100,
-        query: {
-          oneOf: {
-            [`1:${primaryDisplay}`]: fieldState?.value?.map(
-              value => value.primaryDisplay
-            ),
-          },
-        },
-      }).then(response => {
-        const value = multiselect ? multiValue : singleValue
-        selectedOptions = response.rows.filter(row => value.includes(row._id))
+    // We don't want to reorder while the dropdown is open, to avoid UX jumps
+    if (!open) {
+      enrichedOptions = enrichedOptions.sort((a, b) => {
+        const selectedValues = flatten(fieldState?.value) || []
+
+        const aIsSelected = selectedValues.find(v => v === a._id)
+        const bIsSelected = selectedValues.find(v => v === b._id)
+        if (aIsSelected && !bIsSelected) {
+          return -1
+        } else if (!aIsSelected && bIsSelected) {
+          return 1
+        }
+
+        return a[primaryDisplay] > b[primaryDisplay]
+      })
+    }
+  }
+
+  $: fetchRows(searchTerm, primaryDisplay)
+
+  const fetchRows = (searchTerm, primaryDisplay) => {
+    const allRowsFetched =
+      $fetch.loaded &&
+      !Object.keys($fetch.query?.string || {}).length &&
+      !$fetch.hasNextPage
+    // Don't request until we have the primary display
+    if (!allRowsFetched && primaryDisplay) {
+      fetch.update({
+        query: { string: { [primaryDisplay]: searchTerm } },
       })
     }
   }
@@ -113,7 +120,7 @@
   }
 
   const getDisplayName = row => {
-    return row?.[tableDefinition?.primaryDisplay || "_id"] || "-"
+    return row?.[primaryDisplay] || "-"
   }
 
   const singleHandler = e => {
@@ -136,66 +143,16 @@
 
   const handleChange = value => {
     const changed = fieldApi.setValue(value)
-    selectedOptions = value.map(val => ({
-      _id: val,
-      [primaryDisplay]: options.find(option => option._id === val)[
-        primaryDisplay
-      ],
-    }))
     if (onChange && changed) {
       onChange({ value })
     }
   }
 
-  // Search for rows based on the search string
-  const search = async searchString => {
-    // Reset state if this search is invalid
-    if (!linkedTableId || !searchString) {
-      searchResults = null
-      return
+  const loadMore = () => {
+    if (!$fetch.loading) {
+      fetch.nextPage()
     }
-
-    // If a filter exists, then do a client side search
-    if (hasFilter) {
-      searchResults = $fetch.rows.filter(option =>
-        option[primaryDisplay].startsWith(searchString)
-      )
-      isOpen = true
-      return
-    }
-
-    // Search for results, using IDs to track invocations and ensure we're
-    // handling the latest update
-    lastSearchId = Math.random()
-    searching = true
-    const thisSearchId = lastSearchId
-    const results = await API.searchTable({
-      paginate: false,
-      tableId: linkedTableId,
-      limit: 100,
-      query: {
-        string: {
-          [`1:${primaryDisplay}`]: searchString || "",
-        },
-      },
-    })
-    searching = false
-
-    // In case searching takes longer than our debounced update, abandon these
-    // results
-    if (thisSearchId !== lastSearchId) {
-      return
-    }
-
-    // Process results
-    searchResults = results.rows?.map(row => ({
-      ...row,
-      primaryDisplay: row[primaryDisplay],
-    }))
   }
-
-  // Debounced version of searching
-  const debouncedSearch = Utils.debounce(search, 250)
 </script>
 
 <Field
@@ -210,63 +167,23 @@
   bind:fieldSchema
 >
   {#if fieldState}
-    <div class={autocomplete ? "field-with-search" : ""}>
-      <svelte:component
-        this={component}
-        bind:open={isOpen}
-        {options}
-        autocomplete={false}
-        value={multiselect ? multiValue : singleValue}
-        on:change={multiselect ? multiHandler : singleHandler}
-        id={fieldState.fieldId}
-        disabled={fieldState.disabled}
-        error={fieldState.error}
-        getOptionLabel={getDisplayName}
-        getOptionValue={option => option._id}
-        {placeholder}
-        customPopoverOffsetBelow={autocomplete ? 32 : null}
-        customPopoverMaxHeight={autocomplete ? 240 : null}
-        sort={true}
-      />
-      {#if autocomplete}
-        <div class="search">
-          <Input
-            autofocus
-            quiet
-            type="text"
-            bind:value={searchString}
-            placeholder={primaryDisplay ? `Search by ${primaryDisplay}` : null}
-          />
-          {#if searching}
-            <div>
-              <ProgressCircle size="S" />
-            </div>
-          {/if}
-        </div>
-      {/if}
-    </div>
+    <svelte:component
+      this={component}
+      options={enrichedOptions}
+      {autocomplete}
+      value={selectedValue}
+      on:change={multiselect ? multiHandler : singleHandler}
+      on:loadMore={loadMore}
+      id={fieldState.fieldId}
+      disabled={fieldState.disabled}
+      error={fieldState.error}
+      getOptionLabel={getDisplayName}
+      getOptionValue={option => option._id}
+      {placeholder}
+      bind:searchTerm
+      loading={$fetch.loading}
+      bind:open
+      customPopoverMaxHeight={400}
+    />
   {/if}
 </Field>
-
-<style>
-  .search {
-    flex: 0 0 calc(var(--default-row-height) - 1px);
-    display: flex;
-    align-items: center;
-    margin: 4px var(--cell-padding);
-    width: calc(100% - 2 * var(--cell-padding));
-  }
-  .search :global(.spectrum-Textfield) {
-    min-width: 0;
-    width: 100%;
-  }
-  .search :global(.spectrum-Textfield-input) {
-    font-size: 13px;
-  }
-  .search :global(.spectrum-Form-item) {
-    flex: 1 1 auto;
-  }
-  .field-with-search {
-    min-height: 80px;
-  }
-</style>