Enable uploading a CSV file as a datasource and fix edge cases

packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte

@@ -10,6 +10,10 @@
     Drawer,
     DrawerContent,
     Icon,
+    Modal,
+    ModalContent,
+    CoreDropzone,
+    notifications,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { store, currentAsset } from "builderStore"
@@ -24,6 +28,7 @@
   import IntegrationQueryEditor from "components/integration/index.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { API } from "api"
 
   export let value = {}
   export let otherSources
@@ -39,6 +44,7 @@
   let tmpQueryParams
   let tmpCustomData
   let customDataValid = true
+  let modal
 
   $: text = value?.label ?? "Choose an option"
   $: tables = $tablesStore.list.map(m => ({
@@ -184,6 +190,26 @@
     })
     drawer.hide()
   }
+
+  const promptForCSV = () => {
+    drawer.hide()
+    modal.show()
+  }
+
+  const handleCSV = async e => {
+    try {
+      const csv = await e.detail[0]?.text()
+      if (csv?.length) {
+        const js = await API.csvToJson(csv)
+        tmpCustomData = JSON.stringify(js)
+      }
+    } catch (error) {
+      console.log(error)
+      notifications.error("Failed to parse CSV")
+    }
+    modal.hide()
+    drawer.show()
+  }
 </script>
 
 <div class="container" bind:this={anchorRight}>
@@ -227,12 +253,12 @@
       <Icon hoverable name="Settings" on:click={openCustomDrawer} />
     </div>
     <Drawer title="Custom data" bind:this={drawer}>
-      <Button
-        slot="buttons"
-        cta
-        on:click={saveCustomData}
-        disabled={!customDataValid}>Save</Button
-      >
+      <div slot="buttons" style="display:contents">
+        <Button primary on:click={promptForCSV}>Load CSV</Button>
+        <Button cta on:click={saveCustomData} disabled={!customDataValid}>
+          Save
+        </Button>
+      </div>
       <div slot="description">
         Provide a JavaScript or JSON array to use as data
       </div>
@@ -349,6 +375,12 @@
   </div>
 </Popover>
 
+<Modal bind:this={modal}>
+  <ModalContent title="Choose a CSV">
+    <CoreDropzone compact extensions=".csv" on:change={handleCSV} />
+  </ModalContent>
+</Modal>
+
 <style>
   .container {
     display: flex;

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -146,6 +146,11 @@ export const createActions = context => {
     return getAPI()?.actions.canUseColumn(name)
   }
 
+  // Gets the default number of rows for a single page
+  const getFeatures = () => {
+    return getAPI()?.actions.getFeatures()
+  }
+
   return {
     datasource: {
       ...datasource,
@@ -158,6 +163,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js

@@ -1,4 +1,7 @@
 import { get } from "svelte/store"
+import QueryFetch from "../../../../fetch/QueryFetch"
+
+export const RowPageSize = 100000
 
 export const createActions = context => {
   const { columns, stickyColumn, table, viewV2 } = context
@@ -35,6 +38,11 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
+  const getFeatures = () => {
+    // We don't support any features
+    return {}
+  }
+
   return {
     nonPlus: {
       actions: {
@@ -45,6 +53,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/table.js

@@ -1,7 +1,10 @@
 import { get } from "svelte/store"
+import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
+export const RowPageSize = 100
+
 export const createActions = context => {
   const { API, datasource, columns, stickyColumn } = context
 
@@ -45,6 +48,10 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
+  const getFeatures = () => {
+    return new TableFetch(null).determineFeatureFlags()
+  }
+
   return {
     table: {
       actions: {
@@ -55,6 +62,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -1,7 +1,10 @@
 import { get } from "svelte/store"
+import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
+export const RowPageSize = 100
+
 export const createActions = context => {
   const { API, datasource, columns, stickyColumn } = context
 
@@ -45,6 +48,10 @@ export const createActions = context => {
     )
   }
 
+  const getFeatures = () => {
+    return new ViewV2Fetch(null).determineFeatureFlags()
+  }
+
   return {
     viewV2: {
       actions: {
@@ -55,6 +62,7 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
+        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/rows.js

@@ -2,7 +2,6 @@ import { writable, derived, get } from "svelte/store"
 import { fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
 import { tick } from "svelte"
-import { Helpers } from "@budibase/bbui"
 
 export const createStores = () => {
   const rows = writable([])
@@ -111,6 +110,10 @@ export const createActions = context => {
     const $filter = get(filter)
     const $sort = get(sort)
 
+    // Determine how many rows to fetch per page
+    const features = datasource.actions.getFeatures()
+    const limit = features?.supportsPagination ? RowPageSize : 100000
+
     // Create new fetch model
     const newFetch = fetchData({
       API,
@@ -119,7 +122,7 @@ export const createActions = context => {
         filter: $filter,
         sortColumn: $sort.column,
         sortOrder: $sort.order,
-        limit: RowPageSize,
+        limit,
         paginate: true,
       },
     })

packages/frontend-core/src/fetch/CustomFetch.js

@@ -31,9 +31,9 @@ export default class CustomFetch extends DataFetch {
     if (typeof data === "string") {
       // Try JSON parsing
       try {
-        data = JSON.parse(data)
-        if (Array.isArray(data)) {
-          return data
+        const js = JSON.parse(data)
+        if (Array.isArray(js)) {
+          return js
         }
       } catch (error) {
         // Ignore