Merge pull request #10766 from Budibase/budi-6945/fetch_tables_on_googledatasource_creation

BUDI-6945 - Fetch sheets on creation

packages/builder/src/builderStore/datasource.js

@@ -47,5 +47,6 @@ export async function validateDatasourceConfig(config) {
 
 export async function getDatasourceInfo(config) {
   const datasource = prepareData(config)
-  return await API.fetchInfoForDatasource(datasource)
+  const resp = await API.fetchInfoForDatasource(datasource)
+  return resp
 }

packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte

@@ -5,12 +5,16 @@
     Layout,
     Link,
     notifications,
+    FancyCheckboxGroup,
   } from "@budibase/bbui"
   import { IntegrationNames, IntegrationTypes } from "constants/backend"
   import GoogleButton from "../_components/GoogleButton.svelte"
   import { organisation } from "stores/portal"
   import { onMount } from "svelte"
-  import { validateDatasourceConfig } from "builderStore/datasource"
+  import {
+    validateDatasourceConfig,
+    getDatasourceInfo,
+  } from "builderStore/datasource"
   import cloneDeep from "lodash/cloneDeepWith"
   import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte"
   import { goto } from "@roxi/routify"
@@ -32,8 +36,9 @@
   const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS]
 
   export const GoogleDatasouceConfigStep = {
-    AUTH: "Auth",
+    AUTH: "auth",
-    SET_URL: "Set_url",
+    SET_URL: "set_url",
+    SET_SHEETS: "set_sheets",
   }
 
   let step = continueSetupId
@@ -42,12 +47,34 @@
 
   let isValid = false
 
+  let allSheets
+  let selectedSheets
+
+  const saveDatasourceAndRedirect = async () => {
+    try {
+      const resp = await saveDatasource(datasource, {
+        tablesFilter: selectedSheets,
+      })
+      $goto(`./datasource/${resp._id}`)
+      notifications.success(`Datasource created successfully.`)
+    } catch (err) {
+      notifications.error(err?.message ?? "Error saving datasource")
+      // prevent the modal from closing
+      return false
+    }
+  }
+
   const modalConfig = {
-    [GoogleDatasouceConfigStep.AUTH]: {},
+    [GoogleDatasouceConfigStep.AUTH]: {
+      title: `Connect to ${integrationName}`,
+    },
     [GoogleDatasouceConfigStep.SET_URL]: {
+      title: `Connect your spreadsheet`,
       confirmButtonText: "Connect",
       onConfirm: async () => {
-        if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
+        const checkConnection =
+          integration.features[DatasourceFeature.CONNECTION_CHECKING]
+        if (checkConnection) {
           const resp = await validateDatasourceConfig(datasource)
           if (!resp.connected) {
             notifications.error(`Unable to connect - ${resp.error}`)
@@ -55,22 +82,37 @@
           }
         }
 
-        try {
+        if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) {
-          const resp = await saveDatasource(datasource)
+          saveDatasourceAndRedirect()
-          $goto(`./datasource/${resp._id}`)
+          return
-          notifications.success(`Datasource created successfully.`)
-        } catch (err) {
-          notifications.error(err?.message ?? "Error saving datasource")
-          // prevent the modal from closing
-          return false
         }
+
+        const info = await getDatasourceInfo(datasource)
+        allSheets = info.tableNames
+
+        step = GoogleDatasouceConfigStep.SET_SHEETS
+        notifications.success(
+          checkConnection
+            ? "Connection Successful"
+            : `Datasource created successfully.`
+        )
+
+        // prevent the modal from closing
+        return false
+      },
+    },
+    [GoogleDatasouceConfigStep.SET_SHEETS]: {
+      title: `Choose your sheets`,
+      confirmButtonText: "Fetch sheets",
+      onConfirm: async () => {
+        await saveDatasourceAndRedirect()
       },
     },
   }
 </script>
 
 <ModalContent
-  title={`Connect to ${integrationName}`}
+  title={modalConfig[step].title}
   cancelText="Cancel"
   size="L"
   confirmText={modalConfig[step].confirmButtonText}
@@ -107,4 +149,11 @@
       />
     </Layout>
   {/if}
+  {#if step === GoogleDatasouceConfigStep.SET_SHEETS}
+    <Layout noPadding no>
+      <Body size="S">Select which spreadsheets you want to connect.</Body>
+
+      <FancyCheckboxGroup options={allSheets} bind:selected={selectedSheets} />
+    </Layout>
+  {/if}
 </ModalContent>

packages/server/src/integrations/googlesheets.ts

@@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils"
 import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { cache, configs, context, HTTPError } from "@budibase/backend-core"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, utils } from "@budibase/shared-core"
 import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
 import sdk from "../sdk"
 
@@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async testConnection(): Promise<ConnectionInfo> {
     try {
-      await setupCreationAuth(this.config)
       await this.connect()
       return { connected: true }
     } catch (e: any) {
@@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus {
 
   async connect() {
     try {
+      await setupCreationAuth(this.config)
+
       // Initialise oAuth client
       let googleConfig = await configs.getGoogleDatasourceConfig()
       if (!googleConfig) {
@@ -273,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
   }
 
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
-    // not fully configured yet
-    if (!this.config.auth) {
-      return
-    }
     await this.connect()
     const sheets = this.client.sheetsByIndex
     const tables: Record<string, Table> = {}
-    for (let sheet of sheets) {
+    await utils.parallelForeach(
-      // must fetch rows to determine schema
+      sheets,
-      await sheet.getRows()
+      async sheet => {
+        // must fetch rows to determine schema
+        await sheet.getRows({ limit: 0, offset: 0 })
 
-      const id = buildExternalTableId(datasourceId, sheet.title)
+        const id = buildExternalTableId(datasourceId, sheet.title)
-      tables[sheet.title] = this.getTableSchema(
+        tables[sheet.title] = this.getTableSchema(
-        sheet.title,
+          sheet.title,
-        sheet.headerValues,
+          sheet.headerValues,
-        id
+          id
-      )
+        )
-    }
+      },
+      10
+    )
     const final = finaliseExternalTables(tables, entities)
     this.tables = final.tables
     this.schemaErrors = final.errors

packages/shared-core/src/utils.ts

@@ -4,3 +4,42 @@ export function unreachable(
 ) {
   throw new Error(message)
 }
+
+export async function parallelForeach<T>(
+  items: T[],
+  task: (item: T) => Promise<void>,
+  maxConcurrency: number
+): Promise<void> {
+  const promises: Promise<void>[] = []
+  let index = 0
+
+  const processItem = async (item: T) => {
+    try {
+      await task(item)
+    } finally {
+      processNext()
+    }
+  }
+
+  const processNext = () => {
+    if (index >= items.length) {
+      // No more items to process
+      return
+    }
+
+    const item = items[index]
+    index++
+
+    const promise = processItem(item)
+    promises.push(promise)
+
+    if (promises.length >= maxConcurrency) {
+      Promise.race(promises).then(processNext)
+    } else {
+      processNext()
+    }
+  }
+  processNext()
+
+  await Promise.all(promises)
+}