diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 49250d2628..6f9bd29fd8 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -47,5 +47,6 @@ export async function validateDatasourceConfig(config) { export async function getDatasourceInfo(config) { const datasource = prepareData(config) - return await API.fetchInfoForDatasource(datasource) + const resp = await API.fetchInfoForDatasource(datasource) + return resp } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 7b4808967d..93f8953cb6 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -5,12 +5,16 @@ Layout, Link, notifications, + FancyCheckboxGroup, } from "@budibase/bbui" import { IntegrationNames, IntegrationTypes } from "constants/backend" import GoogleButton from "../_components/GoogleButton.svelte" import { organisation } from "stores/portal" import { onMount } from "svelte" - import { validateDatasourceConfig } from "builderStore/datasource" + import { + validateDatasourceConfig, + getDatasourceInfo, + } from "builderStore/datasource" import cloneDeep from "lodash/cloneDeepWith" import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte" import { goto } from "@roxi/routify" @@ -32,8 +36,9 @@ const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS] export const GoogleDatasouceConfigStep = { - AUTH: "Auth", - SET_URL: "Set_url", + AUTH: "auth", + SET_URL: "set_url", + SET_SHEETS: "set_sheets", } let step = continueSetupId @@ -42,12 +47,34 @@ let isValid = false + let allSheets + let selectedSheets + + const saveDatasourceAndRedirect = async () => { + try { + const resp = await saveDatasource(datasource, { + tablesFilter: selectedSheets, + }) + $goto(`./datasource/${resp._id}`) + notifications.success(`Datasource created successfully.`) + } catch (err) { + notifications.error(err?.message ?? "Error saving datasource") + // prevent the modal from closing + return false + } + } + const modalConfig = { - [GoogleDatasouceConfigStep.AUTH]: {}, + [GoogleDatasouceConfigStep.AUTH]: { + title: `Connect to ${integrationName}`, + }, [GoogleDatasouceConfigStep.SET_URL]: { + title: `Connect your spreadsheet`, confirmButtonText: "Connect", onConfirm: async () => { - if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) { + const checkConnection = + integration.features[DatasourceFeature.CONNECTION_CHECKING] + if (checkConnection) { const resp = await validateDatasourceConfig(datasource) if (!resp.connected) { notifications.error(`Unable to connect - ${resp.error}`) @@ -55,22 +82,37 @@ } } - try { - const resp = await saveDatasource(datasource) - $goto(`./datasource/${resp._id}`) - notifications.success(`Datasource created successfully.`) - } catch (err) { - notifications.error(err?.message ?? "Error saving datasource") - // prevent the modal from closing - return false + if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) { + saveDatasourceAndRedirect() + return } + + const info = await getDatasourceInfo(datasource) + allSheets = info.tableNames + + step = GoogleDatasouceConfigStep.SET_SHEETS + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) + + // prevent the modal from closing + return false + }, + }, + [GoogleDatasouceConfigStep.SET_SHEETS]: { + title: `Choose your sheets`, + confirmButtonText: "Fetch sheets", + onConfirm: async () => { + await saveDatasourceAndRedirect() }, }, } {/if} + {#if step === GoogleDatasouceConfigStep.SET_SHEETS} + + Select which spreadsheets you want to connect. + + + + {/if} diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index a792f49b57..91bc310a44 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils" import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet" import fetch from "node-fetch" import { cache, configs, context, HTTPError } from "@budibase/backend-core" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, utils } from "@budibase/shared-core" import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants" import sdk from "../sdk" @@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus { async testConnection(): Promise { try { - await setupCreationAuth(this.config) await this.connect() return { connected: true } } catch (e: any) { @@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus { async connect() { try { + await setupCreationAuth(this.config) + // Initialise oAuth client let googleConfig = await configs.getGoogleDatasourceConfig() if (!googleConfig) { @@ -273,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus { } async buildSchema(datasourceId: string, entities: Record) { - // not fully configured yet - if (!this.config.auth) { - return - } await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} - for (let sheet of sheets) { - // must fetch rows to determine schema - await sheet.getRows() + await utils.parallelForeach( + sheets, + async sheet => { + // must fetch rows to determine schema + await sheet.getRows({ limit: 0, offset: 0 }) - const id = buildExternalTableId(datasourceId, sheet.title) - tables[sheet.title] = this.getTableSchema( - sheet.title, - sheet.headerValues, - id - ) - } + const id = buildExternalTableId(datasourceId, sheet.title) + tables[sheet.title] = this.getTableSchema( + sheet.title, + sheet.headerValues, + id + ) + }, + 10 + ) const final = finaliseExternalTables(tables, entities) this.tables = final.tables this.schemaErrors = final.errors diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 720027d6a7..46a6585f89 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -4,3 +4,42 @@ export function unreachable( ) { throw new Error(message) } + +export async function parallelForeach( + items: T[], + task: (item: T) => Promise, + maxConcurrency: number +): Promise { + const promises: Promise[] = [] + let index = 0 + + const processItem = async (item: T) => { + try { + await task(item) + } finally { + processNext() + } + } + + const processNext = () => { + if (index >= items.length) { + // No more items to process + return + } + + const item = items[index] + index++ + + const promise = processItem(item) + promises.push(promise) + + if (promises.length >= maxConcurrency) { + Promise.race(promises).then(processNext) + } else { + processNext() + } + } + processNext() + + await Promise.all(promises) +}