Merge pull request #10766 from Budibase/budi-6945/fetch_tables_on_googledatasource_creation
BUDI-6945 - Fetch sheets on creation
This commit is contained in:
commit
7c39946584
4 changed files with 121 additions and 31 deletions
|
@ -47,5 +47,6 @@ export async function validateDatasourceConfig(config) {
|
|||
|
||||
export async function getDatasourceInfo(config) {
|
||||
const datasource = prepareData(config)
|
||||
return await API.fetchInfoForDatasource(datasource)
|
||||
const resp = await API.fetchInfoForDatasource(datasource)
|
||||
return resp
|
||||
}
|
||||
|
|
|
@ -5,12 +5,16 @@
|
|||
Layout,
|
||||
Link,
|
||||
notifications,
|
||||
FancyCheckboxGroup,
|
||||
} from "@budibase/bbui"
|
||||
import { IntegrationNames, IntegrationTypes } from "constants/backend"
|
||||
import GoogleButton from "../_components/GoogleButton.svelte"
|
||||
import { organisation } from "stores/portal"
|
||||
import { onMount } from "svelte"
|
||||
import { validateDatasourceConfig } from "builderStore/datasource"
|
||||
import {
|
||||
validateDatasourceConfig,
|
||||
getDatasourceInfo,
|
||||
} from "builderStore/datasource"
|
||||
import cloneDeep from "lodash/cloneDeepWith"
|
||||
import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte"
|
||||
import { goto } from "@roxi/routify"
|
||||
|
@ -32,8 +36,9 @@
|
|||
const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS]
|
||||
|
||||
export const GoogleDatasouceConfigStep = {
|
||||
AUTH: "Auth",
|
||||
SET_URL: "Set_url",
|
||||
AUTH: "auth",
|
||||
SET_URL: "set_url",
|
||||
SET_SHEETS: "set_sheets",
|
||||
}
|
||||
|
||||
let step = continueSetupId
|
||||
|
@ -42,12 +47,34 @@
|
|||
|
||||
let isValid = false
|
||||
|
||||
let allSheets
|
||||
let selectedSheets
|
||||
|
||||
const saveDatasourceAndRedirect = async () => {
|
||||
try {
|
||||
const resp = await saveDatasource(datasource, {
|
||||
tablesFilter: selectedSheets,
|
||||
})
|
||||
$goto(`./datasource/${resp._id}`)
|
||||
notifications.success(`Datasource created successfully.`)
|
||||
} catch (err) {
|
||||
notifications.error(err?.message ?? "Error saving datasource")
|
||||
// prevent the modal from closing
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
const modalConfig = {
|
||||
[GoogleDatasouceConfigStep.AUTH]: {},
|
||||
[GoogleDatasouceConfigStep.AUTH]: {
|
||||
title: `Connect to ${integrationName}`,
|
||||
},
|
||||
[GoogleDatasouceConfigStep.SET_URL]: {
|
||||
title: `Connect your spreadsheet`,
|
||||
confirmButtonText: "Connect",
|
||||
onConfirm: async () => {
|
||||
if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
|
||||
const checkConnection =
|
||||
integration.features[DatasourceFeature.CONNECTION_CHECKING]
|
||||
if (checkConnection) {
|
||||
const resp = await validateDatasourceConfig(datasource)
|
||||
if (!resp.connected) {
|
||||
notifications.error(`Unable to connect - ${resp.error}`)
|
||||
|
@ -55,22 +82,37 @@
|
|||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await saveDatasource(datasource)
|
||||
$goto(`./datasource/${resp._id}`)
|
||||
notifications.success(`Datasource created successfully.`)
|
||||
} catch (err) {
|
||||
notifications.error(err?.message ?? "Error saving datasource")
|
||||
// prevent the modal from closing
|
||||
return false
|
||||
if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) {
|
||||
saveDatasourceAndRedirect()
|
||||
return
|
||||
}
|
||||
|
||||
const info = await getDatasourceInfo(datasource)
|
||||
allSheets = info.tableNames
|
||||
|
||||
step = GoogleDatasouceConfigStep.SET_SHEETS
|
||||
notifications.success(
|
||||
checkConnection
|
||||
? "Connection Successful"
|
||||
: `Datasource created successfully.`
|
||||
)
|
||||
|
||||
// prevent the modal from closing
|
||||
return false
|
||||
},
|
||||
},
|
||||
[GoogleDatasouceConfigStep.SET_SHEETS]: {
|
||||
title: `Choose your sheets`,
|
||||
confirmButtonText: "Fetch sheets",
|
||||
onConfirm: async () => {
|
||||
await saveDatasourceAndRedirect()
|
||||
},
|
||||
},
|
||||
}
|
||||
</script>
|
||||
|
||||
<ModalContent
|
||||
title={`Connect to ${integrationName}`}
|
||||
title={modalConfig[step].title}
|
||||
cancelText="Cancel"
|
||||
size="L"
|
||||
confirmText={modalConfig[step].confirmButtonText}
|
||||
|
@ -107,4 +149,11 @@
|
|||
/>
|
||||
</Layout>
|
||||
{/if}
|
||||
{#if step === GoogleDatasouceConfigStep.SET_SHEETS}
|
||||
<Layout noPadding no>
|
||||
<Body size="S">Select which spreadsheets you want to connect.</Body>
|
||||
|
||||
<FancyCheckboxGroup options={allSheets} bind:selected={selectedSheets} />
|
||||
</Layout>
|
||||
{/if}
|
||||
</ModalContent>
|
||||
|
|
|
@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils"
|
|||
import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
|
||||
import fetch from "node-fetch"
|
||||
import { cache, configs, context, HTTPError } from "@budibase/backend-core"
|
||||
import { dataFilters } from "@budibase/shared-core"
|
||||
import { dataFilters, utils } from "@budibase/shared-core"
|
||||
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
|
||||
import sdk from "../sdk"
|
||||
|
||||
|
@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
|
|||
|
||||
async testConnection(): Promise<ConnectionInfo> {
|
||||
try {
|
||||
await setupCreationAuth(this.config)
|
||||
await this.connect()
|
||||
return { connected: true }
|
||||
} catch (e: any) {
|
||||
|
@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus {
|
|||
|
||||
async connect() {
|
||||
try {
|
||||
await setupCreationAuth(this.config)
|
||||
|
||||
// Initialise oAuth client
|
||||
let googleConfig = await configs.getGoogleDatasourceConfig()
|
||||
if (!googleConfig) {
|
||||
|
@ -273,24 +274,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
|
|||
}
|
||||
|
||||
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
|
||||
// not fully configured yet
|
||||
if (!this.config.auth) {
|
||||
return
|
||||
}
|
||||
await this.connect()
|
||||
const sheets = this.client.sheetsByIndex
|
||||
const tables: Record<string, Table> = {}
|
||||
for (let sheet of sheets) {
|
||||
// must fetch rows to determine schema
|
||||
await sheet.getRows()
|
||||
await utils.parallelForeach(
|
||||
sheets,
|
||||
async sheet => {
|
||||
// must fetch rows to determine schema
|
||||
await sheet.getRows({ limit: 0, offset: 0 })
|
||||
|
||||
const id = buildExternalTableId(datasourceId, sheet.title)
|
||||
tables[sheet.title] = this.getTableSchema(
|
||||
sheet.title,
|
||||
sheet.headerValues,
|
||||
id
|
||||
)
|
||||
}
|
||||
const id = buildExternalTableId(datasourceId, sheet.title)
|
||||
tables[sheet.title] = this.getTableSchema(
|
||||
sheet.title,
|
||||
sheet.headerValues,
|
||||
id
|
||||
)
|
||||
},
|
||||
10
|
||||
)
|
||||
const final = finaliseExternalTables(tables, entities)
|
||||
this.tables = final.tables
|
||||
this.schemaErrors = final.errors
|
||||
|
|
|
@ -4,3 +4,42 @@ export function unreachable(
|
|||
) {
|
||||
throw new Error(message)
|
||||
}
|
||||
|
||||
export async function parallelForeach<T>(
|
||||
items: T[],
|
||||
task: (item: T) => Promise<void>,
|
||||
maxConcurrency: number
|
||||
): Promise<void> {
|
||||
const promises: Promise<void>[] = []
|
||||
let index = 0
|
||||
|
||||
const processItem = async (item: T) => {
|
||||
try {
|
||||
await task(item)
|
||||
} finally {
|
||||
processNext()
|
||||
}
|
||||
}
|
||||
|
||||
const processNext = () => {
|
||||
if (index >= items.length) {
|
||||
// No more items to process
|
||||
return
|
||||
}
|
||||
|
||||
const item = items[index]
|
||||
index++
|
||||
|
||||
const promise = processItem(item)
|
||||
promises.push(promise)
|
||||
|
||||
if (promises.length >= maxConcurrency) {
|
||||
Promise.race(promises).then(processNext)
|
||||
} else {
|
||||
processNext()
|
||||
}
|
||||
}
|
||||
processNext()
|
||||
|
||||
await Promise.all(promises)
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue