Merge pull request #10766 from Budibase/budi-6945/fetch_tables_on_googledatasource_creation
BUDI-6945 - Fetch sheets on creation
This commit is contained in:
commit
7c39946584
4 changed files with 121 additions and 31 deletions
|
@ -47,5 +47,6 @@ export async function validateDatasourceConfig(config) {
|
||||||
|
|
||||||
export async function getDatasourceInfo(config) {
|
export async function getDatasourceInfo(config) {
|
||||||
const datasource = prepareData(config)
|
const datasource = prepareData(config)
|
||||||
return await API.fetchInfoForDatasource(datasource)
|
const resp = await API.fetchInfoForDatasource(datasource)
|
||||||
|
return resp
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,12 +5,16 @@
|
||||||
Layout,
|
Layout,
|
||||||
Link,
|
Link,
|
||||||
notifications,
|
notifications,
|
||||||
|
FancyCheckboxGroup,
|
||||||
} from "@budibase/bbui"
|
} from "@budibase/bbui"
|
||||||
import { IntegrationNames, IntegrationTypes } from "constants/backend"
|
import { IntegrationNames, IntegrationTypes } from "constants/backend"
|
||||||
import GoogleButton from "../_components/GoogleButton.svelte"
|
import GoogleButton from "../_components/GoogleButton.svelte"
|
||||||
import { organisation } from "stores/portal"
|
import { organisation } from "stores/portal"
|
||||||
import { onMount } from "svelte"
|
import { onMount } from "svelte"
|
||||||
import { validateDatasourceConfig } from "builderStore/datasource"
|
import {
|
||||||
|
validateDatasourceConfig,
|
||||||
|
getDatasourceInfo,
|
||||||
|
} from "builderStore/datasource"
|
||||||
import cloneDeep from "lodash/cloneDeepWith"
|
import cloneDeep from "lodash/cloneDeepWith"
|
||||||
import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte"
|
import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte"
|
||||||
import { goto } from "@roxi/routify"
|
import { goto } from "@roxi/routify"
|
||||||
|
@ -32,8 +36,9 @@
|
||||||
const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS]
|
const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS]
|
||||||
|
|
||||||
export const GoogleDatasouceConfigStep = {
|
export const GoogleDatasouceConfigStep = {
|
||||||
AUTH: "Auth",
|
AUTH: "auth",
|
||||||
SET_URL: "Set_url",
|
SET_URL: "set_url",
|
||||||
|
SET_SHEETS: "set_sheets",
|
||||||
}
|
}
|
||||||
|
|
||||||
let step = continueSetupId
|
let step = continueSetupId
|
||||||
|
@ -42,21 +47,14 @@
|
||||||
|
|
||||||
let isValid = false
|
let isValid = false
|
||||||
|
|
||||||
const modalConfig = {
|
let allSheets
|
||||||
[GoogleDatasouceConfigStep.AUTH]: {},
|
let selectedSheets
|
||||||
[GoogleDatasouceConfigStep.SET_URL]: {
|
|
||||||
confirmButtonText: "Connect",
|
|
||||||
onConfirm: async () => {
|
|
||||||
if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) {
|
|
||||||
const resp = await validateDatasourceConfig(datasource)
|
|
||||||
if (!resp.connected) {
|
|
||||||
notifications.error(`Unable to connect - ${resp.error}`)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
const saveDatasourceAndRedirect = async () => {
|
||||||
try {
|
try {
|
||||||
const resp = await saveDatasource(datasource)
|
const resp = await saveDatasource(datasource, {
|
||||||
|
tablesFilter: selectedSheets,
|
||||||
|
})
|
||||||
$goto(`./datasource/${resp._id}`)
|
$goto(`./datasource/${resp._id}`)
|
||||||
notifications.success(`Datasource created successfully.`)
|
notifications.success(`Datasource created successfully.`)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
@ -64,13 +62,57 @@
|
||||||
// prevent the modal from closing
|
// prevent the modal from closing
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const modalConfig = {
|
||||||
|
[GoogleDatasouceConfigStep.AUTH]: {
|
||||||
|
title: `Connect to ${integrationName}`,
|
||||||
|
},
|
||||||
|
[GoogleDatasouceConfigStep.SET_URL]: {
|
||||||
|
title: `Connect your spreadsheet`,
|
||||||
|
confirmButtonText: "Connect",
|
||||||
|
onConfirm: async () => {
|
||||||
|
const checkConnection =
|
||||||
|
integration.features[DatasourceFeature.CONNECTION_CHECKING]
|
||||||
|
if (checkConnection) {
|
||||||
|
const resp = await validateDatasourceConfig(datasource)
|
||||||
|
if (!resp.connected) {
|
||||||
|
notifications.error(`Unable to connect - ${resp.error}`)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) {
|
||||||
|
saveDatasourceAndRedirect()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const info = await getDatasourceInfo(datasource)
|
||||||
|
allSheets = info.tableNames
|
||||||
|
|
||||||
|
step = GoogleDatasouceConfigStep.SET_SHEETS
|
||||||
|
notifications.success(
|
||||||
|
checkConnection
|
||||||
|
? "Connection Successful"
|
||||||
|
: `Datasource created successfully.`
|
||||||
|
)
|
||||||
|
|
||||||
|
// prevent the modal from closing
|
||||||
|
return false
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[GoogleDatasouceConfigStep.SET_SHEETS]: {
|
||||||
|
title: `Choose your sheets`,
|
||||||
|
confirmButtonText: "Fetch sheets",
|
||||||
|
onConfirm: async () => {
|
||||||
|
await saveDatasourceAndRedirect()
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<ModalContent
|
<ModalContent
|
||||||
title={`Connect to ${integrationName}`}
|
title={modalConfig[step].title}
|
||||||
cancelText="Cancel"
|
cancelText="Cancel"
|
||||||
size="L"
|
size="L"
|
||||||
confirmText={modalConfig[step].confirmButtonText}
|
confirmText={modalConfig[step].confirmButtonText}
|
||||||
|
@ -107,4 +149,11 @@
|
||||||
/>
|
/>
|
||||||
</Layout>
|
</Layout>
|
||||||
{/if}
|
{/if}
|
||||||
|
{#if step === GoogleDatasouceConfigStep.SET_SHEETS}
|
||||||
|
<Layout noPadding no>
|
||||||
|
<Body size="S">Select which spreadsheets you want to connect.</Body>
|
||||||
|
|
||||||
|
<FancyCheckboxGroup options={allSheets} bind:selected={selectedSheets} />
|
||||||
|
</Layout>
|
||||||
|
{/if}
|
||||||
</ModalContent>
|
</ModalContent>
|
||||||
|
|
|
@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils"
|
||||||
import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
|
import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet"
|
||||||
import fetch from "node-fetch"
|
import fetch from "node-fetch"
|
||||||
import { cache, configs, context, HTTPError } from "@budibase/backend-core"
|
import { cache, configs, context, HTTPError } from "@budibase/backend-core"
|
||||||
import { dataFilters } from "@budibase/shared-core"
|
import { dataFilters, utils } from "@budibase/shared-core"
|
||||||
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
|
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
|
||||||
import sdk from "../sdk"
|
import sdk from "../sdk"
|
||||||
|
|
||||||
|
@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
|
||||||
|
|
||||||
async testConnection(): Promise<ConnectionInfo> {
|
async testConnection(): Promise<ConnectionInfo> {
|
||||||
try {
|
try {
|
||||||
await setupCreationAuth(this.config)
|
|
||||||
await this.connect()
|
await this.connect()
|
||||||
return { connected: true }
|
return { connected: true }
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
|
@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus {
|
||||||
|
|
||||||
async connect() {
|
async connect() {
|
||||||
try {
|
try {
|
||||||
|
await setupCreationAuth(this.config)
|
||||||
|
|
||||||
// Initialise oAuth client
|
// Initialise oAuth client
|
||||||
let googleConfig = await configs.getGoogleDatasourceConfig()
|
let googleConfig = await configs.getGoogleDatasourceConfig()
|
||||||
if (!googleConfig) {
|
if (!googleConfig) {
|
||||||
|
@ -273,16 +274,14 @@ class GoogleSheetsIntegration implements DatasourcePlus {
|
||||||
}
|
}
|
||||||
|
|
||||||
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
|
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
|
||||||
// not fully configured yet
|
|
||||||
if (!this.config.auth) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
await this.connect()
|
await this.connect()
|
||||||
const sheets = this.client.sheetsByIndex
|
const sheets = this.client.sheetsByIndex
|
||||||
const tables: Record<string, Table> = {}
|
const tables: Record<string, Table> = {}
|
||||||
for (let sheet of sheets) {
|
await utils.parallelForeach(
|
||||||
|
sheets,
|
||||||
|
async sheet => {
|
||||||
// must fetch rows to determine schema
|
// must fetch rows to determine schema
|
||||||
await sheet.getRows()
|
await sheet.getRows({ limit: 0, offset: 0 })
|
||||||
|
|
||||||
const id = buildExternalTableId(datasourceId, sheet.title)
|
const id = buildExternalTableId(datasourceId, sheet.title)
|
||||||
tables[sheet.title] = this.getTableSchema(
|
tables[sheet.title] = this.getTableSchema(
|
||||||
|
@ -290,7 +289,9 @@ class GoogleSheetsIntegration implements DatasourcePlus {
|
||||||
sheet.headerValues,
|
sheet.headerValues,
|
||||||
id
|
id
|
||||||
)
|
)
|
||||||
}
|
},
|
||||||
|
10
|
||||||
|
)
|
||||||
const final = finaliseExternalTables(tables, entities)
|
const final = finaliseExternalTables(tables, entities)
|
||||||
this.tables = final.tables
|
this.tables = final.tables
|
||||||
this.schemaErrors = final.errors
|
this.schemaErrors = final.errors
|
||||||
|
|
|
@ -4,3 +4,42 @@ export function unreachable(
|
||||||
) {
|
) {
|
||||||
throw new Error(message)
|
throw new Error(message)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function parallelForeach<T>(
|
||||||
|
items: T[],
|
||||||
|
task: (item: T) => Promise<void>,
|
||||||
|
maxConcurrency: number
|
||||||
|
): Promise<void> {
|
||||||
|
const promises: Promise<void>[] = []
|
||||||
|
let index = 0
|
||||||
|
|
||||||
|
const processItem = async (item: T) => {
|
||||||
|
try {
|
||||||
|
await task(item)
|
||||||
|
} finally {
|
||||||
|
processNext()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const processNext = () => {
|
||||||
|
if (index >= items.length) {
|
||||||
|
// No more items to process
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const item = items[index]
|
||||||
|
index++
|
||||||
|
|
||||||
|
const promise = processItem(item)
|
||||||
|
promises.push(promise)
|
||||||
|
|
||||||
|
if (promises.length >= maxConcurrency) {
|
||||||
|
Promise.race(promises).then(processNext)
|
||||||
|
} else {
|
||||||
|
processNext()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
processNext()
|
||||||
|
|
||||||
|
await Promise.all(promises)
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue