CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-09 00:06:05 +12:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/whitelabelling

Browse source
This commit is contained in:
Dean 2023-03-15 10:28:08 +00:00
parent f3b58923cf 75838994d0
commit c7e54947a8
17 changed files with 925 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View file

@ -308,7 +308,7 @@
{ name: "Auto Column", type: AUTO_TYPE }, { name: "Auto Column", type: AUTO_TYPE },
] ]
} else { } else {
return [ let fields = [
FIELDS.STRING, FIELDS.STRING,
FIELDS.BARCODEQR, FIELDS.BARCODEQR,
FIELDS.LONGFORM, FIELDS.LONGFORM,
@ -316,10 +316,13 @@
FIELDS.DATETIME, FIELDS.DATETIME,
FIELDS.NUMBER, FIELDS.NUMBER,
FIELDS.BOOLEAN, FIELDS.BOOLEAN,
FIELDS.ARRAY,
FIELDS.FORMULA, FIELDS.FORMULA,
FIELDS.LINK,
] ]
// no-sql or a spreadsheet
if (!external || table.sql) {
fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
}
return fields
} }
} }

4
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte
View file

@ -35,7 +35,9 @@
await datasources.fetch() await datasources.fetch()
$goto(`../../table/${table._id}`) $goto(`../../table/${table._id}`)
} catch (error) { } catch (error) {
notifications.error("Error saving table") notifications.error(
`Error saving table - ${error?.message || "unknown error"}`
)
} }
} }
</script> </script>

32
packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte
View file

@ -1,15 +1,22 @@
<script> <script>
import { ModalContent, Body, Layout } from "@budibase/bbui" import { ModalContent, Body, Layout, Link } from "@budibase/bbui"
import { IntegrationNames } from "constants/backend" import { IntegrationNames } from "constants/backend"
import cloneDeep from "lodash/cloneDeepWith" import cloneDeep from "lodash/cloneDeepWith"
import GoogleButton from "../_components/GoogleButton.svelte" import GoogleButton from "../_components/GoogleButton.svelte"
import { saveDatasource as save } from "builderStore/datasource" import { saveDatasource as save } from "builderStore/datasource"
import { organisation } from "stores/portal"
import { onMount } from "svelte"
export let integration export let integration
export let modal export let modal
// kill the reference so the input isn't saved // kill the reference so the input isn't saved
let datasource = cloneDeep(integration) let datasource = cloneDeep(integration)
$: isGoogleConfigured = !!$organisation.google
onMount(async () => {
await organisation.init()
})
</script> </script>
<ModalContent <ModalContent
@ -18,12 +25,21 @@
cancelText="Back" cancelText="Back"
size="L" size="L"
> >
<Layout noPadding> <!-- check true and false directly, don't render until flag is set -->
<Body size="XS" {#if isGoogleConfigured === true}
>Authenticate with your google account to use the {IntegrationNames[ <Layout noPadding>
datasource.type <Body size="S"
]} integration.</Body >Authenticate with your google account to use the {IntegrationNames[
datasource.type
]} integration.</Body
>
</Layout>
<GoogleButton preAuthStep={() => save(datasource, true)} />
{:else if isGoogleConfigured === false}
<Body size="S"
>Google authentication is not enabled, please complete Google SSO
configuration.</Body
> >
</Layout> <Link href="/builder/portal/settings/auth">Configure Google SSO</Link>
<GoogleButton preAuthStep={() => save(datasource, true)} /> {/if}
</ModalContent> </ModalContent>

17
packages/builder/src/pages/builder/portal/settings/auth/index.svelte
View file

@ -47,8 +47,9 @@
$: googleCallbackTooltip = $admin.cloud $: googleCallbackTooltip = $admin.cloud
? null ? null
: googleCallbackReadonly : googleCallbackReadonly
? "Vist the organisation page to update the platform URL" ? "Visit the organisation page to update the platform URL"
: "Leave blank to use the default callback URL" : "Leave blank to use the default callback URL"
$: googleSheetsCallbackUrl = `${$organisation.platformUrl}/api/global/auth/datasource/google/callback`
$: GoogleConfigFields = { $: GoogleConfigFields = {
Google: [ Google: [
@ -62,6 +63,14 @@
placeholder: $organisation.googleCallbackUrl, placeholder: $organisation.googleCallbackUrl,
copyButton: true, copyButton: true,
}, },
{
name: "sheetsURL",
label: "Sheets URL",
readonly: googleCallbackReadonly,
tooltip: googleCallbackTooltip,
placeholder: googleSheetsCallbackUrl,
copyButton: true,
},
], ],
} }
@ -396,7 +405,11 @@
</Heading> </Heading>
<Body size="S"> <Body size="S">
To allow users to authenticate using their Google accounts, fill out the To allow users to authenticate using their Google accounts, fill out the
fields below. fields below. Read the <Link
size="M"
href={"https://docs.budibase.com/docs/sso-with-google"}
>documentation</Link
> for more information.
</Body> </Body>
</Layout> </Layout>
<Layout gap="XS" noPadding> <Layout gap="XS" noPadding>

3
packages/server/src/api/controllers/datasource.ts
View file

@ -84,8 +84,9 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
setDefaultDisplayColumns(datasource) setDefaultDisplayColumns(datasource)
const dbResp = await db.put(datasource) const dbResp = await db.put(datasource)
datasource._rev = dbResp.rev datasource._rev = dbResp.rev
const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
const response: any = { datasource } const response: any = { datasource: cleanedDatasource }
if (error) { if (error) {
response.error = error response.error = error
} }

42
packages/server/src/api/controllers/row/external.ts
View file

@ -12,7 +12,7 @@ import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem" import { apiFileReturn } from "../../../utilities/fileSystem"
import { import {
Operation, Operation,
BBContext, UserCtx,
Row, Row,
PaginationJson, PaginationJson,
Table, Table,
@ -21,6 +21,7 @@ import {
SortJson, SortJson,
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import * as utils from "./utils"
const { cleanExportRows } = require("./utils") const { cleanExportRows } = require("./utils")
@ -49,12 +50,19 @@ export async function handleRequest(
) )
} }
export async function patch(ctx: BBContext) { export async function patch(ctx: UserCtx) {
const inputs = ctx.request.body const inputs = ctx.request.body
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const id = inputs._id const id = inputs._id
// don't save the ID to db // don't save the ID to db
delete inputs._id delete inputs._id
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.UPDATE, tableId, { return handleRequest(Operation.UPDATE, tableId, {
id: breakRowIdField(id), id: breakRowIdField(id),
row: inputs, row: inputs,
@ -62,16 +70,23 @@ export async function patch(ctx: BBContext) {
}) })
} }
export async function save(ctx: BBContext) { export async function save(ctx: UserCtx) {
const inputs = ctx.request.body const inputs = ctx.request.body
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.CREATE, tableId, { return handleRequest(Operation.CREATE, tableId, {
row: inputs, row: inputs,
includeSqlRelationships: IncludeRelationship.EXCLUDE, includeSqlRelationships: IncludeRelationship.EXCLUDE,
}) })
} }
export async function fetchView(ctx: BBContext) { export async function fetchView(ctx: UserCtx) {
// there are no views in external datasources, shouldn't ever be called // there are no views in external datasources, shouldn't ever be called
// for now just fetch // for now just fetch
const split = ctx.params.viewName.split("all_") const split = ctx.params.viewName.split("all_")
@ -79,14 +94,14 @@ export async function fetchView(ctx: BBContext) {
return fetch(ctx) return fetch(ctx)
} }
export async function fetch(ctx: BBContext) { export async function fetch(ctx: UserCtx) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, { return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE, includeSqlRelationships: IncludeRelationship.INCLUDE,
}) })
} }
export async function find(ctx: BBContext) { export async function find(ctx: UserCtx) {
const id = ctx.params.rowId const id = ctx.params.rowId
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const response = (await handleRequest(Operation.READ, tableId, { const response = (await handleRequest(Operation.READ, tableId, {
@ -96,7 +111,7 @@ export async function find(ctx: BBContext) {
return response ? response[0] : response return response ? response[0] : response
} }
export async function destroy(ctx: BBContext) { export async function destroy(ctx: UserCtx) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const id = ctx.request.body._id const id = ctx.request.body._id
const { row } = (await handleRequest(Operation.DELETE, tableId, { const { row } = (await handleRequest(Operation.DELETE, tableId, {
@ -106,7 +121,7 @@ export async function destroy(ctx: BBContext) {
return { response: { ok: true }, row } return { response: { ok: true }, row }
} }
export async function bulkDestroy(ctx: BBContext) { export async function bulkDestroy(ctx: UserCtx) {
const { rows } = ctx.request.body const { rows } = ctx.request.body
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
let promises: Promise<Row[] | { row: Row; table: Table }>[] = [] let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
@ -122,7 +137,7 @@ export async function bulkDestroy(ctx: BBContext) {
return { response: { ok: true }, rows: responses.map(resp => resp.row) } return { response: { ok: true }, rows: responses.map(resp => resp.row) }
} }
export async function search(ctx: BBContext) { export async function search(ctx: UserCtx) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params let { bookmark, limit } = params
@ -185,12 +200,7 @@ export async function search(ctx: BBContext) {
} }
} }
export async function validate(ctx: BBContext) { export async function exportRows(ctx: UserCtx) {
// can't validate external right now - maybe in future
return { valid: true }
}
export async function exportRows(ctx: BBContext) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format const format = ctx.query.format
const { columns } = ctx.request.body const { columns } = ctx.request.body
@ -244,7 +254,7 @@ export async function exportRows(ctx: BBContext) {
return apiFileReturn(exporter(headers, exportRows)) return apiFileReturn(exporter(headers, exportRows))
} }
export async function fetchEnrichedRow(ctx: BBContext) { export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId const id = ctx.params.rowId
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const { datasourceId, tableName } = breakExternalTableId(tableId) const { datasourceId, tableName } = breakExternalTableId(tableId)

9
packages/server/src/api/controllers/row/index.ts
View file

@ -2,6 +2,8 @@ import { quotas } from "@budibase/pro"
import * as internal from "./internal" import * as internal from "./internal"
import * as external from "./external" import * as external from "./external"
import { isExternalTable } from "../../../integrations/utils" import { isExternalTable } from "../../../integrations/utils"
import { Ctx } from "@budibase/types"
import * as utils from "./utils"
function pickApi(tableId: any) { function pickApi(tableId: any) {
if (isExternalTable(tableId)) { if (isExternalTable(tableId)) {
@ -129,9 +131,12 @@ export async function search(ctx: any) {
}) })
} }
export async function validate(ctx: any) { export async function validate(ctx: Ctx) {
const tableId = getTableId(ctx) const tableId = getTableId(ctx)
ctx.body = await pickApi(tableId).validate(ctx) ctx.body = await utils.validate({
row: ctx.request.body,
tableId,
})
} }
export async function fetchEnrichedRow(ctx: any) { export async function fetchEnrichedRow(ctx: any) {

7
packages/server/src/api/controllers/row/internal.ts
View file

@ -387,13 +387,6 @@ export async function search(ctx: Ctx) {
return response return response
} }
export async function validate(ctx: Ctx) {
return utils.validate({
tableId: ctx.params.tableId,
row: ctx.request.body,
})
}
export async function exportRows(ctx: Ctx) { export async function exportRows(ctx: Ctx) {
const db = context.getAppDB() const db = context.getAppDB()
const table = await db.get(ctx.params.tableId) const table = await db.get(ctx.params.tableId)

7
packages/server/src/api/controllers/row/utils.ts
View file

@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query" import { makeExternalQuery } from "../../../integrations/base/query"
import { Row, Table } from "@budibase/types" import { Row, Table } from "@budibase/types"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import { Format } from "../view/exporters" import { Format } from "../view/exporters"
import { Ctx } from "@budibase/types" import { Ctx } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
validateJs.extend(validateJs.validators.datetime, { validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) { parse: function (value: string) {
@ -56,8 +56,7 @@ export async function validate({
}) { }) {
let fetchedTable: Table let fetchedTable: Table
if (!table) { if (!table) {
const db = context.getAppDB() fetchedTable = await sdk.tables.getTable(tableId)
fetchedTable = await db.get(tableId)
} else { } else {
fetchedTable = table fetchedTable = table
} }

41
packages/server/src/api/controllers/table/external.ts
View file

@ -7,6 +7,7 @@ import {
generateJunctionTableName, generateJunctionTableName,
foreignKeyStructure, foreignKeyStructure,
hasTypeChanged, hasTypeChanged,
setStaticSchemas,
} from "./utils" } from "./utils"
import { FieldTypes } from "../../../constants" import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query" import { makeExternalQuery } from "../../../integrations/base/query"
@ -20,7 +21,7 @@ import {
Operation, Operation,
RenameColumn, RenameColumn,
FieldSchema, FieldSchema,
BBContext, UserCtx,
TableRequest, TableRequest,
RelationshipTypes, RelationshipTypes,
} from "@budibase/types" } from "@budibase/types"
@ -194,20 +195,20 @@ function isRelationshipSetup(column: FieldSchema) {
return column.foreignKey || column.through return column.foreignKey || column.through
} }
export async function save(ctx: BBContext) { export async function save(ctx: UserCtx) {
const table: TableRequest = ctx.request.body const inputs: TableRequest = ctx.request.body
const renamed = table?._rename const renamed = inputs?._rename
// can't do this right now // can't do this right now
delete table.rows delete inputs.rows
const datasourceId = getDatasourceId(ctx.request.body)! const datasourceId = getDatasourceId(ctx.request.body)!
// table doesn't exist already, note that it is created // table doesn't exist already, note that it is created
if (!table._id) { if (!inputs._id) {
table.created = true inputs.created = true
} }
let tableToSave: TableRequest = { let tableToSave: TableRequest = {
type: "table", type: "table",
_id: buildExternalTableId(datasourceId, table.name), _id: buildExternalTableId(datasourceId, inputs.name),
...table, ...inputs,
} }
let oldTable let oldTable
@ -224,6 +225,10 @@ export async function save(ctx: BBContext) {
if (!datasource.entities) { if (!datasource.entities) {
datasource.entities = {} datasource.entities = {}
} }
// GSheets is a specific case - only ever has a static primary key
tableToSave = setStaticSchemas(datasource, tableToSave)
const oldTables = cloneDeep(datasource.entities) const oldTables = cloneDeep(datasource.entities)
const tables: Record<string, Table> = datasource.entities const tables: Record<string, Table> = datasource.entities
@ -246,7 +251,7 @@ export async function save(ctx: BBContext) {
const junctionTable = generateManyLinkSchema( const junctionTable = generateManyLinkSchema(
datasource, datasource,
schema, schema,
table, tableToSave,
relatedTable relatedTable
) )
if (tables[junctionTable.name]) { if (tables[junctionTable.name]) {
@ -256,10 +261,12 @@ export async function save(ctx: BBContext) {
extraTablesToUpdate.push(junctionTable) extraTablesToUpdate.push(junctionTable)
} else { } else {
const fkTable = const fkTable =
relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable relationType === RelationshipTypes.ONE_TO_MANY
? tableToSave
: relatedTable
const foreignKey = generateLinkSchema( const foreignKey = generateLinkSchema(
schema, schema,
table, tableToSave,
relatedTable, relatedTable,
relationType relationType
) )
@ -271,11 +278,11 @@ export async function save(ctx: BBContext) {
fkTable.constrained.push(foreignKey) fkTable.constrained.push(foreignKey)
} }
// foreign key is in other table, need to save it to external // foreign key is in other table, need to save it to external
if (fkTable._id !== table._id) { if (fkTable._id !== tableToSave._id) {
extraTablesToUpdate.push(fkTable) extraTablesToUpdate.push(fkTable)
} }
} }
generateRelatedSchema(schema, relatedTable, table, relatedColumnName) generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName)
schema.main = true schema.main = true
} }
@ -313,7 +320,7 @@ export async function save(ctx: BBContext) {
return tableToSave return tableToSave
} }
export async function destroy(ctx: BBContext) { export async function destroy(ctx: UserCtx) {
const tableToDelete: TableRequest = await sdk.tables.getTable( const tableToDelete: TableRequest = await sdk.tables.getTable(
ctx.params.tableId ctx.params.tableId
) )
@ -339,7 +346,7 @@ export async function destroy(ctx: BBContext) {
return tableToDelete return tableToDelete
} }
export async function bulkImport(ctx: BBContext) { export async function bulkImport(ctx: UserCtx) {
const table = await sdk.tables.getTable(ctx.params.tableId) const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows }: { rows: unknown } = ctx.request.body const { rows }: { rows: unknown } = ctx.request.body
const schema: unknown = table.schema const schema: unknown = table.schema
@ -348,7 +355,7 @@ export async function bulkImport(ctx: BBContext) {
ctx.throw(400, "Provided data import information is invalid.") ctx.throw(400, "Provided data import information is invalid.")
} }
const parsedRows = await parse(rows, schema) const parsedRows = parse(rows, schema)
await handleRequest(Operation.BULK_CREATE, table._id!, { await handleRequest(Operation.BULK_CREATE, table._id!, {
rows: parsedRows, rows: parsedRows,
}) })

18
packages/server/src/api/controllers/table/index.ts
View file

@ -8,7 +8,7 @@ import {
import { isExternalTable, isSQL } from "../../../integrations/utils" import { isExternalTable, isSQL } from "../../../integrations/utils"
import { getDatasourceParams } from "../../../db/utils" import { getDatasourceParams } from "../../../db/utils"
import { context, events } from "@budibase/backend-core" import { context, events } from "@budibase/backend-core"
import { Table, BBContext } from "@budibase/types" import { Table, UserCtx } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import csv from "csvtojson" import csv from "csvtojson"
@ -25,7 +25,7 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
} }
// covers both internal and external // covers both internal and external
export async function fetch(ctx: BBContext) { export async function fetch(ctx: UserCtx) {
const db = context.getAppDB() const db = context.getAppDB()
const internal = await sdk.tables.getAllInternalTables() const internal = await sdk.tables.getAllInternalTables()
@ -53,12 +53,12 @@ export async function fetch(ctx: BBContext) {
ctx.body = [...internal, ...external] ctx.body = [...internal, ...external]
} }
export async function find(ctx: BBContext) { export async function find(ctx: UserCtx) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
ctx.body = await sdk.tables.getTable(tableId) ctx.body = await sdk.tables.getTable(tableId)
} }
export async function save(ctx: BBContext) { export async function save(ctx: UserCtx) {
const appId = ctx.appId const appId = ctx.appId
const table = ctx.request.body const table = ctx.request.body
const isImport = table.rows const isImport = table.rows
@ -79,7 +79,7 @@ export async function save(ctx: BBContext) {
ctx.body = savedTable ctx.body = savedTable
} }
export async function destroy(ctx: BBContext) { export async function destroy(ctx: UserCtx) {
const appId = ctx.appId const appId = ctx.appId
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const deletedTable = await pickApi({ tableId }).destroy(ctx) const deletedTable = await pickApi({ tableId }).destroy(ctx)
@ -91,7 +91,7 @@ export async function destroy(ctx: BBContext) {
ctx.body = { message: `Table ${tableId} deleted.` } ctx.body = { message: `Table ${tableId} deleted.` }
} }
export async function bulkImport(ctx: BBContext) { export async function bulkImport(ctx: UserCtx) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
await pickApi({ tableId }).bulkImport(ctx) await pickApi({ tableId }).bulkImport(ctx)
// right now we don't trigger anything for bulk import because it // right now we don't trigger anything for bulk import because it
@ -101,7 +101,7 @@ export async function bulkImport(ctx: BBContext) {
ctx.body = { message: `Bulk rows created.` } ctx.body = { message: `Bulk rows created.` }
} }
export async function csvToJson(ctx: BBContext) { export async function csvToJson(ctx: UserCtx) {
const { csvString } = ctx.request.body const { csvString } = ctx.request.body
const result = await csv().fromString(csvString) const result = await csv().fromString(csvString)
@ -110,7 +110,7 @@ export async function csvToJson(ctx: BBContext) {
ctx.body = result ctx.body = result
} }
export async function validateNewTableImport(ctx: BBContext) { export async function validateNewTableImport(ctx: UserCtx) {
const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
if (isRows(rows) && isSchema(schema)) { if (isRows(rows) && isSchema(schema)) {
@ -121,7 +121,7 @@ export async function validateNewTableImport(ctx: BBContext) {
} }
} }
export async function validateExistingTableImport(ctx: BBContext) { export async function validateExistingTableImport(ctx: UserCtx) {
const { rows, tableId }: { rows: unknown; tableId: unknown } = const { rows, tableId }: { rows: unknown; tableId: unknown } =
ctx.request.body ctx.request.body

20
packages/server/src/api/controllers/table/utils.ts
View file

@ -1,7 +1,11 @@
import { parse, isSchema, isRows } from "../../../utilities/schema" import { parse, isSchema, isRows } from "../../../utilities/schema"
import { getRowParams, generateRowID, InternalTables } from "../../../db/utils" import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
import { isEqual } from "lodash" import { isEqual } from "lodash"
import { AutoFieldSubTypes, FieldTypes } from "../../../constants" import {
AutoFieldSubTypes,
FieldTypes,
GOOGLE_SHEETS_PRIMARY_KEY,
} from "../../../constants"
import { import {
inputProcessing, inputProcessing,
cleanupAttachments, cleanupAttachments,
@ -16,7 +20,7 @@ import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro" import { quotas } from "@budibase/pro"
import { events, context } from "@budibase/backend-core" import { events, context } from "@budibase/backend-core"
import { Database } from "@budibase/types" import { Database, Datasource, SourceName, Table } from "@budibase/types"
export async function clearColumns(table: any, columnNames: any) { export async function clearColumns(table: any, columnNames: any) {
const db: Database = context.getAppDB() const db: Database = context.getAppDB()
@ -392,5 +396,17 @@ export function hasTypeChanged(table: any, oldTable: any) {
return false return false
} }
// used for external tables, some of them will have static schemas that need
// to be hard set
export function setStaticSchemas(datasource: Datasource, table: Table) {
// GSheets is a specific case - only ever has a static primary key
if (table && datasource.source === SourceName.GOOGLE_SHEETS) {
table.primary = [GOOGLE_SHEETS_PRIMARY_KEY]
// if there is an id column, remove it, should never exist in GSheets
delete table.schema?.id
}
return table
}
const _TableSaveFunctions = TableSaveFunctions const _TableSaveFunctions = TableSaveFunctions
export { _TableSaveFunctions as TableSaveFunctions } export { _TableSaveFunctions as TableSaveFunctions }

1
packages/server/src/constants/index.ts
View file

@ -180,3 +180,4 @@ export enum AutomationErrors {
// pass through the list from the auth/core lib // pass through the list from the auth/core lib
export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
export const MAX_AUTOMATION_RECURRING_ERRORS = 5 export const MAX_AUTOMATION_RECURRING_ERRORS = 5
export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"

222
packages/server/src/integrations/googlesheets.ts
View file

@ -1,22 +1,25 @@
import { import {
DatasourceFieldType, DatasourceFieldType,
DatasourcePlus, DatasourcePlus,
FieldType,
Integration, Integration,
Operation,
PaginationJson, PaginationJson,
QueryJson, QueryJson,
QueryType, QueryType,
Row,
SearchFilters, SearchFilters,
SortJson, SortJson,
Table, Table,
TableSchema, TableRequest,
} from "@budibase/types" } from "@budibase/types"
import { OAuth2Client } from "google-auth-library" import { OAuth2Client } from "google-auth-library"
import { buildExternalTableId } from "./utils" import { buildExternalTableId, finaliseExternalTables } from "./utils"
import { DataSourceOperation, FieldTypes } from "../constants"
import { GoogleSpreadsheet } from "google-spreadsheet" import { GoogleSpreadsheet } from "google-spreadsheet"
import fetch from "node-fetch" import fetch from "node-fetch"
import { configs, HTTPError } from "@budibase/backend-core" import { configs, HTTPError } from "@budibase/backend-core"
import { dataFilters } from "@budibase/shared-core" import { dataFilters } from "@budibase/shared-core"
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
interface GoogleSheetsConfig { interface GoogleSheetsConfig {
spreadsheetId: string spreadsheetId: string
@ -39,6 +42,17 @@ interface AuthTokenResponse {
access_token: string access_token: string
} }
const ALLOWED_TYPES = [
FieldType.STRING,
FieldType.FORMULA,
FieldType.NUMBER,
FieldType.LONGFORM,
FieldType.DATETIME,
FieldType.OPTIONS,
FieldType.BOOLEAN,
FieldType.BARCODEQR,
]
const SCHEMA: Integration = { const SCHEMA: Integration = {
plus: true, plus: true,
auth: { auth: {
@ -199,73 +213,90 @@ class GoogleSheetsIntegration implements DatasourcePlus {
this.client.useOAuth2Client(oauthClient) this.client.useOAuth2Client(oauthClient)
await this.client.loadInfo() await this.client.loadInfo()
} catch (err) { } catch (err: any) {
// this happens for xlsx imports
if (err.message?.includes("operation is not supported")) {
err.message =
"This operation is not supported - XLSX sheets must be converted."
}
console.error("Error connecting to google sheets", err) console.error("Error connecting to google sheets", err)
throw err throw err
} }
} }
async buildSchema(datasourceId: string) { getTableSchema(title: string, headerValues: string[], id?: string) {
// base table
const table: Table = {
name: title,
primary: [GOOGLE_SHEETS_PRIMARY_KEY],
schema: {},
}
if (id) {
table._id = id
}
// build schema from headers
for (let header of headerValues) {
table.schema[header] = {
name: header,
type: FieldType.STRING,
}
}
return table
}
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
await this.connect() await this.connect()
const sheets = this.client.sheetsByIndex const sheets = this.client.sheetsByIndex
const tables: Record<string, Table> = {} const tables: Record<string, Table> = {}
for (let sheet of sheets) { for (let sheet of sheets) {
// must fetch rows to determine schema // must fetch rows to determine schema
await sheet.getRows() await sheet.getRows()
// build schema
const schema: TableSchema = {}
// build schema from headers const id = buildExternalTableId(datasourceId, sheet.title)
for (let header of sheet.headerValues) { tables[sheet.title] = this.getTableSchema(
schema[header] = { sheet.title,
name: header, sheet.headerValues,
type: FieldTypes.STRING, id
} )
}
// create tables
tables[sheet.title] = {
_id: buildExternalTableId(datasourceId, sheet.title),
name: sheet.title,
primary: ["rowNumber"],
schema,
}
} }
const final = finaliseExternalTables(tables, entities)
this.tables = tables this.tables = final.tables
this.schemaErrors = final.errors
} }
async query(json: QueryJson) { async query(json: QueryJson) {
const sheet = json.endpoint.entityId const sheet = json.endpoint.entityId
switch (json.endpoint.operation) {
const handlers = { case Operation.CREATE:
[DataSourceOperation.CREATE]: () => return this.create({ sheet, row: json.body as Row })
this.create({ sheet, row: json.body }), case Operation.BULK_CREATE:
[DataSourceOperation.READ]: () => this.read({ ...json, sheet }), return this.createBulk({ sheet, rows: json.body as Row[] })
[DataSourceOperation.UPDATE]: () => case Operation.READ:
this.update({ return this.read({ ...json, sheet })
case Operation.UPDATE:
return this.update({
// exclude the header row and zero index // exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet, sheet,
row: json.body, row: json.body,
}), })
[DataSourceOperation.DELETE]: () => case Operation.DELETE:
this.delete({ return this.delete({
// exclude the header row and zero index // exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet, sheet,
}), })
[DataSourceOperation.CREATE_TABLE]: () => case Operation.CREATE_TABLE:
this.createTable(json?.table?.name), return this.createTable(json?.table?.name)
[DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table), case Operation.UPDATE_TABLE:
[DataSourceOperation.DELETE_TABLE]: () => return this.updateTable(json.table!)
this.deleteTable(json?.table?.name), case Operation.DELETE_TABLE:
return this.deleteTable(json?.table?.name)
default:
throw new Error(
`GSheets integration does not support "${json.endpoint.operation}".`
)
} }
// @ts-ignore
const internalQueryMethod = handlers[json.endpoint.operation]
return await internalQueryMethod()
} }
buildRowObject(headers: string[], values: string[], rowNumber: number) { buildRowObject(headers: string[], values: string[], rowNumber: number) {
@ -278,47 +309,70 @@ class GoogleSheetsIntegration implements DatasourcePlus {
} }
async createTable(name?: string) { async createTable(name?: string) {
if (!name) {
throw new Error("Must provide name for new sheet.")
}
try { try {
await this.connect() await this.connect()
return await this.client.addSheet({ title: name, headerValues: ["test"] }) return await this.client.addSheet({ title: name, headerValues: [name] })
} catch (err) { } catch (err) {
console.error("Error creating new table in google sheets", err) console.error("Error creating new table in google sheets", err)
throw err throw err
} }
} }
async updateTable(table?: any) { async updateTable(table: TableRequest) {
try { await this.connect()
await this.connect() const sheet = this.client.sheetsByTitle[table.name]
const sheet = this.client.sheetsByTitle[table.name] await sheet.loadHeaderRow()
await sheet.loadHeaderRow()
if (table._rename) { if (table._rename) {
const headers = [] const headers = []
for (let header of sheet.headerValues) { for (let header of sheet.headerValues) {
if (header === table._rename.old) { if (header === table._rename.old) {
headers.push(table._rename.updated) headers.push(table._rename.updated)
} else { } else {
headers.push(header) headers.push(header)
}
} }
await sheet.setHeaderRow(headers)
} else {
const updatedHeaderValues = [...sheet.headerValues]
const newField = Object.keys(table.schema).find(
key => !sheet.headerValues.includes(key)
)
if (newField) {
updatedHeaderValues.push(newField)
}
await sheet.setHeaderRow(updatedHeaderValues)
} }
} catch (err) { try {
console.error("Error updating table in google sheets", err) await sheet.setHeaderRow(headers)
throw err } catch (err) {
console.error("Error updating column name in google sheets", err)
throw err
}
} else {
const updatedHeaderValues = [...sheet.headerValues]
// add new column - doesn't currently exist
for (let [key, column] of Object.entries(table.schema)) {
if (!ALLOWED_TYPES.includes(column.type)) {
throw new Error(
`Column type: ${column.type} not allowed for GSheets integration.`
)
}
if (
!sheet.headerValues.includes(key) &&
column.type !== FieldType.FORMULA
) {
updatedHeaderValues.push(key)
}
}
// clear out deleted columns
for (let key of sheet.headerValues) {
if (!Object.keys(table.schema).includes(key)) {
const idx = updatedHeaderValues.indexOf(key)
updatedHeaderValues.splice(idx, 1)
}
}
try {
await sheet.setHeaderRow(updatedHeaderValues)
} catch (err) {
console.error("Error updating table in google sheets", err)
throw err
}
} }
} }
@ -349,6 +403,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
} }
} }
async createBulk(query: { sheet: string; rows: any[] }) {
try {
await this.connect()
const sheet = this.client.sheetsByTitle[query.sheet]
let rowsToInsert = []
for (let row of query.rows) {
rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row)
}
const rows = await sheet.addRows(rowsToInsert)
return rows.map(row =>
this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber)
)
} catch (err) {
console.error("Error bulk writing to google sheets", err)
throw err
}
}
async read(query: { async read(query: {
sheet: string sheet: string
filters?: SearchFilters filters?: SearchFilters

20
packages/server/src/integrations/utils.ts
View file

@ -4,6 +4,7 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g const ROW_ID_REGEX = /^\[.*]$/g
const ENCODED_SPACE = encodeURIComponent(" ")
const SQL_NUMBER_TYPE_MAP = { const SQL_NUMBER_TYPE_MAP = {
integer: FieldTypes.NUMBER, integer: FieldTypes.NUMBER,
@ -79,6 +80,10 @@ export function isExternalTable(tableId: string) {
} }
export function buildExternalTableId(datasourceId: string, tableName: string) { export function buildExternalTableId(datasourceId: string, tableName: string) {
// encode spaces
if (tableName.includes(" ")) {
tableName = encodeURIComponent(tableName)
}
return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
} }
@ -90,6 +95,10 @@ export function breakExternalTableId(tableId: string | undefined) {
let datasourceId = parts.shift() let datasourceId = parts.shift()
// if they need joined // if they need joined
let tableName = parts.join(DOUBLE_SEPARATOR) let tableName = parts.join(DOUBLE_SEPARATOR)
// if contains encoded spaces, decode it
if (tableName.includes(ENCODED_SPACE)) {
tableName = decodeURIComponent(tableName)
}
return { datasourceId, tableName } return { datasourceId, tableName }
} }
@ -200,9 +209,9 @@ export function isIsoDateString(str: string) {
* @param column The column to check, to see if it is a valid relationship. * @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist. * @param tableIds The IDs of the tables which currently exist.
*/ */
function shouldCopyRelationship( export function shouldCopyRelationship(
column: { type: string; tableId?: string }, column: { type: string; tableId?: string },
tableIds: [string] tableIds: string[]
) { ) {
return ( return (
column.type === FieldTypes.LINK && column.type === FieldTypes.LINK &&
@ -219,7 +228,7 @@ function shouldCopyRelationship(
* @param column The column to check for options or boolean type. * @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database. * @param fetchedColumn The fetched column to check for the type in the external database.
*/ */
function shouldCopySpecialColumn( export function shouldCopySpecialColumn(
column: { type: string }, column: { type: string },
fetchedColumn: { type: string } | undefined fetchedColumn: { type: string } | undefined
) { ) {
@ -257,9 +266,12 @@ function copyExistingPropsOver(
tableIds: [string] tableIds: [string]
) { ) {
if (entities && entities[tableName]) { if (entities && entities[tableName]) {
if (entities[tableName].primaryDisplay) { if (entities[tableName]?.primaryDisplay) {
table.primaryDisplay = entities[tableName].primaryDisplay table.primaryDisplay = entities[tableName].primaryDisplay
} }
if (entities[tableName]?.created) {
table.created = entities[tableName]?.created
}
const existingTableSchema = entities[tableName].schema const existingTableSchema = entities[tableName].schema
for (let key in existingTableSchema) { for (let key in existingTableSchema) {
if (!existingTableSchema.hasOwnProperty(key)) { if (!existingTableSchema.hasOwnProperty(key)) {

637
packages/server/yarn.lock
View file

File diff suppressed because it is too large Load diff

1
packages/types/src/documents/app/table.ts
View file

@ -76,6 +76,7 @@ export interface Table extends Document {
sql?: boolean sql?: boolean
indexes?: { [key: string]: any } indexes?: { [key: string]: any }
rows?: { [key: string]: any } rows?: { [key: string]: any }
created?: boolean
} }
export interface TableRequest extends Table { export interface TableRequest extends Table {