CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-05 22:40:39 +12:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/whitelabelling

Browse source
This commit is contained in:
Dean 2023-03-15 10:28:08 +00:00
parent f3b58923cf 75838994d0
commit c7e54947a8
17 changed files with 925 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View file

@ -308,7 +308,7 @@
{ name: "Auto Column", type: AUTO_TYPE },
]
} else {
return [
let fields = [
FIELDS.STRING,
FIELDS.BARCODEQR,
FIELDS.LONGFORM,
@ -316,10 +316,13 @@
FIELDS.DATETIME,
FIELDS.NUMBER,
FIELDS.BOOLEAN,
FIELDS.ARRAY,
FIELDS.FORMULA,
FIELDS.LINK,
]
// no-sql or a spreadsheet
if (!external || table.sql) {
fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
}
return fields
}
}

4
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte
View file

@ -35,7 +35,9 @@
await datasources.fetch()
$goto(`../../table/${table._id}`)
} catch (error) {
notifications.error("Error saving table")
notifications.error(
`Error saving table - ${error?.message || "unknown error"}`
)
}
}
</script>

32
packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte
View file

@ -1,15 +1,22 @@
<script>
import { ModalContent, Body, Layout } from "@budibase/bbui"
import { ModalContent, Body, Layout, Link } from "@budibase/bbui"
import { IntegrationNames } from "constants/backend"
import cloneDeep from "lodash/cloneDeepWith"
import GoogleButton from "../_components/GoogleButton.svelte"
import { saveDatasource as save } from "builderStore/datasource"
import { organisation } from "stores/portal"
import { onMount } from "svelte"
export let integration
export let modal
// kill the reference so the input isn't saved
let datasource = cloneDeep(integration)
$: isGoogleConfigured = !!$organisation.google
onMount(async () => {
await organisation.init()
})
</script>
<ModalContent
@ -18,12 +25,21 @@
cancelText="Back"
size="L"
>
<Layout noPadding>
<Body size="XS"
>Authenticate with your google account to use the {IntegrationNames[
datasource.type
]} integration.</Body
<!-- check true and false directly, don't render until flag is set -->
{#if isGoogleConfigured === true}
<Layout noPadding>
<Body size="S"
>Authenticate with your google account to use the {IntegrationNames[
datasource.type
]} integration.</Body
>
</Layout>
<GoogleButton preAuthStep={() => save(datasource, true)} />
{:else if isGoogleConfigured === false}
<Body size="S"
>Google authentication is not enabled, please complete Google SSO
configuration.</Body
>
</Layout>
<GoogleButton preAuthStep={() => save(datasource, true)} />
<Link href="/builder/portal/settings/auth">Configure Google SSO</Link>
{/if}
</ModalContent>

17
packages/builder/src/pages/builder/portal/settings/auth/index.svelte
View file

@ -47,8 +47,9 @@
$: googleCallbackTooltip = $admin.cloud
? null
: googleCallbackReadonly
? "Vist the organisation page to update the platform URL"
? "Visit the organisation page to update the platform URL"
: "Leave blank to use the default callback URL"
$: googleSheetsCallbackUrl = `${$organisation.platformUrl}/api/global/auth/datasource/google/callback`
$: GoogleConfigFields = {
Google: [
@ -62,6 +63,14 @@
placeholder: $organisation.googleCallbackUrl,
copyButton: true,
},
{
name: "sheetsURL",
label: "Sheets URL",
readonly: googleCallbackReadonly,
tooltip: googleCallbackTooltip,
placeholder: googleSheetsCallbackUrl,
copyButton: true,
},
],
}
@ -396,7 +405,11 @@
</Heading>
<Body size="S">
To allow users to authenticate using their Google accounts, fill out the
fields below.
fields below. Read the <Link
size="M"
href={"https://docs.budibase.com/docs/sso-with-google"}
>documentation</Link
> for more information.
</Body>
</Layout>
<Layout gap="XS" noPadding>

3
packages/server/src/api/controllers/datasource.ts
View file

@ -84,8 +84,9 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
setDefaultDisplayColumns(datasource)
const dbResp = await db.put(datasource)
datasource._rev = dbResp.rev
const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
const response: any = { datasource }
const response: any = { datasource: cleanedDatasource }
if (error) {
response.error = error
}

42
packages/server/src/api/controllers/row/external.ts
View file

@ -12,7 +12,7 @@ import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
Operation,
BBContext,
UserCtx,
Row,
PaginationJson,
Table,
@ -21,6 +21,7 @@ import {
SortJson,
} from "@budibase/types"
import sdk from "../../../sdk"
import * as utils from "./utils"
const { cleanExportRows } = require("./utils")
@ -49,12 +50,19 @@ export async function handleRequest(
)
}
export async function patch(ctx: BBContext) {
export async function patch(ctx: UserCtx) {
const inputs = ctx.request.body
const tableId = ctx.params.tableId
const id = inputs._id
// don't save the ID to db
delete inputs._id
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.UPDATE, tableId, {
id: breakRowIdField(id),
row: inputs,
@ -62,16 +70,23 @@ export async function patch(ctx: BBContext) {
})
}
export async function save(ctx: BBContext) {
export async function save(ctx: UserCtx) {
const inputs = ctx.request.body
const tableId = ctx.params.tableId
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.CREATE, tableId, {
row: inputs,
includeSqlRelationships: IncludeRelationship.EXCLUDE,
})
}
export async function fetchView(ctx: BBContext) {
export async function fetchView(ctx: UserCtx) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
@ -79,14 +94,14 @@ export async function fetchView(ctx: BBContext) {
return fetch(ctx)
}
export async function fetch(ctx: BBContext) {
export async function fetch(ctx: UserCtx) {
const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function find(ctx: BBContext) {
export async function find(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId
const response = (await handleRequest(Operation.READ, tableId, {
@ -96,7 +111,7 @@ export async function find(ctx: BBContext) {
return response ? response[0] : response
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const tableId = ctx.params.tableId
const id = ctx.request.body._id
const { row } = (await handleRequest(Operation.DELETE, tableId, {
@ -106,7 +121,7 @@ export async function destroy(ctx: BBContext) {
return { response: { ok: true }, row }
}
export async function bulkDestroy(ctx: BBContext) {
export async function bulkDestroy(ctx: UserCtx) {
const { rows } = ctx.request.body
const tableId = ctx.params.tableId
let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
@ -122,7 +137,7 @@ export async function bulkDestroy(ctx: BBContext) {
return { response: { ok: true }, rows: responses.map(resp => resp.row) }
}
export async function search(ctx: BBContext) {
export async function search(ctx: UserCtx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
@ -185,12 +200,7 @@ export async function search(ctx: BBContext) {
}
}
export async function validate(ctx: BBContext) {
// can't validate external right now - maybe in future
return { valid: true }
}
export async function exportRows(ctx: BBContext) {
export async function exportRows(ctx: UserCtx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format
const { columns } = ctx.request.body
@ -244,7 +254,7 @@ export async function exportRows(ctx: BBContext) {
return apiFileReturn(exporter(headers, exportRows))
}
export async function fetchEnrichedRow(ctx: BBContext) {
export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId
const { datasourceId, tableName } = breakExternalTableId(tableId)

9
packages/server/src/api/controllers/row/index.ts
View file

@ -2,6 +2,8 @@ import { quotas } from "@budibase/pro"
import * as internal from "./internal"
import * as external from "./external"
import { isExternalTable } from "../../../integrations/utils"
import { Ctx } from "@budibase/types"
import * as utils from "./utils"
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
@ -129,9 +131,12 @@ export async function search(ctx: any) {
})
}
export async function validate(ctx: any) {
export async function validate(ctx: Ctx) {
const tableId = getTableId(ctx)
ctx.body = await pickApi(tableId).validate(ctx)
ctx.body = await utils.validate({
row: ctx.request.body,
tableId,
})
}
export async function fetchEnrichedRow(ctx: any) {

7
packages/server/src/api/controllers/row/internal.ts
View file

@ -387,13 +387,6 @@ export async function search(ctx: Ctx) {
return response
}
export async function validate(ctx: Ctx) {
return utils.validate({
tableId: ctx.params.tableId,
row: ctx.request.body,
})
}
export async function exportRows(ctx: Ctx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)

7
packages/server/src/api/controllers/row/utils.ts
View file

@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants"
import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Row, Table } from "@budibase/types"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import { Format } from "../view/exporters"
import { Ctx } from "@budibase/types"
import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
@ -56,8 +56,7 @@ export async function validate({
}) {
let fetchedTable: Table
if (!table) {
const db = context.getAppDB()
fetchedTable = await db.get(tableId)
fetchedTable = await sdk.tables.getTable(tableId)
} else {
fetchedTable = table
}

41
packages/server/src/api/controllers/table/external.ts
View file

@ -7,6 +7,7 @@ import {
generateJunctionTableName,
foreignKeyStructure,
hasTypeChanged,
setStaticSchemas,
} from "./utils"
import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query"
@ -20,7 +21,7 @@ import {
Operation,
RenameColumn,
FieldSchema,
BBContext,
UserCtx,
TableRequest,
RelationshipTypes,
} from "@budibase/types"
@ -194,20 +195,20 @@ function isRelationshipSetup(column: FieldSchema) {
return column.foreignKey || column.through
}
export async function save(ctx: BBContext) {
const table: TableRequest = ctx.request.body
const renamed = table?._rename
export async function save(ctx: UserCtx) {
const inputs: TableRequest = ctx.request.body
const renamed = inputs?._rename
// can't do this right now
delete table.rows
delete inputs.rows
const datasourceId = getDatasourceId(ctx.request.body)!
// table doesn't exist already, note that it is created
if (!table._id) {
table.created = true
if (!inputs._id) {
inputs.created = true
}
let tableToSave: TableRequest = {
type: "table",
_id: buildExternalTableId(datasourceId, table.name),
...table,
_id: buildExternalTableId(datasourceId, inputs.name),
...inputs,
}
let oldTable
@ -224,6 +225,10 @@ export async function save(ctx: BBContext) {
if (!datasource.entities) {
datasource.entities = {}
}
// GSheets is a specific case - only ever has a static primary key
tableToSave = setStaticSchemas(datasource, tableToSave)
const oldTables = cloneDeep(datasource.entities)
const tables: Record<string, Table> = datasource.entities
@ -246,7 +251,7 @@ export async function save(ctx: BBContext) {
const junctionTable = generateManyLinkSchema(
datasource,
schema,
table,
tableToSave,
relatedTable
)
if (tables[junctionTable.name]) {
@ -256,10 +261,12 @@ export async function save(ctx: BBContext) {
extraTablesToUpdate.push(junctionTable)
} else {
const fkTable =
relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable
relationType === RelationshipTypes.ONE_TO_MANY
? tableToSave
: relatedTable
const foreignKey = generateLinkSchema(
schema,
table,
tableToSave,
relatedTable,
relationType
)
@ -271,11 +278,11 @@ export async function save(ctx: BBContext) {
fkTable.constrained.push(foreignKey)
}
// foreign key is in other table, need to save it to external
if (fkTable._id !== table._id) {
if (fkTable._id !== tableToSave._id) {
extraTablesToUpdate.push(fkTable)
}
}
generateRelatedSchema(schema, relatedTable, table, relatedColumnName)
generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName)
schema.main = true
}
@ -313,7 +320,7 @@ export async function save(ctx: BBContext) {
return tableToSave
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const tableToDelete: TableRequest = await sdk.tables.getTable(
ctx.params.tableId
)
@ -339,7 +346,7 @@ export async function destroy(ctx: BBContext) {
return tableToDelete
}
export async function bulkImport(ctx: BBContext) {
export async function bulkImport(ctx: UserCtx) {
const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows }: { rows: unknown } = ctx.request.body
const schema: unknown = table.schema
@ -348,7 +355,7 @@ export async function bulkImport(ctx: BBContext) {
ctx.throw(400, "Provided data import information is invalid.")
}
const parsedRows = await parse(rows, schema)
const parsedRows = parse(rows, schema)
await handleRequest(Operation.BULK_CREATE, table._id!, {
rows: parsedRows,
})

18
packages/server/src/api/controllers/table/index.ts
View file

@ -8,7 +8,7 @@ import {
import { isExternalTable, isSQL } from "../../../integrations/utils"
import { getDatasourceParams } from "../../../db/utils"
import { context, events } from "@budibase/backend-core"
import { Table, BBContext } from "@budibase/types"
import { Table, UserCtx } from "@budibase/types"
import sdk from "../../../sdk"
import csv from "csvtojson"
@ -25,7 +25,7 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
}
// covers both internal and external
export async function fetch(ctx: BBContext) {
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const internal = await sdk.tables.getAllInternalTables()
@ -53,12 +53,12 @@ export async function fetch(ctx: BBContext) {
ctx.body = [...internal, ...external]
}
export async function find(ctx: BBContext) {
export async function find(ctx: UserCtx) {
const tableId = ctx.params.tableId
ctx.body = await sdk.tables.getTable(tableId)
}
export async function save(ctx: BBContext) {
export async function save(ctx: UserCtx) {
const appId = ctx.appId
const table = ctx.request.body
const isImport = table.rows
@ -79,7 +79,7 @@ export async function save(ctx: BBContext) {
ctx.body = savedTable
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const appId = ctx.appId
const tableId = ctx.params.tableId
const deletedTable = await pickApi({ tableId }).destroy(ctx)
@ -91,7 +91,7 @@ export async function destroy(ctx: BBContext) {
ctx.body = { message: `Table ${tableId} deleted.` }
}
export async function bulkImport(ctx: BBContext) {
export async function bulkImport(ctx: UserCtx) {
const tableId = ctx.params.tableId
await pickApi({ tableId }).bulkImport(ctx)
// right now we don't trigger anything for bulk import because it
@ -101,7 +101,7 @@ export async function bulkImport(ctx: BBContext) {
ctx.body = { message: `Bulk rows created.` }
}
export async function csvToJson(ctx: BBContext) {
export async function csvToJson(ctx: UserCtx) {
const { csvString } = ctx.request.body
const result = await csv().fromString(csvString)
@ -110,7 +110,7 @@ export async function csvToJson(ctx: BBContext) {
ctx.body = result
}
export async function validateNewTableImport(ctx: BBContext) {
export async function validateNewTableImport(ctx: UserCtx) {
const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
if (isRows(rows) && isSchema(schema)) {
@ -121,7 +121,7 @@ export async function validateNewTableImport(ctx: BBContext) {
}
}
export async function validateExistingTableImport(ctx: BBContext) {
export async function validateExistingTableImport(ctx: UserCtx) {
const { rows, tableId }: { rows: unknown; tableId: unknown } =
ctx.request.body

20
packages/server/src/api/controllers/table/utils.ts
View file

@ -1,7 +1,11 @@
import { parse, isSchema, isRows } from "../../../utilities/schema"
import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
import { isEqual } from "lodash"
import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
import {
AutoFieldSubTypes,
FieldTypes,
GOOGLE_SHEETS_PRIMARY_KEY,
} from "../../../constants"
import {
inputProcessing,
cleanupAttachments,
@ -16,7 +20,7 @@ import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
import { events, context } from "@budibase/backend-core"
import { Database } from "@budibase/types"
import { Database, Datasource, SourceName, Table } from "@budibase/types"
export async function clearColumns(table: any, columnNames: any) {
const db: Database = context.getAppDB()
@ -392,5 +396,17 @@ export function hasTypeChanged(table: any, oldTable: any) {
return false
}
// used for external tables, some of them will have static schemas that need
// to be hard set
export function setStaticSchemas(datasource: Datasource, table: Table) {
// GSheets is a specific case - only ever has a static primary key
if (table && datasource.source === SourceName.GOOGLE_SHEETS) {
table.primary = [GOOGLE_SHEETS_PRIMARY_KEY]
// if there is an id column, remove it, should never exist in GSheets
delete table.schema?.id
}
return table
}
const _TableSaveFunctions = TableSaveFunctions
export { _TableSaveFunctions as TableSaveFunctions }

1
packages/server/src/constants/index.ts
View file

@ -180,3 +180,4 @@ export enum AutomationErrors {
// pass through the list from the auth/core lib
export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
export const MAX_AUTOMATION_RECURRING_ERRORS = 5
export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"

222
packages/server/src/integrations/googlesheets.ts
View file

@ -1,22 +1,25 @@
import {
DatasourceFieldType,
DatasourcePlus,
FieldType,
Integration,
Operation,
PaginationJson,
QueryJson,
QueryType,
Row,
SearchFilters,
SortJson,
Table,
TableSchema,
TableRequest,
} from "@budibase/types"
import { OAuth2Client } from "google-auth-library"
import { buildExternalTableId } from "./utils"
import { DataSourceOperation, FieldTypes } from "../constants"
import { buildExternalTableId, finaliseExternalTables } from "./utils"
import { GoogleSpreadsheet } from "google-spreadsheet"
import fetch from "node-fetch"
import { configs, HTTPError } from "@budibase/backend-core"
import { dataFilters } from "@budibase/shared-core"
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
interface GoogleSheetsConfig {
spreadsheetId: string
@ -39,6 +42,17 @@ interface AuthTokenResponse {
access_token: string
}
const ALLOWED_TYPES = [
FieldType.STRING,
FieldType.FORMULA,
FieldType.NUMBER,
FieldType.LONGFORM,
FieldType.DATETIME,
FieldType.OPTIONS,
FieldType.BOOLEAN,
FieldType.BARCODEQR,
]
const SCHEMA: Integration = {
plus: true,
auth: {
@ -199,73 +213,90 @@ class GoogleSheetsIntegration implements DatasourcePlus {
this.client.useOAuth2Client(oauthClient)
await this.client.loadInfo()
} catch (err) {
} catch (err: any) {
// this happens for xlsx imports
if (err.message?.includes("operation is not supported")) {
err.message =
"This operation is not supported - XLSX sheets must be converted."
}
console.error("Error connecting to google sheets", err)
throw err
}
}
async buildSchema(datasourceId: string) {
getTableSchema(title: string, headerValues: string[], id?: string) {
// base table
const table: Table = {
name: title,
primary: [GOOGLE_SHEETS_PRIMARY_KEY],
schema: {},
}
if (id) {
table._id = id
}
// build schema from headers
for (let header of headerValues) {
table.schema[header] = {
name: header,
type: FieldType.STRING,
}
}
return table
}
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
await this.connect()
const sheets = this.client.sheetsByIndex
const tables: Record<string, Table> = {}
for (let sheet of sheets) {
// must fetch rows to determine schema
await sheet.getRows()
// build schema
const schema: TableSchema = {}
// build schema from headers
for (let header of sheet.headerValues) {
schema[header] = {
name: header,
type: FieldTypes.STRING,
}
}
// create tables
tables[sheet.title] = {
_id: buildExternalTableId(datasourceId, sheet.title),
name: sheet.title,
primary: ["rowNumber"],
schema,
}
const id = buildExternalTableId(datasourceId, sheet.title)
tables[sheet.title] = this.getTableSchema(
sheet.title,
sheet.headerValues,
id
)
}
this.tables = tables
const final = finaliseExternalTables(tables, entities)
this.tables = final.tables
this.schemaErrors = final.errors
}
async query(json: QueryJson) {
const sheet = json.endpoint.entityId
const handlers = {
[DataSourceOperation.CREATE]: () =>
this.create({ sheet, row: json.body }),
[DataSourceOperation.READ]: () => this.read({ ...json, sheet }),
[DataSourceOperation.UPDATE]: () =>
this.update({
switch (json.endpoint.operation) {
case Operation.CREATE:
return this.create({ sheet, row: json.body as Row })
case Operation.BULK_CREATE:
return this.createBulk({ sheet, rows: json.body as Row[] })
case Operation.READ:
return this.read({ ...json, sheet })
case Operation.UPDATE:
return this.update({
// exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet,
row: json.body,
}),
[DataSourceOperation.DELETE]: () =>
this.delete({
})
case Operation.DELETE:
return this.delete({
// exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet,
}),
[DataSourceOperation.CREATE_TABLE]: () =>
this.createTable(json?.table?.name),
[DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
[DataSourceOperation.DELETE_TABLE]: () =>
this.deleteTable(json?.table?.name),
})
case Operation.CREATE_TABLE:
return this.createTable(json?.table?.name)
case Operation.UPDATE_TABLE:
return this.updateTable(json.table!)
case Operation.DELETE_TABLE:
return this.deleteTable(json?.table?.name)
default:
throw new Error(
`GSheets integration does not support "${json.endpoint.operation}".`
)
}
// @ts-ignore
const internalQueryMethod = handlers[json.endpoint.operation]
return await internalQueryMethod()
}
buildRowObject(headers: string[], values: string[], rowNumber: number) {
@ -278,47 +309,70 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
async createTable(name?: string) {
if (!name) {
throw new Error("Must provide name for new sheet.")
}
try {
await this.connect()
return await this.client.addSheet({ title: name, headerValues: ["test"] })
return await this.client.addSheet({ title: name, headerValues: [name] })
} catch (err) {
console.error("Error creating new table in google sheets", err)
throw err
}
}
async updateTable(table?: any) {
try {
await this.connect()
const sheet = this.client.sheetsByTitle[table.name]
await sheet.loadHeaderRow()
async updateTable(table: TableRequest) {
await this.connect()
const sheet = this.client.sheetsByTitle[table.name]
await sheet.loadHeaderRow()
if (table._rename) {
const headers = []
for (let header of sheet.headerValues) {
if (header === table._rename.old) {
headers.push(table._rename.updated)
} else {
headers.push(header)
}
if (table._rename) {
const headers = []
for (let header of sheet.headerValues) {
if (header === table._rename.old) {
headers.push(table._rename.updated)
} else {
headers.push(header)
}
await sheet.setHeaderRow(headers)
} else {
const updatedHeaderValues = [...sheet.headerValues]
const newField = Object.keys(table.schema).find(
key => !sheet.headerValues.includes(key)
)
if (newField) {
updatedHeaderValues.push(newField)
}
await sheet.setHeaderRow(updatedHeaderValues)
}
} catch (err) {
console.error("Error updating table in google sheets", err)
throw err
try {
await sheet.setHeaderRow(headers)
} catch (err) {
console.error("Error updating column name in google sheets", err)
throw err
}
} else {
const updatedHeaderValues = [...sheet.headerValues]
// add new column - doesn't currently exist
for (let [key, column] of Object.entries(table.schema)) {
if (!ALLOWED_TYPES.includes(column.type)) {
throw new Error(
`Column type: ${column.type} not allowed for GSheets integration.`
)
}
if (
!sheet.headerValues.includes(key) &&
column.type !== FieldType.FORMULA
) {
updatedHeaderValues.push(key)
}
}
// clear out deleted columns
for (let key of sheet.headerValues) {
if (!Object.keys(table.schema).includes(key)) {
const idx = updatedHeaderValues.indexOf(key)
updatedHeaderValues.splice(idx, 1)
}
}
try {
await sheet.setHeaderRow(updatedHeaderValues)
} catch (err) {
console.error("Error updating table in google sheets", err)
throw err
}
}
}
@ -349,6 +403,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
}
async createBulk(query: { sheet: string; rows: any[] }) {
try {
await this.connect()
const sheet = this.client.sheetsByTitle[query.sheet]
let rowsToInsert = []
for (let row of query.rows) {
rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row)
}
const rows = await sheet.addRows(rowsToInsert)
return rows.map(row =>
this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber)
)
} catch (err) {
console.error("Error bulk writing to google sheets", err)
throw err
}
}
async read(query: {
sheet: string
filters?: SearchFilters

20
packages/server/src/integrations/utils.ts
View file

@ -4,6 +4,7 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
const ENCODED_SPACE = encodeURIComponent(" ")
const SQL_NUMBER_TYPE_MAP = {
integer: FieldTypes.NUMBER,
@ -79,6 +80,10 @@ export function isExternalTable(tableId: string) {
}
export function buildExternalTableId(datasourceId: string, tableName: string) {
// encode spaces
if (tableName.includes(" ")) {
tableName = encodeURIComponent(tableName)
}
return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
}
@ -90,6 +95,10 @@ export function breakExternalTableId(tableId: string | undefined) {
let datasourceId = parts.shift()
// if they need joined
let tableName = parts.join(DOUBLE_SEPARATOR)
// if contains encoded spaces, decode it
if (tableName.includes(ENCODED_SPACE)) {
tableName = decodeURIComponent(tableName)
}
return { datasourceId, tableName }
}
@ -200,9 +209,9 @@ export function isIsoDateString(str: string) {
* @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist.
*/
function shouldCopyRelationship(
export function shouldCopyRelationship(
column: { type: string; tableId?: string },
tableIds: [string]
tableIds: string[]
) {
return (
column.type === FieldTypes.LINK &&
@ -219,7 +228,7 @@ function shouldCopyRelationship(
* @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database.
*/
function shouldCopySpecialColumn(
export function shouldCopySpecialColumn(
column: { type: string },
fetchedColumn: { type: string } | undefined
) {
@ -257,9 +266,12 @@ function copyExistingPropsOver(
tableIds: [string]
) {
if (entities && entities[tableName]) {
if (entities[tableName].primaryDisplay) {
if (entities[tableName]?.primaryDisplay) {
table.primaryDisplay = entities[tableName].primaryDisplay
}
if (entities[tableName]?.created) {
table.created = entities[tableName]?.created
}
const existingTableSchema = entities[tableName].schema
for (let key in existingTableSchema) {
if (!existingTableSchema.hasOwnProperty(key)) {

637
packages/server/yarn.lock
View file

File diff suppressed because it is too large Load diff

1
packages/types/src/documents/app/table.ts
View file

@ -76,6 +76,7 @@ export interface Table extends Document {
sql?: boolean
indexes?: { [key: string]: any }
rows?: { [key: string]: any }
created?: boolean
}
export interface TableRequest extends Table {