From 8d1163e50dec40ad7f339c90217b885fa35e5be4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 18:58:19 +0100 Subject: [PATCH] Adding external SDK, updating external controllers. --- .../src/api/controllers/table/external.ts | 368 ++---------------- .../src/sdk/app/tables/external/index.ts | 51 ++- .../src/sdk/app/tables/external/utils.ts | 6 +- packages/server/src/sdk/app/tables/index.ts | 96 +---- packages/server/src/sdk/app/tables/update.ts | 31 ++ 5 files changed, 107 insertions(+), 445 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 967176c2e4..f035822068 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -1,108 +1,20 @@ -import { - breakExternalTableId, - buildExternalTableId, -} from "../../../integrations/utils" -import { - foreignKeyStructure, - generateForeignKey, - generateJunctionTableName, - hasTypeChanged, - setStaticSchemas, -} from "./utils" -import { FieldTypes } from "../../../constants" -import { makeExternalQuery } from "../../../integrations/base/query" +import { breakExternalTableId } from "../../../integrations/utils" import { handleRequest } from "../row/external" -import { context, events } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { isRows, isSchema, parse } from "../../../utilities/schema" import { BulkImportRequest, BulkImportResponse, - Datasource, - FieldSchema, - ManyToManyRelationshipFieldMetadata, - ManyToOneRelationshipFieldMetadata, - OneToManyRelationshipFieldMetadata, Operation, - QueryJson, - RelationshipFieldMetadata, - RelationshipType, - RenameColumn, SaveTableRequest, SaveTableResponse, Table, TableRequest, UserCtx, - ViewV2, } from "@budibase/types" import sdk from "../../../sdk" import { builderSocket } from "../../../websockets" -const { cloneDeep } = require("lodash/fp") - -async function makeTableRequest( - datasource: Datasource, - operation: Operation, - table: Table, - tables: Record, - oldTable?: Table, - renamed?: RenameColumn -) { - const json: QueryJson = { - endpoint: { - datasourceId: datasource._id!, - entityId: table._id!, - operation, - }, - meta: { - tables, - }, - table, - } - if (oldTable) { - json.meta!.table = oldTable - } - if (renamed) { - json.meta!.renamed = renamed - } - return makeExternalQuery(datasource, json) -} - -function cleanupRelationships( - table: Table, - tables: Record, - oldTable?: Table -) { - const tableToIterate = oldTable ? oldTable : table - // clean up relationships in couch table schemas - for (let [key, schema] of Object.entries(tableToIterate.schema)) { - if ( - schema.type === FieldTypes.LINK && - (!oldTable || table.schema[key] == null) - ) { - const schemaTableId = schema.tableId - const relatedTable = Object.values(tables).find( - table => table._id === schemaTableId - ) - const foreignKey = - schema.relationshipType !== RelationshipType.MANY_TO_MANY && - schema.foreignKey - if (!relatedTable || !foreignKey) { - continue - } - for (let [relatedKey, relatedSchema] of Object.entries( - relatedTable.schema - )) { - if ( - relatedSchema.type === FieldTypes.LINK && - relatedSchema.fieldName === foreignKey - ) { - delete relatedTable.schema[relatedKey] - } - } - } - } -} - function getDatasourceId(table: Table) { if (!table) { throw "No table supplied" @@ -113,247 +25,32 @@ function getDatasourceId(table: Table) { return breakExternalTableId(table._id).datasourceId } -function otherRelationshipType(type?: string) { - if (type === RelationshipType.MANY_TO_MANY) { - return RelationshipType.MANY_TO_MANY - } - return type === RelationshipType.ONE_TO_MANY - ? RelationshipType.MANY_TO_ONE - : RelationshipType.ONE_TO_MANY -} - -function generateManyLinkSchema( - datasource: Datasource, - column: ManyToManyRelationshipFieldMetadata, - table: Table, - relatedTable: Table -): Table { - if (!table.primary || !relatedTable.primary) { - throw new Error("Unable to generate many link schema, no primary keys") - } - const primary = table.name + table.primary[0] - const relatedPrimary = relatedTable.name + relatedTable.primary[0] - const jcTblName = generateJunctionTableName(column, table, relatedTable) - // first create the new table - const junctionTable = { - _id: buildExternalTableId(datasource._id!, jcTblName), - name: jcTblName, - primary: [primary, relatedPrimary], - constrained: [primary, relatedPrimary], - schema: { - [primary]: foreignKeyStructure(primary, { - toTable: table.name, - toKey: table.primary[0], - }), - [relatedPrimary]: foreignKeyStructure(relatedPrimary, { - toTable: relatedTable.name, - toKey: relatedTable.primary[0], - }), - }, - } - column.through = junctionTable._id - column.throughFrom = relatedPrimary - column.throughTo = primary - column.fieldName = relatedPrimary - return junctionTable -} - -function generateLinkSchema( - column: - | OneToManyRelationshipFieldMetadata - | ManyToOneRelationshipFieldMetadata, - table: Table, - relatedTable: Table, - type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE -) { - if (!table.primary || !relatedTable.primary) { - throw new Error("Unable to generate link schema, no primary keys") - } - const isOneSide = type === RelationshipType.ONE_TO_MANY - const primary = isOneSide ? relatedTable.primary[0] : table.primary[0] - // generate a foreign key - const foreignKey = generateForeignKey(column, relatedTable) - column.relationshipType = type - column.foreignKey = isOneSide ? foreignKey : primary - column.fieldName = isOneSide ? primary : foreignKey - return foreignKey -} - -function generateRelatedSchema( - linkColumn: RelationshipFieldMetadata, - table: Table, - relatedTable: Table, - columnName: string -) { - // generate column for other table - const relatedSchema = cloneDeep(linkColumn) - const isMany2Many = - linkColumn.relationshipType === RelationshipType.MANY_TO_MANY - // swap them from the main link - if (!isMany2Many && linkColumn.foreignKey) { - relatedSchema.fieldName = linkColumn.foreignKey - relatedSchema.foreignKey = linkColumn.fieldName - } - // is many to many - else if (isMany2Many) { - // don't need to copy through, already got it - relatedSchema.fieldName = linkColumn.throughTo - relatedSchema.throughTo = linkColumn.throughFrom - relatedSchema.throughFrom = linkColumn.throughTo - } - relatedSchema.relationshipType = otherRelationshipType( - linkColumn.relationshipType - ) - relatedSchema.tableId = relatedTable._id - relatedSchema.name = columnName - table.schema[columnName] = relatedSchema -} - -function isRelationshipSetup(column: RelationshipFieldMetadata) { - return (column as any).foreignKey || (column as any).through -} - export async function save(ctx: UserCtx) { const inputs = ctx.request.body - const renamed = inputs?._rename + const renaming = inputs?._rename // can't do this right now delete inputs.rows - const datasourceId = getDatasourceId(ctx.request.body)! + const tableId = ctx.request.body._id + const datasourceId = getDatasourceId(ctx.request.body) // table doesn't exist already, note that it is created if (!inputs._id) { inputs.created = true } - let tableToSave: TableRequest = { - type: "table", - _id: buildExternalTableId(datasourceId, inputs.name), - sourceId: datasourceId, - ...inputs, - } - - let oldTable: Table | undefined - if (ctx.request.body && ctx.request.body._id) { - oldTable = await sdk.tables.getTable(ctx.request.body._id) - } - - if (hasTypeChanged(tableToSave, oldTable)) { - ctx.throw(400, "A column type has changed.") - } - - for (let view in tableToSave.views) { - const tableView = tableToSave.views[view] - if (!tableView || !sdk.views.isV2(tableView)) continue - - tableToSave.views[view] = sdk.views.syncSchema( - oldTable!.views![view] as ViewV2, - tableToSave.schema, - renamed + try { + const { datasource, table } = await sdk.tables.external.save( + datasourceId!, + inputs, + { tableId, renaming } ) - } - - const db = context.getAppDB() - const datasource = await sdk.datasources.get(datasourceId) - if (!datasource.entities) { - datasource.entities = {} - } - - // GSheets is a specific case - only ever has a static primary key - tableToSave = setStaticSchemas(datasource, tableToSave) - - const oldTables = cloneDeep(datasource.entities) - const tables: Record = datasource.entities - - const extraTablesToUpdate = [] - - // check if relations need setup - for (let schema of Object.values(tableToSave.schema)) { - if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) { - continue - } - const schemaTableId = schema.tableId - const relatedTable = Object.values(tables).find( - table => table._id === schemaTableId - ) - if (!relatedTable) { - continue - } - const relatedColumnName = schema.fieldName! - const relationType = schema.relationshipType - if (relationType === RelationshipType.MANY_TO_MANY) { - const junctionTable = generateManyLinkSchema( - datasource, - schema, - tableToSave, - relatedTable - ) - if (tables[junctionTable.name]) { - throw "Junction table already exists, cannot create another relationship." - } - tables[junctionTable.name] = junctionTable - extraTablesToUpdate.push(junctionTable) + builderSocket?.emitDatasourceUpdate(ctx, datasource) + return table + } catch (err: any) { + if (err instanceof Error) { + ctx.throw(400, err.message) } else { - const fkTable = - relationType === RelationshipType.ONE_TO_MANY - ? tableToSave - : relatedTable - const foreignKey = generateLinkSchema( - schema, - tableToSave, - relatedTable, - relationType - ) - fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey) - if (fkTable.constrained == null) { - fkTable.constrained = [] - } - if (fkTable.constrained.indexOf(foreignKey) === -1) { - fkTable.constrained.push(foreignKey) - } - // foreign key is in other table, need to save it to external - if (fkTable._id !== tableToSave._id) { - extraTablesToUpdate.push(fkTable) - } + ctx.throw(err.status || 500, err?.message || err) } - generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName) - schema.main = true } - - cleanupRelationships(tableToSave, tables, oldTable) - - const operation = oldTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest( - datasource, - operation, - tableToSave, - tables, - oldTable, - renamed - ) - // update any extra tables (like foreign keys in other tables) - for (let extraTable of extraTablesToUpdate) { - const oldExtraTable = oldTables[extraTable.name] - let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable) - } - - // make sure the constrained list, all still exist - if (Array.isArray(tableToSave.constrained)) { - tableToSave.constrained = tableToSave.constrained.filter(constraint => - Object.keys(tableToSave.schema).includes(constraint) - ) - } - - // remove the rename prop - delete tableToSave._rename - // store it into couch now for budibase reference - datasource.entities[tableToSave.name] = tableToSave - await db.put(sdk.tables.populateExternalTableSchemas(datasource)) - - // Since tables are stored inside datasources, we need to notify clients - // that the datasource definition changed - const updatedDatasource = await sdk.datasources.get(datasource._id!) - builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) - - return tableToSave } export async function destroy(ctx: UserCtx) { @@ -364,27 +61,20 @@ export async function destroy(ctx: UserCtx) { ctx.throw(400, "Cannot delete tables which weren't created in Budibase.") } const datasourceId = getDatasourceId(tableToDelete) - - const db = context.getAppDB() - const datasource = await sdk.datasources.get(datasourceId!) - const tables = datasource.entities - - const operation = Operation.DELETE_TABLE - if (tables) { - await makeTableRequest(datasource, operation, tableToDelete, tables) - cleanupRelationships(tableToDelete, tables) - delete tables[tableToDelete.name] - datasource.entities = tables + try { + const { datasource, table } = await sdk.tables.external.destroy( + datasourceId!, + tableToDelete + ) + builderSocket?.emitDatasourceUpdate(ctx, datasource) + return table + } catch (err: any) { + if (err instanceof Error) { + ctx.throw(400, err.message) + } else { + ctx.throw(err.status || 500, err.message || err) + } } - - await db.put(sdk.tables.populateExternalTableSchemas(datasource)) - - // Since tables are stored inside datasources, we need to notify clients - // that the datasource definition changed - const updatedDatasource = await sdk.datasources.get(datasource._id!) - builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) - - return tableToDelete } export async function bulkImport( diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 3303ca38d1..1045ef1e33 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -7,25 +7,29 @@ import { ViewV2, } from "@budibase/types" import { context } from "@budibase/backend-core" -import { buildExternalTableId } from "../../../integrations/utils" -import sdk from "../../index" +import { buildExternalTableId } from "../../../../integrations/utils" import { foreignKeyStructure, hasTypeChanged, setStaticSchemas, -} from "../../../api/controllers/table/utils" +} from "../../../../api/controllers/table/utils" import { cloneDeep } from "lodash/fp" -import { FieldTypes } from "../../../constants" -import { makeTableRequest } from "../../../api/controllers/table/ExternalRequest" +import { FieldTypes } from "../../../../constants" +import { makeTableRequest } from "../../../../api/controllers/table/ExternalRequest" import { isRelationshipSetup, cleanupRelationships, generateLinkSchema, generateManyLinkSchema, generateRelatedSchema, -} from "./externalUtils" +} from "./utils" -export async function externalSave( +import { getTable } from "../getters" +import { populateExternalTableSchemas } from "../validation" +import datasourceSdk from "../../datasources" +import * as viewSdk from "../../views" + +export async function save( datasourceId: string, update: Table, opts?: { tableId?: string; renaming?: RenameColumn } @@ -39,7 +43,7 @@ export async function externalSave( let oldTable: Table | undefined if (opts?.tableId) { - oldTable = await sdk.tables.getTable(opts.tableId) + oldTable = await getTable(opts.tableId) } if (hasTypeChanged(tableToSave, oldTable)) { @@ -48,9 +52,9 @@ export async function externalSave( for (let view in tableToSave.views) { const tableView = tableToSave.views[view] - if (!tableView || !sdk.views.isV2(tableView)) continue + if (!tableView || !viewSdk.isV2(tableView)) continue - tableToSave.views[view] = sdk.views.syncSchema( + tableToSave.views[view] = viewSdk.syncSchema( oldTable!.views![view] as ViewV2, tableToSave.schema, opts?.renaming @@ -58,7 +62,7 @@ export async function externalSave( } const db = context.getAppDB() - const datasource = await sdk.datasources.get(datasourceId) + const datasource = await datasourceSdk.get(datasourceId) if (!datasource.entities) { datasource.entities = {} } @@ -155,11 +159,32 @@ export async function externalSave( delete tableToSave._rename // store it into couch now for budibase reference datasource.entities[tableToSave.name] = tableToSave - await db.put(sdk.tables.populateExternalTableSchemas(datasource)) + await db.put(populateExternalTableSchemas(datasource)) // Since tables are stored inside datasources, we need to notify clients // that the datasource definition changed - const updatedDatasource = await sdk.datasources.get(datasource._id!) + const updatedDatasource = await datasourceSdk.get(datasource._id!) return { datasource: updatedDatasource, table: tableToSave } } + +export async function destroy(datasourceId: string, table: Table) { + const db = context.getAppDB() + const datasource = await datasourceSdk.get(datasourceId) + const tables = datasource.entities + + const operation = Operation.DELETE_TABLE + if (tables) { + await makeTableRequest(datasource, operation, table, tables) + cleanupRelationships(table, tables) + delete tables[table.name] + datasource.entities = tables + } + + await db.put(populateExternalTableSchemas(datasource)) + + // Since tables are stored inside datasources, we need to notify clients + // that the datasource definition changed + const updatedDatasource = await datasourceSdk.get(datasource._id!) + return { datasource: updatedDatasource, table } +} diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index 8d22b8da1a..ef280472bf 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -7,13 +7,13 @@ import { RelationshipType, Table, } from "@budibase/types" -import { FieldTypes } from "../../../constants" +import { FieldTypes } from "../../../../constants" import { foreignKeyStructure, generateForeignKey, generateJunctionTableName, -} from "../../../api/controllers/table/utils" -import { buildExternalTableId } from "../../../integrations/utils" +} from "../../../../api/controllers/table/utils" +import { buildExternalTableId } from "../../../../integrations/utils" import { cloneDeep } from "lodash/fp" export function cleanupRelationships( diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 64fcde4bff..8542250517 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,95 +1,11 @@ -import { context } from "@budibase/backend-core" -import { BudibaseInternalDB, getTableParams } from "../../../db/utils" -import { - breakExternalTableId, - isExternalTable, - isSQL, -} from "../../../integrations/utils" -import { - Database, - Table, - TableResponse, - TableViewsResponse, -} from "@budibase/types" -import datasources from "../datasources" import { populateExternalTableSchemas } from "./validation" -import sdk from "../../../sdk" - -async function getAllInternalTables(db?: Database): Promise { - if (!db) { - db = context.getAppDB() - } - const internalTables = await db.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - return internalTables.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, - })) -} - -async function getAllExternalTables( - datasourceId: any -): Promise> { - const datasource = await datasources.get(datasourceId, { enriched: true }) - if (!datasource || !datasource.entities) { - throw "Datasource is not configured fully." - } - return datasource.entities -} - -async function getExternalTable( - datasourceId: any, - tableName: any -): Promise { - const entities = await getAllExternalTables(datasourceId) - return entities[tableName] -} - -async function getTable(tableId: any): Promise
{ - const db = context.getAppDB() - if (isExternalTable(tableId)) { - let { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await datasources.get(datasourceId!) - const table = await getExternalTable(datasourceId, tableName) - return { ...table, sql: isSQL(datasource) } - } else { - return db.get(tableId) - } -} - -function enrichViewSchemas(table: Table): TableResponse { - return { - ...table, - views: Object.values(table.views ?? []) - .map(v => sdk.views.enrichSchema(v, table.schema)) - .reduce((p, v) => { - p[v.name] = v - return p - }, {} as TableViewsResponse), - } -} - -async function saveTable(table: Table) { - const db = context.getAppDB() - if (isExternalTable(table._id!)) { - const datasource = await sdk.datasources.get(table.sourceId!) - datasource.entities![table.name] = table - await db.put(datasource) - } else { - await db.put(table) - } -} +import * as getters from "./getters" +import * as updates from "./update" +import * as utils from "./utils" export default { - getAllInternalTables, - getAllExternalTables, - getExternalTable, - getTable, populateExternalTableSchemas, - enrichViewSchemas, - saveTable, + ...updates, + ...getters, + ...utils, } diff --git a/packages/server/src/sdk/app/tables/update.ts b/packages/server/src/sdk/app/tables/update.ts index e69de29bb2..9bba4a967e 100644 --- a/packages/server/src/sdk/app/tables/update.ts +++ b/packages/server/src/sdk/app/tables/update.ts @@ -0,0 +1,31 @@ +import { Table, RenameColumn } from "@budibase/types" +import { isExternalTable } from "../../../integrations/utils" +import sdk from "../../index" +import { context } from "@budibase/backend-core" +import { isExternal } from "./utils" + +import * as external from "./external" +import * as internal from "./internal" +export * as external from "./external" +export * as internal from "./internal" + +export async function saveTable(table: Table) { + const db = context.getAppDB() + if (isExternalTable(table._id!)) { + const datasource = await sdk.datasources.get(table.sourceId!) + datasource.entities![table.name] = table + await db.put(datasource) + } else { + await db.put(table) + } +} + +export async function update(table: Table, renaming?: RenameColumn) { + const tableId = table._id + if (isExternal({ table })) { + const datasourceId = table.sourceId! + await external.save(datasourceId, table, { tableId, renaming }) + } else { + await internal.save(table, { tableId, renaming }) + } +}