From 069fd3396464346b3d8a9ad519cb403af7b6477e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Oct 2023 19:18:21 +0100 Subject: [PATCH 01/16] Some work towards improving performance of internal DB enrichment, there is a problem with cyclic enrichment due to the outputProcessing, need to decide how to handle formulas on enrichment. --- .../src/api/controllers/row/internal.ts | 67 +++++++---------- packages/server/src/api/controllers/user.ts | 6 +- packages/server/src/db/linkedRows/index.ts | 26 ++++--- .../server/src/db/linkedRows/linkUtils.ts | 4 +- .../src/sdk/app/datasources/datasources.ts | 31 ++++++-- packages/server/src/sdk/app/tables/index.ts | 75 +++++++++++++++---- 6 files changed, 134 insertions(+), 75 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index f33df4536f..f00621ba64 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,6 +1,7 @@ import * as linkRows from "../../../db/linkedRows" import { generateRowID, + getMultiIDParams, getTableIDFromRowID, InternalTables, } from "../../../db/utils" @@ -24,6 +25,8 @@ import { UserCtx, } from "@budibase/types" import sdk from "../../../sdk" +import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" +import { flatten } from "lodash" export async function patch(ctx: UserCtx) { const tableId = utils.getTableId(ctx) @@ -224,57 +227,43 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const db = context.getAppDB() const tableId = utils.getTableId(ctx) const rowId = ctx.params.rowId - // need table to work out where links go in row - let [table, row] = await Promise.all([ + // need table to work out where links go in row, as well as the link docs + let response = await Promise.all([ sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), + linkRows.getLinkDocuments({ tableId, rowId }), ]) - // get the link docs - const linkVals = (await linkRows.getLinkDocuments({ - tableId, - rowId, - })) as LinkDocumentValue[] + const table = response[0] as Table + const row = response[1] as Row + const linkVals = response[2] as LinkDocumentValue[] // look up the actual rows based on the ids - let response = ( - await db.allDocs({ - include_docs: true, - keys: linkVals.map(linkVal => linkVal.id), - }) - ).rows.map(row => row.doc) - // group responses by table - let groups: any = {}, - tables: Record = {} - for (let row of response) { - if (!row.tableId) { - row.tableId = getTableIDFromRowID(row._id) - } - const linkedTableId = row.tableId - if (groups[linkedTableId] == null) { - groups[linkedTableId] = [row] - tables[linkedTableId] = await db.get(linkedTableId) - } else { - groups[linkedTableId].push(row) - } - } - let linkedRows: Row[] = [] - for (let [tableId, rows] of Object.entries(groups)) { - // need to include the IDs in these rows for any links they may have - linkedRows = linkedRows.concat( - await outputProcessing(tables[tableId], rows as Row[]) + const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) + let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc) + + // get the linked tables + const linkTableIds = getLinkedTableIDs(table as Table) + const linkTables = await sdk.tables.getTables(linkTableIds) + + // perform output processing + let final = [] + for (let linkTable of linkTables) { + const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id) + final = final.concat( + outputProcessing(linkTable, relatedRows, { preserveLinks: true }) ) } + // finalise the promises + linkedRows = flatten(await Promise.all(final)) // insert the link rows in the correct place throughout the main row for (let fieldName of Object.keys(table.schema)) { let field = table.schema[fieldName] if (field.type === FieldTypes.LINK) { - // find the links that pertain to this field, get their indexes - const linkIndexes = linkVals - .filter(link => link.fieldName === fieldName) - .map(link => linkVals.indexOf(link)) + // find the links that pertain to this field + const links = linkVals.filter(link => link.fieldName === fieldName) // find the rows that the links state are linked to this field - row[fieldName] = linkedRows.filter((linkRow, index) => - linkIndexes.includes(index) + row[fieldName] = linkedRows.filter(linkRow => + links.find(link => link.id === linkRow._id) ) } } diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index dbbfc5c586..b6c3e7c6bd 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -1,13 +1,11 @@ -import { generateUserFlagID } from "../../db/utils" -import { InternalTables } from "../../db/utils" +import { generateUserFlagID, InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { context } from "@budibase/backend-core" import { Ctx, UserCtx } from "@budibase/types" import sdk from "../../sdk" export async function fetchMetadata(ctx: Ctx) { - const users = await sdk.users.fetchMetadata() - ctx.body = users + ctx.body = await sdk.users.fetchMetadata() } export async function updateSelfMetadata(ctx: UserCtx) { diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 3c5379ac97..1b64e615b0 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -9,13 +9,13 @@ import { getLinkedTable, } from "./linkUtils" import flatten from "lodash/flatten" -import { FieldTypes } from "../../constants" import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, LinkDocumentValue } from "@budibase/types" +import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types" +import sdk from "../../sdk" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -35,7 +35,7 @@ export const EventType = { function clearRelationshipFields(table: Table, rows: Row[]) { for (let [key, field] of Object.entries(table.schema)) { - if (field.type === FieldTypes.LINK) { + if (field.type === FieldType.LINK) { rows = rows.map(row => { delete row[key] return row @@ -45,7 +45,7 @@ function clearRelationshipFields(table: Table, rows: Row[]) { return rows } -async function getLinksForRows(rows: Row[]) { +async function getLinksForRows(rows: Row[]): Promise { const tableIds = [...new Set(rows.map(el => el.tableId))] // start by getting all the link values for performance reasons const promises = tableIds.map(tableId => @@ -153,15 +153,19 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) { if (linkedTableIds.length === 0) { return rows } - // get all the links - const links = (await getLinksForRows(rows)).filter(link => + // get tables and links + let response = await Promise.all([ + getLinksForRows(rows), + sdk.tables.getTables(linkedTableIds), + ]) + const links = (response[0] as LinkDocumentValue[]).filter(link => rows.some(row => row._id === link.thisId) ) + const linkedTables = response[1] as Table[] // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows let linked = await getFullLinkedDocs(links) - const linkedTables: Table[] = [] for (let row of rows) { for (let link of links.filter(link => link.thisId === row._id)) { if (row[link.fieldName] == null) { @@ -171,7 +175,9 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) { if (linkedRow) { const linkedTableId = linkedRow.tableId || getRelatedTableForField(table, link.fieldName) - const linkedTable = await getLinkedTable(linkedTableId, linkedTables) + const linkedTable = linkedTables.find( + table => table._id === linkedTableId + ) if (linkedTable) { row[link.fieldName].push(processFormulas(linkedTable, linkedRow)) } @@ -199,13 +205,13 @@ export async function squashLinksToPrimaryDisplay( // this only fetches the table if its not already in array const rowTable = await getLinkedTable(row.tableId!, linkedTables) for (let [column, schema] of Object.entries(rowTable?.schema || {})) { - if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) { + if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) { continue } const newLinks = [] for (let link of row[column]) { const linkTblId = link.tableId || getRelatedTableForField(table, column) - const linkedTable = await getLinkedTable(linkTblId, linkedTables) + const linkedTable = await getLinkedTable(linkTblId!, linkedTables) const obj: any = { _id: link._id } if (linkedTable?.primaryDisplay && link[linkedTable.primaryDisplay]) { obj.primaryDisplay = link[linkedTable.primaryDisplay] diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index c7db7d522a..3c81843565 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -91,10 +91,10 @@ export function getUniqueByProp(array: any[], prop: string) { }) } -export function getLinkedTableIDs(table: Table) { +export function getLinkedTableIDs(table: Table): string[] { return Object.values(table.schema) .filter((column: FieldSchema) => column.type === FieldTypes.LINK) - .map(column => column.tableId) + .map(column => column.tableId!) } export async function getLinkedTable(id: string, tables: Table[]) { diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 35107fd6b8..fb5d04b03e 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -23,10 +23,13 @@ import { getTableParams, } from "../../../db/utils" import sdk from "../../index" +import datasource from "../../../api/routes/datasource" const ENV_VAR_PREFIX = "env." -export async function fetch() { +export async function fetch(opts?: { + enriched: boolean +}): Promise { // Get internal tables const db = context.getAppDB() const internalTables = await db.allDocs( @@ -44,7 +47,7 @@ export async function fetch() { const bbInternalDb = { ...BudibaseInternalDB, - } + } as Datasource // Get external datasources const datasources = ( @@ -66,7 +69,18 @@ export async function fetch() { } } - return [bbInternalDb, ...datasources] + if (opts?.enriched) { + const envVars = await getEnvironmentVariables() + const promises = datasources.map(datasource => + enrichDatasourceWithValues(datasource, envVars) + ) + const enriched = (await Promise.all(promises)).map( + result => result.datasource + ) + return [bbInternalDb, ...enriched] + } else { + return [bbInternalDb, ...datasources] + } } export function areRESTVariablesValid(datasource: Datasource) { @@ -107,9 +121,12 @@ export function checkDatasourceTypes(schema: Integration, config: any) { return config } -async function enrichDatasourceWithValues(datasource: Datasource) { +async function enrichDatasourceWithValues( + datasource: Datasource, + variables?: Record +) { const cloned = cloneDeep(datasource) - const env = await getEnvironmentVariables() + const env = variables ? variables : await getEnvironmentVariables() //Do not process entities, as we do not want to process formulas const { entities, ...clonedWithoutEntities } = cloned const processed = processObjectSync( @@ -235,9 +252,9 @@ export function mergeConfigs(update: Datasource, old: Datasource) { if (value !== PASSWORD_REPLACEMENT) { continue } - if (old.config?.[key]) { + if (update.config && old.config && old.config?.[key]) { update.config[key] = old.config?.[key] - } else { + } else if (update.config) { delete update.config[key] } } diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 64fcde4bff..f02f46e156 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,11 +1,16 @@ import { context } from "@budibase/backend-core" -import { BudibaseInternalDB, getTableParams } from "../../../db/utils" +import { + BudibaseInternalDB, + getMultiIDParams, + getTableParams, +} from "../../../db/utils" import { breakExternalTableId, isExternalTable, isSQL, } from "../../../integrations/utils" import { + AllDocsResponse, Database, Table, TableResponse, @@ -15,6 +20,14 @@ import datasources from "../datasources" import { populateExternalTableSchemas } from "./validation" import sdk from "../../../sdk" +function processInternalTables(docs: AllDocsResponse): Table[] { + return docs.rows.map((tableDoc: any) => ({ + ...tableDoc.doc, + type: "internal", + sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, + })) +} + async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() @@ -24,15 +37,49 @@ async function getAllInternalTables(db?: Database): Promise { include_docs: true, }) ) - return internalTables.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, - })) + return processInternalTables(internalTables) } -async function getAllExternalTables( - datasourceId: any +async function getAllExternalTables(): Promise { + const datasources = await sdk.datasources.fetch({ enriched: true }) + const allEntities = datasources.map(datasource => datasource.entities) + let final = [] + for (let entities of allEntities) { + final = final.concat(Object.values(entities)) + } + return final +} + +async function getAllTables() { + const [internal, external] = await Promise.all([ + getAllInternalTables(), + getAllExternalTables(), + ]) + return [...internal, external] +} + +async function getTables(tableIds: string[]): Promise { + const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)), + internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId)) + let tables = [] + if (externalTableIds.length) { + const externalTables = await getAllExternalTables() + tables = tables.concat( + externalTables.filter( + table => externalTableIds.indexOf(table._id!) !== -1 + ) + ) + } + if (internalTableIds.length) { + const db = context.getAppDB() + const internalTables = await db.allDocs(getMultiIDParams(internalTableIds)) + tables = tables.concat(processInternalTables(internalTables)) + } + return tables +} + +async function getExternalTablesInDatasource( + datasourceId: string ): Promise> { const datasource = await datasources.get(datasourceId, { enriched: true }) if (!datasource || !datasource.entities) { @@ -42,14 +89,14 @@ async function getAllExternalTables( } async function getExternalTable( - datasourceId: any, - tableName: any + datasourceId: string, + tableName: string ): Promise { - const entities = await getAllExternalTables(datasourceId) + const entities = await getExternalTablesInDatasource(datasourceId) return entities[tableName] } -async function getTable(tableId: any): Promise
{ +async function getTable(tableId: string): Promise
{ const db = context.getAppDB() if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) @@ -86,9 +133,11 @@ async function saveTable(table: Table) { export default { getAllInternalTables, - getAllExternalTables, + getExternalTablesInDatasource, getExternalTable, getTable, + getAllTables, + getTables, populateExternalTableSchemas, enrichViewSchemas, saveTable, From 3e2f9dfc4ee9677f4686d9dafe0626d622f8668a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Oct 2023 18:23:18 +0100 Subject: [PATCH 02/16] Further enhancement, client library sends up the column it wants enriched and then we can ignore everything else, makes a big difference for enriching users (with a lot of relationships). --- .../frontend-core/src/api/relationships.js | 4 ++- .../src/api/controllers/row/internal.ts | 12 ++++++--- packages/server/src/db/linkedRows/index.ts | 25 ++++++++++++++++--- .../server/src/db/linkedRows/linkUtils.ts | 16 +++++++++--- packages/server/src/sdk/app/tables/index.ts | 2 +- .../src/utilities/rowProcessor/index.ts | 6 +++-- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/packages/frontend-core/src/api/relationships.js b/packages/frontend-core/src/api/relationships.js index fbc727f8e1..45595750a8 100644 --- a/packages/frontend-core/src/api/relationships.js +++ b/packages/frontend-core/src/api/relationships.js @@ -9,7 +9,9 @@ export const buildRelationshipEndpoints = API => ({ if (!tableId || !rowId) { return [] } - const response = await API.get({ url: `/api/${tableId}/${rowId}/enrich` }) + const response = await API.get({ + url: `/api/${tableId}/${rowId}/enrich?field=${fieldName}`, + }) if (!fieldName) { return response || [] } else { diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index f00621ba64..b4ea80e72b 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -224,14 +224,15 @@ export async function bulkDestroy(ctx: UserCtx) { } export async function fetchEnrichedRow(ctx: UserCtx) { + const fieldName = ctx.request.query.field as string | undefined const db = context.getAppDB() const tableId = utils.getTableId(ctx) - const rowId = ctx.params.rowId + const rowId = ctx.params.rowId as string // need table to work out where links go in row, as well as the link docs let response = await Promise.all([ sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), - linkRows.getLinkDocuments({ tableId, rowId }), + linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) const table = response[0] as Table const row = response[1] as Row @@ -248,8 +249,13 @@ export async function fetchEnrichedRow(ctx: UserCtx) { let final = [] for (let linkTable of linkTables) { const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id) + // include the row being enriched for performance reasons, don't need to fetch it to include final = final.concat( - outputProcessing(linkTable, relatedRows, { preserveLinks: true }) + outputProcessing(linkTable, relatedRows, { + // have to clone to avoid JSON cycle + fromRow: cloneDeep(row), + squash: true, + }) ) } // finalise the promises diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 1b64e615b0..85d64dda82 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -146,9 +146,14 @@ export async function updateLinks(args: { * This is required for formula fields, this may only be utilised internally (for now). * @param {object} table The table from which the rows originated. * @param {array} rows The rows which are to be enriched. + * @param {object} opts optional - options like passing in a base row to use for enrichment. * @return {Promise<*>} returns the rows with all of the enriched relationships on it. */ -export async function attachFullLinkedDocs(table: Table, rows: Row[]) { +export async function attachFullLinkedDocs( + table: Table, + rows: Row[], + opts?: { fromRow?: Row } +) { const linkedTableIds = getLinkedTableIDs(table) if (linkedTableIds.length === 0) { return rows @@ -158,20 +163,34 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) { getLinksForRows(rows), sdk.tables.getTables(linkedTableIds), ]) + // find the links that pertain to one of the rows that is being enriched const links = (response[0] as LinkDocumentValue[]).filter(link => rows.some(row => row._id === link.thisId) ) + // if fromRow has been passed in, then we don't need to fetch it (optimisation) + let linksWithoutFromRow = links + if (opts?.fromRow) { + linksWithoutFromRow = links.filter(link => link.id !== opts?.fromRow?._id) + } const linkedTables = response[1] as Table[] // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows - let linked = await getFullLinkedDocs(links) + let linked = [] + if (linksWithoutFromRow.length > 0) { + linked = await getFullLinkedDocs(linksWithoutFromRow) + } for (let row of rows) { for (let link of links.filter(link => link.thisId === row._id)) { if (row[link.fieldName] == null) { row[link.fieldName] = [] } - const linkedRow = linked.find(row => row._id === link.id) + let linkedRow: Row + if (opts?.fromRow && opts?.fromRow?._id === link.id) { + linkedRow = opts.fromRow! + } else { + linkedRow = linked.find(row => row._id === link.id) + } if (linkedRow) { const linkedTableId = linkedRow.tableId || getRelatedTableForField(table, link.fieldName) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 3c81843565..1a15246f85 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -35,19 +35,22 @@ export const IncludeDocs = { export async function getLinkDocuments(args: { tableId?: string rowId?: string - includeDocs?: any + fieldName?: string + includeDocs?: boolean }): Promise { - const { tableId, rowId, includeDocs } = args + const { tableId, rowId, fieldName, includeDocs } = args const db = context.getAppDB() let params: any - if (rowId != null) { + if (rowId) { params = { key: [tableId, rowId] } } // only table is known else { params = { startKey: [tableId], endKey: [tableId, {}] } } - params.include_docs = !!includeDocs + if (includeDocs) { + params.include_docs = true + } try { let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows // filter to get unique entries @@ -67,6 +70,11 @@ export async function getLinkDocuments(args: { return unique }) + // filter down to just the required field name + if (fieldName) { + linkRows = linkRows.filter(link => link.value.fieldName === fieldName) + } + // return docs if docs requested, otherwise just the value information if (includeDocs) { return linkRows.map(row => row.doc) as LinkDocument[] } else { diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index f02f46e156..d2b1fe005a 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -104,7 +104,7 @@ async function getTable(tableId: string): Promise
{ const table = await getExternalTable(datasourceId, tableName) return { ...table, sql: isSQL(datasource) } } else { - return db.get(tableId) + return db.get
(tableId) } } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 0bdaaa393e..b3e780491d 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -201,7 +201,7 @@ export async function inputProcessing( export async function outputProcessing( table: Table, rows: T, - opts: { squash?: boolean; preserveLinks?: boolean } = { + opts: { squash?: boolean; preserveLinks?: boolean; fromRow?: Row } = { squash: true, preserveLinks: false, } @@ -216,7 +216,9 @@ export async function outputProcessing( } // attach any linked row information let enriched = !opts.preserveLinks - ? await linkRows.attachFullLinkedDocs(table, safeRows) + ? await linkRows.attachFullLinkedDocs(table, safeRows, { + fromRow: opts?.fromRow, + }) : safeRows // process formulas From ee4a042204acd369c3445309266f1d8d45e3c68e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Oct 2023 18:40:56 +0100 Subject: [PATCH 03/16] Typing fixes - unsure why all of these came up suddenly. --- .../server/src/api/controllers/row/index.ts | 3 +- .../src/api/controllers/row/internal.ts | 6 +- .../src/api/controllers/row/staticFormula.ts | 2 +- .../server/src/api/controllers/row/utils.ts | 128 +----------------- .../server/src/api/controllers/table/index.ts | 4 +- .../server/src/db/linkedRows/linkUtils.ts | 5 +- packages/server/src/sdk/app/rows/utils.ts | 9 +- packages/server/src/sdk/app/tables/index.ts | 20 +-- 8 files changed, 33 insertions(+), 144 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 6e0a6d979e..0ccbf5cacf 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -26,6 +26,7 @@ import { fixRow } from "../public/rows" import sdk from "../../../sdk" import * as exporters from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" +import { Format } from "../view/exporters" export * as views from "./views" function pickApi(tableId: any) { @@ -267,7 +268,7 @@ export const exportRows = async ( async () => { const { fileName, content } = await sdk.rows.exportRows({ tableId, - format, + format: format as Format, rowIds: rows, columns, query, diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index b4ea80e72b..bc5162d6e8 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -157,14 +157,14 @@ export async function destroy(ctx: UserCtx) { if (row.tableId !== tableId) { throw "Supplied tableId doesn't match the row's tableId" } - const table = await sdk.tables.getTable(row.tableId) + const table = await sdk.tables.getTable(tableId) // update the row to include full relationships before deleting them row = await outputProcessing(table, row, { squash: false }) // now remove the relationships await linkRows.updateLinks({ eventType: linkRows.EventType.ROW_DELETE, row, - tableId: row.tableId, + tableId, }) // remove any attachments that were on the row from object storage await cleanupAttachments(table, { row }) @@ -246,7 +246,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const linkTables = await sdk.tables.getTables(linkTableIds) // perform output processing - let final = [] + let final: Promise[] = [] for (let linkTable of linkTables) { const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id) // include the row being enriched for performance reasons, don't need to fetch it to include diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index efe6f8719c..6f426c6fa0 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -149,7 +149,7 @@ export async function finaliseRow( await db.put(table) } catch (err: any) { if (err.status === 409) { - const updatedTable = await sdk.tables.getTable(table._id) + const updatedTable = await sdk.tables.getTable(table._id!) let response = processAutoColumn(null, updatedTable, row, { reprocessing: true, }) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 5f10fd9ad4..ed6ccd4c53 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,26 +1,9 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" import { context } from "@budibase/backend-core" -import { - Ctx, - FieldType, - Row, - SearchFilters, - Table, - UserCtx, -} from "@budibase/types" -import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" -import sdk from "../../../sdk" +import { Ctx, Row, UserCtx } from "@budibase/types" import validateJs from "validate.js" -import { cloneDeep } from "lodash/fp" - -function isForeignKey(key: string, table: Table) { - const relationships = Object.values(table.schema).filter( - column => column.type === FieldType.LINK - ) - return relationships.some(relationship => relationship.foreignKey === key) -} validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -51,7 +34,7 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { return row } -export function getTableId(ctx: Ctx) { +export function getTableId(ctx: Ctx): string { // top priority, use the URL first if (ctx.params?.sourceId) { return ctx.params.sourceId @@ -68,112 +51,7 @@ export function getTableId(ctx: Ctx) { if (ctx.params?.viewName) { return ctx.params.viewName } -} - -export async function validate({ - tableId, - row, - table, -}: { - tableId?: string - row: Row - table?: Table -}) { - let fetchedTable: Table - if (!table) { - fetchedTable = await sdk.tables.getTable(tableId) - } else { - fetchedTable = table - } - const errors: any = {} - for (let fieldName of Object.keys(fetchedTable.schema)) { - const column = fetchedTable.schema[fieldName] - const constraints = cloneDeep(column.constraints) - const type = column.type - // foreign keys are likely to be enriched - if (isForeignKey(fieldName, fetchedTable)) { - continue - } - // formulas shouldn't validated, data will be deleted anyway - if (type === FieldTypes.FORMULA || column.autocolumn) { - continue - } - // special case for options, need to always allow unselected (empty) - if (type === FieldTypes.OPTIONS && constraints?.inclusion) { - constraints.inclusion.push(null as any, "") - } - let res - - // Validate.js doesn't seem to handle array - if (type === FieldTypes.ARRAY && row[fieldName]) { - if (row[fieldName].length) { - if (!Array.isArray(row[fieldName])) { - row[fieldName] = row[fieldName].split(",") - } - row[fieldName].map((val: any) => { - if ( - !constraints?.inclusion?.includes(val) && - constraints?.inclusion?.length !== 0 - ) { - errors[fieldName] = "Field not in list" - } - }) - } else if (constraints?.presence && row[fieldName].length === 0) { - // non required MultiSelect creates an empty array, which should not throw errors - errors[fieldName] = [`${fieldName} is required`] - } - } else if ( - (type === FieldTypes.ATTACHMENT || type === FieldTypes.JSON) && - typeof row[fieldName] === "string" - ) { - // this should only happen if there is an error - try { - const json = JSON.parse(row[fieldName]) - if (type === FieldTypes.ATTACHMENT) { - if (Array.isArray(json)) { - row[fieldName] = json - } else { - errors[fieldName] = [`Must be an array`] - } - } - } catch (err) { - errors[fieldName] = [`Contains invalid JSON`] - } - } else { - res = validateJs.single(row[fieldName], constraints) - } - if (res) errors[fieldName] = res - } - return { valid: Object.keys(errors).length === 0, errors } -} - -// don't do a pure falsy check, as 0 is included -// https://github.com/Budibase/budibase/issues/10118 -export function removeEmptyFilters(filters: SearchFilters) { - for (let filterField of NoEmptyFilterStrings) { - if (!filters[filterField]) { - continue - } - - for (let filterType of Object.keys(filters)) { - if (filterType !== filterField) { - continue - } - // don't know which one we're checking, type could be anything - const value = filters[filterType] as unknown - if (typeof value === "object") { - for (let [key, value] of Object.entries( - filters[filterType] as object - )) { - if (value == null || value === "") { - // @ts-ignore - delete filters[filterField][key] - } - } - } - } - } - return filters + throw new Error("Unable to find table ID in request") } export function isUserMetadataTable(tableId: string) { diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index e7c6ae57b0..d61f593346 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -14,6 +14,7 @@ import { Table, TableResponse, UserCtx, + Row, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" @@ -129,8 +130,7 @@ export async function validateNewTableImport(ctx: UserCtx) { } export async function validateExistingTableImport(ctx: UserCtx) { - const { rows, tableId }: { rows: unknown; tableId: unknown } = - ctx.request.body + const { rows, tableId }: { rows: Row[]; tableId?: string } = ctx.request.body let schema = null if (tableId) { diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 1a15246f85..4579454531 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -72,7 +72,10 @@ export async function getLinkDocuments(args: { // filter down to just the required field name if (fieldName) { - linkRows = linkRows.filter(link => link.value.fieldName === fieldName) + linkRows = linkRows.filter(link => { + const value = link.value as LinkDocumentValue + return value.fieldName === fieldName + }) } // return docs if docs requested, otherwise just the value information if (includeDocs) { diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 51e418c324..d9d2c745d3 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -68,12 +68,15 @@ export async function validate({ valid: boolean errors: Record }> { - let fetchedTable: Table - if (!table) { + let fetchedTable: Table | undefined + if (!table && tableId) { fetchedTable = await sdk.tables.getTable(tableId) - } else { + } else if (table) { fetchedTable = table } + if (fetchedTable === undefined) { + throw new Error("Unable to fetch table for validation") + } const errors: Record = {} for (let fieldName of Object.keys(fetchedTable.schema)) { const column = fetchedTable.schema[fieldName] diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index d2b1fe005a..96cda05396 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -32,20 +32,22 @@ async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() } - const internalTables = await db.allDocs( + const internalTableDocs = await db.allDocs( getTableParams(null, { include_docs: true, }) ) - return processInternalTables(internalTables) + return processInternalTables(internalTableDocs) } async function getAllExternalTables(): Promise { const datasources = await sdk.datasources.fetch({ enriched: true }) const allEntities = datasources.map(datasource => datasource.entities) - let final = [] + let final: Table[] = [] for (let entities of allEntities) { - final = final.concat(Object.values(entities)) + if (entities) { + final = final.concat(Object.values(entities)) + } } return final } @@ -61,7 +63,7 @@ async function getAllTables() { async function getTables(tableIds: string[]): Promise { const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)), internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId)) - let tables = [] + let tables: Table[] = [] if (externalTableIds.length) { const externalTables = await getAllExternalTables() tables = tables.concat( @@ -72,8 +74,10 @@ async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTables = await db.allDocs(getMultiIDParams(internalTableIds)) - tables = tables.concat(processInternalTables(internalTables)) + const internalTableDocs = await db.allDocs( + getMultiIDParams(internalTableIds) + ) + tables = tables.concat(processInternalTables(internalTableDocs)) } return tables } @@ -101,7 +105,7 @@ async function getTable(tableId: string): Promise
{ if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) const datasource = await datasources.get(datasourceId!) - const table = await getExternalTable(datasourceId, tableName) + const table = await getExternalTable(datasourceId!, tableName!) return { ...table, sql: isSQL(datasource) } } else { return db.get
(tableId) From 797677284222bb397a4e83600945d20c9a1e0ede Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Oct 2023 23:19:11 +0100 Subject: [PATCH 04/16] Fixing test case. --- packages/server/src/middleware/tests/trimViewRowInfo.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts index 17b4cc7b93..bf717d5828 100644 --- a/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts +++ b/packages/server/src/middleware/tests/trimViewRowInfo.spec.ts @@ -16,6 +16,7 @@ jest.mock("../../sdk", () => ({ import sdk from "../../sdk" import { Next } from "koa" +const tableId = utils.generateTableID() const mockGetView = sdk.views.get as jest.MockedFunction const mockGetTable = sdk.tables.getTable as jest.MockedFunction< typeof sdk.tables.getTable @@ -41,6 +42,7 @@ class TestConfiguration { body: ctxRequestBody, } this.params.viewId = viewId + this.params.sourceId = tableId return this.middleware( { request: this.request as any, @@ -69,7 +71,7 @@ describe("trimViewRowInfo middleware", () => { }) const table: Table = { - _id: utils.generateTableID(), + _id: tableId, name: generator.word(), type: "table", schema: { From 35e6452ecb369eebe84fd1cd2cde99f2e06cd2f2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 18:57:19 +0100 Subject: [PATCH 05/16] Adding basic structure for the external/internal SDK. --- .../api/controllers/table/ExternalRequest.ts | 36 ++++ .../src/sdk/app/tables/external/index.ts | 165 ++++++++++++++++++ .../src/sdk/app/tables/external/utils.ts | 158 +++++++++++++++++ packages/server/src/sdk/app/tables/getters.ts | 0 .../src/sdk/app/tables/internal/index.ts | 0 .../src/sdk/app/tables/internal/utils.ts | 0 packages/server/src/sdk/app/tables/update.ts | 0 packages/server/src/sdk/app/tables/utils.ts | 0 8 files changed, 359 insertions(+) create mode 100644 packages/server/src/api/controllers/table/ExternalRequest.ts create mode 100644 packages/server/src/sdk/app/tables/external/index.ts create mode 100644 packages/server/src/sdk/app/tables/external/utils.ts create mode 100644 packages/server/src/sdk/app/tables/getters.ts create mode 100644 packages/server/src/sdk/app/tables/internal/index.ts create mode 100644 packages/server/src/sdk/app/tables/internal/utils.ts create mode 100644 packages/server/src/sdk/app/tables/update.ts create mode 100644 packages/server/src/sdk/app/tables/utils.ts diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts new file mode 100644 index 0000000000..65cead3a1d --- /dev/null +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -0,0 +1,36 @@ +import { + Datasource, + Operation, + QueryJson, + RenameColumn, + Table, +} from "@budibase/types" +import { makeExternalQuery } from "../../../integrations/base/query" + +export async function makeTableRequest( + datasource: Datasource, + operation: Operation, + table: Table, + tables: Record, + oldTable?: Table, + renamed?: RenameColumn +) { + const json: QueryJson = { + endpoint: { + datasourceId: datasource._id!, + entityId: table._id!, + operation, + }, + meta: { + tables, + }, + table, + } + if (oldTable) { + json.meta!.table = oldTable + } + if (renamed) { + json.meta!.renamed = renamed + } + return makeExternalQuery(datasource, json) +} diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts new file mode 100644 index 0000000000..3303ca38d1 --- /dev/null +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -0,0 +1,165 @@ +import { + Operation, + RelationshipType, + RenameColumn, + Table, + TableRequest, + ViewV2, +} from "@budibase/types" +import { context } from "@budibase/backend-core" +import { buildExternalTableId } from "../../../integrations/utils" +import sdk from "../../index" +import { + foreignKeyStructure, + hasTypeChanged, + setStaticSchemas, +} from "../../../api/controllers/table/utils" +import { cloneDeep } from "lodash/fp" +import { FieldTypes } from "../../../constants" +import { makeTableRequest } from "../../../api/controllers/table/ExternalRequest" +import { + isRelationshipSetup, + cleanupRelationships, + generateLinkSchema, + generateManyLinkSchema, + generateRelatedSchema, +} from "./externalUtils" + +export async function externalSave( + datasourceId: string, + update: Table, + opts?: { tableId?: string; renaming?: RenameColumn } +) { + let tableToSave: TableRequest = { + type: "table", + _id: buildExternalTableId(datasourceId, update.name), + sourceId: datasourceId, + ...update, + } + + let oldTable: Table | undefined + if (opts?.tableId) { + oldTable = await sdk.tables.getTable(opts.tableId) + } + + if (hasTypeChanged(tableToSave, oldTable)) { + throw new Error("A column type has changed.") + } + + for (let view in tableToSave.views) { + const tableView = tableToSave.views[view] + if (!tableView || !sdk.views.isV2(tableView)) continue + + tableToSave.views[view] = sdk.views.syncSchema( + oldTable!.views![view] as ViewV2, + tableToSave.schema, + opts?.renaming + ) + } + + const db = context.getAppDB() + const datasource = await sdk.datasources.get(datasourceId) + if (!datasource.entities) { + datasource.entities = {} + } + + // GSheets is a specific case - only ever has a static primary key + tableToSave = setStaticSchemas(datasource, tableToSave) + + const oldTables = cloneDeep(datasource.entities) + const tables: Record = datasource.entities + + const extraTablesToUpdate = [] + + // check if relations need setup + for (let schema of Object.values(tableToSave.schema)) { + if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) { + continue + } + const schemaTableId = schema.tableId + const relatedTable = Object.values(tables).find( + table => table._id === schemaTableId + ) + if (!relatedTable) { + continue + } + const relatedColumnName = schema.fieldName! + const relationType = schema.relationshipType + if (relationType === RelationshipType.MANY_TO_MANY) { + const junctionTable = generateManyLinkSchema( + datasource, + schema, + tableToSave, + relatedTable + ) + if (tables[junctionTable.name]) { + throw new Error( + "Junction table already exists, cannot create another relationship." + ) + } + tables[junctionTable.name] = junctionTable + extraTablesToUpdate.push(junctionTable) + } else { + const fkTable = + relationType === RelationshipType.ONE_TO_MANY + ? tableToSave + : relatedTable + const foreignKey = generateLinkSchema( + schema, + tableToSave, + relatedTable, + relationType + ) + fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey) + if (fkTable.constrained == null) { + fkTable.constrained = [] + } + if (fkTable.constrained.indexOf(foreignKey) === -1) { + fkTable.constrained.push(foreignKey) + } + // foreign key is in other table, need to save it to external + if (fkTable._id !== tableToSave._id) { + extraTablesToUpdate.push(fkTable) + } + } + generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName) + schema.main = true + } + + cleanupRelationships(tableToSave, tables, oldTable) + + const operation = oldTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE + await makeTableRequest( + datasource, + operation, + tableToSave, + tables, + oldTable, + opts?.renaming + ) + // update any extra tables (like foreign keys in other tables) + for (let extraTable of extraTablesToUpdate) { + const oldExtraTable = oldTables[extraTable.name] + let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE + await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable) + } + + // make sure the constrained list, all still exist + if (Array.isArray(tableToSave.constrained)) { + tableToSave.constrained = tableToSave.constrained.filter(constraint => + Object.keys(tableToSave.schema).includes(constraint) + ) + } + + // remove the rename prop + delete tableToSave._rename + // store it into couch now for budibase reference + datasource.entities[tableToSave.name] = tableToSave + await db.put(sdk.tables.populateExternalTableSchemas(datasource)) + + // Since tables are stored inside datasources, we need to notify clients + // that the datasource definition changed + const updatedDatasource = await sdk.datasources.get(datasource._id!) + + return { datasource: updatedDatasource, table: tableToSave } +} diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts new file mode 100644 index 0000000000..8d22b8da1a --- /dev/null +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -0,0 +1,158 @@ +import { + Datasource, + ManyToManyRelationshipFieldMetadata, + ManyToOneRelationshipFieldMetadata, + OneToManyRelationshipFieldMetadata, + RelationshipFieldMetadata, + RelationshipType, + Table, +} from "@budibase/types" +import { FieldTypes } from "../../../constants" +import { + foreignKeyStructure, + generateForeignKey, + generateJunctionTableName, +} from "../../../api/controllers/table/utils" +import { buildExternalTableId } from "../../../integrations/utils" +import { cloneDeep } from "lodash/fp" + +export function cleanupRelationships( + table: Table, + tables: Record, + oldTable?: Table +) { + const tableToIterate = oldTable ? oldTable : table + // clean up relationships in couch table schemas + for (let [key, schema] of Object.entries(tableToIterate.schema)) { + if ( + schema.type === FieldTypes.LINK && + (!oldTable || table.schema[key] == null) + ) { + const schemaTableId = schema.tableId + const relatedTable = Object.values(tables).find( + table => table._id === schemaTableId + ) + const foreignKey = + schema.relationshipType !== RelationshipType.MANY_TO_MANY && + schema.foreignKey + if (!relatedTable || !foreignKey) { + continue + } + for (let [relatedKey, relatedSchema] of Object.entries( + relatedTable.schema + )) { + if ( + relatedSchema.type === FieldTypes.LINK && + relatedSchema.fieldName === foreignKey + ) { + delete relatedTable.schema[relatedKey] + } + } + } + } +} + +export function otherRelationshipType(type?: string) { + if (type === RelationshipType.MANY_TO_MANY) { + return RelationshipType.MANY_TO_MANY + } + return type === RelationshipType.ONE_TO_MANY + ? RelationshipType.MANY_TO_ONE + : RelationshipType.ONE_TO_MANY +} + +export function generateManyLinkSchema( + datasource: Datasource, + column: ManyToManyRelationshipFieldMetadata, + table: Table, + relatedTable: Table +): Table { + if (!table.primary || !relatedTable.primary) { + throw new Error("Unable to generate many link schema, no primary keys") + } + const primary = table.name + table.primary[0] + const relatedPrimary = relatedTable.name + relatedTable.primary[0] + const jcTblName = generateJunctionTableName(column, table, relatedTable) + // first create the new table + const junctionTable = { + _id: buildExternalTableId(datasource._id!, jcTblName), + name: jcTblName, + primary: [primary, relatedPrimary], + constrained: [primary, relatedPrimary], + schema: { + [primary]: foreignKeyStructure(primary, { + toTable: table.name, + toKey: table.primary[0], + }), + [relatedPrimary]: foreignKeyStructure(relatedPrimary, { + toTable: relatedTable.name, + toKey: relatedTable.primary[0], + }), + }, + } + column.through = junctionTable._id + column.throughFrom = relatedPrimary + column.throughTo = primary + column.fieldName = relatedPrimary + return junctionTable +} + +export function generateLinkSchema( + column: + | OneToManyRelationshipFieldMetadata + | ManyToOneRelationshipFieldMetadata, + table: Table, + relatedTable: Table, + type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE +) { + if (!table.primary || !relatedTable.primary) { + throw new Error("Unable to generate link schema, no primary keys") + } + const isOneSide = type === RelationshipType.ONE_TO_MANY + const primary = isOneSide ? relatedTable.primary[0] : table.primary[0] + // generate a foreign key + const foreignKey = generateForeignKey(column, relatedTable) + column.relationshipType = type + column.foreignKey = isOneSide ? foreignKey : primary + column.fieldName = isOneSide ? primary : foreignKey + return foreignKey +} + +export function generateRelatedSchema( + linkColumn: RelationshipFieldMetadata, + table: Table, + relatedTable: Table, + columnName: string +) { + // generate column for other table + let relatedSchema: RelationshipFieldMetadata + const isMany2Many = + linkColumn.relationshipType === RelationshipType.MANY_TO_MANY + // swap them from the main link + if (!isMany2Many && linkColumn.foreignKey) { + relatedSchema = cloneDeep(linkColumn) as + | OneToManyRelationshipFieldMetadata + | ManyToOneRelationshipFieldMetadata + relatedSchema.fieldName = linkColumn.foreignKey + relatedSchema.foreignKey = linkColumn.fieldName + } + // is many to many + else { + const manyToManyCol = linkColumn as ManyToManyRelationshipFieldMetadata + relatedSchema = cloneDeep(linkColumn) as ManyToManyRelationshipFieldMetadata + // don't need to copy through, already got it + relatedSchema.fieldName = manyToManyCol.throughTo! + relatedSchema.throughTo = manyToManyCol.throughFrom + relatedSchema.throughFrom = manyToManyCol.throughTo + } + relatedSchema.relationshipType = otherRelationshipType( + linkColumn.relationshipType + ) + relatedSchema.tableId = relatedTable._id! + relatedSchema.name = columnName + table.schema[columnName] = relatedSchema +} + +export function isRelationshipSetup(column: RelationshipFieldMetadata) { + return (column as any).foreignKey || (column as any).through +} diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/server/src/sdk/app/tables/internal/utils.ts b/packages/server/src/sdk/app/tables/internal/utils.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/server/src/sdk/app/tables/update.ts b/packages/server/src/sdk/app/tables/update.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts new file mode 100644 index 0000000000..e69de29bb2 From 291a97a07855603a0e4feb595ddcc376ee98dc18 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 18:57:30 +0100 Subject: [PATCH 06/16] Moving getters. --- packages/server/src/sdk/app/tables/getters.ts | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index e69de29bb2..ef7af67f32 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -0,0 +1,73 @@ +import { context } from "@budibase/backend-core" +import { BudibaseInternalDB, getTableParams } from "../../../db/utils" +import { + breakExternalTableId, + isExternalTable, + isSQL, +} from "../../../integrations/utils" +import { + Database, + Table, + TableResponse, + TableViewsResponse, +} from "@budibase/types" +import datasources from "../datasources" +import sdk from "../../../sdk" + +export async function getAllInternalTables(db?: Database): Promise { + if (!db) { + db = context.getAppDB() + } + const internalTables = await db.allDocs( + getTableParams(null, { + include_docs: true, + }) + ) + return internalTables.rows.map((tableDoc: any) => ({ + ...tableDoc.doc, + type: "internal", + sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, + })) +} + +export async function getAllExternalTables( + datasourceId: any +): Promise> { + const datasource = await datasources.get(datasourceId, { enriched: true }) + if (!datasource || !datasource.entities) { + throw "Datasource is not configured fully." + } + return datasource.entities +} + +export async function getExternalTable( + datasourceId: any, + tableName: any +): Promise
{ + const entities = await getAllExternalTables(datasourceId) + return entities[tableName] +} + +export async function getTable(tableId: any): Promise
{ + const db = context.getAppDB() + if (isExternalTable(tableId)) { + let { datasourceId, tableName } = breakExternalTableId(tableId) + const datasource = await datasources.get(datasourceId!) + const table = await getExternalTable(datasourceId, tableName) + return { ...table, sql: isSQL(datasource) } + } else { + return db.get(tableId) + } +} + +export function enrichViewSchemas(table: Table): TableResponse { + return { + ...table, + views: Object.values(table.views ?? []) + .map(v => sdk.views.enrichSchema(v, table.schema)) + .reduce((p, v) => { + p[v.name] = v + return p + }, {} as TableViewsResponse), + } +} From 6bfa9d407b82c54810d500ed1839477799da326b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 18:57:52 +0100 Subject: [PATCH 07/16] Adding internal SDK, updating internal table controllers. --- .../src/api/controllers/table/internal.ts | 204 +++--------------- .../src/sdk/app/tables/internal/index.ts | 187 ++++++++++++++++ .../src/sdk/app/tables/internal/utils.ts | 20 ++ 3 files changed, 233 insertions(+), 178 deletions(-) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index eeb4a9eb5f..3a8b0b39a8 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -1,14 +1,5 @@ -import { updateLinks, EventType } from "../../../db/linkedRows" -import { getRowParams, generateTableID } from "../../../db/utils" -import { FieldTypes } from "../../../constants" -import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils" -import { context } from "@budibase/backend-core" -import env from "../../../environment" -import { - cleanupAttachments, - fixAutoColumnSubType, -} from "../../../utilities/rowProcessor" -import { runStaticFormulaChecks } from "./bulkFormula" +import { generateTableID } from "../../../db/utils" +import { handleDataImport } from "./utils" import { BulkImportRequest, BulkImportResponse, @@ -17,195 +8,52 @@ import { SaveTableResponse, Table, UserCtx, - ViewStatisticsSchema, - ViewV2, } from "@budibase/types" -import { quotas } from "@budibase/pro" -import isEqual from "lodash/isEqual" -import { cloneDeep } from "lodash/fp" import sdk from "../../../sdk" -function checkAutoColumns(table: Table, oldTable?: Table) { - if (!table.schema) { - return table - } - for (let [key, schema] of Object.entries(table.schema)) { - if (!schema.autocolumn || schema.subtype) { - continue - } - const oldSchema = oldTable && oldTable.schema[key] - if (oldSchema && oldSchema.subtype) { - table.schema[key].subtype = oldSchema.subtype - } else { - table.schema[key] = fixAutoColumnSubType(schema) - } - } - return table -} - export async function save(ctx: UserCtx) { - const db = context.getAppDB() const { rows, ...rest } = ctx.request.body let tableToSave: Table & { - _rename?: { old: string; updated: string } | undefined + _rename?: RenameColumn } = { type: "table", _id: generateTableID(), views: {}, ...rest, } + const renaming = tableToSave._rename + delete tableToSave._rename - // if the table obj had an _id then it will have been retrieved - let oldTable: Table | undefined - if (ctx.request.body && ctx.request.body._id) { - oldTable = await sdk.tables.getTable(ctx.request.body._id) - } - - // check all types are correct - if (hasTypeChanged(tableToSave, oldTable)) { - ctx.throw(400, "A column type has changed.") - } - // check that subtypes have been maintained - tableToSave = checkAutoColumns(tableToSave, oldTable) - - // saving a table is a complex operation, involving many different steps, this - // has been broken out into a utility to make it more obvious/easier to manipulate - const tableSaveFunctions = new TableSaveFunctions({ - user: ctx.user, - oldTable, - importRows: rows, - }) - tableToSave = await tableSaveFunctions.before(tableToSave) - - // make sure that types don't change of a column, have to remove - // the column if you want to change the type - if (oldTable && oldTable.schema) { - for (const propKey of Object.keys(tableToSave.schema)) { - let oldColumn = oldTable.schema[propKey] - if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) { - oldTable.schema[propKey].type = FieldTypes.AUTO - } - } - } - - // Don't rename if the name is the same - let _rename: RenameColumn | undefined = tableToSave._rename - /* istanbul ignore next */ - if (_rename && _rename.old === _rename.updated) { - _rename = undefined - delete tableToSave._rename - } - - // rename row fields when table column is renamed - /* istanbul ignore next */ - if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) { - ctx.throw(400, "Cannot rename a linked column.") - } - - tableToSave = await tableSaveFunctions.mid(tableToSave) - - // update schema of non-statistics views when new columns are added - for (let view in tableToSave.views) { - const tableView = tableToSave.views[view] - if (!tableView) continue - - if (sdk.views.isV2(tableView)) { - tableToSave.views[view] = sdk.views.syncSchema( - oldTable!.views![view] as ViewV2, - tableToSave.schema, - _rename - ) - continue - } - - if ( - (tableView.schema as ViewStatisticsSchema).group || - tableView.schema.field - ) - continue - tableView.schema = tableToSave.schema - } - - // update linked rows try { - const linkResp: any = await updateLinks({ - eventType: oldTable ? EventType.TABLE_UPDATED : EventType.TABLE_SAVE, - table: tableToSave, - oldTable: oldTable, + const { table } = await sdk.tables.internal.save(tableToSave, { + user: ctx.user, + rowsToImport: rows, + tableId: ctx.request.body._id, + renaming: renaming, }) - if (linkResp != null && linkResp._rev) { - tableToSave._rev = linkResp._rev + + return table + } catch (err: any) { + if (err instanceof Error) { + ctx.throw(400, err.message) + } else { + ctx.throw(err.status || 500, err.message || err) } - } catch (err) { - ctx.throw(400, err as string) } - - // don't perform any updates until relationships have been - // checked by the updateLinks function - const updatedRows = tableSaveFunctions.getUpdatedRows() - if (updatedRows && updatedRows.length !== 0) { - await db.bulkDocs(updatedRows) - } - let result = await db.put(tableToSave) - tableToSave._rev = result.rev - const savedTable = cloneDeep(tableToSave) - - tableToSave = await tableSaveFunctions.after(tableToSave) - // the table may be updated as part of the table save after functionality - need to write it - if (!isEqual(savedTable, tableToSave)) { - result = await db.put(tableToSave) - tableToSave._rev = result.rev - } - // has to run after, make sure it has _id - await runStaticFormulaChecks(tableToSave, { oldTable, deletion: false }) - return tableToSave } -export async function destroy(ctx: any) { - const db = context.getAppDB() +export async function destroy(ctx: UserCtx) { const tableToDelete = await sdk.tables.getTable(ctx.params.tableId) - - // Delete all rows for that table - const rowsData = await db.allDocs( - getRowParams(ctx.params.tableId, null, { - include_docs: true, - }) - ) - await db.bulkDocs( - rowsData.rows.map((row: any) => ({ ...row.doc, _deleted: true })) - ) - await quotas.removeRows(rowsData.rows.length, { - tableId: ctx.params.tableId, - }) - - // update linked rows - await updateLinks({ - eventType: EventType.TABLE_DELETE, - table: tableToDelete, - }) - - // don't remove the table itself until very end - await db.remove(tableToDelete._id!, tableToDelete._rev) - - // remove table search index - if (!env.isTest() || env.COUCH_DB_URL) { - const currentIndexes = await db.getIndexes() - const existingIndex = currentIndexes.indexes.find( - (existing: any) => existing.name === `search:${ctx.params.tableId}` - ) - if (existingIndex) { - await db.deleteIndex(existingIndex) + try { + const { table } = await sdk.tables.internal.destroy(tableToDelete) + return table + } catch (err: any) { + if (err instanceof Error) { + ctx.throw(400, err.message) + } else { + ctx.throw(err.status || 500, err.message || err) } } - - // has to run after, make sure it has _id - await runStaticFormulaChecks(tableToDelete, { - deletion: true, - }) - await cleanupAttachments(tableToDelete, { - rows: rowsData.rows.map((row: any) => row.doc), - }) - return tableToDelete } export async function bulkImport( diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index e69de29bb2..c5c7822ffe 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -0,0 +1,187 @@ +import { + RenameColumn, + Table, + ViewStatisticsSchema, + ViewV2, + Row, + ContextUser, +} from "@budibase/types" +import { + hasTypeChanged, + TableSaveFunctions, +} from "../../../../api/controllers/table/utils" +import { FieldTypes } from "../../../../constants" +import { EventType, updateLinks } from "../../../../db/linkedRows" +import { cloneDeep } from "lodash/fp" +import isEqual from "lodash/isEqual" +import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" +import { context } from "@budibase/backend-core" +import { getTable } from "../getters" +import { checkAutoColumns } from "./utils" +import * as viewsSdk from "../../views" +import sdk from "../../../index" +import { getRowParams } from "../../../../db/utils" +import { quotas } from "@budibase/pro" +import env from "../../../../environment" +import { cleanupAttachments } from "../../../../utilities/rowProcessor" + +export async function save( + table: Table, + opts?: { + user?: ContextUser + tableId?: string + rowsToImport?: Row[] + renaming?: RenameColumn + } +) { + const db = context.getAppDB() + + // if the table obj had an _id then it will have been retrieved + let oldTable: Table | undefined + if (opts?.tableId) { + oldTable = await getTable(opts.tableId) + } + + // check all types are correct + if (hasTypeChanged(table, oldTable)) { + throw new Error("A column type has changed.") + } + // check that subtypes have been maintained + table = checkAutoColumns(table, oldTable) + + // saving a table is a complex operation, involving many different steps, this + // has been broken out into a utility to make it more obvious/easier to manipulate + const tableSaveFunctions = new TableSaveFunctions({ + user: opts?.user, + oldTable, + importRows: opts?.rowsToImport, + }) + table = await tableSaveFunctions.before(table) + + // make sure that types don't change of a column, have to remove + // the column if you want to change the type + if (oldTable && oldTable.schema) { + for (const propKey of Object.keys(table.schema)) { + let oldColumn = oldTable.schema[propKey] + if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) { + oldTable.schema[propKey].type = FieldTypes.AUTO + } + } + } + + let renaming = opts?.renaming + if (renaming && renaming.old === renaming.updated) { + renaming = undefined + } + + // rename row fields when table column is renamed + if (renaming && table.schema[renaming.updated].type === FieldTypes.LINK) { + throw new Error("Cannot rename a linked column.") + } + + table = await tableSaveFunctions.mid(table) + + // update schema of non-statistics views when new columns are added + for (let view in table.views) { + const tableView = table.views[view] + if (!tableView) continue + + if (viewsSdk.isV2(tableView)) { + table.views[view] = viewsSdk.syncSchema( + oldTable!.views![view] as ViewV2, + table.schema, + renaming + ) + continue + } + + if ( + (tableView.schema as ViewStatisticsSchema).group || + tableView.schema.field + ) + continue + tableView.schema = table.schema + } + + // update linked rows + try { + const linkResp: any = await updateLinks({ + eventType: oldTable ? EventType.TABLE_UPDATED : EventType.TABLE_SAVE, + table: table, + oldTable: oldTable, + }) + if (linkResp != null && linkResp._rev) { + table._rev = linkResp._rev + } + } catch (err) { + throw new Error(err as string) + } + + // don't perform any updates until relationships have been + // checked by the updateLinks function + const updatedRows = tableSaveFunctions.getUpdatedRows() + if (updatedRows && updatedRows.length !== 0) { + await db.bulkDocs(updatedRows) + } + let result = await db.put(table) + table._rev = result.rev + const savedTable = cloneDeep(table) + + table = await tableSaveFunctions.after(table) + // the table may be updated as part of the table save after functionality - need to write it + if (!isEqual(savedTable, table)) { + result = await db.put(table) + table._rev = result.rev + } + // has to run after, make sure it has _id + await runStaticFormulaChecks(table, { oldTable, deletion: false }) + return { table } +} + +export async function destroy(table: Table) { + const db = context.getAppDB() + const tableId = table._id! + + // Delete all rows for that table + const rowsData = await db.allDocs( + getRowParams(tableId, null, { + include_docs: true, + }) + ) + await db.bulkDocs( + rowsData.rows.map((row: any) => ({ ...row.doc, _deleted: true })) + ) + await quotas.removeRows(rowsData.rows.length, { + tableId, + }) + + // update linked rows + await updateLinks({ + eventType: EventType.TABLE_DELETE, + table: table, + }) + + // don't remove the table itself until very end + await db.remove(tableId, table._rev) + + // remove table search index + if (!env.isTest() || env.COUCH_DB_URL) { + const currentIndexes = await db.getIndexes() + const existingIndex = currentIndexes.indexes.find( + (existing: any) => existing.name === `search:${tableId}` + ) + if (existingIndex) { + await db.deleteIndex(existingIndex) + } + } + + // has to run after, make sure it has _id + await runStaticFormulaChecks(table, { + deletion: true, + }) + await cleanupAttachments(table, { + rows: rowsData.rows.map((row: any) => row.doc), + }) + + return { table } +} diff --git a/packages/server/src/sdk/app/tables/internal/utils.ts b/packages/server/src/sdk/app/tables/internal/utils.ts index e69de29bb2..2d892ac272 100644 --- a/packages/server/src/sdk/app/tables/internal/utils.ts +++ b/packages/server/src/sdk/app/tables/internal/utils.ts @@ -0,0 +1,20 @@ +import { Table } from "@budibase/types" +import { fixAutoColumnSubType } from "../../../../utilities/rowProcessor" + +export function checkAutoColumns(table: Table, oldTable?: Table) { + if (!table.schema) { + return table + } + for (let [key, schema] of Object.entries(table.schema)) { + if (!schema.autocolumn || schema.subtype) { + continue + } + const oldSchema = oldTable && oldTable.schema[key] + if (oldSchema && oldSchema.subtype) { + table.schema[key].subtype = oldSchema.subtype + } else { + table.schema[key] = fixAutoColumnSubType(schema) + } + } + return table +} From 8d1163e50dec40ad7f339c90217b885fa35e5be4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 18:58:19 +0100 Subject: [PATCH 08/16] Adding external SDK, updating external controllers. --- .../src/api/controllers/table/external.ts | 368 ++---------------- .../src/sdk/app/tables/external/index.ts | 51 ++- .../src/sdk/app/tables/external/utils.ts | 6 +- packages/server/src/sdk/app/tables/index.ts | 96 +---- packages/server/src/sdk/app/tables/update.ts | 31 ++ 5 files changed, 107 insertions(+), 445 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 967176c2e4..f035822068 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -1,108 +1,20 @@ -import { - breakExternalTableId, - buildExternalTableId, -} from "../../../integrations/utils" -import { - foreignKeyStructure, - generateForeignKey, - generateJunctionTableName, - hasTypeChanged, - setStaticSchemas, -} from "./utils" -import { FieldTypes } from "../../../constants" -import { makeExternalQuery } from "../../../integrations/base/query" +import { breakExternalTableId } from "../../../integrations/utils" import { handleRequest } from "../row/external" -import { context, events } from "@budibase/backend-core" +import { events } from "@budibase/backend-core" import { isRows, isSchema, parse } from "../../../utilities/schema" import { BulkImportRequest, BulkImportResponse, - Datasource, - FieldSchema, - ManyToManyRelationshipFieldMetadata, - ManyToOneRelationshipFieldMetadata, - OneToManyRelationshipFieldMetadata, Operation, - QueryJson, - RelationshipFieldMetadata, - RelationshipType, - RenameColumn, SaveTableRequest, SaveTableResponse, Table, TableRequest, UserCtx, - ViewV2, } from "@budibase/types" import sdk from "../../../sdk" import { builderSocket } from "../../../websockets" -const { cloneDeep } = require("lodash/fp") - -async function makeTableRequest( - datasource: Datasource, - operation: Operation, - table: Table, - tables: Record, - oldTable?: Table, - renamed?: RenameColumn -) { - const json: QueryJson = { - endpoint: { - datasourceId: datasource._id!, - entityId: table._id!, - operation, - }, - meta: { - tables, - }, - table, - } - if (oldTable) { - json.meta!.table = oldTable - } - if (renamed) { - json.meta!.renamed = renamed - } - return makeExternalQuery(datasource, json) -} - -function cleanupRelationships( - table: Table, - tables: Record, - oldTable?: Table -) { - const tableToIterate = oldTable ? oldTable : table - // clean up relationships in couch table schemas - for (let [key, schema] of Object.entries(tableToIterate.schema)) { - if ( - schema.type === FieldTypes.LINK && - (!oldTable || table.schema[key] == null) - ) { - const schemaTableId = schema.tableId - const relatedTable = Object.values(tables).find( - table => table._id === schemaTableId - ) - const foreignKey = - schema.relationshipType !== RelationshipType.MANY_TO_MANY && - schema.foreignKey - if (!relatedTable || !foreignKey) { - continue - } - for (let [relatedKey, relatedSchema] of Object.entries( - relatedTable.schema - )) { - if ( - relatedSchema.type === FieldTypes.LINK && - relatedSchema.fieldName === foreignKey - ) { - delete relatedTable.schema[relatedKey] - } - } - } - } -} - function getDatasourceId(table: Table) { if (!table) { throw "No table supplied" @@ -113,247 +25,32 @@ function getDatasourceId(table: Table) { return breakExternalTableId(table._id).datasourceId } -function otherRelationshipType(type?: string) { - if (type === RelationshipType.MANY_TO_MANY) { - return RelationshipType.MANY_TO_MANY - } - return type === RelationshipType.ONE_TO_MANY - ? RelationshipType.MANY_TO_ONE - : RelationshipType.ONE_TO_MANY -} - -function generateManyLinkSchema( - datasource: Datasource, - column: ManyToManyRelationshipFieldMetadata, - table: Table, - relatedTable: Table -): Table { - if (!table.primary || !relatedTable.primary) { - throw new Error("Unable to generate many link schema, no primary keys") - } - const primary = table.name + table.primary[0] - const relatedPrimary = relatedTable.name + relatedTable.primary[0] - const jcTblName = generateJunctionTableName(column, table, relatedTable) - // first create the new table - const junctionTable = { - _id: buildExternalTableId(datasource._id!, jcTblName), - name: jcTblName, - primary: [primary, relatedPrimary], - constrained: [primary, relatedPrimary], - schema: { - [primary]: foreignKeyStructure(primary, { - toTable: table.name, - toKey: table.primary[0], - }), - [relatedPrimary]: foreignKeyStructure(relatedPrimary, { - toTable: relatedTable.name, - toKey: relatedTable.primary[0], - }), - }, - } - column.through = junctionTable._id - column.throughFrom = relatedPrimary - column.throughTo = primary - column.fieldName = relatedPrimary - return junctionTable -} - -function generateLinkSchema( - column: - | OneToManyRelationshipFieldMetadata - | ManyToOneRelationshipFieldMetadata, - table: Table, - relatedTable: Table, - type: RelationshipType.ONE_TO_MANY | RelationshipType.MANY_TO_ONE -) { - if (!table.primary || !relatedTable.primary) { - throw new Error("Unable to generate link schema, no primary keys") - } - const isOneSide = type === RelationshipType.ONE_TO_MANY - const primary = isOneSide ? relatedTable.primary[0] : table.primary[0] - // generate a foreign key - const foreignKey = generateForeignKey(column, relatedTable) - column.relationshipType = type - column.foreignKey = isOneSide ? foreignKey : primary - column.fieldName = isOneSide ? primary : foreignKey - return foreignKey -} - -function generateRelatedSchema( - linkColumn: RelationshipFieldMetadata, - table: Table, - relatedTable: Table, - columnName: string -) { - // generate column for other table - const relatedSchema = cloneDeep(linkColumn) - const isMany2Many = - linkColumn.relationshipType === RelationshipType.MANY_TO_MANY - // swap them from the main link - if (!isMany2Many && linkColumn.foreignKey) { - relatedSchema.fieldName = linkColumn.foreignKey - relatedSchema.foreignKey = linkColumn.fieldName - } - // is many to many - else if (isMany2Many) { - // don't need to copy through, already got it - relatedSchema.fieldName = linkColumn.throughTo - relatedSchema.throughTo = linkColumn.throughFrom - relatedSchema.throughFrom = linkColumn.throughTo - } - relatedSchema.relationshipType = otherRelationshipType( - linkColumn.relationshipType - ) - relatedSchema.tableId = relatedTable._id - relatedSchema.name = columnName - table.schema[columnName] = relatedSchema -} - -function isRelationshipSetup(column: RelationshipFieldMetadata) { - return (column as any).foreignKey || (column as any).through -} - export async function save(ctx: UserCtx) { const inputs = ctx.request.body - const renamed = inputs?._rename + const renaming = inputs?._rename // can't do this right now delete inputs.rows - const datasourceId = getDatasourceId(ctx.request.body)! + const tableId = ctx.request.body._id + const datasourceId = getDatasourceId(ctx.request.body) // table doesn't exist already, note that it is created if (!inputs._id) { inputs.created = true } - let tableToSave: TableRequest = { - type: "table", - _id: buildExternalTableId(datasourceId, inputs.name), - sourceId: datasourceId, - ...inputs, - } - - let oldTable: Table | undefined - if (ctx.request.body && ctx.request.body._id) { - oldTable = await sdk.tables.getTable(ctx.request.body._id) - } - - if (hasTypeChanged(tableToSave, oldTable)) { - ctx.throw(400, "A column type has changed.") - } - - for (let view in tableToSave.views) { - const tableView = tableToSave.views[view] - if (!tableView || !sdk.views.isV2(tableView)) continue - - tableToSave.views[view] = sdk.views.syncSchema( - oldTable!.views![view] as ViewV2, - tableToSave.schema, - renamed + try { + const { datasource, table } = await sdk.tables.external.save( + datasourceId!, + inputs, + { tableId, renaming } ) - } - - const db = context.getAppDB() - const datasource = await sdk.datasources.get(datasourceId) - if (!datasource.entities) { - datasource.entities = {} - } - - // GSheets is a specific case - only ever has a static primary key - tableToSave = setStaticSchemas(datasource, tableToSave) - - const oldTables = cloneDeep(datasource.entities) - const tables: Record = datasource.entities - - const extraTablesToUpdate = [] - - // check if relations need setup - for (let schema of Object.values(tableToSave.schema)) { - if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) { - continue - } - const schemaTableId = schema.tableId - const relatedTable = Object.values(tables).find( - table => table._id === schemaTableId - ) - if (!relatedTable) { - continue - } - const relatedColumnName = schema.fieldName! - const relationType = schema.relationshipType - if (relationType === RelationshipType.MANY_TO_MANY) { - const junctionTable = generateManyLinkSchema( - datasource, - schema, - tableToSave, - relatedTable - ) - if (tables[junctionTable.name]) { - throw "Junction table already exists, cannot create another relationship." - } - tables[junctionTable.name] = junctionTable - extraTablesToUpdate.push(junctionTable) + builderSocket?.emitDatasourceUpdate(ctx, datasource) + return table + } catch (err: any) { + if (err instanceof Error) { + ctx.throw(400, err.message) } else { - const fkTable = - relationType === RelationshipType.ONE_TO_MANY - ? tableToSave - : relatedTable - const foreignKey = generateLinkSchema( - schema, - tableToSave, - relatedTable, - relationType - ) - fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey) - if (fkTable.constrained == null) { - fkTable.constrained = [] - } - if (fkTable.constrained.indexOf(foreignKey) === -1) { - fkTable.constrained.push(foreignKey) - } - // foreign key is in other table, need to save it to external - if (fkTable._id !== tableToSave._id) { - extraTablesToUpdate.push(fkTable) - } + ctx.throw(err.status || 500, err?.message || err) } - generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName) - schema.main = true } - - cleanupRelationships(tableToSave, tables, oldTable) - - const operation = oldTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest( - datasource, - operation, - tableToSave, - tables, - oldTable, - renamed - ) - // update any extra tables (like foreign keys in other tables) - for (let extraTable of extraTablesToUpdate) { - const oldExtraTable = oldTables[extraTable.name] - let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable) - } - - // make sure the constrained list, all still exist - if (Array.isArray(tableToSave.constrained)) { - tableToSave.constrained = tableToSave.constrained.filter(constraint => - Object.keys(tableToSave.schema).includes(constraint) - ) - } - - // remove the rename prop - delete tableToSave._rename - // store it into couch now for budibase reference - datasource.entities[tableToSave.name] = tableToSave - await db.put(sdk.tables.populateExternalTableSchemas(datasource)) - - // Since tables are stored inside datasources, we need to notify clients - // that the datasource definition changed - const updatedDatasource = await sdk.datasources.get(datasource._id!) - builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) - - return tableToSave } export async function destroy(ctx: UserCtx) { @@ -364,27 +61,20 @@ export async function destroy(ctx: UserCtx) { ctx.throw(400, "Cannot delete tables which weren't created in Budibase.") } const datasourceId = getDatasourceId(tableToDelete) - - const db = context.getAppDB() - const datasource = await sdk.datasources.get(datasourceId!) - const tables = datasource.entities - - const operation = Operation.DELETE_TABLE - if (tables) { - await makeTableRequest(datasource, operation, tableToDelete, tables) - cleanupRelationships(tableToDelete, tables) - delete tables[tableToDelete.name] - datasource.entities = tables + try { + const { datasource, table } = await sdk.tables.external.destroy( + datasourceId!, + tableToDelete + ) + builderSocket?.emitDatasourceUpdate(ctx, datasource) + return table + } catch (err: any) { + if (err instanceof Error) { + ctx.throw(400, err.message) + } else { + ctx.throw(err.status || 500, err.message || err) + } } - - await db.put(sdk.tables.populateExternalTableSchemas(datasource)) - - // Since tables are stored inside datasources, we need to notify clients - // that the datasource definition changed - const updatedDatasource = await sdk.datasources.get(datasource._id!) - builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource) - - return tableToDelete } export async function bulkImport( diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 3303ca38d1..1045ef1e33 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -7,25 +7,29 @@ import { ViewV2, } from "@budibase/types" import { context } from "@budibase/backend-core" -import { buildExternalTableId } from "../../../integrations/utils" -import sdk from "../../index" +import { buildExternalTableId } from "../../../../integrations/utils" import { foreignKeyStructure, hasTypeChanged, setStaticSchemas, -} from "../../../api/controllers/table/utils" +} from "../../../../api/controllers/table/utils" import { cloneDeep } from "lodash/fp" -import { FieldTypes } from "../../../constants" -import { makeTableRequest } from "../../../api/controllers/table/ExternalRequest" +import { FieldTypes } from "../../../../constants" +import { makeTableRequest } from "../../../../api/controllers/table/ExternalRequest" import { isRelationshipSetup, cleanupRelationships, generateLinkSchema, generateManyLinkSchema, generateRelatedSchema, -} from "./externalUtils" +} from "./utils" -export async function externalSave( +import { getTable } from "../getters" +import { populateExternalTableSchemas } from "../validation" +import datasourceSdk from "../../datasources" +import * as viewSdk from "../../views" + +export async function save( datasourceId: string, update: Table, opts?: { tableId?: string; renaming?: RenameColumn } @@ -39,7 +43,7 @@ export async function externalSave( let oldTable: Table | undefined if (opts?.tableId) { - oldTable = await sdk.tables.getTable(opts.tableId) + oldTable = await getTable(opts.tableId) } if (hasTypeChanged(tableToSave, oldTable)) { @@ -48,9 +52,9 @@ export async function externalSave( for (let view in tableToSave.views) { const tableView = tableToSave.views[view] - if (!tableView || !sdk.views.isV2(tableView)) continue + if (!tableView || !viewSdk.isV2(tableView)) continue - tableToSave.views[view] = sdk.views.syncSchema( + tableToSave.views[view] = viewSdk.syncSchema( oldTable!.views![view] as ViewV2, tableToSave.schema, opts?.renaming @@ -58,7 +62,7 @@ export async function externalSave( } const db = context.getAppDB() - const datasource = await sdk.datasources.get(datasourceId) + const datasource = await datasourceSdk.get(datasourceId) if (!datasource.entities) { datasource.entities = {} } @@ -155,11 +159,32 @@ export async function externalSave( delete tableToSave._rename // store it into couch now for budibase reference datasource.entities[tableToSave.name] = tableToSave - await db.put(sdk.tables.populateExternalTableSchemas(datasource)) + await db.put(populateExternalTableSchemas(datasource)) // Since tables are stored inside datasources, we need to notify clients // that the datasource definition changed - const updatedDatasource = await sdk.datasources.get(datasource._id!) + const updatedDatasource = await datasourceSdk.get(datasource._id!) return { datasource: updatedDatasource, table: tableToSave } } + +export async function destroy(datasourceId: string, table: Table) { + const db = context.getAppDB() + const datasource = await datasourceSdk.get(datasourceId) + const tables = datasource.entities + + const operation = Operation.DELETE_TABLE + if (tables) { + await makeTableRequest(datasource, operation, table, tables) + cleanupRelationships(table, tables) + delete tables[table.name] + datasource.entities = tables + } + + await db.put(populateExternalTableSchemas(datasource)) + + // Since tables are stored inside datasources, we need to notify clients + // that the datasource definition changed + const updatedDatasource = await datasourceSdk.get(datasource._id!) + return { datasource: updatedDatasource, table } +} diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index 8d22b8da1a..ef280472bf 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -7,13 +7,13 @@ import { RelationshipType, Table, } from "@budibase/types" -import { FieldTypes } from "../../../constants" +import { FieldTypes } from "../../../../constants" import { foreignKeyStructure, generateForeignKey, generateJunctionTableName, -} from "../../../api/controllers/table/utils" -import { buildExternalTableId } from "../../../integrations/utils" +} from "../../../../api/controllers/table/utils" +import { buildExternalTableId } from "../../../../integrations/utils" import { cloneDeep } from "lodash/fp" export function cleanupRelationships( diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 64fcde4bff..8542250517 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,95 +1,11 @@ -import { context } from "@budibase/backend-core" -import { BudibaseInternalDB, getTableParams } from "../../../db/utils" -import { - breakExternalTableId, - isExternalTable, - isSQL, -} from "../../../integrations/utils" -import { - Database, - Table, - TableResponse, - TableViewsResponse, -} from "@budibase/types" -import datasources from "../datasources" import { populateExternalTableSchemas } from "./validation" -import sdk from "../../../sdk" - -async function getAllInternalTables(db?: Database): Promise { - if (!db) { - db = context.getAppDB() - } - const internalTables = await db.allDocs( - getTableParams(null, { - include_docs: true, - }) - ) - return internalTables.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, - })) -} - -async function getAllExternalTables( - datasourceId: any -): Promise> { - const datasource = await datasources.get(datasourceId, { enriched: true }) - if (!datasource || !datasource.entities) { - throw "Datasource is not configured fully." - } - return datasource.entities -} - -async function getExternalTable( - datasourceId: any, - tableName: any -): Promise
{ - const entities = await getAllExternalTables(datasourceId) - return entities[tableName] -} - -async function getTable(tableId: any): Promise
{ - const db = context.getAppDB() - if (isExternalTable(tableId)) { - let { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await datasources.get(datasourceId!) - const table = await getExternalTable(datasourceId, tableName) - return { ...table, sql: isSQL(datasource) } - } else { - return db.get(tableId) - } -} - -function enrichViewSchemas(table: Table): TableResponse { - return { - ...table, - views: Object.values(table.views ?? []) - .map(v => sdk.views.enrichSchema(v, table.schema)) - .reduce((p, v) => { - p[v.name] = v - return p - }, {} as TableViewsResponse), - } -} - -async function saveTable(table: Table) { - const db = context.getAppDB() - if (isExternalTable(table._id!)) { - const datasource = await sdk.datasources.get(table.sourceId!) - datasource.entities![table.name] = table - await db.put(datasource) - } else { - await db.put(table) - } -} +import * as getters from "./getters" +import * as updates from "./update" +import * as utils from "./utils" export default { - getAllInternalTables, - getAllExternalTables, - getExternalTable, - getTable, populateExternalTableSchemas, - enrichViewSchemas, - saveTable, + ...updates, + ...getters, + ...utils, } diff --git a/packages/server/src/sdk/app/tables/update.ts b/packages/server/src/sdk/app/tables/update.ts index e69de29bb2..9bba4a967e 100644 --- a/packages/server/src/sdk/app/tables/update.ts +++ b/packages/server/src/sdk/app/tables/update.ts @@ -0,0 +1,31 @@ +import { Table, RenameColumn } from "@budibase/types" +import { isExternalTable } from "../../../integrations/utils" +import sdk from "../../index" +import { context } from "@budibase/backend-core" +import { isExternal } from "./utils" + +import * as external from "./external" +import * as internal from "./internal" +export * as external from "./external" +export * as internal from "./internal" + +export async function saveTable(table: Table) { + const db = context.getAppDB() + if (isExternalTable(table._id!)) { + const datasource = await sdk.datasources.get(table.sourceId!) + datasource.entities![table.name] = table + await db.put(datasource) + } else { + await db.put(table) + } +} + +export async function update(table: Table, renaming?: RenameColumn) { + const tableId = table._id + if (isExternal({ table })) { + const datasourceId = table.sourceId! + await external.save(datasourceId, table, { tableId, renaming }) + } else { + await internal.save(table, { tableId, renaming }) + } +} From 25ad78a94ef591c38ded1ed3f460e9f9a61d7f17 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 18:58:27 +0100 Subject: [PATCH 09/16] Adding utilities. --- packages/server/src/sdk/app/tables/utils.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts index e69de29bb2..88543e7c4c 100644 --- a/packages/server/src/sdk/app/tables/utils.ts +++ b/packages/server/src/sdk/app/tables/utils.ts @@ -0,0 +1,11 @@ +import { Table } from "@budibase/types" +import { isExternalTable } from "../../../integrations/utils" + +export function isExternal(opts: { table?: Table; tableId?: string }): boolean { + if (opts.table && opts.table.type === "external") { + return true + } else if (opts.tableId && isExternalTable(opts.tableId)) { + return true + } + return false +} From 2d5818ae335d2b9f588aa037ebd3afc25ddee3fd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 18 Oct 2023 18:58:58 +0100 Subject: [PATCH 10/16] Updating datasource controller to use SDK rather than calling through controllers. --- packages/server/src/api/controllers/datasource.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 8e6a0620da..b50c2464f0 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -4,7 +4,6 @@ import { getQueryParams, getTableParams, } from "../../db/utils" -import { destroy as tableDestroy } from "./table/internal" import { getIntegration } from "../../integrations" import { invalidateDynamicVariables } from "../../threads/utils" import { context, db as dbCore, events } from "@budibase/backend-core" @@ -325,11 +324,7 @@ async function destroyInternalTablesBySourceId(datasourceId: string) { // Destroy the tables. for (const table of datasourceTableDocs) { - await tableDestroy({ - params: { - tableId: table._id, - }, - }) + await sdk.tables.internal.destroy(table) } } From 41e5bba533b9f8cc916eeb70f7eb78b780410502 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 19 Oct 2023 06:53:47 +0000 Subject: [PATCH 11/16] Bump version to 2.11.39 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9cdf56cbbb..e01e5ae03e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.38", + "version": "2.11.39", "npmClient": "yarn", "packages": [ "packages/*" From 54d976f75088b5ff5d0ec4109162a61887d1b394 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 19 Oct 2023 12:02:49 +0100 Subject: [PATCH 12/16] Fixing PR comments. --- .../src/sdk/app/tables/external/index.ts | 12 ++++++-- .../src/sdk/app/tables/external/utils.ts | 7 +++-- packages/server/src/sdk/app/tables/getters.ts | 12 +++----- .../src/sdk/app/tables/internal/index.ts | 29 +++++-------------- 4 files changed, 25 insertions(+), 35 deletions(-) diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 1045ef1e33..402baada78 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -41,9 +41,10 @@ export async function save( ...update, } + const tableId = opts?.tableId || update._id let oldTable: Table | undefined - if (opts?.tableId) { - oldTable = await getTable(opts.tableId) + if (tableId) { + oldTable = await getTable(tableId) } if (hasTypeChanged(tableToSave, oldTable)) { @@ -114,6 +115,11 @@ export async function save( relatedTable, relationType ) + if (fkTable.schema[foreignKey] != null) { + throw new Error( + `Unable to generate foreign key - column ${foreignKey} already in use.` + ) + } fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey) if (fkTable.constrained == null) { fkTable.constrained = [] @@ -132,7 +138,7 @@ export async function save( cleanupRelationships(tableToSave, tables, oldTable) - const operation = oldTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE + const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE await makeTableRequest( datasource, operation, diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index ef280472bf..10c755a7d6 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -52,7 +52,7 @@ export function cleanupRelationships( } } -export function otherRelationshipType(type?: string) { +export function otherRelationshipType(type: RelationshipType) { if (type === RelationshipType.MANY_TO_MANY) { return RelationshipType.MANY_TO_MANY } @@ -68,7 +68,10 @@ export function generateManyLinkSchema( relatedTable: Table ): Table { if (!table.primary || !relatedTable.primary) { - throw new Error("Unable to generate many link schema, no primary keys") + const noPrimaryName = !table.primary ? table.name : relatedTable.name + throw new Error( + `Unable to generate many link schema, "${noPrimaryName}" does not have a primary key` + ) } const primary = table.name + table.primary[0] const relatedPrimary = relatedTable.name + relatedTable.primary[0] diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 69b2354196..81467bba6c 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -31,16 +31,12 @@ export async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() } - const internalTables = await db.allDocs( + const internalTables = await db.allDocs( getTableParams(null, { include_docs: true, }) ) - return internalTables.rows.map((tableDoc: any) => ({ - ...tableDoc.doc, - type: "internal", - sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id, - })) + return processInternalTables(internalTables) } async function getAllExternalTables(): Promise { @@ -63,7 +59,7 @@ export async function getExternalTable( return entities[tableName] } -export async function getTable(tableId: any): Promise
{ +export async function getTable(tableId: string): Promise
{ const db = context.getAppDB() if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) @@ -80,7 +76,7 @@ export async function getAllTables() { getAllInternalTables(), getAllExternalTables(), ]) - return [...internal, external] + return [...internal, ...external] } export async function getExternalTablesInDatasource( diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index c5c7822ffe..4fb8dce8c9 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -58,17 +58,6 @@ export async function save( }) table = await tableSaveFunctions.before(table) - // make sure that types don't change of a column, have to remove - // the column if you want to change the type - if (oldTable && oldTable.schema) { - for (const propKey of Object.keys(table.schema)) { - let oldColumn = oldTable.schema[propKey] - if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) { - oldTable.schema[propKey].type = FieldTypes.AUTO - } - } - } - let renaming = opts?.renaming if (renaming && renaming.old === renaming.updated) { renaming = undefined @@ -104,17 +93,13 @@ export async function save( } // update linked rows - try { - const linkResp: any = await updateLinks({ - eventType: oldTable ? EventType.TABLE_UPDATED : EventType.TABLE_SAVE, - table: table, - oldTable: oldTable, - }) - if (linkResp != null && linkResp._rev) { - table._rev = linkResp._rev - } - } catch (err) { - throw new Error(err as string) + const linkResp: any = await updateLinks({ + eventType: oldTable ? EventType.TABLE_UPDATED : EventType.TABLE_SAVE, + table: table, + oldTable: oldTable, + }) + if (linkResp != null && linkResp._rev) { + table._rev = linkResp._rev } // don't perform any updates until relationships have been From f072a77d9fcf2e08213f74065b2df217dce44bad Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 19 Oct 2023 12:03:49 +0100 Subject: [PATCH 13/16] Updating throw. --- packages/server/src/sdk/app/tables/getters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 81467bba6c..80524a0413 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -84,7 +84,7 @@ export async function getExternalTablesInDatasource( ): Promise> { const datasource = await datasources.get(datasourceId, { enriched: true }) if (!datasource || !datasource.entities) { - throw "Datasource is not configured fully." + throw new Error("Datasource is not configured fully.") } return datasource.entities } From 470735cc9759fa6929149e43314ceb26e5e5a3d5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 19 Oct 2023 12:26:29 +0100 Subject: [PATCH 14/16] Updating types across old table utilities. --- .../src/api/controllers/table/internal.ts | 6 +- .../server/src/api/controllers/table/utils.ts | 131 +++++++++++------- .../server/src/api/routes/tests/misc.spec.js | 89 ++++++------ .../src/sdk/app/tables/internal/index.ts | 2 +- packages/types/src/documents/app/view.ts | 2 +- 5 files changed, 133 insertions(+), 97 deletions(-) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 3a8b0b39a8..822ff8a75d 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -61,6 +61,10 @@ export async function bulkImport( ) { const table = await sdk.tables.getTable(ctx.params.tableId) const { rows, identifierFields } = ctx.request.body - await handleDataImport(ctx.user, table, rows, identifierFields) + await handleDataImport(table, { + importRows: rows, + identifierFields, + user: ctx.user, + }) return table } diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index bf64bc32ab..da1573715c 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -26,9 +26,16 @@ import { Row, SourceName, Table, + Database, + RenameColumn, + NumberFieldMetadata, + FieldSchema, + View, + RelationshipFieldMetadata, + FieldType, } from "@budibase/types" -export async function clearColumns(table: any, columnNames: any) { +export async function clearColumns(table: Table, columnNames: string[]) { const db = context.getAppDB() const rows = await db.allDocs( getRowParams(table._id, null, { @@ -43,10 +50,13 @@ export async function clearColumns(table: any, columnNames: any) { )) as { id: string; _rev?: string }[] } -export async function checkForColumnUpdates(oldTable: any, updatedTable: any) { +export async function checkForColumnUpdates( + updatedTable: Table, + oldTable?: Table, + columnRename?: RenameColumn +) { const db = context.getAppDB() let updatedRows = [] - const rename = updatedTable._rename let deletedColumns: any = [] if (oldTable && oldTable.schema && updatedTable.schema) { deletedColumns = Object.keys(oldTable.schema).filter( @@ -54,7 +64,7 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) { ) } // check for renaming of columns or deleted columns - if (rename || deletedColumns.length !== 0) { + if (columnRename || deletedColumns.length !== 0) { // Update all rows const rows = await db.allDocs( getRowParams(updatedTable._id, null, { @@ -64,9 +74,9 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) { const rawRows = rows.rows.map(({ doc }: any) => doc) updatedRows = rawRows.map((row: any) => { row = cloneDeep(row) - if (rename) { - row[rename.updated] = row[rename.old] - delete row[rename.old] + if (columnRename) { + row[columnRename.updated] = row[columnRename.old] + delete row[columnRename.old] } else if (deletedColumns.length !== 0) { deletedColumns.forEach((colName: any) => delete row[colName]) } @@ -76,14 +86,13 @@ export async function checkForColumnUpdates(oldTable: any, updatedTable: any) { // cleanup any attachments from object storage for deleted attachment columns await cleanupAttachments(updatedTable, { oldTable, rows: rawRows }) // Update views - await checkForViewUpdates(updatedTable, rename, deletedColumns) - delete updatedTable._rename + await checkForViewUpdates(updatedTable, deletedColumns, columnRename) } return { rows: updatedRows, table: updatedTable } } // makes sure the passed in table isn't going to reset the auto ID -export function makeSureTableUpToDate(table: any, tableToSave: any) { +export function makeSureTableUpToDate(table: Table, tableToSave: Table) { if (!table) { return tableToSave } @@ -99,16 +108,17 @@ export function makeSureTableUpToDate(table: any, tableToSave: any) { column.subtype === AutoFieldSubTypes.AUTO_ID && tableToSave.schema[field] ) { - tableToSave.schema[field].lastID = column.lastID + const tableCol = tableToSave.schema[field] as NumberFieldMetadata + tableCol.lastID = column.lastID } } return tableToSave } export async function importToRows( - data: any[], + data: Row[], table: Table, - user: ContextUser | null = null + user?: ContextUser ) { let originalTable = table let finalData: any = [] @@ -150,19 +160,20 @@ export async function importToRows( } export async function handleDataImport( - user: ContextUser, table: Table, - rows: Row[], - identifierFields: Array = [] + opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] } ) { const schema = table.schema + const identifierFields = opts?.identifierFields || [] + const user = opts?.user + const importRows = opts?.importRows - if (!rows || !isRows(rows) || !isSchema(schema)) { + if (!importRows || !isRows(importRows) || !isSchema(schema)) { return table } const db = context.getAppDB() - const data = parse(rows, schema) + const data = parse(importRows, schema) let finalData: any = await importToRows(data, table, user) @@ -200,7 +211,7 @@ export async function handleDataImport( return table } -export async function handleSearchIndexes(table: any) { +export async function handleSearchIndexes(table: Table) { const db = context.getAppDB() // create relevant search indexes if (table.indexes && table.indexes.length > 0) { @@ -244,13 +255,13 @@ export async function handleSearchIndexes(table: any) { return table } -export function checkStaticTables(table: any) { +export function checkStaticTables(table: Table) { // check user schema has all required elements if (table._id === InternalTables.USER_METADATA) { for (let [key, schema] of Object.entries(USERS_TABLE_SCHEMA.schema)) { // check if the schema exists on the table to be created/updated if (table.schema[key] == null) { - table.schema[key] = schema + table.schema[key] = schema as FieldSchema } } } @@ -258,13 +269,21 @@ export function checkStaticTables(table: any) { } class TableSaveFunctions { - db: any - user: any - oldTable: any - importRows: any - rows: any + db: Database + user?: ContextUser + oldTable?: Table + importRows?: Row[] + rows: Row[] - constructor({ user, oldTable, importRows }: any) { + constructor({ + user, + oldTable, + importRows, + }: { + user?: ContextUser + oldTable?: Table + importRows?: Row[] + }) { this.db = context.getAppDB() this.user = user this.oldTable = oldTable @@ -274,7 +293,7 @@ class TableSaveFunctions { } // before anything is done - async before(table: any) { + async before(table: Table) { if (this.oldTable) { table = makeSureTableUpToDate(this.oldTable, table) } @@ -283,16 +302,23 @@ class TableSaveFunctions { } // when confirmed valid - async mid(table: any) { - let response = await checkForColumnUpdates(this.oldTable, table) + async mid(table: Table, columnRename?: RenameColumn) { + let response = await checkForColumnUpdates( + table, + this.oldTable, + columnRename + ) this.rows = this.rows.concat(response.rows) return table } // after saving - async after(table: any) { + async after(table: Table) { table = await handleSearchIndexes(table) - table = await handleDataImport(this.user, table, this.importRows) + table = await handleDataImport(table, { + importRows: this.importRows, + user: this.user, + }) return table } @@ -302,9 +328,9 @@ class TableSaveFunctions { } export async function checkForViewUpdates( - table: any, - rename: any, - deletedColumns: any + table: Table, + deletedColumns: string[], + columnRename?: RenameColumn ) { const views = await getViews() const tableViews = views.filter(view => view.meta.tableId === table._id) @@ -314,30 +340,30 @@ export async function checkForViewUpdates( let needsUpdated = false // First check for renames, otherwise check for deletions - if (rename) { + if (columnRename) { // Update calculation field if required - if (view.meta.field === rename.old) { - view.meta.field = rename.updated + if (view.meta.field === columnRename.old) { + view.meta.field = columnRename.updated needsUpdated = true } // Update group by field if required - if (view.meta.groupBy === rename.old) { - view.meta.groupBy = rename.updated + if (view.meta.groupBy === columnRename.old) { + view.meta.groupBy = columnRename.updated needsUpdated = true } // Update filters if required if (view.meta.filters) { view.meta.filters.forEach((filter: any) => { - if (filter.key === rename.old) { - filter.key = rename.updated + if (filter.key === columnRename.old) { + filter.key = columnRename.updated needsUpdated = true } }) } } else if (deletedColumns) { - deletedColumns.forEach((column: any) => { + deletedColumns.forEach((column: string) => { // Remove calculation statement if required if (view.meta.field === column) { delete view.meta.field @@ -378,24 +404,29 @@ export async function checkForViewUpdates( if (!newViewTemplate.meta.schema) { newViewTemplate.meta.schema = table.schema } - table.views[view.name] = newViewTemplate.meta + if (table.views?.[view.name]) { + table.views[view.name] = newViewTemplate.meta as View + } } } } -export function generateForeignKey(column: any, relatedTable: any) { +export function generateForeignKey( + column: RelationshipFieldMetadata, + relatedTable: Table +) { return `fk_${relatedTable.name}_${column.fieldName}` } export function generateJunctionTableName( - column: any, - table: any, - relatedTable: any + column: RelationshipFieldMetadata, + table: Table, + relatedTable: Table ) { return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}` } -export function foreignKeyStructure(keyName: any, meta?: any) { +export function foreignKeyStructure(keyName: string, meta?: any) { const structure: any = { type: FieldTypes.NUMBER, constraints: {}, @@ -407,7 +438,7 @@ export function foreignKeyStructure(keyName: any, meta?: any) { return structure } -export function areSwitchableTypes(type1: any, type2: any) { +export function areSwitchableTypes(type1: FieldType, type2: FieldType) { if ( SwitchableTypes.indexOf(type1) === -1 && SwitchableTypes.indexOf(type2) === -1 diff --git a/packages/server/src/api/routes/tests/misc.spec.js b/packages/server/src/api/routes/tests/misc.spec.js index 21ebea637f..2560198d00 100644 --- a/packages/server/src/api/routes/tests/misc.spec.js +++ b/packages/server/src/api/routes/tests/misc.spec.js @@ -12,14 +12,14 @@ describe("run misc tests", () => { }) describe("/bbtel", () => { - it("check if analytics enabled", async () => { - const res = await request - .get(`/api/bbtel`) - .set(config.defaultHeaders()) - .expect("Content-Type", /json/) - .expect(200) - expect(typeof res.body.enabled).toEqual("boolean") - }) + it("check if analytics enabled", async () => { + const res = await request + .get(`/api/bbtel`) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + expect(typeof res.body.enabled).toEqual("boolean") + }) }) describe("/health", () => { @@ -37,7 +37,6 @@ describe("run misc tests", () => { } else { expect(text.split(".").length).toEqual(3) } - }) }) @@ -93,77 +92,79 @@ describe("run misc tests", () => { constraints: { type: "array", presence: { - "allowEmpty": true + allowEmpty: true, }, - inclusion: [ - "One", - "Two", - "Three", - ] + inclusion: ["One", "Two", "Three"], }, name: "Sample Tags", - sortable: false + sortable: false, }, g: { type: "options", constraints: { type: "string", presence: false, - inclusion: [ - "Alpha", - "Beta", - "Gamma" - ] + inclusion: ["Alpha", "Beta", "Gamma"], }, - name: "Sample Opts" - } + name: "Sample Opts", + }, }, }) - + + const importRows = [ + { a: "1", b: "2", c: "3", d: "4", f: "['One']", g: "Alpha" }, + { a: "5", b: "6", c: "7", d: "8", f: "[]", g: undefined }, + { a: "9", b: "10", c: "11", d: "12", f: "['Two','Four']", g: "" }, + { a: "13", b: "14", c: "15", d: "16", g: "Omega" }, + ] // Shift specific row tests to the row spec - await tableUtils.handleDataImport( - { userId: "test" }, - table, - [ - { a: '1', b: '2', c: '3', d: '4', f: "['One']", g: "Alpha" }, - { a: '5', b: '6', c: '7', d: '8', f: "[]", g: undefined}, - { a: '9', b: '10', c: '11', d: '12', f: "['Two','Four']", g: ""}, - { a: '13', b: '14', c: '15', d: '16', g: "Omega"} - ] - ) + await tableUtils.handleDataImport(table, { + importRows, + user: { userId: "test" }, + }) // 4 rows imported, the auto ID starts at 1 // We expect the handleDataImport function to update the lastID - expect(table.schema.e.lastID).toEqual(4); - + expect(table.schema.e.lastID).toEqual(4) + // Array/Multi - should have added a new value to the inclusion. - expect(table.schema.f.constraints.inclusion).toEqual(['Four','One','Three','Two']); + expect(table.schema.f.constraints.inclusion).toEqual([ + "Four", + "One", + "Three", + "Two", + ]) // Options - should have a new value in the inclusion - expect(table.schema.g.constraints.inclusion).toEqual(['Alpha','Beta','Gamma','Omega']); + expect(table.schema.g.constraints.inclusion).toEqual([ + "Alpha", + "Beta", + "Gamma", + "Omega", + ]) const rows = await config.getRows() - expect(rows.length).toEqual(4); + expect(rows.length).toEqual(4) const rowOne = rows.find(row => row.e === 1) expect(rowOne.a).toEqual("1") - expect(rowOne.f).toEqual(['One']) - expect(rowOne.g).toEqual('Alpha') + expect(rowOne.f).toEqual(["One"]) + expect(rowOne.g).toEqual("Alpha") const rowTwo = rows.find(row => row.e === 2) expect(rowTwo.a).toEqual("5") expect(rowTwo.f).toEqual([]) expect(rowTwo.g).toEqual(undefined) - + const rowThree = rows.find(row => row.e === 3) expect(rowThree.a).toEqual("9") - expect(rowThree.f).toEqual(['Two','Four']) + expect(rowThree.f).toEqual(["Two", "Four"]) expect(rowThree.g).toEqual(null) const rowFour = rows.find(row => row.e === 4) expect(rowFour.a).toEqual("13") expect(rowFour.f).toEqual(undefined) - expect(rowFour.g).toEqual('Omega') + expect(rowFour.g).toEqual("Omega") }) }) }) diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index 4fb8dce8c9..1092c0e32d 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -68,7 +68,7 @@ export async function save( throw new Error("Cannot rename a linked column.") } - table = await tableSaveFunctions.mid(table) + table = await tableSaveFunctions.mid(table, renaming) // update schema of non-statistics views when new columns are added for (let view in table.views) { diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 0de5c69123..0d79e2c505 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -2,7 +2,7 @@ import { SearchFilter, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" export interface View { - name: string + name?: string tableId: string field?: string filters: ViewFilter[] From d1b551c6aaadba7a665a0321006fdf607ae2c5dd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 19 Oct 2023 16:56:45 +0100 Subject: [PATCH 15/16] Type fixes. --- packages/server/src/sdk/app/tables/getters.ts | 2 +- packages/server/src/sdk/app/views/external.ts | 3 +-- packages/server/src/sdk/app/views/internal.ts | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 80524a0413..02cef748c5 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -117,7 +117,7 @@ export function enrichViewSchemas(table: Table): TableResponse { views: Object.values(table.views ?? []) .map(v => sdk.views.enrichSchema(v, table.schema)) .reduce((p, v) => { - p[v.name] = v + p[v.name!] = v return p }, {} as TableViewsResponse), } diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index bc240ae17b..47301873f5 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -59,11 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise { const existingView = Object.values(views).find( v => isV2(v) && v.id === view.id ) - if (!existingView) { + if (!existingView || !existingView.name) { throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404) } - console.log("set to", view) delete views[existingView.name] views[view.name] = view await db.put(ds) diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 97b47fbcb5..d1dedd8566 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -51,11 +51,10 @@ export async function update(tableId: string, view: ViewV2): Promise { const existingView = Object.values(table.views).find( v => isV2(v) && v.id === view.id ) - if (!existingView) { + if (!existingView || !existingView.name) { throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404) } - console.log("set to", view) delete table.views[existingView.name] table.views[view.name] = view await db.put(table) From e79726e264506eabae40a55970fc16a7d83fe7d4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 19 Oct 2023 18:00:40 +0100 Subject: [PATCH 16/16] Only check CLA on PRs to master. --- .github/workflows/cla.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index c7a701535b..b12820344e 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -4,6 +4,8 @@ on: types: [created] pull_request_target: types: [opened,closed,synchronize] + branches: + - master jobs: CLAssistant: @@ -33,4 +35,4 @@ jobs: #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) - #use-dco-flag: true - If you are using DCO instead of CLA \ No newline at end of file + #use-dco-flag: true - If you are using DCO instead of CLA