From 9f0ef9f43e9ce8c6bac4c7d7ab3edc4aede93290 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 Aug 2024 10:25:47 +0100 Subject: [PATCH 01/11] Update packages/pro refernce to latest master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 14c89a5b20..f7997d2d27 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 14c89a5b20ee4de07723063458a2437b0dd4f2c9 +Subproject commit f7997d2d27d84e5e0352798eec7dd5a551a5bae5 From 5a8bb2972b3c0f9fc3fb905902707b631d962004 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 28 Aug 2024 11:36:57 +0200 Subject: [PATCH 02/11] Change types --- .../server/src/api/controllers/row/internal.ts | 3 +-- packages/server/src/db/linkedRows/index.ts | 16 +++++++++------- packages/server/src/db/linkedRows/linkUtils.ts | 14 +++++++++----- packages/server/src/db/tests/linkTests.spec.ts | 8 +++++--- .../server/src/utilities/rowProcessor/index.ts | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index b2982a3542..6b9d03f4ef 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -16,7 +16,6 @@ import { PatchRowRequest, PatchRowResponse, Row, - Table, UserCtx, } from "@budibase/types" import sdk from "../../../sdk" @@ -197,7 +196,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { ) // get the linked tables - const linkTableIds = getLinkedTableIDs(table as Table) + const linkTableIds = getLinkedTableIDs(table.schema) const linkTables = await sdk.tables.getTables(linkTableIds) // perform output processing diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2da7e212b9..9a86df7041 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -18,6 +18,7 @@ import { LinkDocumentValue, Row, Table, + TableSchema, } from "@budibase/types" import sdk from "../../sdk" @@ -46,8 +47,8 @@ export const EventType = { TABLE_DELETE: "table:delete", } -function clearRelationshipFields(table: Table, rows: Row[]) { - for (let [key, field] of Object.entries(table.schema)) { +function clearRelationshipFields(schema: TableSchema, rows: Row[]) { + for (let [key, field] of Object.entries(schema)) { if (field.type === FieldType.LINK) { rows = rows.map(row => { delete row[key] @@ -158,11 +159,11 @@ export async function updateLinks(args: { * @return returns the rows with all of the enriched relationships on it. */ export async function attachFullLinkedDocs( - table: Table, + schema: TableSchema, rows: Row[], opts?: { fromRow?: Row } ) { - const linkedTableIds = getLinkedTableIDs(table) + const linkedTableIds = getLinkedTableIDs(schema) if (linkedTableIds.length === 0) { return rows } @@ -182,7 +183,7 @@ export async function attachFullLinkedDocs( } const linkedTables = response[1] as Table[] // clear any existing links that could be dupe'd - rows = clearRelationshipFields(table, rows) + rows = clearRelationshipFields(schema, rows) // now get the docs and combine into the rows let linked: Row[] = [] if (linksWithoutFromRow.length > 0) { @@ -201,7 +202,7 @@ export async function attachFullLinkedDocs( } if (linkedRow) { const linkedTableId = - linkedRow.tableId || getRelatedTableForField(table, link.fieldName) + linkedRow.tableId || getRelatedTableForField(schema, link.fieldName) const linkedTable = linkedTables.find( table => table._id === linkedTableId ) @@ -263,7 +264,8 @@ export async function squashLinksToPrimaryDisplay( } const newLinks = [] for (let link of row[column]) { - const linkTblId = link.tableId || getRelatedTableForField(table, column) + const linkTblId = + link.tableId || getRelatedTableForField(table.schema, column) const linkedTable = await getLinkedTable(linkTblId!, linkedTables) const obj: any = { _id: link._id } obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index c30d62ef35..4cedc4f86a 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -7,6 +7,7 @@ import { LinkDocument, LinkDocumentValue, Table, + TableSchema, } from "@budibase/types" import sdk from "../../sdk" @@ -121,8 +122,8 @@ export function getUniqueByProp(array: any[], prop: string) { return filteredArray } -export function getLinkedTableIDs(table: Table): string[] { - return Object.values(table.schema) +export function getLinkedTableIDs(schema: TableSchema): string[] { + return Object.values(schema) .filter(isRelationshipColumn) .map(column => column.tableId) } @@ -139,13 +140,16 @@ export async function getLinkedTable(id: string, tables: Table[]) { return linkedTable } -export function getRelatedTableForField(table: Table, fieldName: string) { +export function getRelatedTableForField( + schema: TableSchema, + fieldName: string +) { // look to see if its on the table, straight in the schema - const field = table.schema[fieldName] + const field = schema[fieldName] if (field?.type === FieldType.LINK) { return field.tableId } - for (let column of Object.values(table.schema)) { + for (let column of Object.values(schema)) { if (column.type === FieldType.LINK && column.fieldName === fieldName) { return column.tableId } diff --git a/packages/server/src/db/tests/linkTests.spec.ts b/packages/server/src/db/tests/linkTests.spec.ts index 3dbcb32011..2d6636388e 100644 --- a/packages/server/src/db/tests/linkTests.spec.ts +++ b/packages/server/src/db/tests/linkTests.spec.ts @@ -34,7 +34,7 @@ describe("test link functionality", () => { }) describe("getRelatedTableForField", () => { - let link = basicTable() + const link = basicTable() link.schema.link = { name: "link", relationshipType: RelationshipType.ONE_TO_MANY, @@ -44,11 +44,13 @@ describe("test link functionality", () => { } it("should get the field from the table directly", () => { - expect(linkUtils.getRelatedTableForField(link, "link")).toBe("tableID") + expect(linkUtils.getRelatedTableForField(link.schema, "link")).toBe( + "tableID" + ) }) it("should get the field from the link", () => { - expect(linkUtils.getRelatedTableForField(link, "otherLink")).toBe( + expect(linkUtils.getRelatedTableForField(link.schema, "otherLink")).toBe( "tableID" ) }) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 4b2fd83882..a20a3b25e1 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -262,7 +262,7 @@ export async function outputProcessing( } // attach any linked row information let enriched = !opts.preserveLinks - ? await linkRows.attachFullLinkedDocs(table, safeRows, { + ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { fromRow: opts?.fromRow, }) : safeRows From 20afeeb4b7235316d079d55cc7749608393fe1d9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 Aug 2024 10:59:25 +0100 Subject: [PATCH 03/11] Update packages/pro refernce to latest master. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f7997d2d27..51a83b791a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f7997d2d27d84e5e0352798eec7dd5a551a5bae5 +Subproject commit 51a83b791a7a11b1d51c1fdb91f2ac246298279e From 20f8fef15c44f5eab25e7a28314c5512760c0f6e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 23 Aug 2024 12:28:02 +0200 Subject: [PATCH 04/11] Test sqs --- .../server/src/api/routes/tests/row.spec.ts | 23 +++++++++++++++++-- .../src/utilities/rowProcessor/index.ts | 11 ++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 95e714fbd6..08f06d77dc 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -9,7 +9,13 @@ import { import tk from "timekeeper" import emitter from "../../../../src/events" import { outputProcessing } from "../../../utilities/rowProcessor" -import { context, InternalTable, tenancy } from "@budibase/backend-core" +import { + context, + InternalTable, + tenancy, + withEnv as withCoreEnv, + setEnv as setCoreEnv, +} from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { AttachmentFieldMetadata, @@ -69,6 +75,7 @@ async function waitForEvent( describe.each([ ["internal", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], @@ -76,6 +83,7 @@ describe.each([ [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined + const isSqs = providerType === "sqs" const isMSSQL = providerType === DatabaseName.SQL_SERVER const isOracle = providerType === DatabaseName.ORACLE const config = setup.getConfig() @@ -83,9 +91,17 @@ describe.each([ let table: Table let datasource: Datasource | undefined let client: Knex | undefined + let envCleanup: (() => void) | undefined beforeAll(async () => { - await config.init() + await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init()) + if (isSqs) { + envCleanup = setCoreEnv({ + SQS_SEARCH_ENABLE: "true", + SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()], + }) + } + if (dsProvider) { const rawDatasource = await dsProvider datasource = await config.createDatasource({ @@ -97,6 +113,9 @@ describe.each([ afterAll(async () => { setup.afterAll() + if (envCleanup) { + envCleanup() + } }) function saveTableRequest( diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index a20a3b25e1..795f6970ab 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -3,6 +3,7 @@ import { fixAutoColumnSubType, processFormulas } from "./utils" import { cache, context, + db, HTTPError, objectStore, utils, @@ -349,11 +350,19 @@ export async function outputProcessing( } // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) - if (isExternal) { + if (isExternal || db.isSqsEnabledForTenant()) { for (const row of enriched) { for (const key of Object.keys(row)) { if (row[key] === null) { delete row[key] + } else if (row[key] && table.schema[key]?.type === FieldType.LINK) { + for (const link of row[key] || []) { + for (const linkKey of Object.keys(link)) { + if (link[linkKey] === null) { + delete link[linkKey] + } + } + } } } } From bd1018ba2b08393007f8c8587c290e220262c8ae Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 23 Aug 2024 12:35:43 +0200 Subject: [PATCH 05/11] Fix broken sqs tests --- packages/server/src/api/routes/tests/row.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 08f06d77dc..b060b2f0c1 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -83,6 +83,7 @@ describe.each([ [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined + const isLucene = providerType === "lucene" const isSqs = providerType === "sqs" const isMSSQL = providerType === DatabaseName.SQL_SERVER const isOracle = providerType === DatabaseName.ORACLE @@ -365,7 +366,7 @@ describe.each([ expect(ids).toEqual(expect.arrayContaining(sequence)) }) - isInternal && + isLucene && it("row values are coerced", async () => { const str: FieldSchema = { type: FieldType.STRING, From 90af43ed872fdcae308af1f6c78f3afb82ca16f8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 28 Aug 2024 12:11:28 +0200 Subject: [PATCH 06/11] Fix test --- packages/server/src/api/routes/tests/row.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b060b2f0c1..c50b233a83 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -76,11 +76,11 @@ async function waitForEvent( describe.each([ ["internal", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined const isLucene = providerType === "lucene" From 6d8b0881a16a98c749f2d8862b2550f3da33fd5f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 28 Aug 2024 12:17:25 +0200 Subject: [PATCH 07/11] Rename --- .../src/api/controllers/row/external.ts | 8 +++--- .../server/src/api/controllers/row/index.ts | 26 +++++++++---------- .../src/api/controllers/row/internal.ts | 8 +++--- .../src/api/controllers/row/utils/utils.ts | 2 +- .../server/src/middleware/trimViewRowInfo.ts | 4 +-- packages/server/src/websockets/grid.ts | 6 ++--- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 8b95d9c2f3..f596b60101 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -38,7 +38,7 @@ export async function handleRequest( } export async function patch(ctx: UserCtx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const { _id, ...rowData } = ctx.request.body const table = await sdk.tables.getTable(tableId) @@ -93,7 +93,7 @@ export async function patch(ctx: UserCtx) { } export async function destroy(ctx: UserCtx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const _id = ctx.request.body._id const { row } = await handleRequest(Operation.DELETE, tableId, { id: breakRowIdField(_id), @@ -104,7 +104,7 @@ export async function destroy(ctx: UserCtx) { export async function bulkDestroy(ctx: UserCtx) { const { rows } = ctx.request.body - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) let promises: Promise<{ row: Row; table: Table }>[] = [] for (let row of rows) { promises.push( @@ -123,7 +123,7 @@ export async function bulkDestroy(ctx: UserCtx) { export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const { datasourceId, tableName } = breakExternalTableId(tableId) const datasource: Datasource = await sdk.datasources.get(datasourceId) if (!datasource || !datasource.entities) { diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 237c30cc88..17cc9dc339 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -47,7 +47,7 @@ export async function patch( ctx: UserCtx ): Promise { const appId = ctx.appId - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const body = ctx.request.body // if it doesn't have an _id then its save @@ -72,7 +72,7 @@ export async function patch( export const save = async (ctx: UserCtx) => { const appId = ctx.appId - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const body = ctx.request.body // user metadata doesn't exist yet - don't allow creation @@ -98,7 +98,7 @@ export const save = async (ctx: UserCtx) => { } export async function fetchView(ctx: any) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const viewName = decodeURIComponent(ctx.params.viewName) const { calculation, group, field } = ctx.query @@ -111,12 +111,12 @@ export async function fetchView(ctx: any) { } export async function fetch(ctx: any) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) ctx.body = await sdk.rows.fetch(tableId) } export async function find(ctx: UserCtx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const rowId = ctx.params.rowId ctx.body = await sdk.rows.find(tableId, rowId) @@ -132,7 +132,7 @@ function isDeleteRow(input: any): input is DeleteRow { async function processDeleteRowsRequest(ctx: UserCtx) { let request = ctx.request.body as DeleteRows - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const processedRows = request.rows.map(row => { let processedRow: Row = typeof row == "string" ? { _id: row } : row @@ -148,7 +148,7 @@ async function processDeleteRowsRequest(ctx: UserCtx) { } async function deleteRows(ctx: UserCtx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const appId = ctx.appId let deleteRequest = ctx.request.body as DeleteRows @@ -170,7 +170,7 @@ async function deleteRows(ctx: UserCtx) { async function deleteRow(ctx: UserCtx) { const appId = ctx.appId - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const resp = await pickApi(tableId).destroy(ctx) if (!tableId.includes("datasource_plus")) { @@ -204,7 +204,7 @@ export async function destroy(ctx: UserCtx) { } export async function search(ctx: Ctx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) await context.ensureSnippetContext(true) @@ -226,7 +226,7 @@ export async function search(ctx: Ctx) { } export async function validate(ctx: Ctx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) // external tables are hard to validate currently if (isExternalTableID(tableId)) { ctx.body = { valid: true, errors: {} } @@ -239,14 +239,14 @@ export async function validate(ctx: Ctx) { } export async function fetchEnrichedRow(ctx: UserCtx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx) } export const exportRows = async ( ctx: Ctx ) => { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const format = ctx.query.format @@ -279,7 +279,7 @@ export const exportRows = async ( export async function downloadAttachment(ctx: UserCtx) { const { columnName } = ctx.params - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const rowId = ctx.params.rowId const row = await sdk.rows.find(tableId, rowId) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 6b9d03f4ef..47540baca3 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -23,7 +23,7 @@ import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" import { flatten } from "lodash" export async function patch(ctx: UserCtx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const inputs = ctx.request.body const isUserTable = tableId === InternalTables.USER_METADATA let oldRow @@ -97,7 +97,7 @@ export async function patch(ctx: UserCtx) { export async function destroy(ctx: UserCtx) { const db = context.getAppDB() - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const { _id } = ctx.request.body let row = await db.get(_id) let _rev = ctx.request.body._rev || row._rev @@ -136,7 +136,7 @@ export async function destroy(ctx: UserCtx) { } export async function bulkDestroy(ctx: UserCtx) { - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const table = await sdk.tables.getTable(tableId) let { rows } = ctx.request.body @@ -178,7 +178,7 @@ export async function bulkDestroy(ctx: UserCtx) { export async function fetchEnrichedRow(ctx: UserCtx) { const fieldName = ctx.request.query.field as string | undefined const db = context.getAppDB() - const tableId = utils.getTableId(ctx) + const tableId = utils.getSourceId(ctx) const rowId = ctx.params.rowId as string // need table to work out where links go in row, as well as the link docs const [table, links] = await Promise.all([ diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 911cfe8d5b..f52385c73c 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -78,7 +78,7 @@ export async function findRow(tableId: string, rowId: string) { return row } -export function getTableId(ctx: Ctx): string { +export function getSourceId(ctx: Ctx): string { // top priority, use the URL first if (ctx.params?.sourceId) { return ctx.params.sourceId diff --git a/packages/server/src/middleware/trimViewRowInfo.ts b/packages/server/src/middleware/trimViewRowInfo.ts index 4c382e6123..cdcddf2cc9 100644 --- a/packages/server/src/middleware/trimViewRowInfo.ts +++ b/packages/server/src/middleware/trimViewRowInfo.ts @@ -2,13 +2,13 @@ import { Ctx, Row } from "@budibase/types" import * as utils from "../db/utils" import sdk from "../sdk" import { Next } from "koa" -import { getTableId } from "../api/controllers/row/utils" +import { getSourceId } from "../api/controllers/row/utils" export default async (ctx: Ctx, next: Next) => { const { body } = ctx.request let { _viewId: viewId } = body - const possibleViewId = getTableId(ctx) + const possibleViewId = getSourceId(ctx) if (utils.isViewID(possibleViewId)) { viewId = possibleViewId } diff --git a/packages/server/src/websockets/grid.ts b/packages/server/src/websockets/grid.ts index 979a0bf125..a36413454e 100644 --- a/packages/server/src/websockets/grid.ts +++ b/packages/server/src/websockets/grid.ts @@ -4,7 +4,7 @@ import { BaseSocket } from "./websocket" import { auth, permissions } from "@budibase/backend-core" import http from "http" import Koa from "koa" -import { getTableId } from "../api/controllers/row/utils" +import { getSourceId } from "../api/controllers/row/utils" import { Row, Table, View, ViewV2 } from "@budibase/types" import { Socket } from "socket.io" import { GridSocketEvent } from "@budibase/shared-core" @@ -80,7 +80,7 @@ export default class GridSocket extends BaseSocket { } emitRowUpdate(ctx: any, row: Row) { - const resourceId = ctx.params?.viewId || getTableId(ctx) + const resourceId = ctx.params?.viewId || getSourceId(ctx) const room = `${ctx.appId}-${resourceId}` this.emitToRoom(ctx, room, GridSocketEvent.RowChange, { id: row._id, @@ -89,7 +89,7 @@ export default class GridSocket extends BaseSocket { } emitRowDeletion(ctx: any, row: Row) { - const resourceId = ctx.params?.viewId || getTableId(ctx) + const resourceId = ctx.params?.viewId || getSourceId(ctx) const room = `${ctx.appId}-${resourceId}` this.emitToRoom(ctx, room, GridSocketEvent.RowChange, { id: row._id, From 9459dd1820ceeef7cd98b4f93d8e3f0910446c29 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 28 Aug 2024 12:41:42 +0200 Subject: [PATCH 08/11] getSourceId returns table & view data --- .../src/api/controllers/row/external.ts | 8 +++--- .../server/src/api/controllers/row/index.ts | 27 +++++++++---------- .../src/api/controllers/row/internal.ts | 8 +++--- .../src/api/controllers/row/utils/utils.ts | 25 +++++++++-------- .../server/src/middleware/trimViewRowInfo.ts | 14 ++-------- packages/server/src/sdk/app/rows/search.ts | 3 +-- .../src/sdk/app/rows/search/external.ts | 8 ------ 7 files changed, 38 insertions(+), 55 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index f596b60101..e14e2d454a 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -38,7 +38,7 @@ export async function handleRequest( } export async function patch(ctx: UserCtx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const { _id, ...rowData } = ctx.request.body const table = await sdk.tables.getTable(tableId) @@ -93,7 +93,7 @@ export async function patch(ctx: UserCtx) { } export async function destroy(ctx: UserCtx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const _id = ctx.request.body._id const { row } = await handleRequest(Operation.DELETE, tableId, { id: breakRowIdField(_id), @@ -104,7 +104,7 @@ export async function destroy(ctx: UserCtx) { export async function bulkDestroy(ctx: UserCtx) { const { rows } = ctx.request.body - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) let promises: Promise<{ row: Row; table: Table }>[] = [] for (let row of rows) { promises.push( @@ -123,7 +123,7 @@ export async function bulkDestroy(ctx: UserCtx) { export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const { datasourceId, tableName } = breakExternalTableId(tableId) const datasource: Datasource = await sdk.datasources.get(datasourceId) if (!datasource || !datasource.entities) { diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 17cc9dc339..7d6448ca6b 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -47,7 +47,7 @@ export async function patch( ctx: UserCtx ): Promise { const appId = ctx.appId - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const body = ctx.request.body // if it doesn't have an _id then its save @@ -72,7 +72,7 @@ export async function patch( export const save = async (ctx: UserCtx) => { const appId = ctx.appId - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const body = ctx.request.body // user metadata doesn't exist yet - don't allow creation @@ -98,12 +98,11 @@ export const save = async (ctx: UserCtx) => { } export async function fetchView(ctx: any) { - const tableId = utils.getSourceId(ctx) const viewName = decodeURIComponent(ctx.params.viewName) const { calculation, group, field } = ctx.query - ctx.body = await sdk.rows.fetchView(tableId, viewName, { + ctx.body = await sdk.rows.fetchView(viewName, { calculation, group: calculation ? group : null, field, @@ -111,12 +110,12 @@ export async function fetchView(ctx: any) { } export async function fetch(ctx: any) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) ctx.body = await sdk.rows.fetch(tableId) } export async function find(ctx: UserCtx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const rowId = ctx.params.rowId ctx.body = await sdk.rows.find(tableId, rowId) @@ -132,7 +131,7 @@ function isDeleteRow(input: any): input is DeleteRow { async function processDeleteRowsRequest(ctx: UserCtx) { let request = ctx.request.body as DeleteRows - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const processedRows = request.rows.map(row => { let processedRow: Row = typeof row == "string" ? { _id: row } : row @@ -148,7 +147,7 @@ async function processDeleteRowsRequest(ctx: UserCtx) { } async function deleteRows(ctx: UserCtx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const appId = ctx.appId let deleteRequest = ctx.request.body as DeleteRows @@ -170,7 +169,7 @@ async function deleteRows(ctx: UserCtx) { async function deleteRow(ctx: UserCtx) { const appId = ctx.appId - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const resp = await pickApi(tableId).destroy(ctx) if (!tableId.includes("datasource_plus")) { @@ -204,7 +203,7 @@ export async function destroy(ctx: UserCtx) { } export async function search(ctx: Ctx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) await context.ensureSnippetContext(true) @@ -226,7 +225,7 @@ export async function search(ctx: Ctx) { } export async function validate(ctx: Ctx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) // external tables are hard to validate currently if (isExternalTableID(tableId)) { ctx.body = { valid: true, errors: {} } @@ -239,14 +238,14 @@ export async function validate(ctx: Ctx) { } export async function fetchEnrichedRow(ctx: UserCtx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx) } export const exportRows = async ( ctx: Ctx ) => { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const format = ctx.query.format @@ -279,7 +278,7 @@ export const exportRows = async ( export async function downloadAttachment(ctx: UserCtx) { const { columnName } = ctx.params - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const rowId = ctx.params.rowId const row = await sdk.rows.find(tableId, rowId) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 47540baca3..e698efe981 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -23,7 +23,7 @@ import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" import { flatten } from "lodash" export async function patch(ctx: UserCtx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const inputs = ctx.request.body const isUserTable = tableId === InternalTables.USER_METADATA let oldRow @@ -97,7 +97,7 @@ export async function patch(ctx: UserCtx) { export async function destroy(ctx: UserCtx) { const db = context.getAppDB() - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const { _id } = ctx.request.body let row = await db.get(_id) let _rev = ctx.request.body._rev || row._rev @@ -136,7 +136,7 @@ export async function destroy(ctx: UserCtx) { } export async function bulkDestroy(ctx: UserCtx) { - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const table = await sdk.tables.getTable(tableId) let { rows } = ctx.request.body @@ -178,7 +178,7 @@ export async function bulkDestroy(ctx: UserCtx) { export async function fetchEnrichedRow(ctx: UserCtx) { const fieldName = ctx.request.query.field as string | undefined const db = context.getAppDB() - const tableId = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) const rowId = ctx.params.rowId as string // need table to work out where links go in row, as well as the link docs const [table, links] = await Promise.all([ diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index f52385c73c..46bdbb7662 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,4 +1,4 @@ -import { InternalTables } from "../../../../db/utils" +import * as utils from "../../../../db/utils" import { context } from "@budibase/backend-core" import { @@ -67,7 +67,7 @@ export async function findRow(tableId: string, rowId: string) { const db = context.getAppDB() let row: Row // TODO remove special user case in future - if (tableId === InternalTables.USER_METADATA) { + if (tableId === utils.InternalTables.USER_METADATA) { row = await getFullUser(rowId) } else { row = await db.get(rowId) @@ -78,22 +78,25 @@ export async function findRow(tableId: string, rowId: string) { return row } -export function getSourceId(ctx: Ctx): string { +export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { // top priority, use the URL first if (ctx.params?.sourceId) { - return ctx.params.sourceId + const { sourceId } = ctx.params + if (utils.isViewID(sourceId)) { + return { + tableId: utils.extractViewInfoFromID(sourceId).tableId, + viewId: sourceId, + } + } + return { tableId: ctx.params.sourceId } } // now check for old way of specifying table ID if (ctx.params?.tableId) { - return ctx.params.tableId + return { tableId: ctx.params.tableId } } // check body for a table ID if (ctx.request.body?.tableId) { - return ctx.request.body.tableId - } - // now check if a specific view name - if (ctx.params?.viewName) { - return ctx.params.viewName + return { tableId: ctx.request.body.tableId } } throw new Error("Unable to find table ID in request") } @@ -198,7 +201,7 @@ export async function sqlOutputProcessing( } export function isUserMetadataTable(tableId: string) { - return tableId === InternalTables.USER_METADATA + return tableId === utils.InternalTables.USER_METADATA } export async function enrichArrayContext( diff --git a/packages/server/src/middleware/trimViewRowInfo.ts b/packages/server/src/middleware/trimViewRowInfo.ts index cdcddf2cc9..55efaee732 100644 --- a/packages/server/src/middleware/trimViewRowInfo.ts +++ b/packages/server/src/middleware/trimViewRowInfo.ts @@ -1,33 +1,23 @@ import { Ctx, Row } from "@budibase/types" -import * as utils from "../db/utils" + import sdk from "../sdk" import { Next } from "koa" import { getSourceId } from "../api/controllers/row/utils" export default async (ctx: Ctx, next: Next) => { const { body } = ctx.request - let { _viewId: viewId } = body - - const possibleViewId = getSourceId(ctx) - if (utils.isViewID(possibleViewId)) { - viewId = possibleViewId - } + const viewId = getSourceId(ctx).viewId ?? body._viewId // nothing to do, it is not a view (just a table ID) if (!viewId) { return next() } - const { tableId } = utils.extractViewInfoFromID(viewId) - // don't need to trim delete requests if (ctx?.method?.toLowerCase() !== "delete") { await trimViewFields(ctx.request.body, viewId) } - ctx.params.sourceId = tableId - ctx.params.viewId = viewId - return next() } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 6a4286814d..4aa091ce10 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -122,9 +122,8 @@ export async function fetchRaw(tableId: string): Promise { } export async function fetchView( - tableId: string, viewName: string, params: ViewParams ): Promise { - return pickApi(tableId).fetchView(viewName, params) + return internal.fetchView(viewName, params) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 07181d259b..992596ab34 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -272,11 +272,3 @@ export async function fetchRaw(tableId: string): Promise { }) return response.rows } - -export async function fetchView(viewName: string) { - // there are no views in external datasources, shouldn't ever be called - // for now just fetch - const split = viewName.split("all_") - const tableId = split[1] ? split[1] : split[0] - return fetch(tableId) -} From defbe20a3d719870a622b637e23f9bb4370b20ef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 28 Aug 2024 12:47:34 +0200 Subject: [PATCH 09/11] Undo wrong delete --- packages/server/src/api/routes/tests/row.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index c50b233a83..b060b2f0c1 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -76,11 +76,11 @@ async function waitForEvent( describe.each([ ["internal", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined const isLucene = providerType === "lucene" From 1650cfba298112ab2825947162923cc728748b5b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 23 Aug 2024 12:48:34 +0200 Subject: [PATCH 10/11] Fix test --- .../utilities/rowProcessor/tests/outputProcessing.spec.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index fd48f46016..0d0049a36e 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -10,6 +10,14 @@ import { outputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" import * as bbReferenceProcessor from "../bbReferenceProcessor" +jest.mock("@budibase/backend-core", () => ({ + ...jest.requireActual("@budibase/backend-core"), + db: { + ...jest.requireActual("@budibase/backend-core").db, + isSqsEnabledForTenant: () => true, + }, +})) + jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ processInputBBReference: jest.fn(), processInputBBReferences: jest.fn(), From 54406fd6b56a51c0ea033ef8e92acbf7211a609f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 28 Aug 2024 14:56:40 +0200 Subject: [PATCH 11/11] Rename fetchView to fetchLegacyView --- packages/server/src/api/controllers/row/index.ts | 4 ++-- packages/server/src/api/controllers/view/views.ts | 4 ++-- packages/server/src/api/routes/view.ts | 2 +- packages/server/src/sdk/app/rows/search.ts | 4 ++-- packages/server/src/sdk/app/rows/search/internal/internal.ts | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 7d6448ca6b..46aec4e11c 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -97,12 +97,12 @@ export const save = async (ctx: UserCtx) => { gridSocket?.emitRowUpdate(ctx, row || squashed) } -export async function fetchView(ctx: any) { +export async function fetchLegacyView(ctx: any) { const viewName = decodeURIComponent(ctx.params.viewName) const { calculation, group, field } = ctx.query - ctx.body = await sdk.rows.fetchView(viewName, { + ctx.body = await sdk.rows.fetchLegacyView(viewName, { calculation, group: calculation ? group : null, field, diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index abcc8627f3..b1f1f6c154 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -2,7 +2,7 @@ import viewTemplate from "./viewBuilder" import { apiFileReturn } from "../../../utilities/fileSystem" import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters" import { deleteView, getView, getViews, saveView } from "./utils" -import { fetchView } from "../row" +import { fetchLegacyView } from "../row" import { context, events } from "@budibase/backend-core" import sdk from "../../../sdk" import { @@ -170,7 +170,7 @@ export async function exportView(ctx: Ctx) { ctx.params.viewName = viewName } - await fetchView(ctx) + await fetchLegacyView(ctx) let rows = ctx.body as Row[] let schema: TableSchema = view && view.meta && view.meta.schema diff --git a/packages/server/src/api/routes/view.ts b/packages/server/src/api/routes/view.ts index 545d3016a3..807d8e2f28 100644 --- a/packages/server/src/api/routes/view.ts +++ b/packages/server/src/api/routes/view.ts @@ -46,7 +46,7 @@ router permissions.PermissionType.TABLE, permissions.PermissionLevel.READ ), - rowController.fetchView + rowController.fetchLegacyView ) .get("/api/views", authorized(permissions.BUILDER), viewController.v1.fetch) .delete( diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 4aa091ce10..482621630c 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -121,9 +121,9 @@ export async function fetchRaw(tableId: string): Promise { return pickApi(tableId).fetchRaw(tableId) } -export async function fetchView( +export async function fetchLegacyView( viewName: string, params: ViewParams ): Promise { - return internal.fetchView(viewName, params) + return internal.fetchLegacyView(viewName, params) } diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index acc7033ce1..ae7bca3b0c 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -145,7 +145,7 @@ export async function fetchRaw(tableId: string): Promise { return rows as Row[] } -export async function fetchView( +export async function fetchLegacyView( viewName: string, options: { calculation: string; group: string; field: string } ): Promise {