From 0eb90cfbea9e8dc16ecc917c1994382fdb901000 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 16:35:53 +0100 Subject: [PATCH] Type checks pass, now to find out how much stuff I've broken. --- packages/backend-core/src/db/lucene.ts | 16 +++---- .../backend-core/src/db/tests/lucene.spec.ts | 4 +- .../src/api/controllers/row/external.ts | 2 +- .../src/api/controllers/row/internal.ts | 31 +++++------- .../src/api/controllers/row/staticFormula.ts | 32 +++++-------- .../src/api/controllers/row/utils/utils.ts | 18 +------ .../src/api/controllers/table/external.ts | 5 +- .../server/src/api/controllers/table/utils.ts | 3 +- .../src/api/routes/tests/search.spec.ts | 12 ++--- .../src/api/routes/tests/templates.spec.ts | 2 +- packages/server/src/db/linkedRows/index.ts | 26 +++++----- .../integrations/tests/googlesheets.spec.ts | 22 ++++----- packages/server/src/sdk/app/rows/external.ts | 11 +---- packages/server/src/sdk/app/rows/internal.ts | 47 ++++++++++--------- .../sdk/app/rows/search/internal/internal.ts | 2 +- .../sdk/app/rows/search/internal/lucene.ts | 20 +++++--- .../src/sdk/app/rows/search/internal/sqs.ts | 11 ++++- .../sdk/app/rows/search/tests/search.spec.ts | 10 ++-- .../sdk/app/rows/search/tests/utils.spec.ts | 12 ++--- .../src/sdk/app/rows/tests/utils.spec.ts | 42 ++++++++--------- packages/server/src/sdk/app/rows/utils.ts | 7 +++ packages/server/src/sdk/app/views/index.ts | 3 +- .../src/utilities/rowProcessor/index.ts | 23 +++++++-- .../tests/inputProcessing.spec.ts | 10 ++-- 24 files changed, 183 insertions(+), 188 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index f5ad7e6433..b17c3ddf0d 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -79,8 +79,8 @@ export class QueryBuilder { return this } - setTable(tableId: string) { - this.#query.equal!.tableId = tableId + setSource(sourceId: string) { + this.#query.equal!.tableId = sourceId return this } @@ -638,8 +638,8 @@ async function recursiveSearch( .setSortOrder(params.sortOrder) .setSortType(params.sortType) - if (params.tableId) { - queryBuilder.setTable(params.tableId) + if (params.sourceId) { + queryBuilder.setSource(params.sourceId) } const page = await queryBuilder.run() @@ -672,8 +672,8 @@ export async function paginatedSearch( if (params.version) { search.setVersion(params.version) } - if (params.tableId) { - search.setTable(params.tableId) + if (params.sourceId) { + search.setSource(params.sourceId) } if (params.sort) { search @@ -695,8 +695,8 @@ export async function paginatedSearch( // Try fetching 1 row in the next page to see if another page of results // exists or not search.setBookmark(searchResults.bookmark).setLimit(1) - if (params.tableId) { - search.setTable(params.tableId) + if (params.sourceId) { + search.setSource(params.sourceId) } const nextResults = await search.run() diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index c41bdf88d1..8747f56a4b 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -366,7 +366,7 @@ describe("lucene", () => { }, }, { - tableId: TABLE_ID, + sourceId: TABLE_ID, limit: 1, sort: "property", sortType: SortType.STRING, @@ -390,7 +390,7 @@ describe("lucene", () => { }, }, { - tableId: TABLE_ID, + sourceId: TABLE_ID, query: {}, } ) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 2d4012dfcf..11b6559896 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -45,7 +45,7 @@ export async function patch(ctx: UserCtx) { const table = await utils.getTableFromSource(source) const { _id, ...rowData } = ctx.request.body - const { row: dataToUpdate } = await inputProcessing( + const dataToUpdate = await inputProcessing( ctx.user?._id, cloneDeep(source), rowData diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 33e3c7707b..f5cb42f81d 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -21,18 +21,19 @@ import { import sdk from "../../../sdk" import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" import { flatten } from "lodash" +import { findRow } from "../../../sdk/app/rows/internal" export async function patch(ctx: UserCtx) { - const { tableId, viewId } = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) + const table = sdk.views.isView(source) + ? await sdk.views.getTable(source.id) + : source const inputs = ctx.request.body const isUserTable = tableId === InternalTables.USER_METADATA let oldRow - const dbTable = await sdk.tables.getTable(tableId) try { - oldRow = await outputProcessing( - dbTable, - await utils.findRow(tableId, inputs._id!) - ) + oldRow = await outputProcessing(source, await findRow(tableId, inputs._id!)) } catch (err) { if (isUserTable) { // don't include the rev, it'll be the global rev @@ -48,22 +49,18 @@ export async function patch(ctx: UserCtx) { // need to build up full patch fields before coerce let combinedRow: any = cloneDeep(oldRow) for (let key of Object.keys(inputs)) { - if (!dbTable.schema[key]) continue + if (!table.schema[key]) continue combinedRow[key] = inputs[key] } // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(dbTable) + const tableClone = cloneDeep(table) // this returns the table and row incase they have been updated - let { table, row } = await inputProcessing( - ctx.user?._id, - tableClone, - combinedRow - ) + let row = await inputProcessing(ctx.user?._id, tableClone, combinedRow) const validateResult = await sdk.rows.utils.validate({ row, - table, + source, }) if (!validateResult.valid) { @@ -87,10 +84,8 @@ export async function patch(ctx: UserCtx) { return { row: ctx.body as Row, table, oldRow } } - const result = await finaliseRow(table, row, { - oldTable: dbTable, + const result = await finaliseRow(source, row, { updateFormula: true, - fromViewId: viewId, }) return { ...result, oldRow } @@ -186,7 +181,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { sdk.tables.getTable(tableId), linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) - let row = await utils.findRow(tableId, rowId) + let row = await findRow(tableId, rowId) row = await outputProcessing(table, row) const linkVals = links as LinkDocumentValue[] diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 777379db14..386dee7b4a 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -4,10 +4,11 @@ import { processFormulas, } from "../../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, FormulaType, FieldType } from "@budibase/types" +import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types" import * as linkRows from "../../../db/linkedRows" import isEqual from "lodash/isEqual" import { cloneDeep } from "lodash/fp" +import sdk from "../../../sdk" /** * This function runs through a list of enriched rows, looks at the rows which @@ -121,33 +122,26 @@ export async function updateAllFormulasInTable(table: Table) { * expects the row to be totally enriched/contain all relationships. */ export async function finaliseRow( - table: Table, + source: Table | ViewV2, row: Row, - { - oldTable, - updateFormula, - fromViewId, - }: { oldTable?: Table; updateFormula: boolean; fromViewId?: string } = { - updateFormula: true, - } + opts?: { updateFormula: boolean } ) { const db = context.getAppDB() + const { updateFormula = true } = opts || {} + const table = sdk.views.isView(source) + ? await sdk.views.getTable(source.id) + : source + row.type = "row" // process the row before return, to include relationships - let enrichedRow = (await outputProcessing(table, cloneDeep(row), { + let enrichedRow = await outputProcessing(source, cloneDeep(row), { squash: false, - })) as Row + }) // use enriched row to generate formulas for saving, specifically only use as context row = await processFormulas(table, row, { dynamic: false, contextRows: [enrichedRow], }) - // don't worry about rev, tables handle rev/lastID updates - // if another row has been written since processing this will - // handle the auto ID clash - if (oldTable && !isEqual(oldTable, table)) { - await db.put(table) - } const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev @@ -158,8 +152,6 @@ export async function finaliseRow( if (updateFormula) { await updateRelatedFormula(table, enrichedRow) } - const squashed = await linkRows.squashLinks(table, enrichedRow, { - fromViewId, - }) + const squashed = await linkRows.squashLinks(source, enrichedRow) return { row: enrichedRow, squashed, table } } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index e5397ed4a5..0f565b6951 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,6 +1,6 @@ import * as utils from "../../../../db/utils" -import { context, docIds } from "@budibase/backend-core" +import { docIds } from "@budibase/backend-core" import { Aggregation, Ctx, @@ -20,7 +20,6 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" -import { getFullUser } from "../../../../utilities/users" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -60,21 +59,6 @@ export async function processRelationshipFields( return row } -export async function findRow(tableId: string, rowId: string) { - const db = context.getAppDB() - let row: Row - // TODO remove special user case in future - if (tableId === utils.InternalTables.USER_METADATA) { - row = await getFullUser(rowId) - } else { - row = await db.get(rowId) - } - if (row.tableId !== tableId) { - throw "Supplied tableId does not match the rows tableId" - } - return row -} - export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { // top priority, use the URL first if (ctx.params?.sourceId) { diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index c3356919c8..5b15d3d9c7 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -113,11 +113,10 @@ export async function bulkImport( const processed = await inputProcessing(ctx.user?._id, table, row, { noAutoRelationships: true, }) - parsedRows.push(processed.row) - table = processed.table + parsedRows.push(processed) } - await handleRequest(Operation.BULK_UPSERT, table._id!, { + await handleRequest(Operation.BULK_UPSERT, table, { rows: parsedRows, }) await events.rows.imported(table, parsedRows.length) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 269f079ae8..d568e5f33e 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -139,8 +139,7 @@ export async function importToRows( const processed = await inputProcessing(user?._id, table, row, { noAutoRelationships: true, }) - row = processed.row - table = processed.table + row = processed // However here we must reference the original table, as we want to mutate // the real schema of the table passed in, not the clone used for diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c770c4e460..5788d9195a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -157,7 +157,7 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), this.query) } else { - return config.api.row.search(this.query.tableId, this.query) + return config.api.row.search(this.query.sourceId, this.query) } } @@ -327,8 +327,8 @@ describe.each([ } } - function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, tableId: table._id! }) + function expectSearch(query: Omit) { + return new SearchAssertion({ ...query, sourceId: table._id! }) } function expectQuery(query: SearchFilters) { @@ -1898,7 +1898,7 @@ describe.each([ let { rows: fullRowList } = await config.api.row.search( table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, } ) @@ -1909,7 +1909,7 @@ describe.each([ rowCount: number = 0 do { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, limit: 1, paginate: true, query: {}, @@ -1933,7 +1933,7 @@ describe.each([ // eslint-disable-next-line no-constant-condition while (true) { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, limit: 3, query: {}, bookmark, diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 6f4d468a68..4290b4386f 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -113,7 +113,7 @@ describe("/templates", () => { expect(users.name).toBe("Users") const { rows } = await config.api.row.search(agencyProjects._id!, { - tableId: agencyProjects._id!, + sourceId: agencyProjects._id!, query: {}, }) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 4222f9b5e4..6e65ab36d1 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -255,31 +255,31 @@ export type SquashTableFields = Record * @returns The rows after having their links squashed to only contain the ID and primary display. */ export async function squashLinks( - table: Table, - enriched: T, - options?: { - fromViewId?: string - } + source: Table | ViewV2, + enriched: T ): Promise { const allowRelationshipSchemas = await features.flags.isEnabled( FeatureFlag.ENRICHED_RELATIONSHIPS ) let viewSchema: Record = {} - if (options?.fromViewId) { - const view = Object.values(table.views || {}).find( - (v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId - ) - - if (view && helpers.views.isCalculationView(view)) { + if (sdk.views.isView(source)) { + if (helpers.views.isCalculationView(source)) { return enriched } - if (allowRelationshipSchemas && view) { - viewSchema = view.schema || {} + if (allowRelationshipSchemas) { + viewSchema = source.schema || {} } } + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + // will populate this as we find them const linkedTables = [table] const isArray = Array.isArray(enriched) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 34be1c0c6c..91addf8a50 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -219,7 +219,7 @@ describe("Google Sheets Integration", () => { }) let resp = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, paginate: true, limit: 10, @@ -228,7 +228,7 @@ describe("Google Sheets Integration", () => { while (resp.hasNextPage) { resp = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, paginate: true, limit: 10, @@ -637,7 +637,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with equals filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Foo", @@ -651,7 +651,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not equals filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { notEqual: { name: "Foo", @@ -666,7 +666,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with empty filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { empty: { name: null, @@ -679,7 +679,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not empty filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { notEmpty: { name: null, @@ -692,7 +692,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with one of filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { oneOf: { name: ["Foo", "Bar"], @@ -707,7 +707,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with fuzzy filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { fuzzy: { name: "oo", @@ -721,7 +721,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with range filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { range: { name: { @@ -750,7 +750,7 @@ describe("Google Sheets Integration", () => { }) let response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, @@ -759,7 +759,7 @@ describe("Google Sheets Integration", () => { while (response.hasNextPage) { response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 24ec2302e7..060ef3738a 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -14,7 +14,6 @@ import { outputProcessing, } from "../../../utilities/rowProcessor" import cloneDeep from "lodash/fp/cloneDeep" -import isEqual from "lodash/fp/isEqual" import { tryExtractingTableAndViewId } from "./utils" export async function getRow( @@ -55,11 +54,7 @@ export async function save( source = await sdk.tables.getTable(tableId) } - const { table: updatedTable, row } = await inputProcessing( - userId, - cloneDeep(source), - inputs - ) + const row = await inputProcessing(userId, cloneDeep(source), inputs) const validateResult = await sdk.rows.utils.validate({ row, @@ -73,10 +68,6 @@ export async function save( row, }) - if (sdk.tables.isTable(source) && !isEqual(source, updatedTable)) { - await sdk.tables.saveTable(updatedTable) - } - const rowId = response.row._id if (rowId) { const row = await getRow(source, rowId, { diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index b5b3437e5e..f51bcc37a4 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -1,5 +1,5 @@ import { context, db } from "@budibase/backend-core" -import { Row } from "@budibase/types" +import { Row, Table, ViewV2 } from "@budibase/types" import sdk from "../../../sdk" import cloneDeep from "lodash/fp/cloneDeep" import { finaliseRow } from "../../../api/controllers/row/staticFormula" @@ -10,7 +10,7 @@ import { import * as linkRows from "../../../db/linkedRows" import { InternalTables } from "../../../db/utils" import { getFullUser } from "../../../utilities/users" -import { tryExtractingTableAndViewId } from "./utils" +import { getSource, tryExtractingTableAndViewId } from "./utils" export async function save( tableOrViewId: string, @@ -20,21 +20,28 @@ export async function save( const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) inputs.tableId = tableId + let source: Table | ViewV2 + let table: Table + if (viewId) { + source = await sdk.views.get(viewId) + table = await sdk.views.getTable(viewId) + } else { + source = await sdk.tables.getTable(tableId) + table = source + } + if (!inputs._rev && !inputs._id) { inputs._id = db.generateRowID(inputs.tableId) } - // this returns the table and row incase they have been updated - const dbTable = await sdk.tables.getTable(inputs.tableId) - // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(dbTable) + const sourceClone = cloneDeep(source) - let { table, row } = await inputProcessing(userId, tableClone, inputs) + let row = await inputProcessing(userId, sourceClone, inputs) const validateResult = await sdk.rows.utils.validate({ row, - table, + source, }) if (!validateResult.valid) { @@ -49,24 +56,18 @@ export async function save( table, })) as Row - return finaliseRow(table, row, { - oldTable: dbTable, - updateFormula: true, - fromViewId: viewId, + return finaliseRow(table, row, { updateFormula: true }) +} + +export async function find(sourceId: string, rowId: string): Promise { + const source = await getSource(sourceId) + return await outputProcessing(source, await findRow(sourceId, rowId), { + squash: true, }) } -export async function find(tableOrViewId: string, rowId: string): Promise { - const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) - - const table = await sdk.tables.getTable(tableId) - let row = await findRow(tableId, rowId) - - row = await outputProcessing(table, row, { squash: true, fromViewId: viewId }) - return row -} - -async function findRow(tableId: string, rowId: string) { +export async function findRow(sourceId: string, rowId: string) { + const { tableId } = tryExtractingTableAndViewId(sourceId) const db = context.getAppDB() let row: Row // TODO remove special user case in future diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index 6617fc376c..c9e2aba237 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -64,7 +64,7 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { let searchResponse = await sdk.rows.search({ - tableId, + sourceId: tableId, query, sort, sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 2c149e5b21..24a7de1307 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -8,21 +8,30 @@ import { SortType, Table, User, + ViewV2, } from "@budibase/types" import { getGlobalUsersFromMetadata } from "../../../../../utilities/global" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" +import sdk from "../../../../" export async function search( options: RowSearchParams, - table: Table + source: Table | ViewV2 ): Promise> { - const { tableId } = options + const { sourceId } = options + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } const { paginate, query } = options const params: RowSearchParams = { - tableId: options.tableId, + sourceId: options.sourceId, sort: options.sort, sortOrder: options.sortOrder, sortType: options.sortType, @@ -50,7 +59,7 @@ export async function search( // Enrich search results with relationships if (response.rows && response.rows.length) { // enrich with global users if from users table - if (tableId === InternalTables.USER_METADATA) { + if (sourceId === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) } @@ -59,9 +68,8 @@ export async function search( response.rows = response.rows.map((r: any) => pick(r, fields)) } - response.rows = await outputProcessing(table, response.rows, { + response.rows = await outputProcessing(source, response.rows, { squash: true, - fromViewId: options.viewId, }) } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index b5bf8e752f..90fb082214 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -15,6 +15,7 @@ import { SortType, SqlClient, Table, + ViewV2, } from "@budibase/types" import { buildInternalRelationships, @@ -292,11 +293,18 @@ function resyncDefinitionsRequired(status: number, message: string) { export async function search( options: RowSearchParams, - table: Table, + source: Table | ViewV2, opts?: { retrying?: boolean } ): Promise> { let { paginate, query, ...params } = cloneDeep(options) + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) // make sure we have the mapped/latest table @@ -406,7 +414,6 @@ export async function search( let finalRows = await outputProcessing(table, processed, { preserveLinks: true, squash: true, - fromViewId: options.viewId, aggregations: options.aggregations, }) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index e7fd095865..194f2dd4e3 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -122,7 +122,7 @@ describe.each([ it("querying by fields will always return data attribute columns", async () => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, fields: ["name", "age"], }) @@ -142,7 +142,7 @@ describe.each([ it("will decode _id in oneOf query", async () => { await config.doInContext(config.appId, async () => { const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: { oneOf: { _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], @@ -174,7 +174,7 @@ describe.each([ }, }) const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, }) expect(result.rows).toHaveLength(10) @@ -205,7 +205,7 @@ describe.each([ }, }) const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, fields: ["name", "age"], }) @@ -229,7 +229,7 @@ describe.each([ async (queryFields, expectedRows) => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - tableId: table._id!, + sourceId: table._id!, query: { $or: { conditions: [ diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index 0698f727df..e3f241f15a 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -48,7 +48,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should be able to map ro_ to global user IDs", () => { const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { equal: { "1:user": userMedataId, @@ -61,7 +61,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should handle array of user IDs", () => { const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { oneOf: { "1:user": [userMedataId, globalUserId], @@ -78,7 +78,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't change any other input", () => { const email = "test@example.com" const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { equal: { "1:user": email, @@ -90,10 +90,8 @@ describe.each([tableWithUserCol, tableWithUsersCol])( }) it("shouldn't error if no query supplied", () => { - const params: any = { - tableId, - } - const output = searchInputMapping(col, params) + // @ts-expect-error - intentionally passing in a bad type + const output = searchInputMapping(col, { sourceId: tableId }) expect(output.query).toBeUndefined() }) } diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 55cdf9ea20..548b2b6bc9 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -33,7 +33,7 @@ describe("validate", () => { it("should accept empty values", async () => { const row = {} const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) expect(output.errors).toEqual({}) }) @@ -43,7 +43,7 @@ describe("validate", () => { time: `${hour()}:${minute()}`, } const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -52,7 +52,7 @@ describe("validate", () => { time: `${hour()}:${minute()}:${second()}`, } const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -67,7 +67,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ['"time" is not a valid time'] }) }) @@ -91,7 +91,7 @@ describe("validate", () => { `${generator.integer({ min: 11, max: 23 })}:${minute()}`, ])("should accept values after config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -100,7 +100,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -125,7 +125,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 12 })}:${minute()}`, ])("should accept values before config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -134,7 +134,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:16:17"], @@ -156,7 +156,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -166,7 +166,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -178,7 +178,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], @@ -199,7 +199,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -208,7 +208,7 @@ describe("validate", () => { "should reject values out range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 10:00"], @@ -226,7 +226,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) @@ -237,7 +237,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) @@ -257,7 +257,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -267,7 +267,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -279,7 +279,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], @@ -301,7 +301,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -311,7 +311,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -323,7 +323,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index f17d3e4a03..d5c0560d9b 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -321,3 +321,10 @@ export function tryExtractingTableAndViewId(tableOrViewId: string) { return { tableId: tableOrViewId } } + +export function getSource(tableOrViewId: string) { + if (docIds.isViewId(tableOrViewId)) { + return sdk.views.get(tableOrViewId) + } + return sdk.tables.getTable(tableOrViewId) +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index f911385dc1..4251383712 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -40,7 +40,8 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } -export async function getTable(viewId: string): Promise { +export async function getTable(view: string | ViewV2): Promise
{ + const viewId = typeof view === "string" ? view : view.id const cached = context.getTableForView(viewId) if (cached) { return cached diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 9aa53e18c1..24c8d11bd1 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -73,6 +73,7 @@ export async function processAutoColumn( // check its not user table, or whether any of the processing options have been disabled const shouldUpdateUserFields = !isUserTable && !opts?.reprocessing && !opts?.noAutoRelationships && !noUser + let tableMutated = false for (let [key, schema] of Object.entries(table.schema)) { if (!schema.autocolumn) { continue @@ -105,10 +106,17 @@ export async function processAutoColumn( row[key] = schema.lastID + 1 schema.lastID++ table.schema[key] = schema + tableMutated = true } break } } + + if (tableMutated) { + const db = context.getAppDB() + const resp = await db.put(table) + table._rev = resp.rev + } } async function processDefaultValues(table: Table, row: Row) { @@ -235,8 +243,7 @@ export async function inputProcessing( await processAutoColumn(userId, table, clonedRow, opts) await processDefaultValues(table, clonedRow) - - return { table, row: clonedRow } + return clonedRow } /** @@ -271,6 +278,14 @@ export async function outputProcessing( } else { safeRows = rows } + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + // attach any linked row information let enriched = !opts.preserveLinks ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { @@ -360,9 +375,7 @@ export async function outputProcessing( enriched = await processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { - enriched = await linkRows.squashLinks(table, enriched, { - fromViewId: opts?.fromViewId, - }) + enriched = await linkRows.squashLinks(source, enriched) } // remove null properties to match internal API diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index 244ea3794c..1a75cd6830 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -65,7 +65,7 @@ describe("rowProcessor - inputProcessing", () => { processInputBBReferenceMock.mockResolvedValue(user) - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReference).toHaveBeenCalledTimes( 1 @@ -117,7 +117,7 @@ describe("rowProcessor - inputProcessing", () => { processInputBBReferencesMock.mockResolvedValue(user) - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).toHaveBeenCalledTimes( 1 @@ -164,7 +164,7 @@ describe("rowProcessor - inputProcessing", () => { name: "Jack", } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled() expect(row).toEqual({ ...newRow, user: undefined }) @@ -207,7 +207,7 @@ describe("rowProcessor - inputProcessing", () => { user: userValue, } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) if (userValue === undefined) { // The 'user' field is omitted @@ -262,7 +262,7 @@ describe("rowProcessor - inputProcessing", () => { user: "123", } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled() expect(row).toEqual({