From 51774b3434a344e0ab8e766b96ad2e840ce44033 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 12:30:45 +0100 Subject: [PATCH] Working on plumbing 'source' all the way through our code. --- .../backend-core/src/context/mainContext.ts | 19 +++- packages/backend-core/src/context/types.ts | 3 +- packages/backend-core/src/docIds/params.ts | 14 +-- packages/backend-core/src/sql/sql.ts | 3 - .../api/controllers/row/ExternalRequest.ts | 89 ++++++++++--------- .../src/api/controllers/row/external.ts | 19 ++-- .../src/api/controllers/row/utils/utils.ts | 13 ++- .../server/src/api/controllers/row/views.ts | 7 +- packages/server/src/db/utils.ts | 8 +- .../middleware/triggerRowActionAuthorised.ts | 2 +- .../server/src/sdk/app/permissions/index.ts | 6 +- packages/server/src/sdk/app/rowActions.ts | 6 +- .../server/src/sdk/app/rows/queryUtils.ts | 7 +- packages/server/src/sdk/app/rows/rows.ts | 5 +- packages/server/src/sdk/app/rows/search.ts | 58 +++++++----- .../src/sdk/app/rows/search/external.ts | 7 +- .../server/src/sdk/app/rows/search/utils.ts | 6 +- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 20 ++--- packages/server/src/sdk/app/rows/utils.ts | 35 +++----- packages/server/src/sdk/app/tables/utils.ts | 4 + packages/server/src/sdk/app/views/index.ts | 19 +++- packages/types/src/sdk/row.ts | 5 +- yarn.lock | 64 +------------ 23 files changed, 204 insertions(+), 215 deletions(-) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index a52a17dd53..25b273e51c 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -10,7 +10,7 @@ import { StaticDatabases, DEFAULT_TENANT_ID, } from "../constants" -import { Database, IdentityContext, Snippet, App } from "@budibase/types" +import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types" import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -394,3 +394,20 @@ export function setFeatureFlags(key: string, value: Record) { context.featureFlagCache ??= {} context.featureFlagCache[key] = value } + +export function getTableForView(viewId: string): Table | undefined { + const context = getCurrentContext() + if (!context) { + return + } + return context.viewToTableCache?.[viewId] +} + +export function setTableForView(viewId: string, table: Table) { + const context = getCurrentContext() + if (!context) { + return + } + context.viewToTableCache ??= {} + context.viewToTableCache[viewId] = table +} diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index fe6072e85c..ee84b49459 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,4 @@ -import { IdentityContext, Snippet, VM } from "@budibase/types" +import { IdentityContext, Snippet, Table, VM } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { GoogleSpreadsheet } from "google-spreadsheet" @@ -21,4 +21,5 @@ export type ContextMap = { featureFlagCache?: { [key: string]: Record } + viewToTableCache?: Record } diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts index 093724b55e..016604b69b 100644 --- a/packages/backend-core/src/docIds/params.ts +++ b/packages/backend-core/src/docIds/params.ts @@ -6,7 +6,7 @@ import { ViewName, } from "../constants" import { getProdAppID } from "./conversions" -import { DatabaseQueryOpts } from "@budibase/types" +import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types" /** * If creating DB allDocs/query params with only a single top level ID this can be used, this @@ -66,9 +66,8 @@ export function getQueryIndex(viewName: ViewName) { /** * Check if a given ID is that of a table. - * @returns {boolean} */ -export const isTableId = (id: string) => { +export const isTableId = (id: string): boolean => { // this includes datasource plus tables return ( !!id && @@ -77,13 +76,16 @@ export const isTableId = (id: string) => { ) } +export function isViewId(id: string): boolean { + return !!id && id.startsWith(`${VirtualDocumentType.VIEW}${SEPARATOR}`) +} + /** * Check if a given ID is that of a datasource or datasource plus. - * @returns {boolean} */ -export const isDatasourceId = (id: string) => { +export const isDatasourceId = (id: string): boolean => { // this covers both datasources and datasource plus - return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) + return !!id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) } /** diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 023bfd5d8a..97fc4124fb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1274,9 +1274,6 @@ class InternalBuilder { if (counting) { query = this.addDistinctCount(query) } else if (aggregations.length > 0) { - query = query.select( - this.knex.raw("ROW_NUMBER() OVER (ORDER BY (SELECT 0)) as _id") - ) query = this.addAggregations(query, aggregations) } else { query = query.select(this.generateSelectStatement()) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 9c9bd0b284..a52d7abcd1 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -19,6 +19,7 @@ import { SortJson, SortType, Table, + ViewV2, } from "@budibase/types" import { breakExternalTableId, @@ -159,17 +160,41 @@ function isEditableColumn(column: FieldSchema) { export class ExternalRequest { private readonly operation: T - private readonly tableId: string - private datasource?: Datasource - private tables: { [key: string]: Table } = {} + private readonly source: Table | ViewV2 + private datasource: Datasource - constructor(operation: T, tableId: string, datasource?: Datasource) { - this.operation = operation - this.tableId = tableId - this.datasource = datasource - if (datasource && datasource.entities) { - this.tables = datasource.entities + public static async for( + operation: T, + source: Table | ViewV2, + opts: { datasource?: Datasource } = {} + ) { + if (!opts.datasource) { + if (sdk.views.isView(source)) { + const table = await sdk.views.getTable(source.id) + opts.datasource = await sdk.datasources.get(table.sourceId!) + } else { + opts.datasource = await sdk.datasources.get(source.sourceId!) + } } + + return new ExternalRequest(operation, source, opts.datasource) + } + + private get tables(): { [key: string]: Table } { + if (!this.datasource.entities) { + throw new Error("Datasource does not have entities") + } + return this.datasource.entities + } + + private constructor( + operation: T, + source: Table | ViewV2, + datasource: Datasource + ) { + this.operation = operation + this.source = source + this.datasource = datasource } private prepareFilters( @@ -290,20 +315,6 @@ export class ExternalRequest { return this.tables[tableName] } - // seeds the object with table and datasource information - async retrieveMetadata( - datasourceId: string - ): Promise<{ tables: Record; datasource: Datasource }> { - if (!this.datasource) { - this.datasource = await sdk.datasources.get(datasourceId) - if (!this.datasource || !this.datasource.entities) { - throw "No tables found, fetch tables before query." - } - this.tables = this.datasource.entities - } - return { tables: this.tables, datasource: this.datasource } - } - async getRow(table: Table, rowId: string): Promise { const response = await getDatasourceAndQuery({ endpoint: getEndpoint(table._id!, Operation.READ), @@ -619,24 +630,16 @@ export class ExternalRequest { } async run(config: RunConfig): Promise> { - const { operation, tableId } = this - if (!tableId) { - throw new Error("Unable to run without a table ID") - } - let { datasourceId, tableName } = breakExternalTableId(tableId) - let datasource = this.datasource - if (!datasource) { - const { datasource: ds } = await this.retrieveMetadata(datasourceId) - datasource = ds - } - const tables = this.tables - const table = tables[tableName] - let isSql = isSQL(datasource) - if (!table) { - throw new Error( - `Unable to process query, table "${tableName}" not defined.` - ) + const { operation } = this + let table: Table + if (sdk.views.isView(this.source)) { + table = await sdk.views.getTable(this.source.id) + } else { + table = this.source } + + let isSql = isSQL(this.datasource) + // look for specific components of config which may not be considered acceptable let { id, row, filters, sort, paginate, rows } = cleanupConfig( config, @@ -687,8 +690,8 @@ export class ExternalRequest { } let json: QueryJson = { endpoint: { - datasourceId: datasourceId!, - entityId: tableName, + datasourceId: this.datasource._id!, + entityId: table.name, operation, }, resource: { @@ -714,7 +717,7 @@ export class ExternalRequest { }, meta: { table, - tables: tables, + tables: this.tables, }, } diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index bd5201c05c..fb992059f4 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -17,6 +17,7 @@ import { Row, Table, UserCtx, + ViewV2, } from "@budibase/types" import sdk from "../../../sdk" import * as utils from "./utils" @@ -29,29 +30,29 @@ import { generateIdForRow } from "./utils" export async function handleRequest( operation: T, - tableId: string, + source: Table | ViewV2, opts?: RunConfig ): Promise> { - return new ExternalRequest(operation, tableId, opts?.datasource).run( - opts || {} - ) + return ( + await ExternalRequest.for(operation, source, { + datasource: opts?.datasource, + }) + ).run(opts || {}) } export async function patch(ctx: UserCtx) { - const { tableId, viewId } = utils.getSourceId(ctx) - + const source = await utils.getSource(ctx) const { _id, ...rowData } = ctx.request.body - const table = await sdk.tables.getTable(tableId) const { row: dataToUpdate } = await inputProcessing( ctx.user?._id, - cloneDeep(table), + cloneDeep(source), rowData ) const validateResult = await sdk.rows.utils.validate({ row: dataToUpdate, - tableId, + source, }) if (!validateResult.valid) { throw { validation: validateResult.errors } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 1fc7221294..91c0fc966f 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,6 +1,6 @@ import * as utils from "../../../../db/utils" -import { context } from "@budibase/backend-core" +import { context, docIds } from "@budibase/backend-core" import { Aggregation, Ctx, @@ -9,6 +9,7 @@ import { RelationshipsJson, Row, Table, + ViewV2, } from "@budibase/types" import { processDates, @@ -78,7 +79,7 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { // top priority, use the URL first if (ctx.params?.sourceId) { const { sourceId } = ctx.params - if (utils.isViewID(sourceId)) { + if (docIds.isViewId(sourceId)) { return { tableId: utils.extractViewInfoFromID(sourceId).tableId, viewId: sourceId, @@ -97,6 +98,14 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { throw new Error("Unable to find table ID in request") } +export async function getSource(ctx: Ctx): Promise { + const { tableId, viewId } = getSourceId(ctx) + if (viewId) { + return sdk.views.get(viewId) + } + return sdk.tables.getTable(tableId) +} + export async function validate( opts: { row: Row } & ({ tableId: string } | { table: Table }) ) { diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index f9565f0e82..7008c5e0be 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -84,11 +84,8 @@ export async function searchView( })) const searchOptions: RequiredKeys & - RequiredKeys< - Pick - > = { - tableId: view.tableId, - viewId: view.id, + RequiredKeys> = { + sourceId: view.id, query: enrichedQuery, fields: viewFields, ...getSortOptions(body, view), diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 043394e7a6..6c1065e847 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -1,4 +1,4 @@ -import { context, db as dbCore, utils } from "@budibase/backend-core" +import { context, db as dbCore, docIds, utils } from "@budibase/backend-core" import { DatabaseQueryOpts, Datasource, @@ -318,12 +318,8 @@ export function generateViewID(tableId: string) { }${SEPARATOR}${tableId}${SEPARATOR}${newid()}` } -export function isViewID(viewId: string) { - return viewId?.split(SEPARATOR)[0] === VirtualDocumentType.VIEW -} - export function extractViewInfoFromID(viewId: string) { - if (!isViewID(viewId)) { + if (!docIds.isViewId(viewId)) { throw new Error("Unable to extract table ID, is not a view ID") } const split = viewId.split(SEPARATOR) diff --git a/packages/server/src/middleware/triggerRowActionAuthorised.ts b/packages/server/src/middleware/triggerRowActionAuthorised.ts index 17f22b7000..3e903ba907 100644 --- a/packages/server/src/middleware/triggerRowActionAuthorised.ts +++ b/packages/server/src/middleware/triggerRowActionAuthorised.ts @@ -15,7 +15,7 @@ export function triggerRowActionAuthorised( const rowActionId: string = ctx.params[actionPath] const isTableId = docIds.isTableId(sourceId) - const isViewId = utils.isViewID(sourceId) + const isViewId = docIds.isViewId(sourceId) if (!isTableId && !isViewId) { ctx.throw(400, `'${sourceId}' is not a valid source id`) } diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index a6e81652ee..d5e4aefe3a 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,10 +1,10 @@ -import { db, roles } from "@budibase/backend-core" +import { db, docIds, roles } from "@budibase/backend-core" import { PermissionLevel, PermissionSource, VirtualDocumentType, } from "@budibase/types" -import { extractViewInfoFromID, isViewID } from "../../../db/utils" +import { extractViewInfoFromID } from "../../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, @@ -20,7 +20,7 @@ type ResourcePermissions = Record< export async function getInheritablePermissions( resourceId: string ): Promise { - if (isViewID(resourceId)) { + if (docIds.isViewId(resourceId)) { return await getResourcePerms(extractViewInfoFromID(resourceId).tableId) } } diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts index 9a7d402df0..21c256eacb 100644 --- a/packages/server/src/sdk/app/rowActions.ts +++ b/packages/server/src/sdk/app/rowActions.ts @@ -1,11 +1,11 @@ -import { context, HTTPError, utils } from "@budibase/backend-core" +import { context, docIds, HTTPError, utils } from "@budibase/backend-core" import { AutomationTriggerStepId, SEPARATOR, TableRowActions, VirtualDocumentType, } from "@budibase/types" -import { generateRowActionsID, isViewID } from "../../db/utils" +import { generateRowActionsID } from "../../db/utils" import automations from "./automations" import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInfo" import * as triggers from "../../automations/triggers" @@ -155,7 +155,7 @@ export async function update( async function guardView(tableId: string, viewId: string) { let view - if (isViewID(viewId)) { + if (docIds.isViewId(viewId)) { view = await sdk.views.get(viewId) } if (!view || view.tableId !== tableId) { diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 65f400a1d9..a88763215e 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -53,8 +53,8 @@ export const removeInvalidFilters = ( } export const getQueryableFields = async ( - fields: string[], - table: Table + table: Table, + fields?: string[] ): Promise => { const extractTableFields = async ( table: Table, @@ -110,6 +110,9 @@ export const getQueryableFields = async ( "_id", // Querying by _id is always allowed, even if it's never part of the schema ] + if (fields === undefined) { + fields = Object.keys(table.schema) + } result.push(...(await extractTableFields(table, fields, [table._id!]))) return result diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts index c61b8692ed..ef25d06baf 100644 --- a/packages/server/src/sdk/app/rows/rows.ts +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -1,9 +1,8 @@ -import { db as dbCore, context } from "@budibase/backend-core" +import { db as dbCore, context, docIds } from "@budibase/backend-core" import { Database, Row } from "@budibase/types" import { extractViewInfoFromID, getRowParams, - isViewID, } from "../../../db/utils" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" @@ -26,7 +25,7 @@ export async function getAllInternalRows(appId?: string) { function pickApi(tableOrViewId: string) { let tableId = tableOrViewId - if (isViewID(tableOrViewId)) { + if (docIds.isViewId(tableOrViewId)) { tableId = extractViewInfoFromID(tableOrViewId).tableId } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index dbf0cefd51..fa01a6cf13 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -4,6 +4,8 @@ import { RowSearchParams, SearchResponse, SortOrder, + Table, + ViewV2, } from "@budibase/types" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" @@ -12,7 +14,7 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { features } from "@budibase/backend-core" +import { features, docIds } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" @@ -36,8 +38,7 @@ export async function search( ): Promise> { return await tracer.trace("search", async span => { span?.addTags({ - tableId: options.tableId, - viewId: options.viewId, + sourceId: options.sourceId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, @@ -52,20 +53,18 @@ export async function search( .join(", "), }) - const isExternalTable = isExternalTableID(options.tableId) options.query = dataFilters.cleanupQuery(options.query || {}) options.query = dataFilters.fixupFilterArrays(options.query) - span?.addTags({ + span.addTags({ cleanedQuery: options.query, - isExternalTable, }) if ( !dataFilters.hasFilters(options.query) && options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE ) { - span?.addTags({ emptyQuery: true }) + span.addTags({ emptyQuery: true }) return { rows: [], } @@ -75,34 +74,47 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - const table = await sdk.tables.getTable(options.tableId) - options = searchInputMapping(table, options) + let source: Table | ViewV2 + let table: Table + if (docIds.isTableId(options.sourceId)) { + source = await sdk.tables.getTable(options.sourceId) + table = source + options = searchInputMapping(source, options) + } else if (docIds.isViewId(options.sourceId)) { + source = await sdk.views.get(options.sourceId) + table = await sdk.tables.getTable(source.tableId) + options = searchInputMapping(table, options) - if (options.query) { - const tableFields = Object.keys(table.schema).filter( - f => table.schema[f].visible !== false - ) - - const queriableFields = await getQueryableFields( - options.fields?.filter(f => tableFields.includes(f)) ?? tableFields, - table - ) - options.query = removeInvalidFilters(options.query, queriableFields) + span.addTags({ + tableId: table._id, + }) + } else { + throw new Error(`Invalid source ID: ${options.sourceId}`) } + if (options.query) { + const visibleFields = ( + options.fields || Object.keys(table.schema) + ).filter(field => table.schema[field].visible) + + const queryableFields = await getQueryableFields(table, visibleFields) + options.query = removeInvalidFilters(options.query, queryableFields) + } + + const isExternalTable = isExternalTableID(table._id!) let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) - result = await external.search(options, table) + result = await external.search(options, source) } else if (await features.flags.isEnabled("SQS")) { span?.addTags({ searchType: "sqs" }) - result = await internal.sqs.search(options, table) + result = await internal.sqs.search(options, source) } else { span?.addTags({ searchType: "lucene" }) - result = await internal.lucene.search(options, table) + result = await internal.lucene.search(options, source) } - span?.addTags({ + span.addTags({ foundRows: result.rows.length, totalRows: result.totalRows, }) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 0ff25a00e4..5584b3f110 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -9,6 +9,7 @@ import { SortJson, SortOrder, Table, + ViewV2, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" @@ -60,9 +61,8 @@ function getPaginationAndLimitParameters( export async function search( options: RowSearchParams, - table: Table + source: Table | ViewV2 ): Promise> { - const { tableId } = options const { countRows, paginate, query, ...params } = options const { limit } = params let bookmark = @@ -112,10 +112,9 @@ export async function search( : Promise.resolve(undefined), ]) - let processed = await outputProcessing(table, rows, { + let processed = await outputProcessing(source, rows, { preserveLinks: true, squash: true, - fromViewId: options.viewId, }) let hasNextPage = false diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 5ffc065353..d727e58887 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -9,6 +9,7 @@ import { SearchResponse, Row, RowSearchParams, + ViewV2, } from "@budibase/types" import { db as dbCore, context } from "@budibase/backend-core" import { utils } from "@budibase/shared-core" @@ -83,10 +84,7 @@ function userColumnMapping(column: string, options: RowSearchParams) { // maps through the search parameters to check if any of the inputs are invalid // based on the table schema, converts them to something that is valid. export function searchInputMapping(table: Table, options: RowSearchParams) { - if (!table?.schema) { - return options - } - for (let [key, column] of Object.entries(table.schema)) { + for (let [key, column] of Object.entries(table.schema || {})) { switch (column.type) { case FieldType.BB_REFERENCE_SINGLE: { const subtype = column.subtype diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index aabc359484..f399801f1e 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -203,7 +203,7 @@ describe("query utils", () => { }, }) - const result = await getQueryableFields(Object.keys(table.schema), table) + const result = await getQueryableFields(table) expect(result).toEqual(["_id", "name", "age"]) }) @@ -216,7 +216,7 @@ describe("query utils", () => { }, }) - const result = await getQueryableFields(Object.keys(table.schema), table) + const result = await getQueryableFields(table) expect(result).toEqual(["_id", "name"]) }) @@ -245,7 +245,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -282,7 +282,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"]) }) @@ -313,7 +313,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual(["_id", "name"]) }) @@ -381,7 +381,7 @@ describe("query utils", () => { it("includes nested relationship fields from main table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -398,7 +398,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux 1 table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux1.schema), aux1) + return getQueryableFields(aux1) }) expect(result).toEqual([ "_id", @@ -420,7 +420,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux 2 table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux2.schema), aux2) + return getQueryableFields(aux2) }) expect(result).toEqual([ "_id", @@ -474,7 +474,7 @@ describe("query utils", () => { it("includes nested relationship fields from main table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -488,7 +488,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux.schema), aux) + return getQueryableFields(aux) }) expect(result).toEqual([ "_id", diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bc09116b3b..3899009f13 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -13,16 +13,14 @@ import { TableSchema, SqlClient, ArrayOperator, + ViewV2, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." -import { - extractViewInfoFromID, - isRelationshipColumn, - isViewID, -} from "../../../db/utils" +import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" +import { docIds } from "@budibase/backend-core" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -142,37 +140,32 @@ function isForeignKey(key: string, table: Table) { } export async function validate({ - tableId, + source, row, - table, }: { - tableId?: string + source: Table | ViewV2 row: Row - table?: Table }): Promise<{ valid: boolean errors: Record }> { - let fetchedTable: Table | undefined - if (!table && tableId) { - fetchedTable = await sdk.tables.getTable(tableId) - } else if (table) { - fetchedTable = table - } - if (fetchedTable === undefined) { - throw new Error("Unable to fetch table for validation") + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source } const errors: Record = {} const disallowArrayTypes = [ FieldType.ATTACHMENT_SINGLE, FieldType.BB_REFERENCE_SINGLE, ] - for (let fieldName of Object.keys(fetchedTable.schema)) { - const column = fetchedTable.schema[fieldName] + for (let fieldName of Object.keys(table.schema)) { + const column = table.schema[fieldName] const constraints = cloneDeep(column.constraints) const type = column.type // foreign keys are likely to be enriched - if (isForeignKey(fieldName, fetchedTable)) { + if (isForeignKey(fieldName, table)) { continue } // formulas shouldn't validated, data will be deleted anyway @@ -323,7 +316,7 @@ export function isArrayFilter(operator: any): operator is ArrayOperator { } export function tryExtractingTableAndViewId(tableOrViewId: string) { - if (isViewID(tableOrViewId)) { + if (docIds.isViewId(tableOrViewId)) { return { tableId: extractViewInfoFromID(tableOrViewId).tableId, viewId: tableOrViewId, diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts index b8e3d888af..7a8096fb0a 100644 --- a/packages/server/src/sdk/app/tables/utils.ts +++ b/packages/server/src/sdk/app/tables/utils.ts @@ -9,3 +9,7 @@ export function isExternal(opts: { table?: Table; tableId?: string }): boolean { } return false } + +export function isTable(table: any): table is Table { + return table.type === "table" +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index de3579f7fd..f911385dc1 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -9,7 +9,7 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { HTTPError } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -40,6 +40,23 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } +export async function getTable(viewId: string): Promise
{ + const cached = context.getTableForView(viewId) + if (cached) { + return cached + } + const { tableId } = utils.extractViewInfoFromID(viewId) + const table = await sdk.tables.getTable(tableId) + context.setTableForView(viewId, table) + return table +} + +export function isView(view: any): view is ViewV2 { + return ( + view.version === 2 && "id" in view && "tableId" in view && "name" in view + ) +} + async function guardCalculationViewSchema( table: Table, view: Omit diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index f81d56c082..2b6ff3a6c6 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -10,8 +10,7 @@ export interface Aggregation { } export interface SearchParams { - tableId?: string - viewId?: string + sourceId?: string query?: SearchFilters paginate?: boolean bookmark?: string | number @@ -30,7 +29,7 @@ export interface SearchParams { // when searching for rows we want a more extensive search type that requires certain properties export interface RowSearchParams - extends WithRequired {} + extends WithRequired {} export interface SearchResponse { rows: T[] diff --git a/yarn.lock b/yarn.lock index cd850e833d..01466f66b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17796,21 +17796,11 @@ periscopic@^3.1.0: estree-walker "^3.0.0" is-reference "^3.0.0" -pg-cloudflare@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" - integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== - pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== -pg-connection-string@^2.6.4: - version "2.6.4" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d" - integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA== - pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -17821,21 +17811,11 @@ pg-pool@^3.6.0: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== -pg-pool@^3.6.2: - version "3.6.2" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2" - integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg== - pg-protocol@*, pg-protocol@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== -pg-protocol@^1.6.1: - version "1.6.1" - resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3" - integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg== - pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" @@ -17860,19 +17840,6 @@ pg@8.10.0: pg-types "^2.1.0" pgpass "1.x" -pg@^8.12.0: - version "8.12.0" - resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79" - integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ== - dependencies: - pg-connection-string "^2.6.4" - pg-pool "^3.6.2" - pg-protocol "^1.6.1" - pg-types "^2.1.0" - pgpass "1.x" - optionalDependencies: - pg-cloudflare "^1.1.1" - pgpass@1.x: version "1.0.5" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" @@ -20786,16 +20753,7 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20886,7 +20844,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20900,13 +20858,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22862,7 +22813,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22880,15 +22831,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"