diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts index e0ac85b3df..4c9eb713c8 100644 --- a/packages/backend-core/src/docIds/ids.ts +++ b/packages/backend-core/src/docIds/ids.ts @@ -45,6 +45,11 @@ export function generateGlobalUserID(id?: any) { return `${DocumentType.USER}${SEPARATOR}${id || newid()}` } +const isGlobalUserIDRegex = new RegExp(`^${DocumentType.USER}${SEPARATOR}.+`) +export function isGlobalUserID(id: string) { + return isGlobalUserIDRegex.test(id) +} + /** * Generates a new user ID based on the passed in global ID. * @param {string} globalId The ID of the global user. diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index 6ea2d771e9..43751ad944 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -49,6 +49,15 @@ label: "Long Form Text", value: FIELDS.LONGFORM.type, }, + + { + label: "User", + value: `${FIELDS.USER.type}${FIELDS.USER.subtype}`, + }, + { + label: "Users", + value: `${FIELDS.USERS.type}${FIELDS.USERS.subtype}`, + }, ] $: { @@ -143,7 +152,7 @@
{name} handleChange(name, e)} - options={typeOptions} + options={Object.values(typeOptions)} placeholder={null} getOptionLabel={option => option.label} getOptionValue={option => option.value} diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index f33df4536f..c838208a3b 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -156,7 +156,10 @@ export async function destroy(ctx: UserCtx) { } const table = await sdk.tables.getTable(row.tableId) // update the row to include full relationships before deleting them - row = await outputProcessing(table, row, { squash: false }) + row = await outputProcessing(table, row, { + squash: false, + skipBBReferences: true, + }) // now remove the relationships await linkRows.updateLinks({ eventType: linkRows.EventType.ROW_DELETE, @@ -190,6 +193,7 @@ export async function bulkDestroy(ctx: UserCtx) { // they need to be the full rows (including previous relationships) for automations const processedRows = (await outputProcessing(table, rows, { squash: false, + skipBBReferences: true, })) as Row[] // remove the relationships first diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 327904666d..8917bcaab1 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -15,7 +15,7 @@ import { handleRequest } from "../row/external" import { context, events } from "@budibase/backend-core" import { isRows, isSchema, parse } from "../../../utilities/schema" import { - AutoReason, + BulkImportRequest, Datasource, FieldSchema, Operation, @@ -374,10 +374,10 @@ export async function destroy(ctx: UserCtx) { return tableToDelete } -export async function bulkImport(ctx: UserCtx) { +export async function bulkImport(ctx: UserCtx) { const table = await sdk.tables.getTable(ctx.params.tableId) - const { rows }: { rows: unknown } = ctx.request.body - const schema: unknown = table.schema + const { rows } = ctx.request.body + const schema = table.schema if (!rows || !isRows(rows) || !isSchema(schema)) { ctx.throw(400, "Provided data import information is invalid.") diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index e7c6ae57b0..e1956e33c0 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -8,6 +8,7 @@ import { import { isExternalTable, isSQL } from "../../../integrations/utils" import { events } from "@budibase/backend-core" import { + BulkImportRequest, FetchTablesResponse, SaveTableRequest, SaveTableResponse, @@ -97,7 +98,7 @@ export async function destroy(ctx: UserCtx) { builderSocket?.emitTableDeletion(ctx, deletedTable) } -export async function bulkImport(ctx: UserCtx) { +export async function bulkImport(ctx: UserCtx) { const tableId = ctx.params.tableId await pickApi({ tableId }).bulkImport(ctx) // right now we don't trigger anything for bulk import because it diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index e468848c57..999ea35d37 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -10,6 +10,7 @@ import { } from "../../../utilities/rowProcessor" import { runStaticFormulaChecks } from "./bulkFormula" import { + BulkImportRequest, RenameColumn, SaveTableRequest, SaveTableResponse, @@ -206,7 +207,7 @@ export async function destroy(ctx: any) { return tableToDelete } -export async function bulkImport(ctx: any) { +export async function bulkImport(ctx: UserCtx) { const table = await sdk.tables.getTable(ctx.params.tableId) const { rows, identifierFields } = ctx.request.body await handleDataImport(ctx.user, table, rows, identifierFields) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index e0d564db2a..bf64bc32ab 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -20,7 +20,13 @@ import viewTemplate from "../view/viewBuilder" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" import { events, context } from "@budibase/backend-core" -import { ContextUser, Datasource, SourceName, Table } from "@budibase/types" +import { + ContextUser, + Datasource, + Row, + SourceName, + Table, +} from "@budibase/types" export async function clearColumns(table: any, columnNames: any) { const db = context.getAppDB() @@ -144,12 +150,12 @@ export async function importToRows( } export async function handleDataImport( - user: any, - table: any, - rows: any, + user: ContextUser, + table: Table, + rows: Row[], identifierFields: Array = [] ) { - const schema: unknown = table.schema + const schema = table.schema if (!rows || !isRows(rows) || !isSchema(schema)) { return table diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index e707cad0cc..d6caff6035 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -43,3 +43,7 @@ export enum Format { export function isFormat(format: any): format is Format { return Object.values(Format).includes(format as Format) } + +export function parseCsvExport(value: string) { + return JSON.parse(value?.replace(/'/g, '"')) as T +} diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 4dcc9b3c8c..186326964c 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1578,9 +1578,6 @@ describe.each([ () => config.createUser(), (row: Row) => ({ _id: row._id, - email: row.email, - firstName: row.firstName, - lastName: row.lastName, primaryDisplay: row.email, }), ], diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index f56c6e4e44..aabb64b763 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -1,6 +1,11 @@ import { generator } from "@budibase/backend-core/tests" import { events, context } from "@budibase/backend-core" -import { FieldType, Table, ViewCalculation } from "@budibase/types" +import { + FieldType, + SaveTableRequest, + Table, + ViewCalculation, +} from "@budibase/types" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" const { basicTable } = setup.structures @@ -47,7 +52,7 @@ describe("/tables", () => { }) it("creates a table via data import", async () => { - const table = basicTable() + const table: SaveTableRequest = basicTable() table.rows = [{ name: "test-name", description: "test-desc" }] const res = await createTable(table) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index c7b8998bad..c126530b8d 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -82,9 +82,6 @@ export async function processOutputBBReferences( return users.map(u => ({ _id: u._id, primaryDisplay: u.email, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, })) default: diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index d6aa5e682a..7995fc2545 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -201,9 +201,14 @@ export async function inputProcessing( export async function outputProcessing( table: Table, rows: T, - opts: { squash?: boolean; preserveLinks?: boolean } = { + opts: { + squash?: boolean + preserveLinks?: boolean + skipBBReferences?: boolean + } = { squash: true, preserveLinks: false, + skipBBReferences: false, } ): Promise { let safeRows: Row[] @@ -230,7 +235,10 @@ export async function outputProcessing( attachment.url = objectStore.getAppFileUrl(attachment.key) }) } - } else if (column.type == FieldTypes.BB_REFERENCE) { + } else if ( + !opts.skipBBReferences && + column.type == FieldTypes.BB_REFERENCE + ) { for (let row of enriched) { row[property] = await processOutputBBReferences( row[property], diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts index b6174861d4..10d339f6b4 100644 --- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -180,9 +180,6 @@ describe("bbReferenceProcessor", () => { { _id: user._id, primaryDisplay: user.email, - email: user.email, - firstName: user.firstName, - lastName: user.lastName, }, ]) expect(cacheGetUsersSpy).toBeCalledTimes(1) @@ -207,9 +204,6 @@ describe("bbReferenceProcessor", () => { [user1, user2].map(u => ({ _id: u._id, primaryDisplay: u.email, - email: u.email, - firstName: u.firstName, - lastName: u.lastName, })) ) ) diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index 5ced82d8cf..40e4152b63 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -1,9 +1,13 @@ +import { FieldSubtype } from "@budibase/types" import { FieldTypes } from "../constants" -import { ValidColumnNameRegex } from "@budibase/shared-core" +import { ValidColumnNameRegex, utils } from "@budibase/shared-core" +import { db } from "@budibase/backend-core" +import { parseCsvExport } from "../api/controllers/view/exporters" interface SchemaColumn { readonly name: string readonly type: FieldTypes + readonly subtype: FieldSubtype readonly autocolumn?: boolean readonly constraints?: { presence: boolean @@ -77,8 +81,14 @@ export function validate(rows: Rows, schema: Schema): ValidationResults { rows.forEach(row => { Object.entries(row).forEach(([columnName, columnData]) => { const columnType = schema[columnName]?.type + const columnSubtype = schema[columnName]?.subtype const isAutoColumn = schema[columnName]?.autocolumn + // If the column had an invalid value we don't want to override it + if (results.schemaValidation[columnName] === false) { + return + } + // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array if (typeof columnType !== "string") { results.invalidColumns.push(columnName) @@ -112,6 +122,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults { isNaN(new Date(columnData).getTime()) ) { results.schemaValidation[columnName] = false + } else if ( + columnType === FieldTypes.BB_REFERENCE && + !isValidBBReference(columnData, columnSubtype) + ) { + results.schemaValidation[columnName] = false } else { results.schemaValidation[columnName] = true } @@ -138,6 +153,7 @@ export function parse(rows: Rows, schema: Schema): Rows { } const columnType = schema[columnName].type + const columnSubtype = schema[columnName].subtype if (columnType === FieldTypes.NUMBER) { // If provided must be a valid number @@ -147,6 +163,23 @@ export function parse(rows: Rows, schema: Schema): Rows { parsedRow[columnName] = columnData ? new Date(columnData).toISOString() : columnData + } else if (columnType === FieldTypes.BB_REFERENCE) { + const parsedValues = + !!columnData && parseCsvExport<{ _id: string }[]>(columnData) + if (!parsedValues) { + parsedRow[columnName] = undefined + } else { + switch (columnSubtype) { + case FieldSubtype.USER: + parsedRow[columnName] = parsedValues[0]?._id + break + case FieldSubtype.USERS: + parsedRow[columnName] = parsedValues.map(u => u._id) + break + default: + utils.unreachable(columnSubtype) + } + } } else { parsedRow[columnName] = columnData } @@ -155,3 +188,32 @@ export function parse(rows: Rows, schema: Schema): Rows { return parsedRow }) } + +function isValidBBReference( + columnData: any, + columnSubtype: FieldSubtype +): boolean { + switch (columnSubtype) { + case FieldSubtype.USER: + case FieldSubtype.USERS: + if (typeof columnData !== "string") { + return false + } + const userArray = parseCsvExport<{ _id: string }[]>(columnData) + if (!Array.isArray(userArray)) { + return false + } + + if (columnSubtype === FieldSubtype.USER && userArray.length > 1) { + return false + } + + const constainsWrongId = userArray.find( + user => !db.isGlobalUserID(user._id) + ) + return !constainsWrongId + + default: + throw utils.unreachable(columnSubtype) + } +} diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index ff288811c9..8fb0297a9e 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -1,4 +1,5 @@ import { + Row, Table, TableRequest, TableSchema, @@ -18,6 +19,13 @@ export interface TableResponse extends Table { export type FetchTablesResponse = TableResponse[] -export interface SaveTableRequest extends TableRequest {} +export interface SaveTableRequest extends TableRequest { + rows?: Row[] +} export type SaveTableResponse = Table + +export interface BulkImportRequest { + rows: Row[] + identifierFields?: Array +} diff --git a/packages/types/src/documents/app/table/table.ts b/packages/types/src/documents/app/table/table.ts index 76b2c587b2..5174ec608f 100644 --- a/packages/types/src/documents/app/table/table.ts +++ b/packages/types/src/documents/app/table/table.ts @@ -15,7 +15,6 @@ export interface Table extends Document { constrained?: string[] sql?: boolean indexes?: { [key: string]: any } - rows?: { [key: string]: any } created?: boolean rowHeight?: number }