CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-11 01:06:04 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #11908 from Budibase/BUDI-7403/ds+_user_column_edition

Browse source 
Fixing DS+ save rows
This commit is contained in:
Adria Navarro 2023-09-28 12:17:24 +02:00 committed by GitHub
parent af0aa28bfa 317da512a5
commit 216e22c898
10 changed files with 480 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View file

@ -33,7 +33,7 @@
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
import { ValidColumnNameRegex } from "@budibase/shared-core"
import { FieldSubtype, FieldType } from "@budibase/types"
import { FieldType } from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte"
const AUTO_TYPE = "auto"
@ -43,11 +43,7 @@
const NUMBER_TYPE = FIELDS.NUMBER.type
const JSON_TYPE = FIELDS.JSON.type
const DATE_TYPE = FIELDS.DATETIME.type
const BB_REFERENCE_TYPE = FieldType.BB_REFERENCE
const BB_USER_REFERENCE_TYPE = composeType(
BB_REFERENCE_TYPE,
FieldSubtype.USER
)
const USER_REFRENCE_TYPE = FIELDS.BB_REFERENCE_USER.compositeType
const dispatch = createEventDispatcher()
const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@ -80,33 +76,6 @@
fieldName: $tables.selected.name,
}
const bbRefTypeMapping = {}
function composeType(fieldType, subtype) {
return `${fieldType}_${subtype}`
}
// Handling fields with subtypes
fieldDefinitions = Object.entries(fieldDefinitions).reduce(
(p, [key, field]) => {
if (field.type === BB_REFERENCE_TYPE) {
const composedType = composeType(field.type, field.subtype)
p[key] = {
...field,
type: composedType,
}
bbRefTypeMapping[composedType] = {
type: field.type,
subtype: field.subtype,
}
} else {
p[key] = field
}
return p
},
{}
)
$: if (primaryDisplay) {
editableColumn.constraints.presence = { allowEmpty: false }
}
@ -149,12 +118,8 @@
$tables.selected.primaryDisplay == null ||
$tables.selected.primaryDisplay === editableColumn.name
const mapped = Object.entries(bbRefTypeMapping).find(
([_, v]) => v.type === field.type && v.subtype === field.subtype
)
if (mapped) {
editableColumn.type = mapped[0]
delete editableColumn.subtype
if (editableColumn.type === FieldType.BB_REFERENCE) {
editableColumn.type = `${editableColumn.type}_${editableColumn.subtype}`
}
} else if (!savingColumn) {
let highestNumber = 0
@ -188,8 +153,6 @@
$: initialiseField(field, savingColumn)
$: isBBReference = !!bbRefTypeMapping[editableColumn.type]
$: checkConstraints(editableColumn)
$: required = !!editableColumn?.constraints?.presence || primaryDisplay
$: uneditable =
@ -265,11 +228,12 @@
let saveColumn = cloneDeep(editableColumn)
if (bbRefTypeMapping[saveColumn.type]) {
saveColumn = {
...saveColumn,
...bbRefTypeMapping[saveColumn.type],
}
// Handle types on composite types
const definition = fieldDefinitions[saveColumn.type.toUpperCase()]
if (definition && saveColumn.type === definition.compositeType) {
saveColumn.type = definition.type
saveColumn.subtype = definition.subtype
delete saveColumn.compositeType
}
if (saveColumn.type === AUTO_TYPE) {
@ -352,7 +316,7 @@
editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
} else if (editableColumn.type === FORMULA_TYPE) {
editableColumn.formulaType = "dynamic"
} else if (editableColumn.type === BB_USER_REFERENCE_TYPE) {
} else if (editableColumn.type === USER_REFRENCE_TYPE) {
editableColumn.relationshipType = RelationshipType.ONE_TO_MANY
}
}
@ -410,14 +374,12 @@
FIELDS.BOOLEAN,
FIELDS.FORMULA,
FIELDS.BIGINT,
FIELDS.BB_REFERENCE_USER,
]
// no-sql or a spreadsheet
if (!external || table.sql) {
fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
}
if (fieldDefinitions.USER) {
fields.push(fieldDefinitions.USER)
}
return fields
}
}
@ -426,8 +388,9 @@
if (!fieldToCheck) {
return
}
// most types need this, just make sure its always present
if (fieldToCheck && !fieldToCheck.constraints) {
if (!fieldToCheck.constraints) {
fieldToCheck.constraints = {}
}
// some string types may have been built by server, may not always have constraints
@ -507,7 +470,7 @@
on:change={handleTypeChange}
options={allowedTypes}
getOptionLabel={field => field.name}
getOptionValue={field => field.type}
getOptionValue={field => field.compositeType || field.type}
getOptionIcon={field => field.icon}
isOptionEnabled={option => {
if (option.type == AUTO_TYPE) {
@ -671,7 +634,7 @@
<Button primary text on:click={openJsonSchemaEditor}
>Open schema editor</Button
>
{:else if isBBReference}
{:else if editableColumn.type === USER_REFRENCE_TYPE}
<Toggle
value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
on:change={e =>

3
packages/builder/src/constants/backend/index.js
View file

@ -120,10 +120,11 @@ export const FIELDS = {
presence: false,
},
},
USER: {
BB_REFERENCE_USER: {
name: "User",
type: "bb_reference",
subtype: "user",
compositeType: "bb_reference_user", // Used for working with the subtype on CreateEditColumn as is it was a primary type
icon: "User",
},
}

44
packages/server/src/api/controllers/row/external.ts
View file

@ -18,7 +18,10 @@ import {
import sdk from "../../../sdk"
import * as utils from "./utils"
import { dataFilters } from "@budibase/shared-core"
import { inputProcessing } from "../../../utilities/rowProcessor"
import {
inputProcessing,
outputProcessing,
} from "../../../utilities/rowProcessor"
import { cloneDeep, isEqual } from "lodash"
export async function handleRequest(
@ -46,24 +49,31 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const tableId = utils.getTableId(ctx)
const { _id, ...rowData } = ctx.request.body
const table = await sdk.tables.getTable(tableId)
const { row: dataToUpdate } = await inputProcessing(
ctx.user?._id,
cloneDeep(table),
rowData
)
const validateResult = await sdk.rows.utils.validate({
row: rowData,
row: dataToUpdate,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
const response = await handleRequest(Operation.UPDATE, tableId, {
id: breakRowIdField(_id),
row: rowData,
row: dataToUpdate,
})
const row = await sdk.rows.external.getRow(tableId, _id, {
relationships: true,
})
const table = await sdk.tables.getTable(tableId)
return {
...response,
row,
row: await outputProcessing(table, row),
table,
}
}
@ -71,13 +81,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
export async function save(ctx: UserCtx) {
const inputs = ctx.request.body
const tableId = utils.getTableId(ctx)
const validateResult = await sdk.rows.utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
const table = await sdk.tables.getTable(tableId)
const { table: updatedTable, row } = await inputProcessing(
@ -86,6 +89,14 @@ export async function save(ctx: UserCtx) {
inputs
)
const validateResult = await sdk.rows.utils.validate({
row,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
const response = await handleRequest(Operation.CREATE, tableId, {
row,
})
@ -103,7 +114,7 @@ export async function save(ctx: UserCtx) {
})
return {
...response,
row,
row: await outputProcessing(table, row),
}
} else {
return response
@ -121,7 +132,12 @@ export async function find(ctx: UserCtx): Promise<Row> {
ctx.throw(404)
}
return row
const table = await sdk.tables.getTable(tableId)
// Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
return await outputProcessing(table, row, {
squash: false,
preserveLinks: true,
})
}
export async function destroy(ctx: UserCtx) {

394
packages/server/src/api/routes/tests/row.spec.ts
View file

@ -7,6 +7,7 @@ import { context, InternalTable, roles, tenancy } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
FieldType,
FieldTypeSubtypes,
MonthlyQuotaName,
PermissionLevel,
QuotaUsageType,
@ -17,6 +18,7 @@ import {
SortType,
StaticQuotaName,
Table,
User,
} from "@budibase/types"
import {
expectAnyExternalColsAttributes,
@ -25,6 +27,7 @@ import {
mocks,
structures,
} from "@budibase/backend-core/tests"
import _ from "lodash"
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp)
@ -34,7 +37,7 @@ const { basicRow } = setup.structures
describe.each([
["internal", undefined],
["postgres", databaseTestProviders.postgres],
])("/rows (%s)", (_, dsProvider) => {
])("/rows (%s)", (__, dsProvider) => {
const isInternal = !dsProvider
const request = setup.getRequest()
@ -1511,4 +1514,393 @@ describe.each([
})
})
})
describe("bb reference fields", () => {
let tableId: string
let users: User[]
beforeAll(async () => {
const tableConfig = generateTableConfig()
if (config.datasource) {
tableConfig.sourceId = config.datasource._id
if (config.datasource.plus) {
tableConfig.type = "external"
}
}
const table = await config.api.table.create({
...tableConfig,
schema: {
...tableConfig.schema,
user: {
name: "user",
type: FieldType.BB_REFERENCE,
subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
relationshipType: RelationshipType.ONE_TO_MANY,
},
users: {
name: "users",
type: FieldType.BB_REFERENCE,
subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
relationshipType: RelationshipType.MANY_TO_MANY,
},
},
})
tableId = table._id!
users = [
await config.createUser(),
await config.createUser(),
await config.createUser(),
await config.createUser(),
]
})
it("can save a row when BB reference fields are empty", async () => {
const rowData = {
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
}
const row = await config.api.row.save(tableId, rowData)
expect(row).toEqual({
name: rowData.name,
description: rowData.description,
tableId,
_id: expect.any(String),
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
type: isInternal ? "row" : undefined,
})
})
it("can save a row with a single BB reference field", async () => {
const user = _.sample(users)!
const rowData = {
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
user: user,
}
const row = await config.api.row.save(tableId, rowData)
expect(row).toEqual({
name: rowData.name,
description: rowData.description,
tableId,
user: [
{
_id: user._id,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
primaryDisplay: user.email,
},
],
_id: expect.any(String),
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
type: isInternal ? "row" : undefined,
})
})
it("can save a row with a multiple BB reference field", async () => {
const selectedUsers = _.sampleSize(users, 2)
const rowData = {
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: selectedUsers,
}
const row = await config.api.row.save(tableId, rowData)
expect(row).toEqual({
name: rowData.name,
description: rowData.description,
tableId,
users: selectedUsers.map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
_id: expect.any(String),
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
type: isInternal ? "row" : undefined,
})
})
it("can retrieve rows with no populated BB references", async () => {
const rowData = {
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
}
const row = await config.api.row.save(tableId, rowData)
const { body: retrieved } = await config.api.row.get(tableId, row._id!)
expect(retrieved).toEqual({
name: rowData.name,
description: rowData.description,
tableId,
user: undefined,
users: undefined,
_id: row._id,
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
...defaultRowFields,
})
})
it("can retrieve rows with populated BB references", async () => {
const [user1, user2] = _.sampleSize(users, 2)
const rowData = {
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: [user1, user2],
user: [user1],
}
const row = await config.api.row.save(tableId, rowData)
const { body: retrieved } = await config.api.row.get(tableId, row._id!)
expect(retrieved).toEqual({
name: rowData.name,
description: rowData.description,
tableId,
user: [user1].map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
users: [user1, user2].map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
_id: row._id,
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
...defaultRowFields,
})
})
it("can update an existing populated row", async () => {
const [user1, user2, user3] = _.sampleSize(users, 3)
const rowData = {
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: [user1, user2],
}
const row = await config.api.row.save(tableId, rowData)
const updatedRow = await config.api.row.save(tableId, {
...row,
user: [user3],
users: [user3, user2],
})
expect(updatedRow).toEqual({
name: rowData.name,
description: rowData.description,
tableId,
user: [
{
_id: user3._id,
email: user3.email,
firstName: user3.firstName,
lastName: user3.lastName,
primaryDisplay: user3.email,
},
],
users: [user3, user2].map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
_id: row._id,
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
type: isInternal ? "row" : undefined,
})
})
it("can wipe an existing populated BB references in row", async () => {
const [user1, user2] = _.sampleSize(users, 2)
const rowData = {
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: [user1, user2],
}
const row = await config.api.row.save(tableId, rowData)
const updatedRow = await config.api.row.save(tableId, {
...row,
user: null,
users: null,
})
expect(updatedRow).toEqual({
name: rowData.name,
description: rowData.description,
tableId,
user: isInternal ? null : undefined,
users: isInternal ? null : undefined,
_id: row._id,
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
type: isInternal ? "row" : undefined,
})
})
it("fetch all will populate the BB references", async () => {
const [user1, user2, user3] = _.sampleSize(users, 3)
const rows: {
name: string
description: string
user?: User[]
users?: User[]
tableId: string
}[] = [
{
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: [user1, user2],
},
{
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
user: [user1],
users: [user1, user3],
},
{
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: [user3],
},
]
await config.api.row.save(tableId, rows[0])
await config.api.row.save(tableId, rows[1])
await config.api.row.save(tableId, rows[2])
const res = await config.api.row.fetch(tableId)
expect(res).toEqual(
expect.arrayContaining(
rows.map(r => ({
name: r.name,
description: r.description,
tableId,
user: r.user?.map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
users: r.users?.map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
_id: expect.any(String),
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
...defaultRowFields,
}))
)
)
})
it("search all will populate the BB references", async () => {
const [user1, user2, user3] = _.sampleSize(users, 3)
const rows: {
name: string
description: string
user?: User[]
users?: User[]
tableId: string
}[] = [
{
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: [user1, user2],
},
{
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
user: [user1],
users: [user1, user3],
},
{
...basicRow(tableId),
name: generator.name(),
description: generator.name(),
users: [user3],
},
]
await config.api.row.save(tableId, rows[0])
await config.api.row.save(tableId, rows[1])
await config.api.row.save(tableId, rows[2])
const res = await config.api.row.search(tableId)
expect(res).toEqual({
rows: expect.arrayContaining(
rows.map(r => ({
name: r.name,
description: r.description,
tableId,
user: r.user?.map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
users: r.users?.map(u => ({
_id: u._id,
email: u.email,
firstName: u.firstName,
lastName: u.lastName,
primaryDisplay: u.email,
})),
_id: expect.any(String),
_rev: expect.any(String),
id: isInternal ? undefined : expect.any(Number),
...defaultRowFields,
}))
),
...(isInternal
? {}
: {
hasNextPage: false,
bookmark: null,
}),
})
})
})
})

8
packages/server/src/sdk/app/rows/search/external.ts
View file

@ -17,6 +17,7 @@ import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "../search"
import { HTTPError, db } from "@budibase/backend-core"
import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
export async function search(options: SearchParams) {
const { tableId } = options
@ -75,6 +76,9 @@ export async function search(options: SearchParams) {
rows = rows.map((r: any) => pick(r, fields))
}
const table = await sdk.tables.getTable(tableId)
rows = await outputProcessing(table, rows)
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
} catch (err: any) {
@ -166,9 +170,11 @@ export async function exportRows(
}
export async function fetch(tableId: string) {
return handleRequest(Operation.READ, tableId, {
const response = await handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
const table = await sdk.tables.getTable(tableId)
return await outputProcessing(table, response)
}
export async function fetchView(viewName: string) {

5
packages/server/src/sdk/tests/rows/row.spec.ts
View file

@ -7,9 +7,14 @@ import { HTTPError } from "@budibase/backend-core"
import { Operation } from "@budibase/types"
const mockDatasourcesGet = jest.fn()
const mockTableGet = jest.fn()
sdk.datasources.get = mockDatasourcesGet
sdk.tables.getTable = mockTableGet
jest.mock("../../../api/controllers/row/ExternalRequest")
jest.mock("../../../utilities/rowProcessor", () => ({
outputProcessing: jest.fn((_, rows) => rows),
}))
jest.mock("../../../api/controllers/view/exporters", () => ({
...jest.requireActual("../../../api/controllers/view/exporters"),

16
packages/server/src/tests/utilities/api/row.ts
View file

@ -44,12 +44,12 @@ export class RowAPI extends TestAPI {
}
save = async (
sourceId: string,
tableId: string,
row: SaveRowRequest,
{ expectStatus } = { expectStatus: 200 }
): Promise<Row> => {
const resp = await this.request
.post(`/api/${sourceId}/rows`)
.post(`/api/${tableId}/rows`)
.send(row)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
@ -122,4 +122,16 @@ export class RowAPI extends TestAPI {
.expect(expectStatus)
return request
}
search = async (
sourceId: string,
{ expectStatus } = { expectStatus: 200 }
): Promise<Row[]> => {
const request = this.request
.post(`/api/${sourceId}/search`)
.set(this.config.defaultHeaders())
.expect(expectStatus)
return (await request).body
}
}

4
packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts
View file

@ -6,7 +6,7 @@ import { InvalidBBRefError } from "./errors"
export async function processInputBBReferences(
value: string | string[] | { _id: string } | { _id: string }[],
subtype: FieldSubtype
): Promise<string | undefined> {
): Promise<string | null> {
const referenceIds: string[] = []
if (Array.isArray(value)) {
@ -39,7 +39,7 @@ export async function processInputBBReferences(
throw utils.unreachable(subtype)
}
return referenceIds.join(",") || undefined
return referenceIds.join(",") || null
}
export async function processOutputBBReferences(

9
packages/server/src/utilities/rowProcessor/index.ts
View file

@ -200,7 +200,10 @@ export async function inputProcessing(
export async function outputProcessing<T extends Row[] | Row>(
table: Table,
rows: T,
opts = { squash: true }
opts: { squash?: boolean; preserveLinks?: boolean } = {
squash: true,
preserveLinks: false,
}
): Promise<T> {
let safeRows: Row[]
let wasArray = true
@ -211,7 +214,9 @@ export async function outputProcessing<T extends Row[] | Row>(
safeRows = rows
}
// attach any linked row information
let enriched = await linkRows.attachFullLinkedDocs(table, safeRows)
let enriched = !opts.preserveLinks
? await linkRows.attachFullLinkedDocs(table, safeRows)
: safeRows
// process formulas
enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]

8
packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts
View file

@ -139,20 +139,20 @@ describe("bbReferenceProcessor", () => {
expect(cacheGetUsersSpy).toBeCalledWith(userIds)
})
it("empty strings will return undefined", async () => {
it("empty strings will return null", async () => {
const result = await config.doInTenant(() =>
processInputBBReferences("", FieldSubtype.USER)
)
expect(result).toEqual(undefined)
expect(result).toEqual(null)
})
it("empty arrays will return undefined", async () => {
it("empty arrays will return null", async () => {
const result = await config.doInTenant(() =>
processInputBBReferences([], FieldSubtype.USER)
)
expect(result).toEqual(undefined)
expect(result).toEqual(null)
})
})
})