From 10351c52f8345ad213c475aa5aaec0b67c5d2a09 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 30 Jul 2024 11:52:43 +0200 Subject: [PATCH 1/7] Improve types --- .../types/src/documents/app/table/schema.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 3a2ddf097f..1998ae0401 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -157,6 +157,21 @@ export interface FieldConstraints { } } +export interface OptionsFieldMetadata extends BaseFieldSchema { + type: FieldType.OPTIONS + constraints: FieldConstraints & { + inclusion: string[] + } +} + +export interface ArrayFieldMetadata extends BaseFieldSchema { + type: FieldType.ARRAY + constraints: FieldConstraints & { + type: JsonFieldSubType.ARRAY + inclusion: string[] + } +} + interface BaseFieldSchema extends UIFieldMetadata { type: FieldType name: string @@ -182,6 +197,8 @@ interface OtherFieldMetadata extends BaseFieldSchema { | FieldType.BB_REFERENCE_SINGLE | FieldType.ATTACHMENTS | FieldType.STRING + | FieldType.ARRAY + | FieldType.OPTIONS > } @@ -198,6 +215,8 @@ export type FieldSchema = | JsonFieldMetadata | AttachmentFieldMetadata | BBReferenceSingleFieldMetadata + | ArrayFieldMetadata + | OptionsFieldMetadata export interface TableSchema { [key: string]: FieldSchema From 7d17717895e8fca102ec20b4007f73eb6b50ea87 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 30 Jul 2024 17:48:08 +0200 Subject: [PATCH 2/7] Subtype required for autocolumns --- packages/types/src/documents/app/table/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 1998ae0401..7b89b610be 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -64,7 +64,7 @@ export interface AutoColumnFieldMetadata extends Omit { type: FieldType.AUTO autocolumn: true - subtype?: AutoFieldSubType + subtype: AutoFieldSubType lastID?: number // if the column was turned to an auto-column for SQL, explains why (primary, foreign etc) autoReason?: AutoReason From 21b9bf36fd00da9e28cc44ed47223a3ce4910a26 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 30 Jul 2024 12:38:47 +0200 Subject: [PATCH 3/7] Add api types --- .../server/src/api/controllers/table/index.ts | 22 ++++++++++----- packages/types/src/api/web/app/rows.ts | 2 +- packages/types/src/api/web/app/table.ts | 27 ++++++++++++++++++- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 3148e54af7..ba861064bb 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -14,16 +14,20 @@ import { events, HTTPError } from "@budibase/backend-core" import { BulkImportRequest, BulkImportResponse, + CsvToJsonRequest, + CsvToJsonResponse, FetchTablesResponse, MigrateRequest, MigrateResponse, - Row, SaveTableRequest, SaveTableResponse, Table, TableResponse, TableSourceType, UserCtx, + ValidateNewTableImportRequest, + ValidateTableImportRequest, + ValidateTableImportResponse, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" @@ -144,7 +148,9 @@ export async function bulkImport( ctx.body = { message: `Bulk rows created.` } } -export async function csvToJson(ctx: UserCtx) { +export async function csvToJson( + ctx: UserCtx +) { const { csvString } = ctx.request.body const result = await jsonFromCsvString(csvString) @@ -153,8 +159,10 @@ export async function csvToJson(ctx: UserCtx) { ctx.body = result } -export async function validateNewTableImport(ctx: UserCtx) { - const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body +export async function validateNewTableImport( + ctx: UserCtx +) { + const { rows, schema } = ctx.request.body if (isRows(rows) && isSchema(schema)) { ctx.status = 200 @@ -164,8 +172,10 @@ export async function validateNewTableImport(ctx: UserCtx) { } } -export async function validateExistingTableImport(ctx: UserCtx) { - const { rows, tableId }: { rows: Row[]; tableId?: string } = ctx.request.body +export async function validateExistingTableImport( + ctx: UserCtx +) { + const { rows, tableId } = ctx.request.body let schema = null if (tableId) { diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index c120af0628..ce6f6f672d 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -37,7 +37,7 @@ export interface PaginatedSearchRowResponse PaginationResponse {} export interface ExportRowsRequest { - rows: string[] + rows?: string[] columns?: string[] query?: SearchFilters sort?: string diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index fc382643f1..bb9329f6c1 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -1,4 +1,4 @@ -import { Row, Table, TableRequest, View } from "../../../documents" +import { Row, Table, TableRequest, TableSchema, View } from "../../../documents" import { ViewV2Enriched } from "../../../sdk" export type TableViewsResponse = { [key: string]: View | ViewV2Enriched } @@ -32,3 +32,28 @@ export interface MigrateRequest { export interface MigrateResponse { message: string } + +export interface ValidateNewTableImportRequest { + rows: Row[] + schema: TableSchema +} + +export interface ValidateTableImportRequest { + tableId?: string + rows: Row[] +} + +export interface ValidateTableImportResponse { + schemaValidation: { + [field: string]: boolean + } + allValid: boolean + invalidColumns: Array + errors: Record +} + +export interface CsvToJsonRequest { + csvString: string +} + +export type CsvToJsonResponse = any[] From 9371f9913e7bded6e870d3b272bb63045a83027c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 30 Jul 2024 18:14:54 +0200 Subject: [PATCH 4/7] Types --- .../src/api/routes/tests/datasource.spec.ts | 10 +++++- .../server/src/api/routes/tests/row.spec.ts | 4 ++- .../src/api/routes/tests/search.spec.ts | 6 +++- .../server/src/api/routes/tests/table.spec.ts | 1 + .../src/api/routes/tests/viewV2.spec.ts | 4 +++ .../db/defaultData/datasource_bb_default.ts | 9 ++--- .../server/src/integrations/utils/utils.ts | 36 +++++++++++-------- .../sdk/app/rows/search/tests/search.spec.ts | 9 ++++- 8 files changed, 56 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 6f249f5016..255e46167f 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -15,6 +15,7 @@ import { Table, TableSchema, SupportedSqlTypes, + JsonFieldSubType, } from "@budibase/types" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import { tableForDatasource } from "../../../tests/utilities/structures" @@ -288,7 +289,10 @@ describe("/datasources", () => { name: "options", type: FieldType.OPTIONS, constraints: { - presence: { allowEmpty: false }, + presence: { + allowEmpty: false, + }, + inclusion: [], }, }, [FieldType.NUMBER]: { @@ -302,6 +306,10 @@ describe("/datasources", () => { [FieldType.ARRAY]: { name: "array", type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: [], + }, }, [FieldType.DATETIME]: { name: "datetime", diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9cc53168d0..904ab93e13 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -32,6 +32,7 @@ import { TableSourceType, UpdatedRowEventEmitter, TableSchema, + JsonFieldSubType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import _, { merge } from "lodash" @@ -103,6 +104,7 @@ describe.each([ const defaultSchema: TableSchema = { id: { type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, name: "id", autocolumn: true, constraints: { @@ -384,7 +386,7 @@ describe.each([ const arrayField: FieldSchema = { type: FieldType.ARRAY, constraints: { - type: "array", + type: JsonFieldSubType.ARRAY, presence: false, inclusion: ["One", "Two", "Three"], }, diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index bad512e5fd..2c5756efe4 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -20,6 +20,7 @@ import { Datasource, EmptyFilterOption, FieldType, + JsonFieldSubType, RelationshipType, Row, RowSearchParams, @@ -1494,7 +1495,10 @@ describe.each([ numbers: { name: "numbers", type: FieldType.ARRAY, - constraints: { inclusion: ["one", "two", "three"] }, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["one", "two", "three"], + }, }, }) await createRows([{ numbers: ["one", "two"] }, { numbers: ["three"] }]) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 20c83549d2..a2cff7b395 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -398,6 +398,7 @@ describe.each([ name: "auto", autocolumn: true, type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, }, }, }, diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 3edbc24365..d97ded626d 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -19,6 +19,7 @@ import { ViewV2, SearchResponse, BasicOperator, + AutoFieldSubType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -57,6 +58,7 @@ describe.each([ schema: { id: { type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, name: "id", autocolumn: true, constraints: { @@ -242,6 +244,7 @@ describe.each([ id: { name: "id", type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, autocolumn: true, visible: true, }, @@ -1556,6 +1559,7 @@ describe.each([ id: { name: "id", type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, autocolumn: true, }, name: { diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index fc79b90c00..a393888e51 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -17,6 +17,7 @@ import { AutoFieldSubType, Datasource, FieldType, + JsonFieldSubType, RelationshipType, Row, SourceName, @@ -131,7 +132,7 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = { "Item Tags": { type: FieldType.ARRAY, constraints: { - type: FieldType.ARRAY, + type: JsonFieldSubType.ARRAY, presence: { allowEmpty: false, }, @@ -153,7 +154,7 @@ export const DEFAULT_INVENTORY_TABLE_SCHEMA: Table = { Status: { type: FieldType.ARRAY, constraints: { - type: FieldType.ARRAY, + type: JsonFieldSubType.ARRAY, presence: { allowEmpty: false, }, @@ -291,7 +292,7 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = { "Employee Level": { type: FieldType.ARRAY, constraints: { - type: FieldType.ARRAY, + type: JsonFieldSubType.ARRAY, presence: false, inclusion: ["Manager", "Junior", "Senior", "Apprentice", "Contractor"], }, @@ -535,7 +536,7 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = { "Expense Tags": { type: FieldType.ARRAY, constraints: { - type: FieldType.ARRAY, + type: JsonFieldSubType.ARRAY, presence: { allowEmpty: false, }, diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index b97782ce7e..84742626c1 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -150,22 +150,28 @@ export function generateColumnDefinition(config: { }).internal } - const constraints: { - presence: boolean - inclusion?: string[] - } = { - presence, - } + let schema: FieldSchema if (foundType === FieldType.OPTIONS) { - constraints.inclusion = options - } - - const schema: FieldSchema = { - type: foundType, - externalType, - autocolumn, - name, - constraints, + schema = { + type: foundType, + externalType, + autocolumn, + name, + constraints: { + presence, + inclusion: options!, + }, + } + } else { + schema = { + type: foundType, + externalType, + autocolumn, + name, + constraints: { + presence, + }, + } } if (schema.type === FieldType.DATETIME) { schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 4b80d5bcd3..782b580d68 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -1,4 +1,10 @@ -import { Datasource, FieldType, Row, Table } from "@budibase/types" +import { + AutoFieldSubType, + Datasource, + FieldType, + Row, + Table, +} from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" @@ -56,6 +62,7 @@ describe.each([ id: { name: "id", type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, autocolumn: true, }, name: { From 48349f9ad011e3f806ac021c47653f266376c406 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 30 Jul 2024 18:44:00 +0200 Subject: [PATCH 5/7] Fix tests --- packages/server/src/api/routes/tests/viewV2.spec.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index d97ded626d..2ff5c67852 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -19,7 +19,6 @@ import { ViewV2, SearchResponse, BasicOperator, - AutoFieldSubType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -57,8 +56,7 @@ describe.each([ primary: ["id"], schema: { id: { - type: FieldType.AUTO, - subtype: AutoFieldSubType.AUTO_ID, + type: FieldType.NUMBER, name: "id", autocolumn: true, constraints: { @@ -243,8 +241,7 @@ describe.each([ schema: { id: { name: "id", - type: FieldType.AUTO, - subtype: AutoFieldSubType.AUTO_ID, + type: FieldType.NUMBER, autocolumn: true, visible: true, }, @@ -1558,8 +1555,7 @@ describe.each([ schema: { id: { name: "id", - type: FieldType.AUTO, - subtype: AutoFieldSubType.AUTO_ID, + type: FieldType.NUMBER, autocolumn: true, }, name: { From a9a40516db9c40a2cf9cd9993e9732b8437f3e4f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 30 Jul 2024 18:58:51 +0200 Subject: [PATCH 6/7] Fix test --- packages/server/src/api/routes/tests/row.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 904ab93e13..8871841ee7 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -103,8 +103,7 @@ describe.each([ ): SaveTableRequest { const defaultSchema: TableSchema = { id: { - type: FieldType.AUTO, - subtype: AutoFieldSubType.AUTO_ID, + type: FieldType.NUMBER, name: "id", autocolumn: true, constraints: { From a172a4474178966a1d3491274f1756005d1b4559 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 30 Jul 2024 19:08:00 +0200 Subject: [PATCH 7/7] Fix test --- .../src/sdk/app/rows/search/tests/search.spec.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 782b580d68..47124d6667 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -1,10 +1,4 @@ -import { - AutoFieldSubType, - Datasource, - FieldType, - Row, - Table, -} from "@budibase/types" +import { Datasource, FieldType, Row, Table } from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" @@ -61,8 +55,7 @@ describe.each([ schema: { id: { name: "id", - type: FieldType.AUTO, - subtype: AutoFieldSubType.AUTO_ID, + type: FieldType.NUMBER, autocolumn: true, }, name: {