From 608a38489fdc90b91507621f7a986978cef39793 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 25 Apr 2023 12:34:50 +0100 Subject: [PATCH 1/6] Export undefineds as empty values in csv, instead of empty strings --- packages/server/src/api/controllers/row/internal.ts | 5 ++++- packages/server/src/api/controllers/row/utils.ts | 4 ++-- packages/server/src/api/controllers/view/exporters.ts | 4 +++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index a0bebc2490..cc1450060c 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -37,7 +37,7 @@ import { Table, } from "@budibase/types" -const { cleanExportRows } = require("./utils") +import { cleanExportRows } from "./utils" const CALCULATION_TYPES = { SUM: "sum", @@ -391,6 +391,9 @@ export async function exportRows(ctx: UserCtx) { const table = await db.get(ctx.params.tableId) const rowIds = ctx.request.body.rows let format = ctx.query.format + if (typeof format !== "string") { + ctx.throw(400, "Format parameter is not valid") + } const { columns, query } = ctx.request.body let result diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index e96a4fe6ee..a7c467ea61 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -137,8 +137,8 @@ export function cleanExportRows( delete schema[column] }) - // Intended to avoid 'undefined' in export if (format === Format.CSV) { + // Intended to append empty values in export const schemaKeys = Object.keys(schema) for (let key of schemaKeys) { if (columns?.length && columns.indexOf(key) > 0) { @@ -146,7 +146,7 @@ export function cleanExportRows( } for (let row of cleanRows) { if (row[key] == null) { - row[key] = "" + row[key] = undefined } } } diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index 4d927bca27..ec0aca95a9 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -10,7 +10,9 @@ export function csv(headers: string[], rows: Row[]) { val = typeof val === "object" && !(val instanceof Date) ? `"${JSON.stringify(val).replace(/"/g, "'")}"` - : `"${val}"` + : val !== undefined + ? `"${val}"` + : "" return val.trim() }) .join(",")}` From 29df12c247a473944c436953e470088bd51f2d94 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 2 May 2023 10:34:45 +0100 Subject: [PATCH 2/6] Add csvutils --- packages/server/src/api/controllers/table/index.ts | 4 ++-- packages/server/src/utilities/csv.ts | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 packages/server/src/utilities/csv.ts diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 2ab7ad7b38..bc967a90f4 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils" import { context, events } from "@budibase/backend-core" import { Table, UserCtx } from "@budibase/types" import sdk from "../../../sdk" -import csv from "csvtojson" +import { jsonFromCsvString } from "../../../utilities/csv" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && !tableId) { @@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) { export async function csvToJson(ctx: UserCtx) { const { csvString } = ctx.request.body - const result = await csv().fromString(csvString) + const result = await jsonFromCsvString(csvString) ctx.status = 200 ctx.body = result diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts new file mode 100644 index 0000000000..0fab14db45 --- /dev/null +++ b/packages/server/src/utilities/csv.ts @@ -0,0 +1,6 @@ +import csv from "csvtojson" + +export async function jsonFromCsvString(csvString: string) { + const result = await csv().fromString(csvString) + return result +} From c87cc39cea5fa4e379eecb9156a58846a5788811 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 2 May 2023 10:44:25 +0100 Subject: [PATCH 3/6] Add basic tests --- packages/server/src/utilities/tests/csv.spec.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 packages/server/src/utilities/tests/csv.spec.ts diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts new file mode 100644 index 0000000000..aab5917a9a --- /dev/null +++ b/packages/server/src/utilities/tests/csv.spec.ts @@ -0,0 +1,16 @@ +import { jsonFromCsvString } from "../csv" + +describe("csv", () => { + describe("jsonFromCsvString", () => { + test("multiple lines csv can be casted", async () => { + const csvString = '"id","title"\n"1","aaa"\n"2","bbb"' + + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", title: "aaa" }, + { id: "2", title: "bbb" }, + ]) + }) + }) +}) From 650cbc1f011b0fbb8011efa87aca802f19b9fc8c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 2 May 2023 11:57:18 +0100 Subject: [PATCH 4/6] Handle cast as undefineds --- packages/server/src/utilities/csv.ts | 15 ++++++++++++++- packages/server/src/utilities/tests/csv.spec.ts | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts index 0fab14db45..454f467a9a 100644 --- a/packages/server/src/utilities/csv.ts +++ b/packages/server/src/utilities/csv.ts @@ -1,6 +1,19 @@ import csv from "csvtojson" export async function jsonFromCsvString(csvString: string) { - const result = await csv().fromString(csvString) + const castedWithEmptyStrings = await csv().fromString(csvString) + if (!castedWithEmptyStrings.length) { + return [] + } + + // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion. + // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values + const emptyObject = Object.keys(castedWithEmptyStrings[0]).reduce( + (p, v) => ({ ...p, [v]: undefined }), + {} + ) + + let result = await csv({ ignoreEmpty: true }).fromString(csvString) + result = result.map(r => ({ ...emptyObject, ...r })) return result } diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts index aab5917a9a..9ee0ae4c87 100644 --- a/packages/server/src/utilities/tests/csv.spec.ts +++ b/packages/server/src/utilities/tests/csv.spec.ts @@ -11,6 +11,23 @@ describe("csv", () => { { id: "1", title: "aaa" }, { id: "2", title: "bbb" }, ]) + result.forEach(r => expect(Object.keys(r)).toEqual(["id", "title"])) + }) + + test("empty values are casted as undefined", async () => { + const csvString = + '"id","optional","title"\n1,,"aaa"\n2,"value","bbb"\n3,,"ccc"' + + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", optional: undefined, title: "aaa" }, + { id: "2", optional: "value", title: "bbb" }, + { id: "3", optional: undefined, title: "ccc" }, + ]) + result.forEach(r => + expect(Object.keys(r)).toEqual(["id", "optional", "title"]) + ) }) }) }) From 6e7c78362ea5ee6cc91e37d58f447c45db8c57e4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 2 May 2023 12:46:53 +0100 Subject: [PATCH 5/6] Return null instead of undefined --- packages/server/src/utilities/csv.ts | 23 +++++++++++-------- .../server/src/utilities/tests/csv.spec.ts | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts index 454f467a9a..477ebb896d 100644 --- a/packages/server/src/utilities/csv.ts +++ b/packages/server/src/utilities/csv.ts @@ -1,19 +1,22 @@ import csv from "csvtojson" export async function jsonFromCsvString(csvString: string) { - const castedWithEmptyStrings = await csv().fromString(csvString) - if (!castedWithEmptyStrings.length) { - return [] - } + const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString( + csvString + ) // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion. // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values - const emptyObject = Object.keys(castedWithEmptyStrings[0]).reduce( - (p, v) => ({ ...p, [v]: undefined }), - {} - ) + const result = await csv({ ignoreEmpty: false }).fromString(csvString) + result.forEach((r, i) => { + for (const [key] of Object.entries(r).filter( + ([key, value]) => value === "" + )) { + if (castedWithEmptyValues[i][key] === undefined) { + r[key] = null + } + } + }) - let result = await csv({ ignoreEmpty: true }).fromString(csvString) - result = result.map(r => ({ ...emptyObject, ...r })) return result } diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts index 9ee0ae4c87..14063d0e8e 100644 --- a/packages/server/src/utilities/tests/csv.spec.ts +++ b/packages/server/src/utilities/tests/csv.spec.ts @@ -21,9 +21,9 @@ describe("csv", () => { const result = await jsonFromCsvString(csvString) expect(result).toEqual([ - { id: "1", optional: undefined, title: "aaa" }, + { id: "1", optional: null, title: "aaa" }, { id: "2", optional: "value", title: "bbb" }, - { id: "3", optional: undefined, title: "ccc" }, + { id: "3", optional: null, title: "ccc" }, ]) result.forEach(r => expect(Object.keys(r)).toEqual(["id", "optional", "title"]) From 1d041a3dd5c857817e065f5e3f8e41d372694f2b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 2 May 2023 12:48:05 +0100 Subject: [PATCH 6/6] Support nulls when optional --- packages/server/src/utilities/schema.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index f3d98d35c9..7b5de7898a 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -4,6 +4,9 @@ interface SchemaColumn { readonly name: string readonly type: FieldTypes readonly autocolumn?: boolean + readonly constraints?: { + presence: boolean + } } interface Schema { @@ -76,6 +79,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults { // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array if (typeof columnType !== "string") { results.invalidColumns.push(columnName) + } else if ( + columnData == null && + !schema[columnName].constraints?.presence + ) { + results.schemaValidation[columnName] = true } else if ( // If there's no data for this field don't bother with further checks // If the field is already marked as invalid there's no need for further checks