diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index a0bebc2490..cc1450060c 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -37,7 +37,7 @@ import { Table, } from "@budibase/types" -const { cleanExportRows } = require("./utils") +import { cleanExportRows } from "./utils" const CALCULATION_TYPES = { SUM: "sum", @@ -391,6 +391,9 @@ export async function exportRows(ctx: UserCtx) { const table = await db.get(ctx.params.tableId) const rowIds = ctx.request.body.rows let format = ctx.query.format + if (typeof format !== "string") { + ctx.throw(400, "Format parameter is not valid") + } const { columns, query } = ctx.request.body let result diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index e96a4fe6ee..a7c467ea61 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -137,8 +137,8 @@ export function cleanExportRows( delete schema[column] }) - // Intended to avoid 'undefined' in export if (format === Format.CSV) { + // Intended to append empty values in export const schemaKeys = Object.keys(schema) for (let key of schemaKeys) { if (columns?.length && columns.indexOf(key) > 0) { @@ -146,7 +146,7 @@ export function cleanExportRows( } for (let row of cleanRows) { if (row[key] == null) { - row[key] = "" + row[key] = undefined } } } diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 2ab7ad7b38..bc967a90f4 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils" import { context, events } from "@budibase/backend-core" import { Table, UserCtx } from "@budibase/types" import sdk from "../../../sdk" -import csv from "csvtojson" +import { jsonFromCsvString } from "../../../utilities/csv" function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { if (table && !tableId) { @@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) { export async function csvToJson(ctx: UserCtx) { const { csvString } = ctx.request.body - const result = await csv().fromString(csvString) + const result = await jsonFromCsvString(csvString) ctx.status = 200 ctx.body = result diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts index 4d927bca27..ec0aca95a9 100644 --- a/packages/server/src/api/controllers/view/exporters.ts +++ b/packages/server/src/api/controllers/view/exporters.ts @@ -10,7 +10,9 @@ export function csv(headers: string[], rows: Row[]) { val = typeof val === "object" && !(val instanceof Date) ? `"${JSON.stringify(val).replace(/"/g, "'")}"` - : `"${val}"` + : val !== undefined + ? `"${val}"` + : "" return val.trim() }) .join(",")}` diff --git a/packages/server/src/utilities/csv.ts b/packages/server/src/utilities/csv.ts new file mode 100644 index 0000000000..477ebb896d --- /dev/null +++ b/packages/server/src/utilities/csv.ts @@ -0,0 +1,22 @@ +import csv from "csvtojson" + +export async function jsonFromCsvString(csvString: string) { + const castedWithEmptyValues = await csv({ ignoreEmpty: true }).fromString( + csvString + ) + + // By default the csvtojson library casts empty values as empty strings. This is causing issues on conversion. + // ignoreEmpty will remove the key completly if empty, so creating this empty object will ensure we return the values with the keys but empty values + const result = await csv({ ignoreEmpty: false }).fromString(csvString) + result.forEach((r, i) => { + for (const [key] of Object.entries(r).filter( + ([key, value]) => value === "" + )) { + if (castedWithEmptyValues[i][key] === undefined) { + r[key] = null + } + } + }) + + return result +} diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index f3d98d35c9..7b5de7898a 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -4,6 +4,9 @@ interface SchemaColumn { readonly name: string readonly type: FieldTypes readonly autocolumn?: boolean + readonly constraints?: { + presence: boolean + } } interface Schema { @@ -76,6 +79,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults { // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array if (typeof columnType !== "string") { results.invalidColumns.push(columnName) + } else if ( + columnData == null && + !schema[columnName].constraints?.presence + ) { + results.schemaValidation[columnName] = true } else if ( // If there's no data for this field don't bother with further checks // If the field is already marked as invalid there's no need for further checks diff --git a/packages/server/src/utilities/tests/csv.spec.ts b/packages/server/src/utilities/tests/csv.spec.ts new file mode 100644 index 0000000000..14063d0e8e --- /dev/null +++ b/packages/server/src/utilities/tests/csv.spec.ts @@ -0,0 +1,33 @@ +import { jsonFromCsvString } from "../csv" + +describe("csv", () => { + describe("jsonFromCsvString", () => { + test("multiple lines csv can be casted", async () => { + const csvString = '"id","title"\n"1","aaa"\n"2","bbb"' + + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", title: "aaa" }, + { id: "2", title: "bbb" }, + ]) + result.forEach(r => expect(Object.keys(r)).toEqual(["id", "title"])) + }) + + test("empty values are casted as undefined", async () => { + const csvString = + '"id","optional","title"\n1,,"aaa"\n2,"value","bbb"\n3,,"ccc"' + + const result = await jsonFromCsvString(csvString) + + expect(result).toEqual([ + { id: "1", optional: null, title: "aaa" }, + { id: "2", optional: "value", title: "bbb" }, + { id: "3", optional: null, title: "ccc" }, + ]) + result.forEach(r => + expect(Object.keys(r)).toEqual(["id", "optional", "title"]) + ) + }) + }) +})