From 14ec6ac205cd877930595b67b64557048a7d53a5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jan 2022 19:06:08 +0000 Subject: [PATCH 1/7] Fixing an issue where old apps did not send up the values of the default parameters, the server should be able to handle this, enrich them in if they aren't there. --- .../integration/QueryBindingBuilder.svelte | 2 -- .../server/src/api/controllers/query/index.js | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/integration/QueryBindingBuilder.svelte b/packages/builder/src/components/integration/QueryBindingBuilder.svelte index 34d4d4219b..9bcf9d36f8 100644 --- a/packages/builder/src/components/integration/QueryBindingBuilder.svelte +++ b/packages/builder/src/components/integration/QueryBindingBuilder.svelte @@ -15,8 +15,6 @@ queryBindings = [...queryBindings, {}] } - $: console.log(bindings) - function deleteQueryBinding(idx) { queryBindings.splice(idx, 1) queryBindings = queryBindings diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js index 21db1eebbf..9cf7612e8a 100644 --- a/packages/server/src/api/controllers/query/index.js +++ b/packages/server/src/api/controllers/query/index.js @@ -141,6 +141,16 @@ async function execute(ctx, opts = { rowsOnly: false }) { const query = await db.get(ctx.params.queryId) const datasource = await db.get(query.datasourceId) + const enrichedParameters = ctx.request.body.parameters || {} + // make sure parameters are fully enriched with defaults + if (query && query.parameters) { + for (let parameter of query.parameters) { + if (!enrichedParameters[parameter.name]) { + enrichedParameters[parameter.name] = parameter.default + } + } + } + // call the relevant CRUD method on the integration class try { const { rows, pagination, extra } = await Runner.run({ @@ -149,7 +159,7 @@ async function execute(ctx, opts = { rowsOnly: false }) { queryVerb: query.queryVerb, fields: query.fields, pagination: ctx.request.body.pagination, - parameters: ctx.request.body.parameters, + parameters: enrichedParameters, transformer: query.transformer, queryId: ctx.params.queryId, }) @@ -178,8 +188,9 @@ const removeDynamicVariables = async (db, queryId) => { if (dynamicVariables) { // delete dynamic variables from the datasource - const newVariables = dynamicVariables.filter(dv => dv.queryId !== queryId) - datasource.config.dynamicVariables = newVariables + datasource.config.dynamicVariables = dynamicVariables.filter( + dv => dv.queryId !== queryId + ) await db.put(datasource) // invalidate the deleted variables From 3c50a83de731f1a2d220ddab830c8a26f878c483 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 18 Jan 2022 13:19:41 +0000 Subject: [PATCH 2/7] Derive safe array-like value as the default value for multi-select fields --- .../components/app/forms/MultiFieldSelect.svelte | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/MultiFieldSelect.svelte b/packages/client/src/components/app/forms/MultiFieldSelect.svelte index cecc569b6f..d09a8730ae 100644 --- a/packages/client/src/components/app/forms/MultiFieldSelect.svelte +++ b/packages/client/src/components/app/forms/MultiFieldSelect.svelte @@ -19,6 +19,7 @@ let fieldApi let fieldSchema + $: safeDefaultValue = getSafeDefaultValue(defaultValue) $: flatOptions = optionsSource == null || optionsSource === "schema" $: options = getOptions( optionsSource, @@ -28,6 +29,16 @@ valueColumn, customOptions ) + + const getSafeDefaultValue = value => { + if (value == null || value === "") { + return [] + } + if (!Array.isArray(value)) { + return [value] + } + return value + } Date: Tue, 18 Jan 2022 13:20:06 +0000 Subject: [PATCH 3/7] Transform the output of JS expressions to be actual types rather than strings --- .../src/helpers/javascript.js | 6 ++- .../src/processors/postprocessor.js | 5 ++ .../string-templates/test/javascript.spec.js | 48 ++++++++++++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.js index 9231283e89..0173be0b54 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.js @@ -1,5 +1,6 @@ const { atob } = require("../utilities") const { cloneDeep } = require("lodash/fp") +const { LITERAL_MARKER } = require("../helpers/constants") // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.cjs or index.mjs). @@ -46,8 +47,9 @@ module.exports.processJS = (handlebars, context) => { $: path => getContextValue(path, cloneDeep(context)), } - // Create a sandbox with out context and run the JS - return runJS(js, sandboxContext) + // Create a sandbox with our context and run the JS + const res = { data: runJS(js, sandboxContext) } + return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` } catch (error) { return "Error while executing JS" } diff --git a/packages/string-templates/src/processors/postprocessor.js b/packages/string-templates/src/processors/postprocessor.js index 4d1c84013a..7fc3f663fe 100644 --- a/packages/string-templates/src/processors/postprocessor.js +++ b/packages/string-templates/src/processors/postprocessor.js @@ -36,6 +36,11 @@ module.exports.processors = [ return value === "true" case "object": return JSON.parse(value) + case "js_result": + // We use the literal helper to process the result of JS expressions + // as we want to be able to return any types. + // We wrap the value in an abject to be able to use undefined properly. + return JSON.parse(value).data } return value }), diff --git a/packages/string-templates/test/javascript.spec.js b/packages/string-templates/test/javascript.spec.js index 05cc80331a..5363f37e02 100644 --- a/packages/string-templates/test/javascript.spec.js +++ b/packages/string-templates/test/javascript.spec.js @@ -7,7 +7,7 @@ const processJS = (js, context) => { describe("Test the JavaScript helper", () => { it("should execute a simple expression", () => { const output = processJS(`return 1 + 2`) - expect(output).toBe("3") + expect(output).toBe(3) }) it("should be able to use primitive bindings", () => { @@ -50,6 +50,52 @@ describe("Test the JavaScript helper", () => { expect(output).toBe("shazbat") }) + it("should be able to return an object", () => { + const output = processJS(`return $("foo")`, { + foo: { + bar: { + baz: "shazbat", + }, + }, + }) + expect(output.bar.baz).toBe("shazbat") + }) + + it("should be able to return an array", () => { + const output = processJS(`return $("foo")`, { + foo: ["a", "b", "c"], + }) + expect(output[2]).toBe("c") + }) + + it("should be able to return null", () => { + const output = processJS(`return $("foo")`, { + foo: null, + }) + expect(output).toBe(null) + }) + + it("should be able to return undefined", () => { + const output = processJS(`return $("foo")`, { + foo: undefined, + }) + expect(output).toBe(undefined) + }) + + it("should be able to return 0", () => { + const output = processJS(`return $("foo")`, { + foo: 0, + }) + expect(output).toBe(0) + }) + + it("should be able to return an empty string", () => { + const output = processJS(`return $("foo")`, { + foo: "", + }) + expect(output).toBe("") + }) + it("should be able to use a deep array binding", () => { const output = processJS(`return $("foo.0.bar")`, { foo: [ From c578dedd516249eedd751d70155af7e4c8823bbe Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 18 Jan 2022 13:20:28 +0000 Subject: [PATCH 4/7] Fix issue with array field validation --- .../server/src/api/controllers/row/utils.js | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils.js b/packages/server/src/api/controllers/row/utils.js index 71b22375f7..e18aee582f 100644 --- a/packages/server/src/api/controllers/row/utils.js +++ b/packages/server/src/api/controllers/row/utils.js @@ -52,21 +52,29 @@ exports.validate = async ({ appId, tableId, row, table }) => { const constraints = cloneDeep(table.schema[fieldName].constraints) const type = table.schema[fieldName].type // special case for options, need to always allow unselected (null) - if ( - (type === FieldTypes.OPTIONS || type === FieldTypes.ARRAY) && - constraints.inclusion - ) { + if (type === FieldTypes.OPTIONS && constraints.inclusion) { constraints.inclusion.push(null) } let res // Validate.js doesn't seem to handle array - if (type === FieldTypes.ARRAY && row[fieldName] && row[fieldName].length) { - row[fieldName].map(val => { - if (!constraints.inclusion.includes(val)) { - errors[fieldName] = "Field not in list" - } - }) + if (type === FieldTypes.ARRAY) { + const hasValues = + Array.isArray(row[fieldName]) && row[fieldName].length > 0 + + // Check values are valid if values are specified + if (hasValues) { + row[fieldName].map(val => { + if (!constraints.inclusion.includes(val)) { + errors[fieldName] = "Value not in list" + } + }) + } + + // Check for required constraint + if (constraints.presence === true && !hasValues) { + errors[fieldName] = "Required field" + } } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") { // this should only happen if there is an error try { From 858d7b4b5a0b02d21d2b84791acae5e9a90f8919 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 18 Jan 2022 15:25:43 +0000 Subject: [PATCH 5/7] Revert changes to MultiFieldSelect --- .../components/app/forms/MultiFieldSelect.svelte | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/client/src/components/app/forms/MultiFieldSelect.svelte b/packages/client/src/components/app/forms/MultiFieldSelect.svelte index d09a8730ae..cecc569b6f 100644 --- a/packages/client/src/components/app/forms/MultiFieldSelect.svelte +++ b/packages/client/src/components/app/forms/MultiFieldSelect.svelte @@ -19,7 +19,6 @@ let fieldApi let fieldSchema - $: safeDefaultValue = getSafeDefaultValue(defaultValue) $: flatOptions = optionsSource == null || optionsSource === "schema" $: options = getOptions( optionsSource, @@ -29,16 +28,6 @@ valueColumn, customOptions ) - - const getSafeDefaultValue = value => { - if (value == null || value === "") { - return [] - } - if (!Array.isArray(value)) { - return [value] - } - return value - } Date: Tue, 18 Jan 2022 15:34:10 +0000 Subject: [PATCH 6/7] Fix data fetch for nested providers, JSON arrays or array fields not working --- packages/client/src/utils/fetch/DataFetch.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/client/src/utils/fetch/DataFetch.js b/packages/client/src/utils/fetch/DataFetch.js index 2333991ac9..90808026d5 100644 --- a/packages/client/src/utils/fetch/DataFetch.js +++ b/packages/client/src/utils/fetch/DataFetch.js @@ -111,12 +111,6 @@ export default class DataFetch { */ async getInitialData() { const { datasource, filter, sortColumn, paginate } = this.options - const tableId = datasource?.tableId - - // Ensure table ID exists - if (!tableId) { - return - } // Fetch datasource definition and determine feature flags const definition = await this.constructor.getDefinition(datasource) From 358aed6d4f0a8056cea4fd07926657247cd0c66d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 20 Jan 2022 19:37:01 +0000 Subject: [PATCH 7/7] Update {{ now }} HBS helper to floor to the second instead of millisecond --- packages/string-templates/src/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 820b8da290..d824d5f1db 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -112,9 +112,10 @@ module.exports.processStringSync = (string, context, opts) => { const template = instance.compile(string, { strict: false, }) + const now = Math.floor(Date.now() / 1000) * 1000 return processors.postprocess( template({ - now: new Date().toISOString(), + now: new Date(now).toISOString(), ...context, }) )