From f1b9490f224d2229760a5db13a94a51c1e91d8dd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Jan 2022 17:55:28 +0000 Subject: [PATCH 01/25] Adding method for client/builder to detect that a table supports SQL filtering. --- packages/server/src/api/controllers/table/index.js | 3 ++- packages/server/src/api/controllers/table/utils.js | 5 ++++- packages/server/src/integrations/utils.ts | 11 ++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js index abbb4d6ff9..15e869fc75 100644 --- a/packages/server/src/api/controllers/table/index.js +++ b/packages/server/src/api/controllers/table/index.js @@ -2,7 +2,7 @@ const CouchDB = require("../../../db") const internal = require("./internal") const external = require("./external") const csvParser = require("../../../utilities/csvParser") -const { isExternalTable } = require("../../../integrations/utils") +const { isExternalTable, isSQL } = require("../../../integrations/utils") const { getTableParams, getDatasourceParams, @@ -49,6 +49,7 @@ exports.fetch = async function (ctx) { ...entity, type: "external", sourceId: row.doc._id, + sql: isSQL(row.doc), })) }) diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index e4086e8071..459566ce91 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -12,6 +12,7 @@ const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants") const { isExternalTable, breakExternalTableId, + isSQL, } = require("../../../integrations/utils") const { getViews, saveView } = require("../view/utils") const viewTemplate = require("../view/viewBuilder") @@ -242,7 +243,9 @@ exports.getTable = async (appId, tableId) => { const db = new CouchDB(appId) if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) - return exports.getExternalTable(appId, datasourceId, tableName) + const datasource = await db.get(datasourceId) + const table = await exports.getExternalTable(appId, datasourceId, tableName) + return { ...table, sql: isSQL(datasource) } } else { return db.get(tableId) } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 97380b1b5b..b1b1faf776 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,6 +1,6 @@ -import { SqlQuery } from "../definitions/datasource" +import { SourceNames, SqlQuery } from "../definitions/datasource" import { Datasource, Table } from "../definitions/common" -import { SourceNames } from "../definitions/datasource" + const { DocumentTypes, SEPARATOR } = require("../db/utils") const { FieldTypes, @@ -131,7 +131,12 @@ export function isSQL(datasource: Datasource): boolean { if (!datasource || !datasource.source) { return false } - const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL] + const SQL = [ + SourceNames.POSTGRES, + SourceNames.SQL_SERVER, + SourceNames.MYSQL, + SourceNames.ORACLE, + ] return SQL.indexOf(datasource.source) !== -1 } From 13e9929ca39ae912336ad9359538778d86f7f275 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 13 Jan 2022 16:09:07 +0100 Subject: [PATCH 02/25] v1.0.32 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 6943b316f3..d0dec72bf6 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.27-alpha.16", + "version": "1.0.32", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 4d9d87a1a7..7151f831ee 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index b59fdc16bc..bb6b5f7ed2 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 65cfbb4617..07307aa3ef 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.27-alpha.16", - "@budibase/client": "^1.0.27-alpha.16", + "@budibase/bbui": "^1.0.32", + "@budibase/client": "^1.0.32", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.27-alpha.16", + "@budibase/string-templates": "^1.0.32", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index a91ae0f25c..7480d9bf78 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 7e65d983cd..5cae36f95d 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.27-alpha.16", + "@budibase/bbui": "^1.0.32", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.27-alpha.16", + "@budibase/string-templates": "^1.0.32", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 040da37cfa..404c78d62e 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.27-alpha.16", - "@budibase/client": "^1.0.27-alpha.16", - "@budibase/string-templates": "^1.0.27-alpha.16", + "@budibase/backend-core": "^1.0.32", + "@budibase/client": "^1.0.32", + "@budibase/string-templates": "^1.0.32", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 31643f6ea3..ea822e7c42 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 102097eacf..ff8090fd41 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.27-alpha.16", + "version": "1.0.32", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.27-alpha.16", - "@budibase/string-templates": "^1.0.27-alpha.16", + "@budibase/backend-core": "^1.0.32", + "@budibase/string-templates": "^1.0.32", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From 0329fcb49e06de34e279c5ede44b1875fcfd4cb3 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 13 Jan 2022 16:10:07 +0100 Subject: [PATCH 03/25] v1.0.38 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index d0dec72bf6..1a6a12d1ea 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.32", + "version": "1.0.38", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 7151f831ee..57a71b139d 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.32", + "version": "1.0.38", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index bb6b5f7ed2..d894aea045 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.32", + "version": "1.0.38", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 07307aa3ef..7a528ca3c3 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.32", + "version": "1.0.38", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.32", - "@budibase/client": "^1.0.32", + "@budibase/bbui": "^1.0.38", + "@budibase/client": "^1.0.38", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.32", + "@budibase/string-templates": "^1.0.38", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 7480d9bf78..05f687797d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.32", + "version": "1.0.38", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 5cae36f95d..cae75a159a 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.32", + "version": "1.0.38", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.32", + "@budibase/bbui": "^1.0.38", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.32", + "@budibase/string-templates": "^1.0.38", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 404c78d62e..4b7f8a2108 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.32", + "version": "1.0.38", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.32", - "@budibase/client": "^1.0.32", - "@budibase/string-templates": "^1.0.32", + "@budibase/backend-core": "^1.0.38", + "@budibase/client": "^1.0.38", + "@budibase/string-templates": "^1.0.38", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index ea822e7c42..15b3d59739 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.32", + "version": "1.0.38", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index ff8090fd41..bfd4ea185f 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.32", + "version": "1.0.38", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.32", - "@budibase/string-templates": "^1.0.32", + "@budibase/backend-core": "^1.0.38", + "@budibase/string-templates": "^1.0.38", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From 03ff594973b5967ab170037db4decec069cab5e4 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 13 Jan 2022 15:17:06 +0000 Subject: [PATCH 04/25] v1.0.39 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 1a6a12d1ea..dcd409e116 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.38", + "version": "1.0.39", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 57a71b139d..22f819d83f 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.38", + "version": "1.0.39", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index d894aea045..bfb92cb2ef 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.38", + "version": "1.0.39", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 7a528ca3c3..8f3b64d347 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.38", + "version": "1.0.39", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.38", - "@budibase/client": "^1.0.38", + "@budibase/bbui": "^1.0.39", + "@budibase/client": "^1.0.39", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.38", + "@budibase/string-templates": "^1.0.39", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 05f687797d..ce5c98f5ab 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.38", + "version": "1.0.39", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index cae75a159a..dd33c49f88 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.38", + "version": "1.0.39", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.38", + "@budibase/bbui": "^1.0.39", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.38", + "@budibase/string-templates": "^1.0.39", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 4b7f8a2108..684e4f01c3 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.38", + "version": "1.0.39", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.38", - "@budibase/client": "^1.0.38", - "@budibase/string-templates": "^1.0.38", + "@budibase/backend-core": "^1.0.39", + "@budibase/client": "^1.0.39", + "@budibase/string-templates": "^1.0.39", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 15b3d59739..95215e355c 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.38", + "version": "1.0.39", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index bfd4ea185f..20b00ae1ff 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.38", + "version": "1.0.39", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.38", - "@budibase/string-templates": "^1.0.38", + "@budibase/backend-core": "^1.0.39", + "@budibase/string-templates": "^1.0.39", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From 90859cfe18e4ef64cd7e439bafec8fa5f555bb7a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jan 2022 15:59:04 +0000 Subject: [PATCH 05/25] Basic frontend work, getting it sending up search fields from tables with links. --- .../FilterEditor/FilterDrawer.svelte | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index d9425c961d..31a7307911 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -14,6 +14,7 @@ import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { generate } from "shortid" import { getValidOperatorsForType, OperatorOptions } from "constants/lucene" + import { tables } from "stores/backend" export let schemaFields export let filters = [] @@ -23,9 +24,34 @@ const BannedTypes = ["link", "attachment", "formula", "json", "jsonarray"] - $: fieldOptions = (schemaFields ?? []) - .filter(field => !BannedTypes.includes(field.type)) - .map(field => field.name) + function getTableFields(linkField) { + const table = $tables.list.find(table => table._id === linkField.tableId) + if (!table) { + return [] + } + const linkFields = getFields(Object.values(table.schema), { + allowLinks: false, + }) + return linkFields.map(field => ({ + ...field, + name: `${linkField.name}.${field.name}`, + })) + } + + function getFields(fields, { allowLinks } = { allowLinks: true }) { + let fieldNames = fields.filter(field => !BannedTypes.includes(field.type)) + if (allowLinks) { + const linkFields = fields.filter(field => field.type === "link") + for (let linkField of linkFields) { + // only allow one depth of SQL relationship filtering + fieldNames = fieldNames.concat(getTableFields(linkField)) + } + } + return fieldNames + } + + $: enrichedSchemaFields = getFields(schemaFields || []) + $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] const addFilter = () => { @@ -53,7 +79,7 @@ const onFieldChange = (expression, field) => { // Update the field type - expression.type = schemaFields.find(x => x.name === field)?.type + expression.type = enrichedSchemaFields.find(x => x.name === field)?.type // Ensure a valid operator is set const validOperators = getValidOperatorsForType(expression.type).map( @@ -85,7 +111,7 @@ } const getFieldOptions = field => { - const schema = schemaFields.find(x => x.name === field) + const schema = enrichedSchemaFields.find(x => x.name === field) return schema?.constraints?.inclusion || [] } From 41b9c942372bd372e392f3b8b430387e1ad4edaf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jan 2022 16:48:19 +0000 Subject: [PATCH 06/25] Quick fix, only allow using SQL. --- .../PropertyControls/FilterEditor/FilterDrawer.svelte | 2 +- packages/server/src/integrations/utils.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index 31a7307911..a1cdc48806 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -26,7 +26,7 @@ function getTableFields(linkField) { const table = $tables.list.find(table => table._id === linkField.tableId) - if (!table) { + if (!table || !table.sql) { return [] } const linkFields = getFields(Object.values(table.schema), { diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index c0c5877c7a..46bec0e33e 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,6 +1,5 @@ import { SourceNames, SqlQuery } from "../definitions/datasource" import { Datasource, Table } from "../definitions/common" -import { SourceNames } from "../definitions/datasource" import { DocumentTypes, SEPARATOR } from "../db/utils" import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants" From 4433ebc1f274b998ef17da5c89c8ebe29b680126 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 13 Jan 2022 17:24:52 +0000 Subject: [PATCH 07/25] Handle 403 when loading templates --- packages/builder/src/builderStore/api.js | 7 +++++++ packages/server/src/integrations/s3.ts | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/builderStore/api.js b/packages/builder/src/builderStore/api.js index 4bcb9b74c6..62ddfaeaa5 100644 --- a/packages/builder/src/builderStore/api.js +++ b/packages/builder/src/builderStore/api.js @@ -1,6 +1,7 @@ import { store } from "./index" import { get as svelteGet } from "svelte/store" import { removeCookie, Cookies } from "./cookies" +import { notifications } from "@budibase/bbui" const apiCall = method => @@ -13,6 +14,12 @@ const apiCall = headers, }) if (resp.status === 403) { + if (url.includes("/api/templates")) { + notifications.error( + "There was a problem loading quick start templates." + ) + return { json: () => [] } + } removeCookie(Cookies.Auth) // reload after removing cookie, go to login if (!url.includes("self") && !url.includes("login")) { diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 25b439fd58..273f221575 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -38,7 +38,7 @@ module S3Module { signatureVersion: { type: "string", required: false, - default: "v4" + default: "v4", }, }, query: { From 479220652ebd15e2ae17c2169459ac4e0759e5fe Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 13 Jan 2022 17:40:11 +0000 Subject: [PATCH 08/25] Adjusting SQL system to handle relationship filtering. --- packages/server/src/integrations/base/sql.ts | 21 +++++++++++++------- packages/server/src/integrations/s3.ts | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index df4f2d511b..c1641e8626 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -72,16 +72,22 @@ class InternalBuilder { // right now we only do filters on the specific table being queried addFilters( - tableName: string, query: KnexQuery, - filters: SearchFilters | undefined + filters: SearchFilters | undefined, + opts: { relationship?: boolean; tableName?: string } ): KnexQuery { function iterate( structure: { [key: string]: any }, fn: (key: string, value: any) => void ) { for (let [key, value] of Object.entries(structure)) { - fn(`${tableName}.${key}`, value) + const isRelationshipField = key.includes(".") + if (!opts.relationship && !isRelationshipField) { + fn(`${opts.tableName}.${key}`, value) + } + if (opts.relationship && isRelationshipField) { + fn(key, value) + } } } if (!filters) { @@ -272,7 +278,7 @@ class InternalBuilder { if (foundOffset) { query = query.offset(foundOffset) } - query = this.addFilters(tableName, query, filters) + query = this.addFilters(query, filters, { tableName }) // add sorting to pre-query query = this.addSorting(query, json) // @ts-ignore @@ -285,20 +291,21 @@ class InternalBuilder { preQuery = this.addSorting(preQuery, json) } // handle joins - return this.addRelationships( + query = this.addRelationships( knex, preQuery, selectStatement, tableName, relationships ) + return this.addFilters(query, filters, { relationship: true }) } update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { const { endpoint, body, filters } = json let query: KnexQuery = knex(endpoint.entityId) const parsedBody = parseBody(body) - query = this.addFilters(endpoint.entityId, query, filters) + query = this.addFilters(query, filters, { tableName: endpoint.entityId }) // mysql can't use returning if (opts.disableReturning) { return query.update(parsedBody) @@ -310,7 +317,7 @@ class InternalBuilder { delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { const { endpoint, filters } = json let query: KnexQuery = knex(endpoint.entityId) - query = this.addFilters(endpoint.entityId, query, filters) + query = this.addFilters(query, filters, { tableName: endpoint.entityId }) // mysql can't use returning if (opts.disableReturning) { return query.delete() diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 25b439fd58..273f221575 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -38,7 +38,7 @@ module S3Module { signatureVersion: { type: "string", required: false, - default: "v4" + default: "v4", }, }, query: { From d2e13da831b9d02de60b435baf7af2dafef092af Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 13 Jan 2022 17:46:57 +0000 Subject: [PATCH 09/25] Test fix --- packages/builder/src/builderStore/api.js | 4 ---- packages/builder/src/pages/builder/portal/apps/index.svelte | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/builderStore/api.js b/packages/builder/src/builderStore/api.js index 62ddfaeaa5..897d3a74db 100644 --- a/packages/builder/src/builderStore/api.js +++ b/packages/builder/src/builderStore/api.js @@ -1,7 +1,6 @@ import { store } from "./index" import { get as svelteGet } from "svelte/store" import { removeCookie, Cookies } from "./cookies" -import { notifications } from "@budibase/bbui" const apiCall = method => @@ -15,9 +14,6 @@ const apiCall = }) if (resp.status === 403) { if (url.includes("/api/templates")) { - notifications.error( - "There was a problem loading quick start templates." - ) return { json: () => [] } } removeCookie(Cookies.Auth) diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte index c85ad79d45..ac10b5317f 100644 --- a/packages/builder/src/pages/builder/portal/apps/index.svelte +++ b/packages/builder/src/pages/builder/portal/apps/index.svelte @@ -274,6 +274,9 @@ onMount(async () => { await apps.load() await templates.load() + if ($templates?.length === 0) { + notifications.error("There was a problem loading quick start templates.") + } // if the portal is loaded from an external URL with a template param const initInfo = await auth.getInitInfo() if (initInfo?.init_template) { From 5256a1cf9f557f4031a45cd968e02248dcf66c14 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Thu, 13 Jan 2022 18:35:02 +0000 Subject: [PATCH 10/25] v1.0.40 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index dcd409e116..ec3ffdbe77 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.39", + "version": "1.0.40", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 22f819d83f..5b1e0493b7 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.39", + "version": "1.0.40", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index bfb92cb2ef..519d3a3763 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.39", + "version": "1.0.40", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 8f3b64d347..54e4666a30 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.39", + "version": "1.0.40", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.39", - "@budibase/client": "^1.0.39", + "@budibase/bbui": "^1.0.40", + "@budibase/client": "^1.0.40", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.39", + "@budibase/string-templates": "^1.0.40", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index ce5c98f5ab..4c36f54eb2 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.39", + "version": "1.0.40", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index dd33c49f88..6775a5402f 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.39", + "version": "1.0.40", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.39", + "@budibase/bbui": "^1.0.40", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.39", + "@budibase/string-templates": "^1.0.40", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 684e4f01c3..795590bb11 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.39", + "version": "1.0.40", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.39", - "@budibase/client": "^1.0.39", - "@budibase/string-templates": "^1.0.39", + "@budibase/backend-core": "^1.0.40", + "@budibase/client": "^1.0.40", + "@budibase/string-templates": "^1.0.40", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 95215e355c..fd6e027b86 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.39", + "version": "1.0.40", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 20b00ae1ff..083c82489c 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.39", + "version": "1.0.40", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.39", - "@budibase/string-templates": "^1.0.39", + "@budibase/backend-core": "^1.0.40", + "@budibase/string-templates": "^1.0.40", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From 6a3882cb1f5c6ac3dda08da62e4fff261096fa60 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 13 Jan 2022 20:47:16 +0100 Subject: [PATCH 11/25] sameSite and secure cookie settings --- packages/backend-core/src/environment.js | 8 ++++++++ packages/backend-core/src/utils.js | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/packages/backend-core/src/environment.js b/packages/backend-core/src/environment.js index c26ad1c199..3db59ab321 100644 --- a/packages/backend-core/src/environment.js +++ b/packages/backend-core/src/environment.js @@ -6,6 +6,13 @@ function isTest() { ) } +function isDev() { + return ( + process.env.NODE_ENV !== "production" && + process.env.BUDIBASE_ENVIRONMENT !== "production" + ) +} + module.exports = { JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL, @@ -27,6 +34,7 @@ module.exports = { COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, PLATFORM_URL: process.env.PLATFORM_URL, isTest, + isDev, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index 8c00f2a8b8..37193885f1 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -23,6 +23,7 @@ const { getUserSessions, invalidateSessions } = require("./security/sessions") const { migrateIfRequired } = require("./migrations") const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS +const { isDev, isTest } = require("./environment") const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -108,6 +109,11 @@ exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => { overwrite: true, } + if (!isDev() && !isTest()) { + config.sameSite = "none" + config.secure = true + } + if (environment.COOKIE_DOMAIN) { config.domain = environment.COOKIE_DOMAIN } From 95c4edad08303d5715c164ff5223e91282e3a7de Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Fri, 14 Jan 2022 09:39:56 +0000 Subject: [PATCH 12/25] v1.0.41 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index ec3ffdbe77..9c19e33343 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.40", + "version": "1.0.41", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 5b1e0493b7..03b215be56 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.40", + "version": "1.0.41", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 519d3a3763..02a1f3fdb5 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.40", + "version": "1.0.41", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 54e4666a30..555bb0701f 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.40", + "version": "1.0.41", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.40", - "@budibase/client": "^1.0.40", + "@budibase/bbui": "^1.0.41", + "@budibase/client": "^1.0.41", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.40", + "@budibase/string-templates": "^1.0.41", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 4c36f54eb2..bb8c650e51 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.40", + "version": "1.0.41", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 6775a5402f..5679e723e5 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.40", + "version": "1.0.41", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.40", + "@budibase/bbui": "^1.0.41", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.40", + "@budibase/string-templates": "^1.0.41", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 795590bb11..7e50569440 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.40", + "version": "1.0.41", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.40", - "@budibase/client": "^1.0.40", - "@budibase/string-templates": "^1.0.40", + "@budibase/backend-core": "^1.0.41", + "@budibase/client": "^1.0.41", + "@budibase/string-templates": "^1.0.41", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index fd6e027b86..a5d387ee8a 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.40", + "version": "1.0.41", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 083c82489c..48301e40dc 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.40", + "version": "1.0.41", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.40", - "@budibase/string-templates": "^1.0.40", + "@budibase/backend-core": "^1.0.41", + "@budibase/string-templates": "^1.0.41", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From aae2dc86d32abb72425a0976234d273b06a922f3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jan 2022 17:42:14 +0000 Subject: [PATCH 13/25] Getting client partially working, having an issue with search fields not updating for a table block. --- .../FilterEditor/FilterDrawer.svelte | 30 +---------- .../PropertyControls/SearchFieldSelect.svelte | 52 +++++++++++++++++++ .../PropertyControls/componentSettings.js | 2 + .../builder/src/constants/backend/index.js | 8 +++ packages/builder/src/helpers/searchFields.js | 31 +++++++++++ packages/client/manifest.json | 2 +- .../components/app/blocks/TableBlock.svelte | 44 +++++++++++++--- packages/string-templates/test/basic.spec.js | 11 ++++ 8 files changed, 144 insertions(+), 36 deletions(-) create mode 100644 packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte create mode 100644 packages/builder/src/helpers/searchFields.js diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index a1cdc48806..8d7f50a527 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -14,7 +14,7 @@ import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { generate } from "shortid" import { getValidOperatorsForType, OperatorOptions } from "constants/lucene" - import { tables } from "stores/backend" + import { getFields } from "helpers/searchFields" export let schemaFields export let filters = [] @@ -22,34 +22,6 @@ export let panel = ClientBindingPanel export let allowBindings = true - const BannedTypes = ["link", "attachment", "formula", "json", "jsonarray"] - - function getTableFields(linkField) { - const table = $tables.list.find(table => table._id === linkField.tableId) - if (!table || !table.sql) { - return [] - } - const linkFields = getFields(Object.values(table.schema), { - allowLinks: false, - }) - return linkFields.map(field => ({ - ...field, - name: `${linkField.name}.${field.name}`, - })) - } - - function getFields(fields, { allowLinks } = { allowLinks: true }) { - let fieldNames = fields.filter(field => !BannedTypes.includes(field.type)) - if (allowLinks) { - const linkFields = fields.filter(field => field.type === "link") - for (let linkField of linkFields) { - // only allow one depth of SQL relationship filtering - fieldNames = fieldNames.concat(getTableFields(linkField)) - } - } - return fieldNames - } - $: enrichedSchemaFields = getFields(schemaFields || []) $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte new file mode 100644 index 0000000000..ad8fad8a87 --- /dev/null +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte @@ -0,0 +1,52 @@ + + + diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js index e752240302..5e27cdce28 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js @@ -7,6 +7,7 @@ import ColorPicker from "./ColorPicker.svelte" import { IconSelect } from "./IconSelect" import FieldSelect from "./FieldSelect.svelte" import MultiFieldSelect from "./MultiFieldSelect.svelte" +import SearchFieldSelect from "./SearchFieldSelect.svelte" import SchemaSelect from "./SchemaSelect.svelte" import SectionSelect from "./SectionSelect.svelte" import NavigationEditor from "./NavigationEditor/NavigationEditor.svelte" @@ -30,6 +31,7 @@ const componentMap = { icon: IconSelect, field: FieldSelect, multifield: MultiFieldSelect, + searchfield: SearchFieldSelect, options: OptionsEditor, schema: SchemaSelect, section: SectionSelect, diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index c5fb294f80..c1eea5b0ef 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -229,3 +229,11 @@ export const PaginationLocations = [ { label: "Query parameters", value: "query" }, { label: "Request body", value: "body" }, ] + +export const BannedSearchTypes = [ + "link", + "attachment", + "formula", + "json", + "jsonarray", +] diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js new file mode 100644 index 0000000000..3f3ca1ce7e --- /dev/null +++ b/packages/builder/src/helpers/searchFields.js @@ -0,0 +1,31 @@ +import { tables } from "../stores/backend" +import { BannedSearchTypes } from "../constants/backend" +import { get } from "svelte/store" + +export function getTableFields(linkField) { + const table = get(tables).list.find(table => table._id === linkField.tableId) + if (!table || !table.sql) { + return [] + } + const linkFields = getFields(Object.values(table.schema), { + allowLinks: false, + }) + return linkFields.map(field => ({ + ...field, + name: `${table.name}.${field.name}`, + })) +} + +export function getFields(fields, { allowLinks } = { allowLinks: true }) { + let fieldNames = fields.filter( + field => !BannedSearchTypes.includes(field.type) + ) + if (allowLinks) { + const linkFields = fields.filter(field => field.type === "link") + for (let linkField of linkFields) { + // only allow one depth of SQL relationship filtering + fieldNames = fieldNames.concat(getTableFields(linkField)) + } + } + return fieldNames +} diff --git a/packages/client/manifest.json b/packages/client/manifest.json index dfe9ae5a91..310cec7f4b 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -2811,7 +2811,7 @@ "key": "dataSource" }, { - "type": "multifield", + "type": "searchfield", "label": "Search Columns", "key": "searchColumns", "placeholder": "Choose search columns" diff --git a/packages/client/src/components/app/blocks/TableBlock.svelte b/packages/client/src/components/app/blocks/TableBlock.svelte index 936a8d1734..fef6be5c50 100644 --- a/packages/client/src/components/app/blocks/TableBlock.svelte +++ b/packages/client/src/components/app/blocks/TableBlock.svelte @@ -41,9 +41,11 @@ let dataProviderId let schema let schemaLoaded = false + let enrichedSearchColumns + let enrichedSearchColumnsLoaded = false $: fetchSchema(dataSource) - $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema) + $: enrichSearchColumns(searchColumns, schema) $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId) $: titleButtonAction = [ { @@ -71,9 +73,9 @@ } // Determine data types for search fields and only use those that are valid - const enrichSearchColumns = (searchColumns, schema) => { + const enrichSearchColumns = async (searchColumns, schema) => { let enrichedColumns = [] - searchColumns?.forEach(column => { + const addType = column => { const schemaType = schema?.[column]?.type const componentType = schemaComponentMap[schemaType] if (componentType) { @@ -82,9 +84,39 @@ componentType, type: schemaType, }) + return true } - }) - return enrichedColumns.slice(0, 3) + return false + } + for (let column of searchColumns || []) { + // if addType returns false, it didn't find one, look for SQL relationships + if (!addType(column) && column.includes(".")) { + const [tableName, linkColumn] = column.split(".") + for (let colSchema of Object.values(schema || {})) { + // found the related table + if ( + colSchema.type === "link" && + colSchema.tableId && + colSchema.tableId.endsWith(tableName) + ) { + try { + const linkSchema = await fetchDatasourceSchema({ + ...dataSource, + tableId: colSchema.tableId, + }) + if (linkSchema) { + schema[column] = linkSchema[linkColumn] + addType(column) + } + } catch (err) { + // ignore the error, couldn't get table + } + } + } + } + } + enrichedSearchColumns = enrichedColumns.slice(0, 3) + enrichedSearchColumnsLoaded = true } // Load the datasource schema so we can determine column types @@ -96,7 +128,7 @@ } -{#if schemaLoaded} +{#if schemaLoaded && enrichedSearchColumnsLoaded}
diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.js index b437e386fc..2fd6505410 100644 --- a/packages/string-templates/test/basic.spec.js +++ b/packages/string-templates/test/basic.spec.js @@ -146,3 +146,14 @@ describe("check manifest", () => { ) }) }) + +describe("check full stops that are safe", () => { + it("should allow using an escaped full stop", async () => { + const data = { + "c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI": { "persons.firstname": "1" } + } + const template = "{{ [c53a4a604fa754d33baaafd5bca4d3658-YXuUBqt5vI].[persons.firstname] }}" + const output = await processString(template, data) + expect(output).toEqual("1") + }) +}) From c39bbf46fb0ecd1e3a4b6b9ddef401ce669ca357 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 17 Jan 2022 09:39:22 +0100 Subject: [PATCH 14/25] Revert "sameSite and secure cookie settings" --- packages/backend-core/src/environment.js | 8 -------- packages/backend-core/src/utils.js | 6 ------ 2 files changed, 14 deletions(-) diff --git a/packages/backend-core/src/environment.js b/packages/backend-core/src/environment.js index 3db59ab321..c26ad1c199 100644 --- a/packages/backend-core/src/environment.js +++ b/packages/backend-core/src/environment.js @@ -6,13 +6,6 @@ function isTest() { ) } -function isDev() { - return ( - process.env.NODE_ENV !== "production" && - process.env.BUDIBASE_ENVIRONMENT !== "production" - ) -} - module.exports = { JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL, @@ -34,7 +27,6 @@ module.exports = { COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, PLATFORM_URL: process.env.PLATFORM_URL, isTest, - isDev, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index 37193885f1..8c00f2a8b8 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -23,7 +23,6 @@ const { getUserSessions, invalidateSessions } = require("./security/sessions") const { migrateIfRequired } = require("./migrations") const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS -const { isDev, isTest } = require("./environment") const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -109,11 +108,6 @@ exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => { overwrite: true, } - if (!isDev() && !isTest()) { - config.sameSite = "none" - config.secure = true - } - if (environment.COOKIE_DOMAIN) { config.domain = environment.COOKIE_DOMAIN } From 2a8cbb8805983154b23d59ba0f453379b0cbb1fb Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 17 Jan 2022 08:48:49 +0000 Subject: [PATCH 15/25] v1.0.42 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 9c19e33343..960c7fb249 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.41", + "version": "1.0.42", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 03b215be56..ef6e7354c5 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.41", + "version": "1.0.42", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 02a1f3fdb5..3e0c504a9a 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.41", + "version": "1.0.42", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 555bb0701f..07c3299066 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.41", + "version": "1.0.42", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.41", - "@budibase/client": "^1.0.41", + "@budibase/bbui": "^1.0.42", + "@budibase/client": "^1.0.42", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.41", + "@budibase/string-templates": "^1.0.42", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index bb8c650e51..ee01a63b9d 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.41", + "version": "1.0.42", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 5679e723e5..c66e260ff2 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.41", + "version": "1.0.42", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.41", + "@budibase/bbui": "^1.0.42", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.41", + "@budibase/string-templates": "^1.0.42", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 7e50569440..df04080f5c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.41", + "version": "1.0.42", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.41", - "@budibase/client": "^1.0.41", - "@budibase/string-templates": "^1.0.41", + "@budibase/backend-core": "^1.0.42", + "@budibase/client": "^1.0.42", + "@budibase/string-templates": "^1.0.42", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index a5d387ee8a..48b232db77 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.41", + "version": "1.0.42", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 48301e40dc..85f5e274e5 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.41", + "version": "1.0.42", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.41", - "@budibase/string-templates": "^1.0.41", + "@budibase/backend-core": "^1.0.42", + "@budibase/string-templates": "^1.0.42", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From 0502e62e1caf68873759dc3d23e71d5c428c2d5d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 17 Jan 2022 14:03:47 +0000 Subject: [PATCH 16/25] Add optional enrichment of relationship fields when determining datasource schema and update block filters to properly reference relationship fields --- packages/client/manifest.json | 2 +- .../components/app/blocks/CardsBlock.svelte | 7 ++- .../components/app/blocks/TableBlock.svelte | 51 ++++--------------- packages/client/src/utils/fetch/DataFetch.js | 5 +- packages/client/src/utils/schema.js | 35 +++++++++++-- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 310cec7f4b..9431129fa0 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -2958,7 +2958,7 @@ "key": "dataSource" }, { - "type": "multifield", + "type": "searchfield", "label": "Search Columns", "key": "searchColumns", "placeholder": "Choose search columns" diff --git a/packages/client/src/components/app/blocks/CardsBlock.svelte b/packages/client/src/components/app/blocks/CardsBlock.svelte index 301b440ab3..f0892ca447 100644 --- a/packages/client/src/components/app/blocks/CardsBlock.svelte +++ b/packages/client/src/components/app/blocks/CardsBlock.svelte @@ -71,12 +71,13 @@ const enrichFilter = (filter, columns, formId) => { let enrichedFilter = [...(filter || [])] columns?.forEach(column => { + const safePath = column.name.split(".").map(safe).join(".") enrichedFilter.push({ field: column.name, operator: column.type === "string" ? "string" : "equal", type: column.type === "string" ? "string" : "number", valueType: "Binding", - value: `{{ [${formId}].[${column.name}] }}`, + value: `{{ ${safe(formId)}.${safePath} }}`, }) }) return enrichedFilter @@ -112,7 +113,9 @@ // Load the datasource schema so we can determine column types const fetchSchema = async dataSource => { if (dataSource) { - schema = await fetchDatasourceSchema(dataSource) + schema = await fetchDatasourceSchema(dataSource, { + enrichRelationships: true, + }) } schemaLoaded = true } diff --git a/packages/client/src/components/app/blocks/TableBlock.svelte b/packages/client/src/components/app/blocks/TableBlock.svelte index fef6be5c50..3de4497731 100644 --- a/packages/client/src/components/app/blocks/TableBlock.svelte +++ b/packages/client/src/components/app/blocks/TableBlock.svelte @@ -41,11 +41,9 @@ let dataProviderId let schema let schemaLoaded = false - let enrichedSearchColumns - let enrichedSearchColumnsLoaded = false $: fetchSchema(dataSource) - $: enrichSearchColumns(searchColumns, schema) + $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema) $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId) $: titleButtonAction = [ { @@ -61,21 +59,22 @@ const enrichFilter = (filter, columns, formId) => { let enrichedFilter = [...(filter || [])] columns?.forEach(column => { + const safePath = column.name.split(".").map(safe).join(".") enrichedFilter.push({ field: column.name, operator: column.type === "string" ? "string" : "equal", type: column.type === "string" ? "string" : "number", valueType: "Binding", - value: `{{ ${safe(formId)}.${safe(column.name)} }}`, + value: `{{ ${safe(formId)}.${safePath} }}`, }) }) return enrichedFilter } // Determine data types for search fields and only use those that are valid - const enrichSearchColumns = async (searchColumns, schema) => { + const enrichSearchColumns = (searchColumns, schema) => { let enrichedColumns = [] - const addType = column => { + searchColumns?.forEach(column => { const schemaType = schema?.[column]?.type const componentType = schemaComponentMap[schemaType] if (componentType) { @@ -84,51 +83,23 @@ componentType, type: schemaType, }) - return true } - return false - } - for (let column of searchColumns || []) { - // if addType returns false, it didn't find one, look for SQL relationships - if (!addType(column) && column.includes(".")) { - const [tableName, linkColumn] = column.split(".") - for (let colSchema of Object.values(schema || {})) { - // found the related table - if ( - colSchema.type === "link" && - colSchema.tableId && - colSchema.tableId.endsWith(tableName) - ) { - try { - const linkSchema = await fetchDatasourceSchema({ - ...dataSource, - tableId: colSchema.tableId, - }) - if (linkSchema) { - schema[column] = linkSchema[linkColumn] - addType(column) - } - } catch (err) { - // ignore the error, couldn't get table - } - } - } - } - } - enrichedSearchColumns = enrichedColumns.slice(0, 3) - enrichedSearchColumnsLoaded = true + }) + return enrichedColumns.slice(0, 3) } // Load the datasource schema so we can determine column types const fetchSchema = async dataSource => { if (dataSource) { - schema = await fetchDatasourceSchema(dataSource) + schema = await fetchDatasourceSchema(dataSource, { + enrichRelationships: true, + }) } schemaLoaded = true } -{#if schemaLoaded && enrichedSearchColumnsLoaded} +{#if schemaLoaded}
diff --git a/packages/client/src/utils/fetch/DataFetch.js b/packages/client/src/utils/fetch/DataFetch.js index 2333991ac9..884e12feb1 100644 --- a/packages/client/src/utils/fetch/DataFetch.js +++ b/packages/client/src/utils/fetch/DataFetch.js @@ -67,7 +67,6 @@ export default class DataFetch { this.getPage = this.getPage.bind(this) this.getInitialData = this.getInitialData.bind(this) this.determineFeatureFlags = this.determineFeatureFlags.bind(this) - this.enrichSchema = this.enrichSchema.bind(this) this.refresh = this.refresh.bind(this) this.update = this.update.bind(this) this.hasNextPage = this.hasNextPage.bind(this) @@ -129,7 +128,7 @@ export default class DataFetch { // Fetch and enrich schema let schema = this.constructor.getSchema(datasource, definition) - schema = this.enrichSchema(schema) + schema = DataFetch.enrichSchema(schema) if (!schema) { return } @@ -248,7 +247,7 @@ export default class DataFetch { * @param schema the datasource schema * @return {object} the enriched datasource schema */ - enrichSchema(schema) { + static enrichSchema(schema) { if (schema == null) { return null } diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index e333bb616e..dc00430fcd 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -6,13 +6,19 @@ import RelationshipFetch from "./fetch/RelationshipFetch.js" import NestedProviderFetch from "./fetch/NestedProviderFetch.js" import FieldFetch from "./fetch/FieldFetch.js" import JSONArrayFetch from "./fetch/JSONArrayFetch.js" +import DataFetch from "./fetch/DataFetch.js" /** * Fetches the schema of any kind of datasource. * All datasource fetch classes implement their own functionality to get the * schema of a datasource of their respective types. + * @param datasource the datasource to fetch the schema for + * @param options options for enriching the schema */ -export const fetchDatasourceSchema = async datasource => { +export const fetchDatasourceSchema = async ( + datasource, + options = { enrichRelationships: false } +) => { const handler = { table: TableFetch, view: ViewFetch, @@ -28,7 +34,7 @@ export const fetchDatasourceSchema = async datasource => { // Get the datasource definition and then schema const definition = await handler.getDefinition(datasource) - const schema = handler.getSchema(datasource, definition) + let schema = handler.getSchema(datasource, definition) if (!schema) { return null } @@ -49,5 +55,28 @@ export const fetchDatasourceSchema = async datasource => { }) } }) - return { ...schema, ...jsonAdditions } + schema = { ...schema, ...jsonAdditions } + + // Check for any relationship fields if required + if (options?.enrichRelationships) { + let relationshipAdditions = {} + for (let fieldKey of Object.keys(schema)) { + const fieldSchema = schema[fieldKey] + if (fieldSchema?.type === "link") { + const linkSchema = await fetchDatasourceSchema({ + type: "table", + tableId: fieldSchema?.tableId, + }) + Object.keys(linkSchema || {}).forEach(linkKey => { + relationshipAdditions[`${fieldKey}.${linkKey}`] = { + type: linkSchema[linkKey].type, + } + }) + } + } + schema = { ...schema, ...relationshipAdditions } + } + + // Ensure schema structure is correct + return DataFetch.enrichSchema(schema) } From 1f78d6dc0d71cd10cfe44779a99a9ed8885da468 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 17 Jan 2022 15:57:31 +0100 Subject: [PATCH 17/25] readding query thread timeout --- packages/server/src/api/controllers/query/index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/query/index.js b/packages/server/src/api/controllers/query/index.js index 4096dd433f..21db1eebbf 100644 --- a/packages/server/src/api/controllers/query/index.js +++ b/packages/server/src/api/controllers/query/index.js @@ -9,8 +9,11 @@ const { Thread, ThreadType } = require("../../../threads") const { save: saveDatasource } = require("../datasource") const { RestImporter } = require("./import") const { invalidateDynamicVariables } = require("../../../threads/utils") +const environment = require("../../../environment") -const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 }) +const Runner = new Thread(ThreadType.QUERY, { + timeoutMs: environment.QUERY_THREAD_TIMEOUT || 10000, +}) // simple function to append "readable" to all read queries function enrichQueries(input) { From e3d8302194dc39b61a37a15c5ede61dfb3cda1fb Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 17 Jan 2022 16:42:48 +0000 Subject: [PATCH 18/25] v1.0.43 --- lerna.json | 2 +- packages/backend-core/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 6 +++--- packages/server/package.json | 8 ++++---- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lerna.json b/lerna.json index 960c7fb249..42925e163b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.42", + "version": "1.0.43", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index ef6e7354c5..ac5288687d 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.42", + "version": "1.0.43", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 3e0c504a9a..af9e518b7b 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.42", + "version": "1.0.43", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index 07c3299066..189497dd73 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.42", + "version": "1.0.43", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.42", - "@budibase/client": "^1.0.42", + "@budibase/bbui": "^1.0.43", + "@budibase/client": "^1.0.43", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.42", + "@budibase/string-templates": "^1.0.43", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index ee01a63b9d..5f3f2786bd 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.42", + "version": "1.0.43", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index c66e260ff2..aeafd8d1c3 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.42", + "version": "1.0.43", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.42", + "@budibase/bbui": "^1.0.43", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.42", + "@budibase/string-templates": "^1.0.43", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index df04080f5c..70904e148f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.42", + "version": "1.0.43", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.42", - "@budibase/client": "^1.0.42", - "@budibase/string-templates": "^1.0.42", + "@budibase/backend-core": "^1.0.43", + "@budibase/client": "^1.0.43", + "@budibase/string-templates": "^1.0.43", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 48b232db77..e0bcd641c8 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.42", + "version": "1.0.43", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 85f5e274e5..9c2238501d 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.42", + "version": "1.0.43", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.42", - "@budibase/string-templates": "^1.0.42", + "@budibase/backend-core": "^1.0.43", + "@budibase/string-templates": "^1.0.43", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0", From cde801d99b128f1dc70c67d31cfa3b3277cf27bf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 17 Jan 2022 16:48:49 +0000 Subject: [PATCH 19/25] Adding dynamic filter capabilities, also updating search field select in builder to make sure it removes banned search field types. --- .../PropertyControls/SearchFieldSelect.svelte | 15 ++---- packages/builder/src/helpers/searchFields.js | 6 +-- .../src/components/app/DataProvider.svelte | 1 + .../app/dynamic-filter/DynamicFilter.svelte | 52 +++++++++++++------ packages/client/src/utils/schema.js | 2 +- 5 files changed, 45 insertions(+), 31 deletions(-) diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte index ad8fad8a87..474fbc676c 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/SearchFieldSelect.svelte @@ -7,7 +7,7 @@ import { currentAsset } from "builderStore" import { tables } from "stores/backend" import { createEventDispatcher } from "svelte" - import { getTableFields } from "helpers/searchFields" + import { getFields } from "helpers/searchFields" export let componentInstance = {} export let value = "" @@ -20,19 +20,14 @@ $: boundValue = getSelectedOption(value, options) function getOptions(ds, dsSchema) { - let base = Object.keys(dsSchema) + let base = Object.values(dsSchema) if (!ds?.tableId) { return base } const currentTable = $tables.list.find(table => table._id === ds.tableId) - if (currentTable && currentTable.sql) { - for (let column of Object.values(currentTable.schema)) { - if (column.type === "link") { - base = base.concat(getTableFields(column).map(field => field.name)) - } - } - } - return base + return getFields(base, { allowLinks: currentTable.sql }).map( + field => field.name + ) } function getSelectedOption(selectedOptions, allOptions) { diff --git a/packages/builder/src/helpers/searchFields.js b/packages/builder/src/helpers/searchFields.js index 3f3ca1ce7e..650e04a680 100644 --- a/packages/builder/src/helpers/searchFields.js +++ b/packages/builder/src/helpers/searchFields.js @@ -17,15 +17,15 @@ export function getTableFields(linkField) { } export function getFields(fields, { allowLinks } = { allowLinks: true }) { - let fieldNames = fields.filter( + let filteredFields = fields.filter( field => !BannedSearchTypes.includes(field.type) ) if (allowLinks) { const linkFields = fields.filter(field => field.type === "link") for (let linkField of linkFields) { // only allow one depth of SQL relationship filtering - fieldNames = fieldNames.concat(getTableFields(linkField)) + filteredFields = filteredFields.concat(getTableFields(linkField)) } } - return fieldNames + return filteredFields } diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index 71c54db4da..2341eef3b2 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -67,6 +67,7 @@ $: dataContext = { rows: $fetch.rows, info: $fetch.info, + datasource: dataSource || {}, schema: $fetch.schema, rowsLength: $fetch.rows.length, diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte index 20909d011c..6a114afe3e 100644 --- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte +++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte @@ -11,11 +11,14 @@ export let size = "M" const component = getContext("component") - const { builderStore, ActionTypes, getAction } = getContext("sdk") + const { builderStore, ActionTypes, getAction, fetchDatasourceSchema } = + getContext("sdk") let modal let tmpFilters = [] let filters = [] + let schemaLoaded = false, + schema $: dataProviderId = dataProvider?.id $: addExtension = getAction( @@ -26,7 +29,7 @@ dataProviderId, ActionTypes.RemoveDataProviderQueryExtension ) - $: schema = dataProvider?.schema + $: fetchSchema(dataProvider || {}) $: schemaFields = getSchemaFields(schema, allowedFields) // Add query extension to data provider @@ -39,7 +42,20 @@ } } - const getSchemaFields = (schema, allowedFields) => { + async function fetchSchema(dataProvider) { + const datasource = dataProvider?.datasource + if (datasource) { + schema = await fetchDatasourceSchema(datasource, { + enrichRelationships: true, + }) + } + schemaLoaded = true + } + + function getSchemaFields(schema, allowedFields) { + if (!schemaLoaded) { + return {} + } let clonedSchema = {} if (!allowedFields?.length) { clonedSchema = schema @@ -68,18 +84,20 @@ }) -