From 71a55eb288efea9dbf5e856616517f87e01862fb Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 26 Jul 2022 10:53:53 +0100 Subject: [PATCH 01/76] Support ObjectId in operator blocks --- packages/server/src/integrations/mongodb.ts | 6 ++-- packages/server/yarn.lock | 35 ++++++++++++--------- packages/worker/yarn.lock | 35 ++++++++++++--------- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 802696ff40..70b4be9e14 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -92,12 +92,12 @@ module MongoDBModule { if (json[field] instanceof Object) { json[field] = self.createObjectIds(json[field]) } - if (field === "_id" && typeof json[field] === "string") { - const id = json["_id"].match( + if ((field === "_id" || field?.startsWith("$")) && typeof json[field] === "string") { + const id = json[field].match( /(?<=objectid\(['"]).*(?=['"]\))/gi )?.[0] if (id) { - json["_id"] = ObjectID.createFromHexString(id) + json[field] = ObjectID.createFromHexString(id) } } } diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 2018d4ec5a..fe229cc137 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1094,12 +1094,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.1.21": - version "1.1.21" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.21.tgz#660e0023f674189b25ca5eabbaff67aebcdf03b8" - integrity sha512-qEf9ggMX6NpVOTe1xfp6NZnbdBV0h0ls9qGGTcMjPakhatB4+3YKSTEDSYYRzY/OLNoKPhcd3aErXuFRR7WKrg== +"@budibase/backend-core@1.1.22-alpha.0": + version "1.1.22-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.22-alpha.0.tgz#2a090a723d35ce4a2e377ac0927fe127213bcbb3" + integrity sha512-77gxcPrjejdqaMaMkbrCS0glYA1jdGo74NpCxdadWx+suU4SMTmOt2jgnEZg20aeKcky2xTpVbjDxIq+Fb2J+g== dependencies: - "@budibase/types" "^1.1.21" + "@budibase/types" "^1.1.22-alpha.0" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -1177,13 +1177,13 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.1.21": - version "1.1.21" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.21.tgz#c9d21bc82740e9407673088dc8451273368943d9" - integrity sha512-aE20qZmVg/sPp4bV++xwdUnp8+c9wq7Df34P9hYvlhklgh+fxd+2RyautOAfMADRMIZIbtP4/gal95bE/KTPNQ== +"@budibase/pro@1.1.22-alpha.0": + version "1.1.22-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.22-alpha.0.tgz#5718188fbc76ce3c3cf3c633c99c6e59f5846310" + integrity sha512-QL5bhT/BJnoKVV5XH5+s3jSCBzV/JGOy8YQObEjZgCrtTWUjdvMrm5pFinflFGriCfoArmvR3Q51FBpNJQfaag== dependencies: - "@budibase/backend-core" "1.1.21" - "@budibase/types" "1.1.21" + "@budibase/backend-core" "1.1.22-alpha.0" + "@budibase/types" "1.1.22-alpha.0" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": @@ -1204,10 +1204,15 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/types@1.1.21", "@budibase/types@^1.1.21": - version "1.1.21" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.21.tgz#4f46cd52f3e52c804e5dba06b0520825da81f84e" - integrity sha512-fR8783evr6SKZggu/QZRgcZfd8SAuG2U+xO8lL0x/pLNZI1vOeTyQXASoPLLzj6uA8bWnPIy8BGd9PK4Mw3XVQ== +"@budibase/types@1.1.22-alpha.0": + version "1.1.22-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.22-alpha.0.tgz#85fb7f37773c710e8232c95104095c26a8dd22ca" + integrity sha512-1gRwAtjEl7Ug1jrYwD9Iudbfgs37nndEBEB6yVdNPKA5SpjG+Fwx30zp6R961zlx1vsSu4iMdwM8IbMsCM8p1g== + +"@budibase/types@^1.1.22-alpha.0": + version "1.1.22" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.22.tgz#a73a8315ad6a04bf0709f2b51cb35058442ce723" + integrity sha512-24sun/hZ2OJZdavhiJt+S8Aip+RdlKeTyUrkf7OuRCUqbxgEIR2J9QOKBSEumuckwTcpzJHBAm4P4G3dXX5Neg== "@bull-board/api@3.7.0": version "3.7.0" diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 60bbdef6b8..1fa3281b49 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -291,12 +291,12 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.1.21": - version "1.1.21" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.21.tgz#660e0023f674189b25ca5eabbaff67aebcdf03b8" - integrity sha512-qEf9ggMX6NpVOTe1xfp6NZnbdBV0h0ls9qGGTcMjPakhatB4+3YKSTEDSYYRzY/OLNoKPhcd3aErXuFRR7WKrg== +"@budibase/backend-core@1.1.22-alpha.0": + version "1.1.22-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.22-alpha.0.tgz#2a090a723d35ce4a2e377ac0927fe127213bcbb3" + integrity sha512-77gxcPrjejdqaMaMkbrCS0glYA1jdGo74NpCxdadWx+suU4SMTmOt2jgnEZg20aeKcky2xTpVbjDxIq+Fb2J+g== dependencies: - "@budibase/types" "^1.1.21" + "@budibase/types" "^1.1.22-alpha.0" "@techpass/passport-openidconnect" "0.3.2" aws-sdk "2.1030.0" bcrypt "5.0.1" @@ -324,19 +324,24 @@ uuid "8.3.2" zlib "1.0.5" -"@budibase/pro@1.1.21": - version "1.1.21" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.21.tgz#c9d21bc82740e9407673088dc8451273368943d9" - integrity sha512-aE20qZmVg/sPp4bV++xwdUnp8+c9wq7Df34P9hYvlhklgh+fxd+2RyautOAfMADRMIZIbtP4/gal95bE/KTPNQ== +"@budibase/pro@1.1.22-alpha.0": + version "1.1.22-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.22-alpha.0.tgz#5718188fbc76ce3c3cf3c633c99c6e59f5846310" + integrity sha512-QL5bhT/BJnoKVV5XH5+s3jSCBzV/JGOy8YQObEjZgCrtTWUjdvMrm5pFinflFGriCfoArmvR3Q51FBpNJQfaag== dependencies: - "@budibase/backend-core" "1.1.21" - "@budibase/types" "1.1.21" + "@budibase/backend-core" "1.1.22-alpha.0" + "@budibase/types" "1.1.22-alpha.0" node-fetch "^2.6.1" -"@budibase/types@1.1.21", "@budibase/types@^1.1.21": - version "1.1.21" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.21.tgz#4f46cd52f3e52c804e5dba06b0520825da81f84e" - integrity sha512-fR8783evr6SKZggu/QZRgcZfd8SAuG2U+xO8lL0x/pLNZI1vOeTyQXASoPLLzj6uA8bWnPIy8BGd9PK4Mw3XVQ== +"@budibase/types@1.1.22-alpha.0": + version "1.1.22-alpha.0" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.22-alpha.0.tgz#85fb7f37773c710e8232c95104095c26a8dd22ca" + integrity sha512-1gRwAtjEl7Ug1jrYwD9Iudbfgs37nndEBEB6yVdNPKA5SpjG+Fwx30zp6R961zlx1vsSu4iMdwM8IbMsCM8p1g== + +"@budibase/types@^1.1.22-alpha.0": + version "1.1.22" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.22.tgz#a73a8315ad6a04bf0709f2b51cb35058442ce723" + integrity sha512-24sun/hZ2OJZdavhiJt+S8Aip+RdlKeTyUrkf7OuRCUqbxgEIR2J9QOKBSEumuckwTcpzJHBAm4P4G3dXX5Neg== "@cspotcode/source-map-consumer@0.8.0": version "0.8.0" From 4cc0a327a052270da83a943a9ca502b41237fca3 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Tue, 26 Jul 2022 11:38:34 +0100 Subject: [PATCH 02/76] lint --- packages/server/src/integrations/mongodb.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 70b4be9e14..e300715781 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -92,7 +92,10 @@ module MongoDBModule { if (json[field] instanceof Object) { json[field] = self.createObjectIds(json[field]) } - if ((field === "_id" || field?.startsWith("$")) && typeof json[field] === "string") { + if ( + (field === "_id" || field?.startsWith("$")) && + typeof json[field] === "string" + ) { const id = json[field].match( /(?<=objectid\(['"]).*(?=['"]\))/gi )?.[0] From 5909ce62f7fac1281defe9b26830d2a1c9b08cef Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 5 Aug 2022 15:16:04 +0100 Subject: [PATCH 03/76] Removed query filtering on dynamic variables in binding drawer --- .../[selectedDatasource]/rest/[query]/index.svelte | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte index 5ccc173318..8ecd88243e 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte @@ -231,13 +231,10 @@ ] // convert dynamic variables list to simple key/val object - const getDynamicVariables = (datasource, queryId) => { + const getDynamicVariables = datasource => { const variablesList = datasource?.config?.dynamicVariables if (variablesList && variablesList.length > 0) { - const filtered = queryId - ? variablesList.filter(variable => variable.queryId === queryId) - : variablesList - return filtered.reduce( + return variablesList.reduce( (acc, next) => ({ ...acc, [next.name]: next.value }), {} ) @@ -367,7 +364,7 @@ if (query && !query.fields.pagination) { query.fields.pagination = {} } - dynamicVariables = getDynamicVariables(datasource, query._id) + dynamicVariables = getDynamicVariables(datasource) prettifyQueryRequestBody( query, From de0e8b3d1e50125a89edb54ef66f5903581d1418 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 8 Aug 2022 10:33:39 +0100 Subject: [PATCH 04/76] Walked back previous change. Ensured filtering of global bindings surfaces appropriate values. Filtering logic separated for global and local dynamic variables. --- .../rest/[query]/index.svelte | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte index 8ecd88243e..b0a93f8eec 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte @@ -55,13 +55,16 @@ let saveId, url let response, schema, enabledHeaders let authConfigId - let dynamicVariables, addVariableModal, varBinding + let dynamicVariables, addVariableModal, varBinding, globalDynamicBindings let restBindings = getRestBindings() $: staticVariables = datasource?.config?.staticVariables || {} $: customRequestBindings = toBindingsArray(requestBindings, "Binding") - $: dynamicRequestBindings = toBindingsArray(dynamicVariables, "Dynamic") + $: globalDynamicRequestBindings = toBindingsArray( + globalDynamicBindings, + "Dynamic" + ) $: dataSourceStaticBindings = toBindingsArray( staticVariables, "Datasource.Static" @@ -70,7 +73,7 @@ $: mergedBindings = [ ...restBindings, ...customRequestBindings, - ...dynamicRequestBindings, + ...globalDynamicRequestBindings, ...dataSourceStaticBindings, ] @@ -231,10 +234,13 @@ ] // convert dynamic variables list to simple key/val object - const getDynamicVariables = datasource => { + const getDynamicVariables = (datasource, queryId, matchFn) => { const variablesList = datasource?.config?.dynamicVariables if (variablesList && variablesList.length > 0) { - return variablesList.reduce( + const filtered = queryId + ? variablesList.filter(variable => matchFn(variable, queryId)) + : variablesList + return filtered.reduce( (acc, next) => ({ ...acc, [next.name]: next.value }), {} ) @@ -364,12 +370,21 @@ if (query && !query.fields.pagination) { query.fields.pagination = {} } - dynamicVariables = getDynamicVariables(datasource) + dynamicVariables = getDynamicVariables( + datasource, + query._id, + (variable, queryId) => variable.queryId === queryId + ) + globalDynamicBindings = getDynamicVariables( + datasource, + query._id, + (variable, queryId) => variable.queryId !== queryId + ) prettifyQueryRequestBody( query, requestBindings, - dynamicVariables, + globalDynamicBindings, staticVariables, restBindings ) @@ -434,7 +449,7 @@ valuePlaceholder="Default" bindings={[ ...restBindings, - ...dynamicRequestBindings, + ...globalDynamicRequestBindings, ...dataSourceStaticBindings, ]} bindingDrawerLeft="260px" From 0b8e0309f6e86edde792a82321b0b07a75e71b84 Mon Sep 17 00:00:00 2001 From: NEOLPAR Date: Mon, 8 Aug 2022 12:12:56 +0100 Subject: [PATCH 05/76] filtering datasources for execute query actions --- .../ButtonActionEditor/actions/ExecuteQuery.svelte | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte index 462ee71cbe..042a55d761 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte @@ -3,6 +3,7 @@ import { datasources, integrations, queries } from "stores/backend" import BindingBuilder from "components/integration/QueryBindingBuilder.svelte" import IntegrationQueryEditor from "components/integration/index.svelte" + import { IntegrationTypes } from "constants/backend" export let parameters export let bindings = [] @@ -11,6 +12,12 @@ $: datasource = $datasources.list.find( ds => ds._id === parameters.datasourceId ) + // Executequery action just works on PostgreSQL and MongoDB datasources + $: executeQueryDatasources = $datasources.list.filter( + x => + x.source === IntegrationTypes.POSTGRES || + x.source === IntegrationTypes.MONGODB + ) function fetchQueryDefinition(query) { const source = $datasources.list.find( @@ -24,7 +31,7 @@