From da7153dd64313144a78291b9f004be92c2830c96 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 14 Dec 2021 17:59:02 +0000 Subject: [PATCH] Removing outgoing webhook functionality from automations and adding XML response/sending, as well as form data/encoded data. --- .../components/common/CodeMirrorEditor.svelte | 4 +- .../src/components/integration/codemirror.js | 1 + .../builder/src/constants/backend/index.js | 2 + .../_components/RestBodyInput.svelte | 22 +++- packages/server/package.json | 2 + .../server/src/api/controllers/automation.js | 12 +- .../src/automations/steps/outgoingWebhook.js | 5 +- packages/server/src/automations/utils.js | 11 ++ packages/server/src/integrations/rest.ts | 105 ++++++++++++------ packages/server/yarn.lock | 47 ++++---- 10 files changed, 145 insertions(+), 66 deletions(-) diff --git a/packages/builder/src/components/common/CodeMirrorEditor.svelte b/packages/builder/src/components/common/CodeMirrorEditor.svelte index 65dad9d015..9d036e21ce 100644 --- a/packages/builder/src/components/common/CodeMirrorEditor.svelte +++ b/packages/builder/src/components/common/CodeMirrorEditor.svelte @@ -8,6 +8,9 @@ name: "javascript", json: true, }, + XML: { + name: "xml", + }, SQL: { name: "sql", }, @@ -17,7 +20,6 @@ }, Text: { name: "text/html", - json: false, }, } diff --git a/packages/builder/src/components/integration/codemirror.js b/packages/builder/src/components/integration/codemirror.js index c4afd8488c..78ff8e15c0 100644 --- a/packages/builder/src/components/integration/codemirror.js +++ b/packages/builder/src/components/integration/codemirror.js @@ -4,6 +4,7 @@ import "codemirror/lib/codemirror.css" // Modes import "codemirror/mode/javascript/javascript" import "codemirror/mode/sql/sql" +import "codemirror/mode/xml/xml" import "codemirror/mode/css/css" import "codemirror/mode/handlebars/handlebars" diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index d3c8d7d08d..1f49b0b26f 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -200,6 +200,7 @@ export const RawRestBodyTypes = { ENCODED: "encoded", JSON: "json", TEXT: "text", + XML: "xml", } export const RestBodyTypes = [ @@ -207,5 +208,6 @@ export const RestBodyTypes = [ { name: "form-data", value: "form" }, { name: "x-www-form-encoded", value: "encoded" }, { name: "raw (JSON)", value: "json" }, + { name: "raw (XML)", value: "xml" }, { name: "raw (Text)", value: "text" }, ] diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte index a70b0f02f9..c387f0b492 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte @@ -7,7 +7,11 @@ } from "components/common/CodeMirrorEditor.svelte" const objectTypes = [RawRestBodyTypes.FORM, RawRestBodyTypes.ENCODED] - const textTypes = [RawRestBodyTypes.JSON, RawRestBodyTypes.TEXT] + const textTypes = [ + RawRestBodyTypes.JSON, + RawRestBodyTypes.XML, + RawRestBodyTypes.TEXT, + ] export let query export let bodyType @@ -25,6 +29,18 @@ query.fields.requestBody = "" } } + + function editorMode(type) { + switch (type) { + case RawRestBodyTypes.JSON: + return EditorModes.JSON + case RawRestBodyTypes.XML: + return EditorModes.XML + default: + case RawRestBodyTypes.TEXT: + return EditorModes.Text + } + }
@@ -41,9 +57,7 @@ {:else if textTypes.includes(bodyType)} (query.fields.requestBody = e.detail)} diff --git a/packages/server/package.json b/packages/server/package.json index df9d23a6be..1f30a67695 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -90,6 +90,7 @@ "dotenv": "8.2.0", "download": "8.0.0", "fix-path": "3.0.0", + "form-data": "^4.0.0", "fs-extra": "8.1.0", "jimp": "0.16.1", "joi": "17.2.1", @@ -126,6 +127,7 @@ "validate.js": "0.13.1", "vm2": "^3.9.3", "worker-farm": "^1.7.0", + "xml2js": "^0.4.23", "yargs": "13.2.4", "zlib": "1.0.5" }, diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index 841473a4ff..05337579a0 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils") const { checkForWebhooks, updateTestHistory, + removeDeprecated, } = require("../../automations/utils") const { deleteEntityMetadata } = require("../../utilities") const { MetadataTypes } = require("../../constants") const { setTestFlag, clearTestFlag } = require("../../utilities/redis") +const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS) +const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS) + /************************* * * * BUILDER FUNCTIONS * @@ -155,17 +159,17 @@ exports.destroy = async function (ctx) { } exports.getActionList = async function (ctx) { - ctx.body = actions.ACTION_DEFINITIONS + ctx.body = ACTION_DEFS } exports.getTriggerList = async function (ctx) { - ctx.body = triggers.TRIGGER_DEFINITIONS + ctx.body = TRIGGER_DEFS } module.exports.getDefinitionList = async function (ctx) { ctx.body = { - trigger: triggers.TRIGGER_DEFINITIONS, - action: actions.ACTION_DEFINITIONS, + trigger: TRIGGER_DEFS, + action: ACTION_DEFS, } } diff --git a/packages/server/src/automations/steps/outgoingWebhook.js b/packages/server/src/automations/steps/outgoingWebhook.js index f0637c3351..01d1e8d6be 100644 --- a/packages/server/src/automations/steps/outgoingWebhook.js +++ b/packages/server/src/automations/steps/outgoingWebhook.js @@ -13,12 +13,11 @@ const RequestType = { const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH] /** - * Note, there is some functionality in this that is not currently exposed as it - * is complex and maybe better to be opinionated here. - * GET/DELETE requests cannot handle body elements so they will not be sent if configured. + * NOTE: this functionality is deprecated - it no longer should be used. */ exports.definition = { + deprecated: true, name: "Outgoing webhook", tagline: "Send a {{inputs.requestMethod}} request", icon: "Send", diff --git a/packages/server/src/automations/utils.js b/packages/server/src/automations/utils.js index 6e58e9aae0..d9c96963ab 100644 --- a/packages/server/src/automations/utils.js +++ b/packages/server/src/automations/utils.js @@ -7,6 +7,7 @@ const newid = require("../db/newid") const { updateEntityMetadata } = require("../utilities") const { MetadataTypes } = require("../constants") const { getDeployedAppID } = require("@budibase/auth/db") +const { cloneDeep } = require("lodash/fp") const WH_STEP_ID = definitions.WEBHOOK.stepId const CRON_STEP_ID = definitions.CRON.stepId @@ -42,6 +43,16 @@ exports.updateTestHistory = async (appId, automation, history) => { ) } +exports.removeDeprecated = definitions => { + const base = cloneDeep(definitions) + for (let key of Object.keys(base)) { + if (base[key].deprecated) { + delete base[key] + } + } + return base +} + // end the repetition and the job itself exports.disableAllCrons = async appId => { const promises = [] diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index bba3c87d68..4fa43b9f84 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -10,6 +10,7 @@ import { IntegrationBase } from "./base/IntegrationBase" const BodyTypes = { NONE: "none", FORM_DATA: "form", + XML: "xml", ENCODED: "encoded", JSON: "json", TEXT: "text", @@ -42,6 +43,9 @@ module RestModule { const fetch = require("node-fetch") const { formatBytes } = require("../utilities") const { performance } = require("perf_hooks") + const FormData = require("form-data") + const { URLSearchParams } = require("url") + const xmlParser = require("xml2js").parseStringPromise const SCHEMA: Integration = { docs: "https://github.com/node-fetch/node-fetch", @@ -107,13 +111,26 @@ module RestModule { async parseResponse(response: any) { let data, raw, headers - const contentType = response.headers.get("content-type") - if (contentType && contentType.indexOf("application/json") !== -1) { - data = await response.json() - raw = JSON.stringify(data) - } else { - data = await response.text() - raw = data + const contentType = response.headers.get("content-type") || "" + try { + if (contentType.includes("application/json")) { + data = await response.json() + raw = JSON.stringify(data) + } else if (contentType.includes("text/xml") || contentType.includes("application/xml")) { + const rawXml = await response.text() + data = await xmlParser(rawXml, { explicitArray: false, trim: true, explicitRoot: false }) || {} + // there is only one structure, its an array, return the array so it appears as rows + const keys = Object.keys(data) + if (keys.length === 1 && Array.isArray(data[keys[0]])) { + data = data[keys[0]] + } + raw = rawXml + } else { + data = await response.text() + raw = data + } + } catch (err) { + throw "Failed to parse response body." } const size = formatBytes( response.headers.get("content-length") || Buffer.byteLength(raw, "utf8") @@ -149,6 +166,50 @@ module RestModule { return complete } + addBody(bodyType: string, body: string | any, input: any) { + let error, object, string + try { + string = typeof body !== "string" ? JSON.stringify(body) : body + object = typeof body === "object" ? body : JSON.parse(body) + } catch (err) { + error = err + } + switch (bodyType) { + case BodyTypes.TEXT: + // content type defaults to plaintext + input.body = string + break + case BodyTypes.ENCODED: + const params = new URLSearchParams() + for (let [key, value] of Object.entries(object)) { + params.append(key, value) + } + input.body = params + break + case BodyTypes.FORM_DATA: + const form = new FormData() + for (let [key, value] of Object.entries(object)) { + form.append(key, value) + } + input.body = form + break + case BodyTypes.XML: + input.body = string + input.headers["Content-Type"] = "text/xml" + break + default: + case BodyTypes.JSON: + // if JSON error, throw it + if (error) { + throw "Invalid JSON for request body" + } + input.body = object + input.headers["Content-Type"] = "application/json" + break + } + return input + } + async _req(query: RestQuery) { const { path = "", @@ -172,35 +233,9 @@ module RestModule { } } - const input: any = { method, headers: this.headers } + let input: any = { method, headers: this.headers } if (requestBody) { - switch (bodyType) { - case BodyTypes.TEXT: - const text = - typeof requestBody !== "string" - ? JSON.stringify(requestBody) - : requestBody - // content type defaults to plaintext - input.body = text - break - default: - case BodyTypes.JSON: - try { - // confirm its json - const json = JSON.parse(requestBody) - if ( - json && - typeof json === "object" && - Object.keys(json).length > 0 - ) { - input.body = requestBody - input.headers["Content-Type"] = "application/json" - } - } catch (err) { - throw "Invalid JSON for request body" - } - break - } + input = this.addBody(bodyType, requestBody, input) } this.startTimeMs = performance.now() diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index ff2f63d531..8e2c9db352 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -983,10 +983,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/auth@^1.0.18": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.19.tgz#b5a8ad51170443d2136d244f51cfe7dbcc0db116" - integrity sha512-6H1K80KX8RUseLXD307tKRc+b0B7/b2SZmAYYGq5qrUSdUotydZaZ90pt5pXVdE754duxyc8DlrwmRfri5xu+A== +"@budibase/auth@^1.0.19-alpha.1": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2" + integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1056,10 +1056,10 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/bbui@^1.0.19": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.19.tgz#d79c99e8c0adcf24d9b83f00a15eb262ad73a7e2" - integrity sha512-GhsyqkDjHMvU1MCr7oXKIZi6NOhmkunJ6eAoob8obCLDm+LXC/1Q8ymSuJicctQpDpraaFS7zqQ6vYY9v7kpiQ== +"@budibase/bbui@^1.0.22": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462" + integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A== dependencies: "@adobe/spectrum-css-workflow-icons" "^1.2.1" "@spectrum-css/actionbutton" "^1.0.1" @@ -1106,14 +1106,14 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/client@^1.0.18": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.19.tgz#50ba2ad91ac2fd57c51306b80fbab24a26ba1403" - integrity sha512-8vAsD7VkLfq9ZrD+QPXGUcj/2D3vGO++IPr0zIKGNVG5FlOLFceQ9b7itExSFWutyVAjK/e/yq56tugnf0S+Fg== +"@budibase/client@^1.0.19-alpha.1": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25" + integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA== dependencies: - "@budibase/bbui" "^1.0.19" + "@budibase/bbui" "^1.0.22" "@budibase/standard-components" "^0.9.139" - "@budibase/string-templates" "^1.0.19" + "@budibase/string-templates" "^1.0.22" regexparam "^1.3.0" shortid "^2.2.15" svelte-spa-router "^3.0.5" @@ -1163,10 +1163,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@^1.0.18", "@budibase/string-templates@^1.0.19": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.19.tgz#4b476dfc5d317e56d84a24dffd34715cc74c7b37" - integrity sha512-MmSHF2HK3JS3goyNr3mUQi3azt5vSWlmSGlYFyw473jplRVYkmI8wXrP8gVy9mNJ4vksn3bkgFPI8Hi9RoNSbA== +"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117" + integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw== dependencies: "@budibase/handlebars-helpers" "^0.11.7" dayjs "^1.10.4" @@ -5829,6 +5829,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + formidable@^1.1.1, formidable@^1.2.0: version "1.2.6" resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168" @@ -13081,7 +13090,7 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" -xml2js@^0.4.19, xml2js@^0.4.5: +xml2js@^0.4.19, xml2js@^0.4.23, xml2js@^0.4.5: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==