CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-09 00:06:05 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3785 from Budibase/feature/rest-bodies

Browse source 
REST body types
This commit is contained in:
Michael Drury 2021-12-15 15:28:41 +00:00 committed by GitHub
parent 841feac5fa 1f97ae259e
commit f86b44c36d
12 changed files with 278 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
packages/builder/src/components/common/CodeMirrorEditor.svelte
View file

@ -8,6 +8,9 @@
name: "javascript",
json: true,
},
XML: {
name: "xml",
},
SQL: {
name: "sql",
},
@ -40,11 +43,12 @@
let editor
// Keep editor up to date with value
$: editor?.setOption("mode", mode)
$: editor?.setValue(value || "")
// Creates an instance of a code mirror editor
async function createEditor(mode, value) {
if (!CodeMirror || !textarea || editor) {
if (!CodeMirror || !textarea) {
return
}

1
packages/builder/src/components/integration/codemirror.js
View file

@ -4,6 +4,7 @@ import "codemirror/lib/codemirror.css"
// Modes
import "codemirror/mode/javascript/javascript"
import "codemirror/mode/sql/sql"
import "codemirror/mode/xml/xml"
import "codemirror/mode/css/css"
import "codemirror/mode/handlebars/handlebars"

2
packages/builder/src/constants/backend/index.js
View file

@ -200,6 +200,7 @@ export const RawRestBodyTypes = {
ENCODED: "encoded",
JSON: "json",
TEXT: "text",
XML: "xml",
}
export const RestBodyTypes = [
@ -207,5 +208,6 @@ export const RestBodyTypes = [
{ name: "form-data", value: "form" },
{ name: "x-www-form-encoded", value: "encoded" },
{ name: "raw (JSON)", value: "json" },
{ name: "raw (XML)", value: "xml" },
{ name: "raw (Text)", value: "text" },
]

22
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte
View file

@ -7,7 +7,11 @@
} from "components/common/CodeMirrorEditor.svelte"
const objectTypes = [RawRestBodyTypes.FORM, RawRestBodyTypes.ENCODED]
const textTypes = [RawRestBodyTypes.JSON, RawRestBodyTypes.TEXT]
const textTypes = [
RawRestBodyTypes.JSON,
RawRestBodyTypes.XML,
RawRestBodyTypes.TEXT,
]
export let query
export let bodyType
@ -25,6 +29,18 @@
query.fields.requestBody = ""
}
}
function editorMode(type) {
switch (type) {
case RawRestBodyTypes.JSON:
return EditorModes.JSON
case RawRestBodyTypes.XML:
return EditorModes.XML
default:
case RawRestBodyTypes.TEXT:
return EditorModes.Text
}
}
</script>
<div class="margin">
@ -41,9 +57,7 @@
{:else if textTypes.includes(bodyType)}
<CodeMirrorEditor
height={200}
mode={bodyType === RawRestBodyTypes.JSON
? EditorModes.JSON
: EditorModes.Text}
mode={editorMode(bodyType)}
value={query.fields.requestBody}
resize="vertical"
on:change={e => (query.fields.requestBody = e.detail)}

5
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
View file

@ -48,7 +48,7 @@
let breakQs = {},
bindings = {}
let url = ""
let saveId
let saveId, isGet
let response, schema, enabledHeaders
let datasourceType, integrationInfo, queryConfig, responseSuccess
let authConfigId
@ -58,6 +58,7 @@
$: queryConfig = integrationInfo?.query
$: url = buildUrl(url, breakQs)
$: checkQueryName(url)
$: isGet = query?.queryVerb === "read"
$: responseSuccess =
response?.info?.code >= 200 && response?.info?.code <= 206
$: authConfigs = buildAuthConfigs(datasource)
@ -262,7 +263,7 @@
<Tab title="Body">
<RadioGroup
bind:value={query.fields.bodyType}
options={bodyTypes}
options={isGet ? [bodyTypes[0]] : bodyTypes}
direction="horizontal"
getOptionLabel={option => option.name}
getOptionValue={option => option.value}

2
packages/server/package.json
View file

@ -90,6 +90,7 @@
"dotenv": "8.2.0",
"download": "8.0.0",
"fix-path": "3.0.0",
"form-data": "^4.0.0",
"fs-extra": "8.1.0",
"jimp": "0.16.1",
"joi": "17.2.1",
@ -126,6 +127,7 @@
"validate.js": "0.13.1",
"vm2": "^3.9.3",
"worker-farm": "^1.7.0",
"xml2js": "^0.4.23",
"yargs": "13.2.4",
"zlib": "1.0.5"
},

12
packages/server/src/api/controllers/automation.js
View file

@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils")
const {
checkForWebhooks,
updateTestHistory,
removeDeprecated,
} = require("../../automations/utils")
const { deleteEntityMetadata } = require("../../utilities")
const { MetadataTypes } = require("../../constants")
const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
/*************************
* *
* BUILDER FUNCTIONS *
@ -155,17 +159,17 @@ exports.destroy = async function (ctx) {
}
exports.getActionList = async function (ctx) {
ctx.body = actions.ACTION_DEFINITIONS
ctx.body = ACTION_DEFS
}
exports.getTriggerList = async function (ctx) {
ctx.body = triggers.TRIGGER_DEFINITIONS
ctx.body = TRIGGER_DEFS
}
module.exports.getDefinitionList = async function (ctx) {
ctx.body = {
trigger: triggers.TRIGGER_DEFINITIONS,
action: actions.ACTION_DEFINITIONS,
trigger: TRIGGER_DEFS,
action: ACTION_DEFS,
}
}

5
packages/server/src/automations/steps/outgoingWebhook.js
View file

@ -13,12 +13,11 @@ const RequestType = {
const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
/**
* Note, there is some functionality in this that is not currently exposed as it
* is complex and maybe better to be opinionated here.
* GET/DELETE requests cannot handle body elements so they will not be sent if configured.
* NOTE: this functionality is deprecated - it no longer should be used.
*/
exports.definition = {
deprecated: true,
name: "Outgoing webhook",
tagline: "Send a {{inputs.requestMethod}} request",
icon: "Send",

11
packages/server/src/automations/utils.js
View file

@ -7,6 +7,7 @@ const newid = require("../db/newid")
const { updateEntityMetadata } = require("../utilities")
const { MetadataTypes } = require("../constants")
const { getDeployedAppID } = require("@budibase/auth/db")
const { cloneDeep } = require("lodash/fp")
const WH_STEP_ID = definitions.WEBHOOK.stepId
const CRON_STEP_ID = definitions.CRON.stepId
@ -42,6 +43,16 @@ exports.updateTestHistory = async (appId, automation, history) => {
)
}
exports.removeDeprecated = definitions => {
const base = cloneDeep(definitions)
for (let key of Object.keys(base)) {
if (base[key].deprecated) {
delete base[key]
}
}
return base
}
// end the repetition and the job itself
exports.disableAllCrons = async appId => {
const promises = []

125
packages/server/src/integrations/rest.ts
View file

@ -6,13 +6,14 @@ import {
RestQueryFields as RestQuery,
AuthType,
BasicAuthConfig,
BearerAuthConfig
BearerAuthConfig,
} from "../definitions/datasource"
import { IntegrationBase } from "./base/IntegrationBase"
const BodyTypes = {
NONE: "none",
FORM_DATA: "form",
XML: "xml",
ENCODED: "encoded",
JSON: "json",
TEXT: "text",
@ -45,6 +46,9 @@ module RestModule {
const fetch = require("node-fetch")
const { formatBytes } = require("../utilities")
const { performance } = require("perf_hooks")
const FormData = require("form-data")
const { URLSearchParams } = require("url")
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
const SCHEMA: Integration = {
docs: "https://github.com/node-fetch/node-fetch",
@ -110,15 +114,38 @@ module RestModule {
async parseResponse(response: any) {
let data, raw, headers
const contentType = response.headers.get("content-type")
if (contentType && contentType.indexOf("application/json") !== -1) {
data = await response.json()
raw = JSON.stringify(data)
} else {
data = await response.text()
raw = data
const contentType = response.headers.get("content-type") || ""
try {
if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
} else if (
contentType.includes("text/xml") ||
contentType.includes("application/xml")
) {
const rawXml = await response.text()
data =
(await xmlParser(rawXml, {
explicitArray: false,
trim: true,
explicitRoot: false,
})) || {}
// there is only one structure, its an array, return the array so it appears as rows
const keys = Object.keys(data)
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
data = data[keys[0]]
}
raw = rawXml
} else {
data = await response.text()
raw = data
}
} catch (err) {
throw "Failed to parse response body."
}
const size = formatBytes(response.headers.get("content-length") || Buffer.byteLength(raw, "utf8"))
const size = formatBytes(
response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
)
const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
headers = response.headers.raw()
for (let [key, value] of Object.entries(headers)) {
@ -150,7 +177,59 @@ module RestModule {
return complete
}
getAuthHeaders(authConfigId: string): { [key: string]: any }{
addBody(bodyType: string, body: string | any, input: any) {
let error, object, string
try {
string = typeof body !== "string" ? JSON.stringify(body) : body
object = typeof body === "object" ? body : JSON.parse(body)
} catch (err) {
error = err
}
if (!input.headers) {
input.headers = {}
}
switch (bodyType) {
case BodyTypes.NONE:
break
case BodyTypes.TEXT:
// content type defaults to plaintext
input.body = string
break
case BodyTypes.ENCODED:
const params = new URLSearchParams()
for (let [key, value] of Object.entries(object)) {
params.append(key, value)
}
input.body = params
break
case BodyTypes.FORM_DATA:
const form = new FormData()
for (let [key, value] of Object.entries(object)) {
form.append(key, value)
}
input.body = form
break
case BodyTypes.XML:
if (object != null) {
string = (new XmlBuilder()).buildObject(object)
}
input.body = string
input.headers["Content-Type"] = "application/xml"
break
default:
case BodyTypes.JSON:
// if JSON error, throw it
if (error) {
throw "Invalid JSON for request body"
}
input.body = string
input.headers["Content-Type"] = "application/json"
break
}
return input
}
getAuthHeaders(authConfigId: string): { [key: string]: any } {
let headers: any = {}
if (this.config.authConfigs && authConfigId) {
@ -180,7 +259,16 @@ module RestModule {
}
async _req(query: RestQuery) {
const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody, authConfigId } = query
const {
path = "",
queryString = "",
headers = {},
method = "GET",
disabledHeaders,
bodyType,
requestBody,
authConfigId,
} = query
const authHeaders = this.getAuthHeaders(authConfigId)
this.headers = {
@ -197,18 +285,9 @@ module RestModule {
}
}
let json
if (bodyType === BodyTypes.JSON && requestBody) {
try {
json = JSON.parse(requestBody)
} catch (err) {
throw "Invalid JSON for request body"
}
}
const input: any = { method, headers: this.headers }
if (json && typeof json === "object" && Object.keys(json).length > 0) {
input.body = JSON.stringify(json)
let input: any = { method, headers: this.headers }
if (requestBody) {
input = this.addBody(bodyType, requestBody, input)
}
this.startTimeMs = performance.now()

108
packages/server/src/integrations/tests/rest.spec.js
View file

@ -14,6 +14,11 @@ const fetch = require("node-fetch")
const RestIntegration = require("../rest")
const { AuthType } = require("../rest")
const HEADERS = {
"Accept": "application/json",
"Content-Type": "application/json"
}
class TestConfiguration {
constructor(config = {}) {
this.integration = new RestIntegration.integration(config)
@ -35,9 +40,7 @@ describe("REST Integration", () => {
const query = {
path: "api",
queryString: "test=1",
headers: {
Accept: "application/json",
},
headers: HEADERS,
bodyType: "json",
requestBody: JSON.stringify({
name: "test",
@ -47,9 +50,7 @@ describe("REST Integration", () => {
expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
method: "POST",
body: '{"name":"test"}',
headers: {
Accept: "application/json",
},
headers: HEADERS,
})
})
@ -86,9 +87,7 @@ describe("REST Integration", () => {
expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
method: "PUT",
body: '{"name":"test"}',
headers: {
Accept: "application/json",
},
headers: HEADERS,
})
})
@ -107,13 +106,98 @@ describe("REST Integration", () => {
const response = await config.integration.delete(query)
expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
method: "DELETE",
headers: {
Accept: "application/json",
},
headers: HEADERS,
body: '{"name":"test"}',
})
})
describe("request body", () => {
const input = { a: 1, b: 2 }
it("should allow no body", () => {
const output = config.integration.addBody("none", null, {})
expect(output.body).toBeUndefined()
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow text body", () => {
const output = config.integration.addBody("text", "hello world", {})
expect(output.body).toEqual("hello world")
// gets added by fetch
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow form data", () => {
const FormData = require("form-data")
const output = config.integration.addBody("form", input, {})
expect(output.body instanceof FormData).toEqual(true)
expect(output.body._valueLength).toEqual(2)
// gets added by fetch
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow encoded form data", () => {
const { URLSearchParams } = require("url")
const output = config.integration.addBody("encoded", input, {})
expect(output.body instanceof URLSearchParams).toEqual(true)
expect(output.body.toString()).toEqual("a=1&b=2")
// gets added by fetch
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow JSON", () => {
const output = config.integration.addBody("json", input, {})
expect(output.body).toEqual(JSON.stringify(input))
expect(output.headers["Content-Type"]).toEqual("application/json")
})
it("should allow XML", () => {
const output = config.integration.addBody("xml", input, {})
expect(output.body.includes("<a>1</a>")).toEqual(true)
expect(output.body.includes("<b>2</b>")).toEqual(true)
expect(output.headers["Content-Type"]).toEqual("application/xml")
})
})
describe("response", () => {
function buildInput(json, text, header) {
return {
status: 200,
json: json ? async () => json : undefined,
text: text ? async () => text : undefined,
headers: { get: key => key === "content-length" ? 100 : header, raw: () => ({ "content-type": header }) }
}
}
it("should be able to parse JSON response", async () => {
const input = buildInput({a: 1}, null, "application/json")
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual({a: 1})
expect(output.info.code).toEqual(200)
expect(output.info.size).toEqual("100B")
expect(output.extra.raw).toEqual(JSON.stringify({a: 1}))
expect(output.extra.headers["content-type"]).toEqual("application/json")
})
it("should be able to parse text response", async () => {
const text = "hello world"
const input = buildInput(null, text, "text/plain")
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual(text)
expect(output.extra.raw).toEqual(text)
expect(output.extra.headers["content-type"]).toEqual("text/plain")
})
it("should be able to parse XML response", async () => {
const text = "<root><a>1</a><b>2</b></root>"
const input = buildInput(null, text, "application/xml")
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual({a: "1", b: "2"})
expect(output.extra.raw).toEqual(text)
expect(output.extra.headers["content-type"]).toEqual("application/xml")
})
})
describe("authentication", () => {
const basicAuth = {
_id: "c59c14bd1898a43baa08da68959b24686",

47
packages/server/yarn.lock
View file

@ -983,10 +983,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/auth@^1.0.18":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.19.tgz#b5a8ad51170443d2136d244f51cfe7dbcc0db116"
integrity sha512-6H1K80KX8RUseLXD307tKRc+b0B7/b2SZmAYYGq5qrUSdUotydZaZ90pt5pXVdE754duxyc8DlrwmRfri5xu+A==
"@budibase/auth@^1.0.19-alpha.1":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2"
integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1056,10 +1056,10 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/bbui@^1.0.19":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.19.tgz#d79c99e8c0adcf24d9b83f00a15eb262ad73a7e2"
integrity sha512-GhsyqkDjHMvU1MCr7oXKIZi6NOhmkunJ6eAoob8obCLDm+LXC/1Q8ymSuJicctQpDpraaFS7zqQ6vYY9v7kpiQ==
"@budibase/bbui@^1.0.22":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462"
integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A==
dependencies:
"@adobe/spectrum-css-workflow-icons" "^1.2.1"
"@spectrum-css/actionbutton" "^1.0.1"
@ -1106,14 +1106,14 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/client@^1.0.18":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.19.tgz#50ba2ad91ac2fd57c51306b80fbab24a26ba1403"
integrity sha512-8vAsD7VkLfq9ZrD+QPXGUcj/2D3vGO++IPr0zIKGNVG5FlOLFceQ9b7itExSFWutyVAjK/e/yq56tugnf0S+Fg==
"@budibase/client@^1.0.19-alpha.1":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25"
integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA==
dependencies:
"@budibase/bbui" "^1.0.19"
"@budibase/bbui" "^1.0.22"
"@budibase/standard-components" "^0.9.139"
"@budibase/string-templates" "^1.0.19"
"@budibase/string-templates" "^1.0.22"
regexparam "^1.3.0"
shortid "^2.2.15"
svelte-spa-router "^3.0.5"
@ -1163,10 +1163,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/string-templates@^1.0.18", "@budibase/string-templates@^1.0.19":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.19.tgz#4b476dfc5d317e56d84a24dffd34715cc74c7b37"
integrity sha512-MmSHF2HK3JS3goyNr3mUQi3azt5vSWlmSGlYFyw473jplRVYkmI8wXrP8gVy9mNJ4vksn3bkgFPI8Hi9RoNSbA==
"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117"
integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw==
dependencies:
"@budibase/handlebars-helpers" "^0.11.7"
dayjs "^1.10.4"
@ -5829,6 +5829,15 @@ form-data@^3.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
form-data@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
dependencies:
asynckit "^0.4.0"
combined-stream "^1.0.8"
mime-types "^2.1.12"
formidable@^1.1.1, formidable@^1.2.0:
version "1.2.6"
resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168"
@ -13081,7 +13090,7 @@ xml2js@0.4.19:
sax ">=0.6.0"
xmlbuilder "~9.0.1"
xml2js@^0.4.19, xml2js@^0.4.5:
xml2js@^0.4.19, xml2js@^0.4.23, xml2js@^0.4.5:
version "0.4.23"
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==