CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-06-28 11:00:55 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of github.com:budibase/budibase into fix-arm-image-with-isolated-vm

Browse source
This commit is contained in:
Sam Rose 2024-02-16 11:07:51 +00:00
parent f4d7762d8b 101caf323a
commit 89557d5407
No known key found for this signature in database
42 changed files with 578 additions and 1668 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
hosting/docker-compose.yaml
View file

@ -98,7 +98,6 @@ services:
couchdb-service:
restart: unless-stopped
image: budibase/couchdb
pull_policy: always
environment:
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
- COUCHDB_USER=${COUCH_DB_USER}

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.19.3",
"version": "2.19.5",
"npmClient": "yarn",
"packages": [
"packages/*",

2
package.json
View file

@ -58,7 +58,7 @@
"lint": "yarn run lint:eslint && yarn run lint:prettier",
"lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core",
"lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
"lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
"lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier",
"build:specs": "lerna run --stream specs",
"build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
"build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",

7
packages/backend-core/src/logging/correlation/correlation.ts
View file

@ -2,11 +2,12 @@ import { Header } from "../../constants"
const correlator = require("correlation-id")
export const setHeader = (headers: any) => {
export const setHeader = (headers: Record<string, string>) => {
const correlationId = correlator.getId()
if (correlationId) {
headers[Header.CORRELATION_ID] = correlationId
if (!correlationId) {
return
}
headers[Header.CORRELATION_ID] = correlationId
}
export function getId() {

BIN
packages/builder/assets/n8n_square.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 12 KiB

1507
packages/builder/src/builderStore/store/frontend.js
View file

File diff suppressed because it is too large Load diff

6
packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte
View file

@ -128,10 +128,10 @@
>
<div class="item-body">
<img
width="20"
height="20"
width={20}
height={20}
src={externalActions[action.stepId].icon}
alt="zapier"
alt={externalActions[action.stepId].name}
/>
<span class="icon-spacing">
<Body size="XS">

2
packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js
View file

@ -1,5 +1,6 @@
import DiscordLogo from "assets/discord.svg"
import ZapierLogo from "assets/zapier.png"
import n8nLogo from "assets/n8n_square.png"
import MakeLogo from "assets/make.svg"
import SlackLogo from "assets/slack.svg"
@ -8,4 +9,5 @@ export const externalActions = {
discord: { name: "discord", icon: DiscordLogo },
slack: { name: "slack", icon: SlackLogo },
integromat: { name: "integromat", icon: MakeLogo },
n8n: { name: "n8n", icon: n8nLogo },
}

8
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View file

@ -79,6 +79,7 @@
disableWrapping: true,
})
$: editingJs = codeMode === EditorModes.JS
$: requiredProperties = block.schema.inputs.required || []
$: stepCompletions =
codeMode === EditorModes.Handlebars
@ -359,6 +360,11 @@
)
}
function getFieldLabel(key, value) {
const requiredSuffix = requiredProperties.includes(key) ? "*" : ""
return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}`
}
onMount(async () => {
try {
await environment.loadVariables()
@ -376,7 +382,7 @@
<Label
tooltip={value.title === "Binding / Value"
? "If using the String input type, please use a comma or newline separated string"
: null}>{value.title || (key === "row" ? "Table" : key)}</Label
: null}>{getFieldLabel(key, value)}</Label
>
{/if}
<div class:field-width={shouldRenderField(value)}>

2
packages/builder/src/components/integration/RestQueryViewer.svelte
View file

@ -159,7 +159,7 @@
newQuery.fields.queryString = queryString
newQuery.fields.authConfigId = authConfigId
newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
newQuery.schema = schema
newQuery.schema = schema || {}
return newQuery
}

1
packages/builder/src/constants/backend/automations.js
View file

@ -27,6 +27,7 @@ export const ActionStepID = {
slack: "slack",
zapier: "zapier",
integromat: "integromat",
n8n: "n8n",
}
export const Features = {

34
packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte
View file

@ -3,6 +3,7 @@
import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
export let component
export let opened
$: definition = componentStore.getDefinition(component?._component)
$: noPaste = !$componentStore.componentToPaste
@ -85,6 +86,39 @@
>
Paste
</MenuItem>
{#if component?._children?.length}
<MenuItem
icon="TreeExpand"
keyBind="!ArrowRight"
on:click={() => keyboardEvent("ArrowRight", false)}
disabled={opened}
>
Expand
</MenuItem>
<MenuItem
icon="TreeCollapse"
keyBind="!ArrowLeft"
on:click={() => keyboardEvent("ArrowLeft", false)}
disabled={!opened}
>
Collapse
</MenuItem>
<MenuItem
icon="TreeExpandAll"
keyBind="Ctrl+!ArrowRight"
on:click={() => keyboardEvent("ArrowRight", true)}
>
Expand All
</MenuItem>
<MenuItem
icon="TreeCollapseAll"
keyBind="Ctrl+!ArrowLeft"
on:click={() => keyboardEvent("ArrowLeft", true)}
>
Collapse All
</MenuItem>
{/if}
</ActionMenu>
<style>

35
packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte
View file

@ -9,6 +9,7 @@
import { goto, isActive } from "@roxi/routify"
import { notifications } from "@budibase/bbui"
import ConfirmDialog from "components/common/ConfirmDialog.svelte"
import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
let confirmDeleteDialog
let confirmEjectDialog
@ -61,6 +62,40 @@
["ArrowDown"]: () => {
componentStore.selectNext()
},
["ArrowRight"]: component => {
componentTreeNodesStore.expandNode(component._id)
},
["ArrowLeft"]: component => {
componentTreeNodesStore.collapseNode(component._id)
},
["Ctrl+ArrowRight"]: component => {
componentTreeNodesStore.expandNode(component._id)
const expandChildren = component => {
const children = component._children ?? []
children.forEach(child => {
componentTreeNodesStore.expandNode(child._id)
expandChildren(child)
})
}
expandChildren(component)
},
["Ctrl+ArrowLeft"]: component => {
componentTreeNodesStore.collapseNode(component._id)
const collapseChildren = component => {
const children = component._children ?? []
children.forEach(child => {
componentTreeNodesStore.collapseNode(child._id)
collapseChildren(child)
})
}
collapseChildren(component)
},
["Escape"]: () => {
if ($isActive(`./:componentId/new`)) {
$goto(`./${$componentStore.selectedComponentId}`)

32
packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte
View file

@ -17,11 +17,12 @@
} from "helpers/components"
import { get } from "svelte/store"
import { dndStore } from "./dndStore"
import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
export let components = []
export let level = 0
let closedNodes = {}
$: openNodes = $componentTreeNodesStore
$: filteredComponents = components?.filter(component => {
return (
@ -54,15 +55,6 @@
return componentSupportsChildren(component) && component._children?.length
}
function toggleNodeOpen(componentId) {
if (closedNodes[componentId]) {
delete closedNodes[componentId]
} else {
closedNodes[componentId] = true
}
closedNodes = closedNodes
}
const onDrop = async e => {
e.stopPropagation()
try {
@ -72,14 +64,14 @@
}
}
const isOpen = (component, selectedComponentPath, closedNodes) => {
const isOpen = (component, selectedComponentPath, openNodes) => {
if (!component?._children?.length) {
return false
}
if (selectedComponentPath.includes(component._id)) {
if (selectedComponentPath.slice(0, -1).includes(component._id)) {
return true
}
return !closedNodes[component._id]
return openNodes[`nodeOpen-${component._id}`]
}
const isChildOfSelectedComponent = component => {
@ -96,7 +88,7 @@
<ul>
{#each filteredComponents || [] as component, index (component._id)}
{@const opened = isOpen(component, $selectedComponentPath, closedNodes)}
{@const opened = isOpen(component, $selectedComponentPath, openNodes)}
<li
on:click|stopPropagation={() => {
componentStore.select(component._id)
@ -110,7 +102,7 @@
on:dragend={dndStore.actions.reset}
on:dragstart={() => dndStore.actions.dragstart(component)}
on:dragover={dragover(component, index)}
on:iconClick={() => toggleNodeOpen(component._id)}
on:iconClick={() => componentTreeNodesStore.toggleNode(component._id)}
on:drop={onDrop}
hovering={$hoverStore.componentId === component._id}
on:mouseenter={() => hover(component._id)}
@ -125,8 +117,9 @@
highlighted={isChildOfSelectedComponent(component)}
selectedBy={$userSelectedResourceMap[component._id]}
>
<ComponentDropdownMenu {component} />
<ComponentDropdownMenu {opened} {component} />
</NavItem>
{#if opened}
<svelte:self
components={component._children}
@ -144,13 +137,6 @@
padding-left: 0;
margin: 0;
}
ul :global(.icon.arrow) {
transition: opacity 130ms ease-out;
opacity: 0;
}
ul:hover :global(.icon.arrow) {
opacity: 1;
}
ul,
li {
min-width: max-content;

19
packages/builder/src/stores/builder/components.js
View file

@ -29,6 +29,7 @@ import {
} from "constants/backend"
import BudiStore from "./BudiStore"
import { Utils } from "@budibase/frontend-core"
import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
export const INITIAL_COMPONENTS_STATE = {
components: {},
@ -662,6 +663,7 @@ export class ComponentStore extends BudiStore {
const screen = get(selectedScreen)
const parent = findComponentParent(screen.props, componentId)
const index = parent?._children.findIndex(x => x._id === componentId)
const componentTreeNodes = get(componentTreeNodesStore)
// Check for screen and navigation component edge cases
const screenComponentId = `${screen._id}-screen`
@ -680,9 +682,15 @@ export class ComponentStore extends BudiStore {
if (index > 0) {
// If sibling before us accepts children, select a descendant
const previousSibling = parent._children[index - 1]
if (previousSibling._children?.length) {
if (
previousSibling._children?.length &&
componentTreeNodes[`nodeOpen-${previousSibling._id}`]
) {
let target = previousSibling
while (target._children?.length) {
while (
target._children?.length &&
componentTreeNodes[`nodeOpen-${target._id}`]
) {
target = target._children[target._children.length - 1]
}
return target._id
@ -703,6 +711,7 @@ export class ComponentStore extends BudiStore {
const screen = get(selectedScreen)
const parent = findComponentParent(screen.props, componentId)
const index = parent?._children.findIndex(x => x._id === componentId)
const componentTreeNodes = get(componentTreeNodesStore)
// Check for screen and navigation component edge cases
const screenComponentId = `${screen._id}-screen`
@ -712,7 +721,11 @@ export class ComponentStore extends BudiStore {
}
// If we have children, select first child
if (component._children?.length) {
if (
component._children?.length &&
(state.selectedComponentId === navComponentId ||
componentTreeNodes[`nodeOpen-${component._id}`])
) {
return component._children[0]._id
} else if (!parent) {
return null

3
packages/builder/src/stores/builder/index.js
View file

@ -92,12 +92,13 @@ const resetBuilderHistory = () => {
export const initialise = async pkg => {
const { application } = pkg
// must be first operation to make sure subsequent requests have correct app ID
appStore.syncAppPackage(pkg)
await Promise.all([
appStore.syncAppRoutes(),
componentStore.refreshDefinitions(application?.appId),
])
builderStore.init(application)
appStore.syncAppPackage(pkg)
navigationStore.syncAppNavigation(application?.navigation)
themeStore.syncAppTheme(application)
screenStore.syncAppScreens(pkg)

36
packages/builder/src/stores/portal/componentTreeNodesStore.js Normal file
View file

@ -0,0 +1,36 @@
import { createSessionStorageStore } from "@budibase/frontend-core"
const baseStore = createSessionStorageStore("openNodes", {})
const toggleNode = componentId => {
baseStore.update(openNodes => {
openNodes[`nodeOpen-${componentId}`] = !openNodes[`nodeOpen-${componentId}`]
return openNodes
})
}
const expandNode = componentId => {
baseStore.update(openNodes => {
openNodes[`nodeOpen-${componentId}`] = true
return openNodes
})
}
const collapseNode = componentId => {
baseStore.update(openNodes => {
openNodes[`nodeOpen-${componentId}`] = false
return openNodes
})
}
const store = {
subscribe: baseStore.subscribe,
toggleNode,
expandNode,
collapseNode,
}
export default store

7
packages/cli/src/hosting/update.ts
View file

@ -13,7 +13,12 @@ import { COMPOSE_PATH } from "./makeFiles"
import { info, success } from "../utils"
import { start } from "./start"
const BB_COMPOSE_SERVICES = ["app-service", "worker-service", "proxy-service"]
const BB_COMPOSE_SERVICES = [
"app-service",
"worker-service",
"proxy-service",
"couchdb-service",
]
const BB_SINGLE_SERVICE = ["budibase"]
export async function update() {

1
packages/frontend-core/src/stores/index.js
View file

@ -1 +1,2 @@
export { createLocalStorageStore } from "./localStorage"
export { createSessionStorageStore } from "./sessionStorage"

46
packages/frontend-core/src/stores/sessionStorage.js Normal file
View file

@ -0,0 +1,46 @@
import { get, writable } from "svelte/store"
export const createSessionStorageStore = (sessionStorageKey, initialValue) => {
const store = writable(initialValue, () => {
// Hydrate from session storage when we get a new subscriber
hydrate()
// Listen for session storage changes and keep store in sync
const storageListener = ({ key }) => {
return key === sessionStorageKey && hydrate()
}
window.addEventListener("storage", storageListener)
return () => window.removeEventListener("storage", storageListener)
})
// New store setter which updates the store and sessionstorage
const set = value => {
store.set(value)
sessionStorage.setItem(sessionStorageKey, JSON.stringify(value))
}
// New store updater which updates the store and sessionstorage
const update = updaterFn => set(updaterFn(get(store)))
// Hydrates the store from sessionstorage
const hydrate = () => {
const sessionValue = sessionStorage.getItem(sessionStorageKey)
if (sessionValue == null) {
set(initialValue)
} else {
try {
store.set(JSON.parse(sessionValue))
} catch {
set(initialValue)
}
}
}
// Patch the default svelte store functions with our overrides
return {
...store,
set,
update,
}
}

1
packages/server/Dockerfile
View file

@ -66,6 +66,7 @@ COPY packages/server/dist/ dist/
COPY packages/server/docker_run.sh .
COPY packages/server/builder/ builder/
COPY packages/server/client/ client/
COPY packages/server/pm2.config.js .
ARG BUDIBASE_VERSION
ARG GIT_COMMIT_SHA

32
packages/server/src/api/controllers/automation.ts
View file

@ -1,4 +1,3 @@
import * as actions from "../../automations/actions"
import * as triggers from "../../automations/triggers"
import {
getAutomationParams,
@ -20,11 +19,12 @@ import {
Automation,
AutomationActionStepId,
AutomationResults,
BBContext,
UserCtx,
} from "@budibase/types"
import { getActionDefinitions as actionDefs } from "../../automations/actions"
import sdk from "../../sdk"
import { builderSocket } from "../../websockets"
import env from "../../environment"
async function getActionDefinitions() {
return removeDeprecated(await actionDefs())
@ -72,7 +72,7 @@ function cleanAutomationInputs(automation: Automation) {
return automation
}
export async function create(ctx: BBContext) {
export async function create(ctx: UserCtx) {
const db = context.getAppDB()
let automation = ctx.request.body
automation.appId = ctx.appId
@ -141,7 +141,7 @@ export async function handleStepEvents(
}
}
export async function update(ctx: BBContext) {
export async function update(ctx: UserCtx) {
const db = context.getAppDB()
let automation = ctx.request.body
automation.appId = ctx.appId
@ -192,7 +192,7 @@ export async function update(ctx: BBContext) {
builderSocket?.emitAutomationUpdate(ctx, automation)
}
export async function fetch(ctx: BBContext) {
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const response = await db.allDocs(
getAutomationParams(null, {
@ -202,12 +202,12 @@ export async function fetch(ctx: BBContext) {
ctx.body = response.rows.map(row => row.doc)
}
export async function find(ctx: BBContext) {
export async function find(ctx: UserCtx) {
const db = context.getAppDB()
ctx.body = await db.get(ctx.params.id)
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const db = context.getAppDB()
const automationId = ctx.params.id
const oldAutomation = await db.get<Automation>(automationId)
@ -221,11 +221,11 @@ export async function destroy(ctx: BBContext) {
builderSocket?.emitAutomationDeletion(ctx, automationId)
}
export async function logSearch(ctx: BBContext) {
export async function logSearch(ctx: UserCtx) {
ctx.body = await automations.logs.logSearch(ctx.request.body)
}
export async function clearLogError(ctx: BBContext) {
export async function clearLogError(ctx: UserCtx) {
const { automationId, appId } = ctx.request.body
await context.doInAppContext(appId, async () => {
const db = context.getProdAppDB()
@ -244,15 +244,15 @@ export async function clearLogError(ctx: BBContext) {
})
}
export async function getActionList(ctx: BBContext) {
export async function getActionList(ctx: UserCtx) {
ctx.body = await getActionDefinitions()
}
export async function getTriggerList(ctx: BBContext) {
export async function getTriggerList(ctx: UserCtx) {
ctx.body = getTriggerDefinitions()
}
export async function getDefinitionList(ctx: BBContext) {
export async function getDefinitionList(ctx: UserCtx) {
ctx.body = {
trigger: getTriggerDefinitions(),
action: await getActionDefinitions(),
@ -265,7 +265,7 @@ export async function getDefinitionList(ctx: BBContext) {
* *
*********************/
export async function trigger(ctx: BBContext) {
export async function trigger(ctx: UserCtx) {
const db = context.getAppDB()
let automation = await db.get<Automation>(ctx.params.id)
@ -275,7 +275,9 @@ export async function trigger(ctx: BBContext) {
automation,
{
fields: ctx.request.body.fields,
timeout: ctx.request.body.timeout * 1000 || 120000,
timeout:
ctx.request.body.timeout * 1000 ||
env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
},
{ getResponses: true }
)
@ -310,7 +312,7 @@ function prepareTestInput(input: any) {
return input
}
export async function test(ctx: BBContext) {
export async function test(ctx: UserCtx) {
const db = context.getAppDB()
let automation = await db.get<Automation>(ctx.params.id)
await setTestFlag(automation._id!)

11
packages/server/src/api/controllers/dev.ts
View file

@ -1,7 +1,7 @@
import fetch from "node-fetch"
import env from "../../environment"
import { checkSlashesInUrl } from "../../utilities"
import { request } from "../../utilities/workerRequests"
import { createRequest } from "../../utilities/workerRequests"
import { clearLock as redisClearLock } from "../../utilities/redis"
import { DocumentType } from "../../db/utils"
import {
@ -13,14 +13,19 @@ import {
} from "@budibase/backend-core"
import { App } from "@budibase/types"
async function redirect(ctx: any, method: string, path: string = "global") {
async function redirect(
ctx: any,
method: "GET" | "POST" | "DELETE",
path: string = "global"
) {
const { devPath } = ctx.params
const queryString = ctx.originalUrl.split("?")[1] || ""
const response = await fetch(
checkSlashesInUrl(
`${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
),
request(ctx, {
createRequest({
ctx,
method,
body: ctx.request.body,
})

2
packages/server/src/api/controllers/query/index.ts
View file

@ -22,7 +22,7 @@ import {
import { ValidQueryNameRegex } from "@budibase/shared-core"
const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
timeoutMs: env.QUERY_THREAD_TIMEOUT,
})
// simple function to append "readable" to all read queries

3
packages/server/src/automations/actions.ts
View file

@ -9,6 +9,7 @@ import * as serverLog from "./steps/serverLog"
import * as discord from "./steps/discord"
import * as slack from "./steps/slack"
import * as zapier from "./steps/zapier"
import * as n8n from "./steps/n8n"
import * as make from "./steps/make"
import * as filter from "./steps/filter"
import * as delay from "./steps/delay"
@ -48,6 +49,7 @@ const ACTION_IMPLS: Record<
slack: slack.run,
zapier: zapier.run,
integromat: make.run,
n8n: n8n.run,
}
export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
{
@ -70,6 +72,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
slack: slack.definition,
zapier: zapier.definition,
integromat: make.definition,
n8n: n8n.definition,
}
// don't add the bash script/definitions unless in self host

2
packages/server/src/automations/bullboard.ts
View file

@ -15,7 +15,7 @@ const PATH_PREFIX = "/bulladmin"
export async function init() {
// Set up queues for bull board admin
const backupQueue = await backups.getBackupQueue()
const backupQueue = backups.getBackupQueue()
const queues = [automationQueue]
if (backupQueue) {
queues.push(backupQueue)

2
packages/server/src/automations/steps/bash.ts
View file

@ -65,7 +65,7 @@ export async function run({ inputs, context }: AutomationStepInput) {
success = true
try {
stdout = execSync(command, {
timeout: environment.QUERY_THREAD_TIMEOUT || 500,
timeout: environment.QUERY_THREAD_TIMEOUT,
}).toString()
} catch (err: any) {
stdout = err.message

2
packages/server/src/automations/steps/executeQuery.ts
View file

@ -10,6 +10,8 @@ import {
AutomationStepSchema,
AutomationStepType,
} from "@budibase/types"
import { utils } from "@budibase/backend-core"
import env from "../../environment"
export const definition: AutomationStepSchema = {
name: "External Data Connector",

22
packages/server/src/automations/steps/make.ts
View file

@ -34,28 +34,8 @@ export const definition: AutomationStepSchema = {
type: AutomationIOType.JSON,
title: "Payload",
},
value1: {
type: AutomationIOType.STRING,
title: "Input Value 1",
},
value2: {
type: AutomationIOType.STRING,
title: "Input Value 2",
},
value3: {
type: AutomationIOType.STRING,
title: "Input Value 3",
},
value4: {
type: AutomationIOType.STRING,
title: "Input Value 4",
},
value5: {
type: AutomationIOType.STRING,
title: "Input Value 5",
},
},
required: ["url", "value1", "value2", "value3", "value4", "value5"],
required: ["url", "body"],
},
outputs: {
properties: {

125
packages/server/src/automations/steps/n8n.ts Normal file
View file

@ -0,0 +1,125 @@
import fetch, { HeadersInit } from "node-fetch"
import { getFetchResponse } from "./utils"
import {
AutomationActionStepId,
AutomationStepSchema,
AutomationStepInput,
AutomationStepType,
AutomationIOType,
AutomationFeature,
HttpMethod,
} from "@budibase/types"
export const definition: AutomationStepSchema = {
name: "n8n Integration",
stepTitle: "n8n",
tagline: "Trigger an n8n workflow",
description:
"Performs a webhook call to n8n and gets the response (if configured)",
icon: "ri-shut-down-line",
stepId: AutomationActionStepId.n8n,
type: AutomationStepType.ACTION,
internal: false,
features: {
[AutomationFeature.LOOPING]: true,
},
inputs: {},
schema: {
inputs: {
properties: {
url: {
type: AutomationIOType.STRING,
title: "Webhook URL",
},
method: {
type: AutomationIOType.STRING,
title: "Method",
enum: Object.values(HttpMethod),
},
authorization: {
type: AutomationIOType.STRING,
title: "Authorization",
},
body: {
type: AutomationIOType.JSON,
title: "Payload",
},
},
required: ["url", "method"],
},
outputs: {
properties: {
success: {
type: AutomationIOType.BOOLEAN,
description: "Whether call was successful",
},
httpStatus: {
type: AutomationIOType.NUMBER,
description: "The HTTP status code returned",
},
response: {
type: AutomationIOType.OBJECT,
description: "The webhook response - this can have properties",
},
},
required: ["success", "response"],
},
},
}
export async function run({ inputs }: AutomationStepInput) {
const { url, body, method, authorization } = inputs
let payload = {}
try {
payload = body?.value ? JSON.parse(body?.value) : {}
} catch (err) {
return {
httpStatus: 400,
response: "Invalid payload JSON",
success: false,
}
}
if (!url?.trim()?.length) {
return {
httpStatus: 400,
response: "Missing Webhook URL",
success: false,
}
}
let response
let request: {
method: string
headers: HeadersInit
body?: string
} = {
method: method || HttpMethod.GET,
headers: {
"Content-Type": "application/json",
Authorization: authorization,
},
}
if (!["GET", "HEAD"].includes(request.method)) {
request.body = JSON.stringify({
...payload,
})
}
try {
response = await fetch(url, request)
} catch (err: any) {
return {
httpStatus: 400,
response: err.message,
success: false,
}
}
const { status, message } = await getFetchResponse(response)
return {
httpStatus: status,
success: status === 200,
response: message,
}
}

6
packages/server/src/automations/steps/triggerAutomationRun.ts
View file

@ -9,8 +9,9 @@ import {
AutomationCustomIOType,
} from "@budibase/types"
import * as triggers from "../triggers"
import { db as dbCore, context } from "@budibase/backend-core"
import { context } from "@budibase/backend-core"
import { features } from "@budibase/pro"
import env from "../../environment"
export const definition: AutomationStepSchema = {
name: "Trigger an automation",
@ -76,7 +77,8 @@ export async function run({ inputs }: AutomationStepInput) {
automation,
{
fields: { ...fieldParams },
timeout: inputs.timeout * 1000 || 120000,
timeout:
inputs.timeout * 1000 || env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
},
{ getResponses: true }
)

20
packages/server/src/automations/steps/zapier.ts
View file

@ -32,26 +32,6 @@ export const definition: AutomationStepSchema = {
type: AutomationIOType.JSON,
title: "Payload",
},
value1: {
type: AutomationIOType.STRING,
title: "Payload Value 1",
},
value2: {
type: AutomationIOType.STRING,
title: "Payload Value 2",
},
value3: {
type: AutomationIOType.STRING,
title: "Payload Value 3",
},
value4: {
type: AutomationIOType.STRING,
title: "Payload Value 4",
},
value5: {
type: AutomationIOType.STRING,
title: "Payload Value 5",
},
},
required: ["url"],
},

68
packages/server/src/automations/tests/n8n.spec.ts Normal file
View file

@ -0,0 +1,68 @@
import { getConfig, afterAll, runStep, actions } from "./utilities"
describe("test the outgoing webhook action", () => {
let config = getConfig()
beforeAll(async () => {
await config.init()
})
afterAll()
it("should be able to run the action and default to 'get'", async () => {
const res = await runStep(actions.n8n.stepId, {
url: "http://www.example.com",
body: {
test: "IGNORE_ME",
},
})
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("GET")
expect(res.response.body).toBeUndefined()
expect(res.success).toEqual(true)
})
it("should add the payload props when a JSON string is provided", async () => {
const payload = `{ "name": "Adam", "age": 9 }`
const res = await runStep(actions.n8n.stepId, {
body: {
value: payload,
},
method: "POST",
url: "http://www.example.com",
})
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("POST")
expect(res.response.body).toEqual(`{"name":"Adam","age":9}`)
expect(res.success).toEqual(true)
})
it("should return a 400 if the JSON payload string is malformed", async () => {
const payload = `{ value1 1 }`
const res = await runStep(actions.n8n.stepId, {
value1: "ONE",
body: {
value: payload,
},
method: "POST",
url: "http://www.example.com",
})
expect(res.httpStatus).toEqual(400)
expect(res.response).toEqual("Invalid payload JSON")
expect(res.success).toEqual(false)
})
it("should not append the body if the method is HEAD", async () => {
const res = await runStep(actions.n8n.stepId, {
url: "http://www.example.com",
method: "HEAD",
body: {
test: "IGNORE_ME",
},
})
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("HEAD")
expect(res.response.body).toBeUndefined()
expect(res.success).toEqual(true)
})
})

13
packages/server/src/automations/tests/triggerAutomationRun.spec.ts
View file

@ -3,6 +3,7 @@ jest.spyOn(global.console, "error")
import * as setup from "./utilities"
import * as automation from "../index"
import { serverLogAutomation } from "../../tests/utilities/structures"
import env from "../../environment"
describe("Test triggering an automation from another automation", () => {
let config = setup.getConfig()
@ -22,7 +23,10 @@ describe("Test triggering an automation from another automation", () => {
let newAutomation = await config.createAutomation(automation)
const inputs: any = {
automation: { automationId: newAutomation._id, timeout: 12000 },
automation: {
automationId: newAutomation._id,
timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
},
}
const res = await setup.runStep(
setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
@ -33,7 +37,12 @@ describe("Test triggering an automation from another automation", () => {
})
it("should fail gracefully if the automation id is incorrect", async () => {
const inputs: any = { automation: { automationId: null, timeout: 12000 } }
const inputs: any = {
automation: {
automationId: null,
timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
},
}
const res = await setup.runStep(
setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
inputs

52
packages/server/src/environment.ts
View file

@ -18,6 +18,21 @@ function parseIntSafe(number?: string) {
}
}
const DEFAULTS = {
QUERY_THREAD_TIMEOUT: 10000,
AUTOMATION_THREAD_TIMEOUT: 12000,
AUTOMATION_SYNC_TIMEOUT: 120000,
AUTOMATION_MAX_ITERATIONS: 200,
JS_PER_EXECUTION_TIME_LIMIT_MS: 1000,
TEMPLATE_REPOSITORY: "app",
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
}
const QUERY_THREAD_TIMEOUT =
parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) ||
DEFAULTS.QUERY_THREAD_TIMEOUT
const environment = {
// features
APP_FEATURES: process.env.APP_FEATURES,
@ -42,7 +57,8 @@ const environment = {
JEST_WORKER_ID: process.env.JEST_WORKER_ID,
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
TEMPLATE_REPOSITORY: process.env.TEMPLATE_REPOSITORY || "app",
TEMPLATE_REPOSITORY:
process.env.TEMPLATE_REPOSITORY || DEFAULTS.TEMPLATE_REPOSITORY,
DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
// minor
@ -50,14 +66,20 @@ const environment = {
LOGGER: process.env.LOGGER,
ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
AUTOMATION_MAX_ITERATIONS:
parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) ||
DEFAULTS.AUTOMATION_MAX_ITERATIONS,
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),
QUERY_THREAD_TIMEOUT: QUERY_THREAD_TIMEOUT,
AUTOMATION_THREAD_TIMEOUT:
parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) ||
DEFAULTS.AUTOMATION_THREAD_TIMEOUT > QUERY_THREAD_TIMEOUT
? DEFAULTS.AUTOMATION_THREAD_TIMEOUT
: QUERY_THREAD_TIMEOUT,
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
OPENAI_API_KEY: process.env.OPENAI_API_KEY,
MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
@ -70,12 +92,21 @@ const environment = {
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
SELF_HOSTED: process.env.SELF_HOSTED,
HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
FORKED_PROCESS_NAME:
process.env.FORKED_PROCESS_NAME || DEFAULTS.FORKED_PROCESS_NAME,
JS_PER_INVOCATION_TIMEOUT_MS:
parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) ||
DEFAULTS.JS_PER_EXECUTION_TIME_LIMIT_MS,
JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe(
process.env.JS_PER_REQUEST_TIME_LIMIT_MS
),
TOP_LEVEL_PATH:
process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
JS_RUNNER_MEMORY_LIMIT:
parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) ||
DEFAULTS.JS_RUNNER_MEMORY_LIMIT,
LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
// old
CLIENT_ID: process.env.CLIENT_ID,
_set(key: string, value: any) {
@ -92,12 +123,9 @@ const environment = {
isInThread: () => {
return process.env.FORKED_PROCESS
},
TOP_LEVEL_PATH:
process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
JS_RUNNER_MEMORY_LIMIT:
parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64,
LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
getDefaults: () => {
return DEFAULTS
},
}
// clean up any environment variable edge cases

23
packages/server/src/integrations/rest.ts
View file

@ -10,6 +10,7 @@ import {
RestAuthType,
RestBasicAuthConfig,
RestBearerAuthConfig,
HttpMethod,
} from "@budibase/types"
import get from "lodash/get"
import * as https from "https"
@ -86,30 +87,30 @@ const SCHEMA: Integration = {
query: {
create: {
readable: true,
displayName: "POST",
displayName: HttpMethod.POST,
type: QueryType.FIELDS,
fields: coreFields,
},
read: {
displayName: "GET",
displayName: HttpMethod.GET,
readable: true,
type: QueryType.FIELDS,
fields: coreFields,
},
update: {
displayName: "PUT",
displayName: HttpMethod.PUT,
readable: true,
type: QueryType.FIELDS,
fields: coreFields,
},
patch: {
displayName: "PATCH",
displayName: HttpMethod.PATCH,
readable: true,
type: QueryType.FIELDS,
fields: coreFields,
},
delete: {
displayName: "DELETE",
displayName: HttpMethod.DELETE,
type: QueryType.FIELDS,
fields: coreFields,
},
@ -358,7 +359,7 @@ class RestIntegration implements IntegrationBase {
path = "",
queryString = "",
headers = {},
method = "GET",
method = HttpMethod.GET,
disabledHeaders,
bodyType,
requestBody,
@ -413,23 +414,23 @@ class RestIntegration implements IntegrationBase {
}
async create(opts: RestQuery) {
return this._req({ ...opts, method: "POST" })
return this._req({ ...opts, method: HttpMethod.POST })
}
async read(opts: RestQuery) {
return this._req({ ...opts, method: "GET" })
return this._req({ ...opts, method: HttpMethod.GET })
}
async update(opts: RestQuery) {
return this._req({ ...opts, method: "PUT" })
return this._req({ ...opts, method: HttpMethod.PUT })
}
async patch(opts: RestQuery) {
return this._req({ ...opts, method: "PATCH" })
return this._req({ ...opts, method: HttpMethod.PATCH })
}
async delete(opts: RestQuery) {
return this._req({ ...opts, method: "DELETE" })
return this._req({ ...opts, method: HttpMethod.DELETE })
}
}

4
packages/server/src/threads/automation.ts
View file

@ -303,7 +303,7 @@ class Orchestrator {
if (timeout) {
setTimeout(() => {
timeoutFlag = true
}, timeout || 12000)
}, timeout || env.AUTOMATION_THREAD_TIMEOUT)
}
stepCount++
@ -621,7 +621,7 @@ export async function executeInThread(job: Job<AutomationData>) {
const timeoutPromise = new Promise((resolve, reject) => {
setTimeout(() => {
reject(new Error("Timeout exceeded"))
}, job.data.event.timeout || 12000)
}, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT)
})
return await context.doInAppContext(appId, async () => {

95
packages/server/src/utilities/workerRequests.ts
View file

@ -1,4 +1,10 @@
import { Response, default as fetch } from "node-fetch"
import {
Response,
default as fetch,
type RequestInit,
Headers,
HeadersInit,
} from "node-fetch"
import env from "../environment"
import { checkSlashesInUrl } from "./index"
import {
@ -7,36 +13,62 @@ import {
tenancy,
logging,
env as coreEnv,
utils,
} from "@budibase/backend-core"
import { Ctx, User, EmailInvite } from "@budibase/types"
export function request(ctx?: Ctx, request?: any) {
if (!request.headers) {
request.headers = {}
interface Request {
ctx?: Ctx
method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH"
headers?: { [key: string]: string }
body?: { [key: string]: any }
}
export function createRequest(request: Request): RequestInit {
const headers: Record<string, string> = {}
const requestInit: RequestInit = {
method: request.method,
}
if (!ctx) {
request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
if (tenancy.isTenantIdSet()) {
request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
const ctx = request.ctx
if (!ctx && coreEnv.INTERNAL_API_KEY) {
headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
} else if (ctx && ctx.headers) {
// copy all Budibase utilised headers over - copying everything can have
// side effects like requests being rejected due to odd content types etc
for (let header of Object.values(constants.Header)) {
const value = ctx.headers[header]
if (value === undefined) {
continue
}
headers[header] = Array.isArray(value) ? value[0] : value
}
// be specific about auth headers
const cookie = ctx.headers[constants.Header.COOKIE],
apiKey = ctx.headers[constants.Header.API_KEY]
if (cookie) {
headers[constants.Header.COOKIE] = cookie
} else if (apiKey) {
headers[constants.Header.API_KEY] = Array.isArray(apiKey)
? apiKey[0]
: apiKey
}
}
// apply tenancy if its available
if (tenancy.isTenantIdSet()) {
headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
}
if (request.body && Object.keys(request.body).length > 0) {
request.headers["Content-Type"] = "application/json"
request.body =
typeof request.body === "object"
? JSON.stringify(request.body)
: request.body
} else {
delete request.body
}
if (ctx && ctx.headers) {
request.headers = ctx.headers
headers["Content-Type"] = "application/json"
requestInit.body = JSON.stringify(request.body)
}
// add x-budibase-correlation-id header
logging.correlation.setHeader(request.headers)
return request
logging.correlation.setHeader(headers)
requestInit.headers = headers
return requestInit
}
async function checkResponse(
@ -54,7 +86,7 @@ async function checkResponse(
}
const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
if (ctx) {
ctx.throw(msg, response.status)
ctx.throw(response.status || 500, msg)
} else {
throw msg
}
@ -85,7 +117,7 @@ export async function sendSmtpEmail({
// tenant ID will be set in header
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
request(undefined, {
createRequest({
method: "POST",
body: {
email: to,
@ -107,7 +139,8 @@ export async function removeAppFromUserRoles(ctx: Ctx, appId: string) {
const prodAppId = dbCore.getProdAppID(appId)
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${prodAppId}`),
request(ctx, {
createRequest({
ctx,
method: "DELETE",
})
)
@ -118,7 +151,7 @@ export async function allGlobalUsers(ctx: Ctx) {
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
// we don't want to use API key when getting self
request(ctx, { method: "GET" })
createRequest({ ctx, method: "GET" })
)
return checkResponse(response, "get users", { ctx })
}
@ -127,7 +160,7 @@ export async function saveGlobalUser(ctx: Ctx) {
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
// we don't want to use API key when getting self
request(ctx, { method: "POST", body: ctx.request.body })
createRequest({ ctx, method: "POST", body: ctx.request.body })
)
return checkResponse(response, "save user", { ctx })
}
@ -138,7 +171,7 @@ export async function deleteGlobalUser(ctx: Ctx) {
env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
),
// we don't want to use API key when getting self
request(ctx, { method: "DELETE" })
createRequest({ ctx, method: "DELETE" })
)
return checkResponse(response, "delete user", { ctx })
}
@ -149,7 +182,7 @@ export async function readGlobalUser(ctx: Ctx): Promise<User> {
env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
),
// we don't want to use API key when getting self
request(ctx, { method: "GET" })
createRequest({ ctx, method: "GET" })
)
return checkResponse(response, "get user", { ctx })
}
@ -159,7 +192,7 @@ export async function getChecklist(): Promise<{
}> {
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + "/api/global/configs/checklist"),
request(undefined, { method: "GET" })
createRequest({ method: "GET" })
)
return checkResponse(response, "get checklist")
}
@ -167,7 +200,7 @@ export async function getChecklist(): Promise<{
export async function generateApiKey(userId: string) {
const response = await fetch(
checkSlashesInUrl(env.WORKER_URL + "/api/global/self/api_key"),
request(undefined, { method: "POST", body: { userId } })
createRequest({ method: "POST", body: { userId } })
)
return checkResponse(response, "generate API key")
}

1
packages/shared-core/src/constants/api.ts
View file

@ -16,4 +16,5 @@ export enum Header {
CORRELATION_ID = "x-budibase-correlation-id",
AUTHORIZATION = "authorization",
MIGRATING_APP = "x-budibase-migrating-app",
COOKIE = "cookie",
}

1
packages/types/src/documents/app/automation.ts
View file

@ -69,6 +69,7 @@ export enum AutomationActionStepId {
slack = "slack",
zapier = "zapier",
integromat = "integromat",
n8n = "n8n",
}
export interface EmailInvite {

9
packages/types/src/documents/app/query.ts
View file

@ -64,3 +64,12 @@ export interface ExecuteQueryRequest {
export interface ExecuteQueryResponse {
data: Row[]
}
export enum HttpMethod {
GET = "GET",
POST = "POST",
PATCH = "PATCH",
PUT = "PUT",
HEAD = "HEAD",
DELETE = "DELETE",
}

1
packages/worker/Dockerfile
View file

@ -37,6 +37,7 @@ RUN apk del .gyp \
COPY packages/worker/dist/ dist/
COPY packages/worker/docker_run.sh .
COPY packages/server/pm2.config.js .
EXPOSE 4001