From 400a112a4bda18fe0a78f160eea16f5fca91f36b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 6 Apr 2023 18:11:44 +0100 Subject: [PATCH 01/12] Refactoring automation steps to be better defined in Typescript, which allows writing the validator for automations. --- packages/backend-core/src/plugin/utils.ts | 66 +++++++++++++++++- packages/server/src/automations/steps/bash.ts | 15 ++-- .../server/src/automations/steps/createRow.ts | 27 ++++---- .../server/src/automations/steps/delay.ts | 3 +- .../server/src/automations/steps/deleteRow.ts | 21 +++--- .../server/src/automations/steps/discord.ts | 18 ++--- .../src/automations/steps/executeQuery.ts | 21 +++--- .../src/automations/steps/executeScript.ts | 15 ++-- .../server/src/automations/steps/filter.ts | 14 ++-- .../src/automations/steps/integromat.ts | 22 +++--- packages/server/src/automations/steps/loop.ts | 26 ++++--- .../src/automations/steps/outgoingWebhook.ts | 25 ++++--- .../server/src/automations/steps/queryRows.ts | 33 +++++---- .../src/automations/steps/sendSmtpEmail.ts | 20 +++--- .../server/src/automations/steps/serverLog.ts | 10 +-- .../server/src/automations/steps/slack.ts | 14 ++-- .../server/src/automations/steps/updateRow.ts | 27 ++++---- .../server/src/automations/steps/zapier.ts | 20 +++--- .../server/src/automations/triggerInfo/app.ts | 13 ++-- .../src/automations/triggerInfo/cron.ts | 11 +-- .../src/automations/triggerInfo/rowDeleted.ts | 13 ++-- .../src/automations/triggerInfo/rowSaved.ts | 17 +++-- .../src/automations/triggerInfo/rowUpdated.ts | 17 +++-- .../src/automations/triggerInfo/webhook.ts | 15 ++-- .../types/src/documents/app/automation.ts | 68 ++++++++++++++++--- packages/types/src/documents/global/plugin.ts | 1 + 26 files changed, 364 insertions(+), 188 deletions(-) diff --git a/packages/backend-core/src/plugin/utils.ts b/packages/backend-core/src/plugin/utils.ts index 7b62248bb5..8e71685279 100644 --- a/packages/backend-core/src/plugin/utils.ts +++ b/packages/backend-core/src/plugin/utils.ts @@ -1,4 +1,12 @@ -import { DatasourceFieldType, QueryType, PluginType } from "@budibase/types" +import { + DatasourceFieldType, + QueryType, + PluginType, + AutomationStepType, + AutomationStepIdArray, + AutomationIOType, + AutomationCustomIOType, +} from "@budibase/types" import joi from "joi" const DATASOURCE_TYPES = [ @@ -19,7 +27,7 @@ function runJoi(validator: joi.Schema, schema: any) { function validateComponent(schema: any) { const validator = joi.object({ - type: joi.string().allow("component").required(), + type: joi.string().allow(PluginType.COMPONENT).required(), metadata: joi.object().unknown(true).required(), hash: joi.string().optional(), version: joi.string().optional(), @@ -53,7 +61,7 @@ function validateDatasource(schema: any) { .required() const validator = joi.object({ - type: joi.string().allow("datasource").required(), + type: joi.string().allow(PluginType.DATASOURCE).required(), metadata: joi.object().unknown(true).required(), hash: joi.string().optional(), version: joi.string().optional(), @@ -82,6 +90,55 @@ function validateDatasource(schema: any) { runJoi(validator, schema) } +function validateAutomation(schema: any) { + const basePropsValidator = joi.object({ + type: joi + .string() + .allow(...Object.values(AutomationIOType)) + .required(), + customType: joi.string().allow(...Object.values(AutomationCustomIOType)), + title: joi.string(), + description: joi.string(), + enum: joi.array().items(joi.string()), + pretty: joi.array().items(joi.string()), + }) + const stepSchemaValidator = joi + .object({ + properties: basePropsValidator, + required: joi.array().items(joi.string()), + }) + .concat(basePropsValidator) + .required() + const validator = joi.object({ + type: joi.string().allow(PluginType.AUTOMATION).required(), + metadata: joi.object().unknown(true).required(), + hash: joi.string().optional(), + version: joi.string().optional(), + schema: joi.object({ + name: joi.string().required(), + tagline: joi.string().required(), + icon: joi.string().required(), + description: joi.string().required(), + type: joi + .string() + .allow(AutomationStepType.ACTION, AutomationStepType.LOGIC) + .required(), + stepId: joi + .string() + .disallow(...AutomationStepIdArray) + .required(), + inputs: joi.object().optional(), + schema: joi + .object({ + inputs: stepSchemaValidator, + outputs: stepSchemaValidator, + }) + .required(), + }), + }) + runJoi(validator, schema) +} + export function validate(schema: any) { switch (schema?.type) { case PluginType.COMPONENT: @@ -90,6 +147,9 @@ export function validate(schema: any) { case PluginType.DATASOURCE: validateDatasource(schema) break + case PluginType.AUTOMATION: + validateAutomation(schema) + break default: throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`) } diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts index e6deb8c38f..b653095338 100644 --- a/packages/server/src/automations/steps/bash.ts +++ b/packages/server/src/automations/steps/bash.ts @@ -4,8 +4,11 @@ import * as automationUtils from "../automationUtils" import environment from "../../environment" import { AutomationActionStepId, - AutomationStepSchema, + AutomationCustomIOType, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -13,7 +16,7 @@ export const definition: AutomationStepSchema = { tagline: "Execute a bash command", icon: "JourneyEvent", description: "Run a bash script", - type: "ACTION", + type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.EXECUTE_BASH, inputs: {}, @@ -21,8 +24,8 @@ export const definition: AutomationStepSchema = { inputs: { properties: { code: { - type: "string", - customType: "code", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.CODE, title: "Code", }, }, @@ -31,11 +34,11 @@ export const definition: AutomationStepSchema = { outputs: { properties: { stdout: { - type: "string", + type: AutomationIOType.STRING, description: "Standard output of your bash command or script", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the command was successful", }, }, diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index d529127360..dac929f1ee 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -3,8 +3,11 @@ import { cleanUpRow, getError } from "../automationUtils" import { buildCtx } from "./utils" import { AutomationActionStepId, - AutomationStepSchema, + AutomationCustomIOType, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = { tagline: "Create a {{inputs.enriched.table.name}} row", icon: "TableRowAddBottom", description: "Add a row to your database", - type: "ACTION", + type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.CREATE_ROW, inputs: {}, @@ -20,14 +23,14 @@ export const definition: AutomationStepSchema = { inputs: { properties: { row: { - type: "object", + type: AutomationIOType.OBJECT, properties: { tableId: { - type: "string", - customType: "table", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, }, }, - customType: "row", + customType: AutomationCustomIOType.ROW, title: "Table", required: ["tableId"], }, @@ -37,24 +40,24 @@ export const definition: AutomationStepSchema = { outputs: { properties: { row: { - type: "object", - customType: "row", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, description: "The new row", }, response: { - type: "object", + type: AutomationIOType.OBJECT, description: "The response from the table", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the row creation was successful", }, id: { - type: "string", + type: AutomationIOType.STRING, description: "The identifier of the new row", }, revision: { - type: "string", + type: AutomationIOType.STRING, description: "The revision of the new row", }, }, diff --git a/packages/server/src/automations/steps/delay.ts b/packages/server/src/automations/steps/delay.ts index 58ca383ac1..9eeebb1667 100644 --- a/packages/server/src/automations/steps/delay.ts +++ b/packages/server/src/automations/steps/delay.ts @@ -3,6 +3,7 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -33,7 +34,7 @@ export const definition: AutomationStepSchema = { required: ["success"], }, }, - type: "LOGIC", + type: AutomationStepType.LOGIC, } export async function run({ inputs }: AutomationStepInput) { diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts index 540d95b94d..86c7703491 100644 --- a/packages/server/src/automations/steps/deleteRow.ts +++ b/packages/server/src/automations/steps/deleteRow.ts @@ -3,8 +3,11 @@ import { buildCtx } from "./utils" import { getError } from "../automationUtils" import { AutomationActionStepId, - AutomationStepSchema, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, + AutomationIOType, + AutomationCustomIOType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = { icon: "TableRowRemoveCenter", name: "Delete Row", tagline: "Delete a {{inputs.enriched.table.name}} row", - type: "ACTION", + type: AutomationStepType.ACTION, stepId: AutomationActionStepId.DELETE_ROW, internal: true, inputs: {}, @@ -20,12 +23,12 @@ export const definition: AutomationStepSchema = { inputs: { properties: { tableId: { - type: "string", - customType: "table", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, title: "Table", }, id: { - type: "string", + type: AutomationIOType.STRING, title: "Row ID", }, }, @@ -34,16 +37,16 @@ export const definition: AutomationStepSchema = { outputs: { properties: { row: { - type: "object", - customType: "row", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, description: "The deleted row", }, response: { - type: "object", + type: AutomationIOType.OBJECT, description: "The response from the table", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the deletion was successful", }, }, diff --git a/packages/server/src/automations/steps/discord.ts b/packages/server/src/automations/steps/discord.ts index 5d7487ed3b..c46220c3b2 100644 --- a/packages/server/src/automations/steps/discord.ts +++ b/packages/server/src/automations/steps/discord.ts @@ -4,6 +4,8 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, + AutomationIOType, } from "@budibase/types" const DEFAULT_USERNAME = "Budibase Automate" @@ -15,26 +17,26 @@ export const definition: AutomationStepSchema = { description: "Send a message to a Discord server", icon: "ri-discord-line", stepId: AutomationActionStepId.discord, - type: "ACTION", + type: AutomationStepType.ACTION, internal: false, inputs: {}, schema: { inputs: { properties: { url: { - type: "string", + type: AutomationIOType.STRING, title: "Discord Webhook URL", }, username: { - type: "string", + type: AutomationIOType.STRING, title: "Bot Name", }, avatar_url: { - type: "string", + type: AutomationIOType.STRING, title: "Bot Avatar URL", }, content: { - type: "string", + type: AutomationIOType.STRING, title: "Message", }, }, @@ -43,15 +45,15 @@ export const definition: AutomationStepSchema = { outputs: { properties: { httpStatus: { - type: "number", + type: AutomationIOType.NUMBER, description: "The HTTP status code of the request", }, response: { - type: "string", + type: AutomationIOType.STRING, description: "The response from the Discord Webhook", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the message sent successfully", }, }, diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index 72fb69b96c..5edc300264 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -3,8 +3,11 @@ import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { AutomationActionStepId, - AutomationStepSchema, + AutomationCustomIOType, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = { tagline: "Execute Data Connector", icon: "Data", description: "Execute a query in an external data connector", - type: "ACTION", + type: AutomationStepType.ACTION, stepId: AutomationActionStepId.EXECUTE_QUERY, internal: true, inputs: {}, @@ -20,14 +23,14 @@ export const definition: AutomationStepSchema = { inputs: { properties: { query: { - type: "object", + type: AutomationIOType.OBJECT, properties: { queryId: { - type: "string", - customType: "query", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.QUERY, }, }, - customType: "queryParams", + customType: AutomationCustomIOType.QUERY_PARAMS, title: "Parameters", required: ["queryId"], }, @@ -37,16 +40,16 @@ export const definition: AutomationStepSchema = { outputs: { properties: { response: { - type: "object", + type: AutomationIOType.OBJECT, description: "The response from the datasource execution", }, info: { - type: "object", + type: AutomationIOType.OBJECT, description: "Some query types may return extra data, like headers from a REST query", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the action was successful", }, }, diff --git a/packages/server/src/automations/steps/executeScript.ts b/packages/server/src/automations/steps/executeScript.ts index 84bdb0e2d5..ac43deb5ac 100644 --- a/packages/server/src/automations/steps/executeScript.ts +++ b/packages/server/src/automations/steps/executeScript.ts @@ -3,8 +3,11 @@ import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { AutomationActionStepId, - AutomationStepSchema, + AutomationCustomIOType, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = { tagline: "Execute JavaScript Code", icon: "Code", description: "Run a piece of JavaScript code in your automation", - type: "ACTION", + type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.EXECUTE_SCRIPT, inputs: {}, @@ -20,8 +23,8 @@ export const definition: AutomationStepSchema = { inputs: { properties: { code: { - type: "string", - customType: "code", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.CODE, title: "Code", }, }, @@ -30,11 +33,11 @@ export const definition: AutomationStepSchema = { outputs: { properties: { value: { - type: "string", + type: AutomationIOType.STRING, description: "The result of the return statement", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the action was successful", }, }, diff --git a/packages/server/src/automations/steps/filter.ts b/packages/server/src/automations/steps/filter.ts index 18914ddca6..c7ab4210ec 100644 --- a/packages/server/src/automations/steps/filter.ts +++ b/packages/server/src/automations/steps/filter.ts @@ -2,6 +2,8 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, + AutomationIOType, } from "@budibase/types" export const FilterConditions = { @@ -24,7 +26,7 @@ export const definition: AutomationStepSchema = { icon: "Branch2", description: "Conditionally halt automations which do not meet certain conditions", - type: "LOGIC", + type: AutomationStepType.LOGIC, internal: true, stepId: AutomationActionStepId.FILTER, inputs: { @@ -34,17 +36,17 @@ export const definition: AutomationStepSchema = { inputs: { properties: { field: { - type: "string", + type: AutomationIOType.STRING, title: "Reference Value", }, condition: { - type: "string", + type: AutomationIOType.STRING, title: "Condition", enum: Object.values(FilterConditions), pretty: Object.values(PrettyFilterConditions), }, value: { - type: "string", + type: AutomationIOType.STRING, title: "Comparison Value", }, }, @@ -53,11 +55,11 @@ export const definition: AutomationStepSchema = { outputs: { properties: { success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the action was successful", }, result: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the logic block passed", }, }, diff --git a/packages/server/src/automations/steps/integromat.ts b/packages/server/src/automations/steps/integromat.ts index 811c0a3d91..d7c78a6dd8 100644 --- a/packages/server/src/automations/steps/integromat.ts +++ b/packages/server/src/automations/steps/integromat.ts @@ -4,6 +4,8 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, + AutomationIOType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -13,34 +15,34 @@ export const definition: AutomationStepSchema = { "Performs a webhook call to Integromat and gets the response (if configured)", icon: "ri-shut-down-line", stepId: AutomationActionStepId.integromat, - type: "ACTION", + type: AutomationStepType.ACTION, internal: false, inputs: {}, schema: { inputs: { properties: { url: { - type: "string", + type: AutomationIOType.STRING, title: "Webhook URL", }, value1: { - type: "string", + type: AutomationIOType.STRING, title: "Input Value 1", }, value2: { - type: "string", + type: AutomationIOType.STRING, title: "Input Value 2", }, value3: { - type: "string", + type: AutomationIOType.STRING, title: "Input Value 3", }, value4: { - type: "string", + type: AutomationIOType.STRING, title: "Input Value 4", }, value5: { - type: "string", + type: AutomationIOType.STRING, title: "Input Value 5", }, }, @@ -49,15 +51,15 @@ export const definition: AutomationStepSchema = { outputs: { properties: { success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether call was successful", }, httpStatus: { - type: "number", + type: AutomationIOType.NUMBER, description: "The HTTP status code returned", }, response: { - type: "object", + type: AutomationIOType.OBJECT, description: "The webhook response - this can have properties", }, }, diff --git a/packages/server/src/automations/steps/loop.ts b/packages/server/src/automations/steps/loop.ts index 72087ae357..fdbec83f2e 100644 --- a/packages/server/src/automations/steps/loop.ts +++ b/packages/server/src/automations/steps/loop.ts @@ -1,4 +1,10 @@ -import { AutomationActionStepId, AutomationStepSchema } from "@budibase/types" +import { + AutomationActionStepId, + AutomationCustomIOType, + AutomationIOType, + AutomationStepSchema, + AutomationStepType, +} from "@budibase/types" export const definition: AutomationStepSchema = { name: "Looping", @@ -12,19 +18,19 @@ export const definition: AutomationStepSchema = { inputs: { properties: { option: { - customType: "loopOption", + customType: AutomationCustomIOType.LOOP_OPTION, title: "Input type", }, binding: { - type: "string", + type: AutomationIOType.STRING, title: "Binding / Value", }, iterations: { - type: "number", + type: AutomationIOType.NUMBER, title: "Max loop iterations", }, failure: { - type: "string", + type: AutomationIOType.STRING, title: "Failure Condition", }, }, @@ -33,20 +39,20 @@ export const definition: AutomationStepSchema = { outputs: { properties: { items: { - customType: "item", + customType: AutomationCustomIOType.ITEM, description: "The item currently being executed", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the message loop was successfully", }, iterations: { - type: "number", - descriptions: "The amount of times the block ran", + type: AutomationIOType.NUMBER, + description: "The amount of times the block ran", }, }, required: ["success", "items", "iterations"], }, }, - type: "LOGIC", + type: AutomationStepType.LOGIC, } diff --git a/packages/server/src/automations/steps/outgoingWebhook.ts b/packages/server/src/automations/steps/outgoingWebhook.ts index ea1ffeb339..f174935195 100644 --- a/packages/server/src/automations/steps/outgoingWebhook.ts +++ b/packages/server/src/automations/steps/outgoingWebhook.ts @@ -3,8 +3,11 @@ import { getFetchResponse } from "./utils" import * as automationUtils from "../automationUtils" import { AutomationActionStepId, - AutomationStepSchema, + AutomationCustomIOType, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, } from "@budibase/types" enum RequestType { @@ -27,7 +30,7 @@ export const definition: AutomationStepSchema = { tagline: "Send a {{inputs.requestMethod}} request", icon: "Send", description: "Send a request of specified method to a URL", - type: "ACTION", + type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.OUTGOING_WEBHOOK, inputs: { @@ -40,23 +43,23 @@ export const definition: AutomationStepSchema = { inputs: { properties: { requestMethod: { - type: "string", + type: AutomationIOType.STRING, enum: Object.values(RequestType), title: "Request method", }, url: { - type: "string", + type: AutomationIOType.STRING, title: "URL", }, requestBody: { - type: "string", + type: AutomationIOType.STRING, title: "JSON Body", - customType: "wide", + customType: AutomationCustomIOType.WIDE, }, headers: { - type: "string", + type: AutomationIOType.STRING, title: "Headers", - customType: "wide", + customType: AutomationCustomIOType.WIDE, }, }, required: ["requestMethod", "url"], @@ -64,15 +67,15 @@ export const definition: AutomationStepSchema = { outputs: { properties: { response: { - type: "object", + type: AutomationIOType.OBJECT, description: "The response from the webhook", }, httpStatus: { - type: "number", + type: AutomationIOType.NUMBER, description: "The HTTP status code returned", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the action was successful", }, }, diff --git a/packages/server/src/automations/steps/queryRows.ts b/packages/server/src/automations/steps/queryRows.ts index 6de518e931..1d7e8a419e 100644 --- a/packages/server/src/automations/steps/queryRows.ts +++ b/packages/server/src/automations/steps/queryRows.ts @@ -5,8 +5,11 @@ import { buildCtx } from "./utils" import * as automationUtils from "../automationUtils" import { AutomationActionStepId, - AutomationStepSchema, + AutomationCustomIOType, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, SearchFilters, Table, } from "@budibase/types" @@ -36,7 +39,7 @@ export const definition: AutomationStepSchema = { icon: "Search", name: "Query rows", tagline: "Query rows from {{inputs.enriched.table.name}} table", - type: "ACTION", + type: AutomationStepType.ACTION, stepId: AutomationActionStepId.QUERY_ROWS, internal: true, inputs: {}, @@ -44,35 +47,35 @@ export const definition: AutomationStepSchema = { inputs: { properties: { tableId: { - type: "string", - customType: "table", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, title: "Table", }, filters: { - type: "object", - customType: "filters", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.FILTERS, title: "Filtering", }, sortColumn: { - type: "string", + type: AutomationIOType.STRING, title: "Sort Column", - customType: "column", + customType: AutomationCustomIOType.COLUMN, }, sortOrder: { - type: "string", + type: AutomationIOType.STRING, title: "Sort Order", enum: Object.values(SortOrder), pretty: Object.values(SortOrderPretty), }, limit: { - type: "number", + type: AutomationIOType.NUMBER, title: "Limit", - customType: "queryLimit", + customType: AutomationCustomIOType.QUERY_LIMIT, }, onEmptyFilter: { pretty: Object.values(EmptyFilterOptionPretty), enum: Object.values(EmptyFilterOption), - type: "string", + type: AutomationIOType.STRING, title: "When Filter Empty", }, }, @@ -81,12 +84,12 @@ export const definition: AutomationStepSchema = { outputs: { properties: { rows: { - type: "array", - customType: "rows", + type: AutomationIOType.ARRAY, + customType: AutomationCustomIOType.ROWS, description: "The rows that were found", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the query was successful", }, }, diff --git a/packages/server/src/automations/steps/sendSmtpEmail.ts b/packages/server/src/automations/steps/sendSmtpEmail.ts index 67516c803d..c4af9aeaa9 100644 --- a/packages/server/src/automations/steps/sendSmtpEmail.ts +++ b/packages/server/src/automations/steps/sendSmtpEmail.ts @@ -4,6 +4,8 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, + AutomationIOType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -11,7 +13,7 @@ export const definition: AutomationStepSchema = { tagline: "Send SMTP email to {{inputs.to}}", icon: "Email", name: "Send Email (SMTP)", - type: "ACTION", + type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.SEND_EMAIL_SMTP, inputs: {}, @@ -19,27 +21,27 @@ export const definition: AutomationStepSchema = { inputs: { properties: { to: { - type: "string", + type: AutomationIOType.STRING, title: "Send To", }, from: { - type: "string", + type: AutomationIOType.STRING, title: "Send From", }, cc: { - type: "string", + type: AutomationIOType.STRING, title: "CC", }, bcc: { - type: "string", + type: AutomationIOType.STRING, title: "BCC", }, subject: { - type: "string", + type: AutomationIOType.STRING, title: "Email Subject", }, contents: { - type: "string", + type: AutomationIOType.STRING, title: "HTML Contents", }, }, @@ -48,11 +50,11 @@ export const definition: AutomationStepSchema = { outputs: { properties: { success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the email was sent", }, response: { - type: "object", + type: AutomationIOType.OBJECT, description: "A response from the email client, this may be an error", }, }, diff --git a/packages/server/src/automations/steps/serverLog.ts b/packages/server/src/automations/steps/serverLog.ts index bb2f49ede8..382f7d4efc 100644 --- a/packages/server/src/automations/steps/serverLog.ts +++ b/packages/server/src/automations/steps/serverLog.ts @@ -2,6 +2,8 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, + AutomationIOType, } from "@budibase/types" /** @@ -15,7 +17,7 @@ export const definition: AutomationStepSchema = { tagline: "Console log a value in the backend", icon: "Monitoring", description: "Logs the given text to the server (using console.log)", - type: "ACTION", + type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.SERVER_LOG, inputs: { @@ -25,7 +27,7 @@ export const definition: AutomationStepSchema = { inputs: { properties: { text: { - type: "string", + type: AutomationIOType.STRING, title: "Log", }, }, @@ -34,11 +36,11 @@ export const definition: AutomationStepSchema = { outputs: { properties: { success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the action was successful", }, message: { - type: "string", + type: AutomationIOType.STRING, description: "What was output", }, }, diff --git a/packages/server/src/automations/steps/slack.ts b/packages/server/src/automations/steps/slack.ts index 0c9320a699..21ee6ed742 100644 --- a/packages/server/src/automations/steps/slack.ts +++ b/packages/server/src/automations/steps/slack.ts @@ -4,6 +4,8 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, + AutomationIOType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -12,18 +14,18 @@ export const definition: AutomationStepSchema = { description: "Send a message to Slack", icon: "ri-slack-line", stepId: AutomationActionStepId.slack, - type: "ACTION", + type: AutomationStepType.ACTION, internal: false, inputs: {}, schema: { inputs: { properties: { url: { - type: "string", + type: AutomationIOType.STRING, title: "Incoming Webhook URL", }, text: { - type: "string", + type: AutomationIOType.STRING, title: "Message", }, }, @@ -32,15 +34,15 @@ export const definition: AutomationStepSchema = { outputs: { properties: { httpStatus: { - type: "number", + type: AutomationIOType.NUMBER, description: "The HTTP status code of the request", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the message sent successfully", }, response: { - type: "string", + type: AutomationIOType.STRING, description: "The response from the Slack Webhook", }, }, diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index 9e905a0706..96ab1d1642 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -3,8 +3,11 @@ import * as automationUtils from "../automationUtils" import { buildCtx } from "./utils" import { AutomationActionStepId, - AutomationStepSchema, + AutomationCustomIOType, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, + AutomationStepType, } from "@budibase/types" export const definition: AutomationStepSchema = { @@ -12,7 +15,7 @@ export const definition: AutomationStepSchema = { tagline: "Update a {{inputs.enriched.table.name}} row", icon: "Refresh", description: "Update a row in your database", - type: "ACTION", + type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.UPDATE_ROW, inputs: {}, @@ -20,16 +23,16 @@ export const definition: AutomationStepSchema = { inputs: { properties: { meta: { - type: "object", + type: AutomationIOType.OBJECT, title: "Field settings", }, row: { - type: "object", - customType: "row", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, title: "Table", }, rowId: { - type: "string", + type: AutomationIOType.STRING, title: "Row ID", }, }, @@ -38,24 +41,24 @@ export const definition: AutomationStepSchema = { outputs: { properties: { row: { - type: "object", - customType: "row", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, description: "The updated row", }, response: { - type: "object", + type: AutomationIOType.OBJECT, description: "The response from the table", }, success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the action was successful", }, id: { - type: "string", + type: AutomationIOType.STRING, description: "The identifier of the updated row", }, revision: { - type: "string", + type: AutomationIOType.STRING, description: "The revision of the updated row", }, }, diff --git a/packages/server/src/automations/steps/zapier.ts b/packages/server/src/automations/steps/zapier.ts index 90068e685d..75a21deaae 100644 --- a/packages/server/src/automations/steps/zapier.ts +++ b/packages/server/src/automations/steps/zapier.ts @@ -4,12 +4,14 @@ import { AutomationActionStepId, AutomationStepSchema, AutomationStepInput, + AutomationStepType, + AutomationIOType, } from "@budibase/types" export const definition: AutomationStepSchema = { name: "Zapier Webhook", stepId: AutomationActionStepId.zapier, - type: "ACTION", + type: AutomationStepType.ACTION, internal: false, description: "Trigger a Zapier Zap via webhooks", tagline: "Trigger a Zapier Zap", @@ -19,27 +21,27 @@ export const definition: AutomationStepSchema = { inputs: { properties: { url: { - type: "string", + type: AutomationIOType.STRING, title: "Webhook URL", }, value1: { - type: "string", + type: AutomationIOType.STRING, title: "Payload Value 1", }, value2: { - type: "string", + type: AutomationIOType.STRING, title: "Payload Value 2", }, value3: { - type: "string", + type: AutomationIOType.STRING, title: "Payload Value 3", }, value4: { - type: "string", + type: AutomationIOType.STRING, title: "Payload Value 4", }, value5: { - type: "string", + type: AutomationIOType.STRING, title: "Payload Value 5", }, }, @@ -48,11 +50,11 @@ export const definition: AutomationStepSchema = { outputs: { properties: { httpStatus: { - type: "number", + type: AutomationIOType.NUMBER, description: "The HTTP status code of the request", }, response: { - type: "string", + type: AutomationIOType.STRING, description: "The response from Zapier", }, }, diff --git a/packages/server/src/automations/triggerInfo/app.ts b/packages/server/src/automations/triggerInfo/app.ts index fca9acaef8..abc1463f1a 100644 --- a/packages/server/src/automations/triggerInfo/app.ts +++ b/packages/server/src/automations/triggerInfo/app.ts @@ -1,4 +1,7 @@ import { + AutomationCustomIOType, + AutomationIOType, + AutomationStepType, AutomationTriggerSchema, AutomationTriggerStepId, } from "@budibase/types" @@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = { inputs: { properties: { fields: { - type: "object", - customType: "triggerSchema", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.TRIGGER_SCHEMA, title: "Fields", }, }, @@ -25,13 +28,13 @@ export const definition: AutomationTriggerSchema = { outputs: { properties: { fields: { - type: "object", + type: AutomationIOType.OBJECT, description: "Fields submitted from the app frontend", - customType: "triggerSchema", + customType: AutomationCustomIOType.TRIGGER_SCHEMA, }, }, required: ["fields"], }, }, - type: "TRIGGER", + type: AutomationStepType.TRIGGER, } diff --git a/packages/server/src/automations/triggerInfo/cron.ts b/packages/server/src/automations/triggerInfo/cron.ts index 91b41f7243..1c47aeaeec 100644 --- a/packages/server/src/automations/triggerInfo/cron.ts +++ b/packages/server/src/automations/triggerInfo/cron.ts @@ -1,4 +1,7 @@ import { + AutomationCustomIOType, + AutomationIOType, + AutomationStepType, AutomationTriggerSchema, AutomationTriggerStepId, } from "@budibase/types" @@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = { inputs: { properties: { cron: { - type: "string", - customType: "cron", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.CRON, title: "Expression", }, }, @@ -25,12 +28,12 @@ export const definition: AutomationTriggerSchema = { outputs: { properties: { timestamp: { - type: "number", + type: AutomationIOType.NUMBER, description: "Timestamp the cron was executed", }, }, required: ["timestamp"], }, }, - type: "TRIGGER", + type: AutomationStepType.TRIGGER, } diff --git a/packages/server/src/automations/triggerInfo/rowDeleted.ts b/packages/server/src/automations/triggerInfo/rowDeleted.ts index de4a1b0412..e1014f3dbc 100644 --- a/packages/server/src/automations/triggerInfo/rowDeleted.ts +++ b/packages/server/src/automations/triggerInfo/rowDeleted.ts @@ -1,4 +1,7 @@ import { + AutomationCustomIOType, + AutomationIOType, + AutomationStepType, AutomationTriggerSchema, AutomationTriggerStepId, } from "@budibase/types" @@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = { inputs: { properties: { tableId: { - type: "string", - customType: "table", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, title: "Table", }, }, @@ -25,13 +28,13 @@ export const definition: AutomationTriggerSchema = { outputs: { properties: { row: { - type: "object", - customType: "row", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, description: "The row that was deleted", }, }, required: ["row"], }, }, - type: "TRIGGER", + type: AutomationStepType.TRIGGER, } diff --git a/packages/server/src/automations/triggerInfo/rowSaved.ts b/packages/server/src/automations/triggerInfo/rowSaved.ts index c1dde25eef..faa32ef96e 100644 --- a/packages/server/src/automations/triggerInfo/rowSaved.ts +++ b/packages/server/src/automations/triggerInfo/rowSaved.ts @@ -1,4 +1,7 @@ import { + AutomationCustomIOType, + AutomationIOType, + AutomationStepType, AutomationTriggerSchema, AutomationTriggerStepId, } from "@budibase/types" @@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = { inputs: { properties: { tableId: { - type: "string", - customType: "table", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, title: "Table", }, }, @@ -25,21 +28,21 @@ export const definition: AutomationTriggerSchema = { outputs: { properties: { row: { - type: "object", - customType: "row", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, description: "The new row that was created", }, id: { - type: "string", + type: AutomationIOType.STRING, description: "Row ID - can be used for updating", }, revision: { - type: "string", + type: AutomationIOType.STRING, description: "Revision of row", }, }, required: ["row", "id"], }, }, - type: "TRIGGER", + type: AutomationStepType.TRIGGER, } diff --git a/packages/server/src/automations/triggerInfo/rowUpdated.ts b/packages/server/src/automations/triggerInfo/rowUpdated.ts index 1bc8811d54..5e60015808 100644 --- a/packages/server/src/automations/triggerInfo/rowUpdated.ts +++ b/packages/server/src/automations/triggerInfo/rowUpdated.ts @@ -1,4 +1,7 @@ import { + AutomationCustomIOType, + AutomationIOType, + AutomationStepType, AutomationTriggerSchema, AutomationTriggerStepId, } from "@budibase/types" @@ -15,8 +18,8 @@ export const definition: AutomationTriggerSchema = { inputs: { properties: { tableId: { - type: "string", - customType: "table", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.TABLE, title: "Table", }, }, @@ -25,21 +28,21 @@ export const definition: AutomationTriggerSchema = { outputs: { properties: { row: { - type: "object", - customType: "row", + type: AutomationIOType.OBJECT, + customType: AutomationCustomIOType.ROW, description: "The row that was updated", }, id: { - type: "string", + type: AutomationIOType.STRING, description: "Row ID - can be used for updating", }, revision: { - type: "string", + type: AutomationIOType.STRING, description: "Revision of row", }, }, required: ["row", "id"], }, }, - type: "TRIGGER", + type: AutomationStepType.TRIGGER, } diff --git a/packages/server/src/automations/triggerInfo/webhook.ts b/packages/server/src/automations/triggerInfo/webhook.ts index 906967a02a..310dd66b41 100644 --- a/packages/server/src/automations/triggerInfo/webhook.ts +++ b/packages/server/src/automations/triggerInfo/webhook.ts @@ -1,4 +1,7 @@ import { + AutomationCustomIOType, + AutomationIOType, + AutomationStepType, AutomationTriggerSchema, AutomationTriggerStepId, } from "@budibase/types" @@ -15,13 +18,13 @@ export const definition: AutomationTriggerSchema = { inputs: { properties: { schemaUrl: { - type: "string", - customType: "webhookUrl", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.WEBHOOK_URL, title: "Schema URL", }, triggerUrl: { - type: "string", - customType: "webhookUrl", + type: AutomationIOType.STRING, + customType: AutomationCustomIOType.WEBHOOK_URL, title: "Trigger URL", }, }, @@ -30,12 +33,12 @@ export const definition: AutomationTriggerSchema = { outputs: { properties: { body: { - type: "object", + type: AutomationIOType.OBJECT, description: "Body of the request which hit the webhook", }, }, required: ["body"], }, }, - type: "TRIGGER", + type: AutomationStepType.TRIGGER, } diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index 0c431b1f93..bf95fdbc1f 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -1,6 +1,32 @@ import { Document } from "../document" import { EventEmitter } from "events" +export enum AutomationIOType { + OBJECT = "object", + STRING = "string", + BOOLEAN = "boolean", + NUMBER = "number", + ARRAY = "array", +} + +export enum AutomationCustomIOType { + TABLE = "table", + ROW = "row", + ROWS = "rows", + WIDE = "wide", + QUERY = "query", + QUERY_PARAMS = "queryParams", + QUERY_LIMIT = "queryLimit", + LOOP_OPTION = "loopOption", + ITEM = "item", + CODE = "code", + FILTERS = "filters", + COLUMN = "column", + TRIGGER_SCHEMA = "triggerSchema", + CRON = "cron", + WEBHOOK_URL = "webhookUrl", +} + export enum AutomationTriggerStepId { ROW_SAVED = "ROW_SAVED", ROW_UPDATED = "ROW_UPDATED", @@ -10,6 +36,12 @@ export enum AutomationTriggerStepId { CRON = "CRON", } +export enum AutomationStepType { + LOGIC = "LOGIC", + ACTION = "ACTION", + TRIGGER = "TRIGGER", +} + export enum AutomationActionStepId { SEND_EMAIL_SMTP = "SEND_EMAIL_SMTP", CREATE_ROW = "CREATE_ROW", @@ -31,6 +63,11 @@ export enum AutomationActionStepId { integromat = "integromat", } +export const AutomationStepIdArray = [ + ...Object.values(AutomationActionStepId), + ...Object.values(AutomationTriggerStepId), +] + export interface Automation extends Document { definition: { steps: AutomationStep[] @@ -41,12 +78,32 @@ export interface Automation extends Document { name: string } +interface BaseIOStructure { + type?: AutomationIOType + customType?: AutomationCustomIOType + title?: string + description?: string + enum?: string[] + pretty?: string[] + properties?: { + [key: string]: BaseIOStructure + } + required?: string[] +} + +interface InputOutputBlock { + properties: { + [key: string]: BaseIOStructure + } + required?: string[] +} + export interface AutomationStepSchema { name: string tagline: string icon: string description: string - type: string + type: AutomationStepType internal?: boolean deprecated?: boolean stepId: AutomationTriggerStepId | AutomationActionStepId @@ -55,13 +112,8 @@ export interface AutomationStepSchema { [key: string]: any } schema: { - inputs: { - [key: string]: any - } - outputs: { - [key: string]: any - } - required?: string[] + inputs: InputOutputBlock + outputs: InputOutputBlock } } diff --git a/packages/types/src/documents/global/plugin.ts b/packages/types/src/documents/global/plugin.ts index 17a8426d85..dd96bc20db 100644 --- a/packages/types/src/documents/global/plugin.ts +++ b/packages/types/src/documents/global/plugin.ts @@ -3,6 +3,7 @@ import { Document } from "../document" export enum PluginType { DATASOURCE = "datasource", COMPONENT = "component", + AUTOMATION = "automation", } export enum PluginSource { From 4badd04e334daee5265276dc6729dc4829a1d214 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Mon, 10 Apr 2023 22:33:53 +0100 Subject: [PATCH 02/12] Some automation fixes and adding option to disable logging for CLI. --- packages/backend-core/src/logging/index.ts | 2 +- .../backend-core/src/logging/pino/logger.ts | 22 +++++++++++++++++-- packages/cli/src/index.ts | 2 ++ packages/cli/src/plugins/index.ts | 2 +- .../server/src/automations/steps/delay.ts | 7 +++--- .../src/automations/steps/executeScript.ts | 2 +- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/logging/index.ts b/packages/backend-core/src/logging/index.ts index 276a8d627c..b229f47dea 100644 --- a/packages/backend-core/src/logging/index.ts +++ b/packages/backend-core/src/logging/index.ts @@ -1,5 +1,5 @@ export * as correlation from "./correlation/correlation" -export { default as logger } from "./pino/logger" +export { logger, disableLogger } from "./pino/logger" export * from "./alerts" // turn off or on context logging i.e. tenantId, appId etc diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index c82876f49a..dd4b505d30 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -5,6 +5,17 @@ import * as correlation from "../correlation" import { IdentityType } from "@budibase/types" import { LOG_CONTEXT } from "../index" +// CORE LOGGERS - for disabling + +const BUILT_INS = { + log: console.log, + error: console.error, + info: console.info, + warn: console.warn, + trace: console.trace, + debug: console.debug, +} + // LOGGER const pinoOptions: LoggerOptions = { @@ -31,6 +42,15 @@ if (env.isDev()) { export const logger = pino(pinoOptions) +export function disableLogger() { + console.log = BUILT_INS.log + console.error = BUILT_INS.error + console.info = BUILT_INS.info + console.warn = BUILT_INS.warn + console.trace = BUILT_INS.trace + console.debug = BUILT_INS.debug +} + // CONSOLE OVERRIDES interface MergingObject { @@ -166,5 +186,3 @@ const getIdentity = () => { } return identity } - -export default logger diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 77e4d6da8e..d64f25f497 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,4 +1,6 @@ #!/usr/bin/env node +import { logging } from "@budibase/backend-core" +logging.disableLogger() import "./prebuilds" import "./environment" import { getCommands } from "./options" diff --git a/packages/cli/src/plugins/index.ts b/packages/cli/src/plugins/index.ts index cfafd67720..45e024be7f 100644 --- a/packages/cli/src/plugins/index.ts +++ b/packages/cli/src/plugins/index.ts @@ -54,7 +54,7 @@ async function init(opts: PluginOpts) { if (!type || !PLUGIN_TYPE_ARR.includes(type)) { console.log( error( - "Please provide a type to init, either 'component' or 'datasource'." + "Please provide a type to init, either 'component', 'datasource' or 'automation'." ) ) return diff --git a/packages/server/src/automations/steps/delay.ts b/packages/server/src/automations/steps/delay.ts index 9eeebb1667..9cf578805c 100644 --- a/packages/server/src/automations/steps/delay.ts +++ b/packages/server/src/automations/steps/delay.ts @@ -1,8 +1,9 @@ import { wait } from "../../utilities" import { AutomationActionStepId, - AutomationStepSchema, + AutomationIOType, AutomationStepInput, + AutomationStepSchema, AutomationStepType, } from "@budibase/types" @@ -18,7 +19,7 @@ export const definition: AutomationStepSchema = { inputs: { properties: { time: { - type: "number", + type: AutomationIOType.NUMBER, title: "Delay in milliseconds", }, }, @@ -27,7 +28,7 @@ export const definition: AutomationStepSchema = { outputs: { properties: { success: { - type: "boolean", + type: AutomationIOType.BOOLEAN, description: "Whether the delay was successful", }, }, diff --git a/packages/server/src/automations/steps/executeScript.ts b/packages/server/src/automations/steps/executeScript.ts index ac43deb5ac..9bd0eaed18 100644 --- a/packages/server/src/automations/steps/executeScript.ts +++ b/packages/server/src/automations/steps/executeScript.ts @@ -41,8 +41,8 @@ export const definition: AutomationStepSchema = { description: "Whether the action was successful", }, }, + required: ["success"], }, - required: ["success"], }, } From 1c7c06e9830c8697680ce3ac6fd86eb71d221259 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Mon, 10 Apr 2023 23:48:54 +0100 Subject: [PATCH 03/12] Bulk of the automation plugin system, supporting custom plugins. --- .../src/logging/pino/middleware.ts | 2 +- packages/backend-core/src/plugin/utils.ts | 2 +- .../FlowChart/ActionModal.svelte | 4 +- .../server/src/api/controllers/automation.ts | 18 ++++++--- .../src/api/routes/tests/automation.spec.js | 4 +- packages/server/src/automations/actions.ts | 22 ++++++++-- packages/server/src/automations/index.ts | 2 +- packages/server/src/automations/steps/bash.ts | 2 +- .../src/automations/steps/executeQuery.ts | 2 +- .../src/automations/tests/utilities/index.ts | 4 +- .../server/src/tests/utilities/structures.ts | 40 ++++++++++--------- packages/server/src/threads/automation.ts | 4 +- .../types/src/documents/app/automation.ts | 5 +++ 13 files changed, 71 insertions(+), 40 deletions(-) diff --git a/packages/backend-core/src/logging/pino/middleware.ts b/packages/backend-core/src/logging/pino/middleware.ts index e9d37ab692..569420c5f2 100644 --- a/packages/backend-core/src/logging/pino/middleware.ts +++ b/packages/backend-core/src/logging/pino/middleware.ts @@ -1,5 +1,5 @@ import env from "../../environment" -import logger from "./logger" +import { logger } from "./logger" import { IncomingMessage } from "http" const pino = require("koa-pino-logger") import { Options } from "pino-http" diff --git a/packages/backend-core/src/plugin/utils.ts b/packages/backend-core/src/plugin/utils.ts index 8e71685279..f73ded0659 100644 --- a/packages/backend-core/src/plugin/utils.ts +++ b/packages/backend-core/src/plugin/utils.ts @@ -91,7 +91,7 @@ function validateDatasource(schema: any) { } function validateAutomation(schema: any) { - const basePropsValidator = joi.object({ + const basePropsValidator = joi.object().pattern(joi.string(), { type: joi .string() .allow(...Object.values(AutomationIOType)) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte index f30b49eb39..e058c48557 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte @@ -26,7 +26,7 @@ const external = actions.reduce((acc, elm) => { const [k, v] = elm - if (!v.internal) { + if (!v.internal && !v.custom) { acc[k] = v } return acc @@ -34,7 +34,7 @@ const internal = actions.reduce((acc, elm) => { const [k, v] = elm - if (v.internal) { + if (v.internal || v.custom) { acc[k] = v } delete acc.LOOP diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 8ccd3fb3d0..ae49cd83f1 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -16,9 +16,15 @@ import { setTestFlag, clearTestFlag } from "../../utilities/redis" import { context, cache, events } from "@budibase/backend-core" import { automations } from "@budibase/pro" import { Automation, BBContext } from "@budibase/types" +import { getActionDefinitions as actionDefs } from "../../automations/actions" -const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS) -const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS) +async function getActionDefinitions() { + return removeDeprecated(await actionDefs()) +} + +function getTriggerDefinitions() { + return removeDeprecated(triggers.TRIGGER_DEFINITIONS) +} /************************* * * @@ -228,17 +234,17 @@ export async function clearLogError(ctx: BBContext) { } export async function getActionList(ctx: BBContext) { - ctx.body = ACTION_DEFS + ctx.body = await getActionDefinitions() } export async function getTriggerList(ctx: BBContext) { - ctx.body = TRIGGER_DEFS + ctx.body = getTriggerDefinitions() } export async function getDefinitionList(ctx: BBContext) { ctx.body = { - trigger: TRIGGER_DEFS, - action: ACTION_DEFS, + trigger: getTriggerDefinitions(), + action: await getActionDefinitions(), } } diff --git a/packages/server/src/api/routes/tests/automation.spec.js b/packages/server/src/api/routes/tests/automation.spec.js index 2eae5e2a61..84fcbbbcf6 100644 --- a/packages/server/src/api/routes/tests/automation.spec.js +++ b/packages/server/src/api/routes/tests/automation.spec.js @@ -7,7 +7,7 @@ const { const setup = require("./utilities") const { basicAutomation, newAutomation, automationTrigger, automationStep } = setup.structures const MAX_RETRIES = 4 -const { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } = require("../../../automations") +const { TRIGGER_DEFINITIONS, BUILTIN_ACTION_DEFINITIONS } = require("../../../automations") const { events } = require("@budibase/backend-core") @@ -52,7 +52,7 @@ describe("/automations", () => { .expect('Content-Type', /json/) .expect(200) - let definitionsLength = Object.keys(ACTION_DEFINITIONS).length + let definitionsLength = Object.keys(BUILTIN_ACTION_DEFINITIONS).length definitionsLength-- // OUTGOING_WEBHOOK is deprecated expect(Object.keys(res.body.action).length).toBeGreaterThanOrEqual(definitionsLength) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 456399bc68..a82be686c0 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -15,7 +15,8 @@ import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" import * as loop from "./steps/loop" import env from "../environment" -import { AutomationStepSchema, AutomationStepInput } from "@budibase/types" +import { AutomationStepSchema, AutomationStepInput, PluginType, AutomationStep } from "@budibase/types" +import { getPlugins } from "../api/controllers/plugin" const ACTION_IMPLS: Record< string, @@ -38,7 +39,7 @@ const ACTION_IMPLS: Record< zapier: zapier.run, integromat: integromat.run, } -export const ACTION_DEFINITIONS: Record = { +export const BUILTIN_ACTION_DEFINITIONS: Record = { SEND_EMAIL_SMTP: sendSmtpEmail.definition, CREATE_ROW: createRow.definition, UPDATE_ROW: updateRow.definition, @@ -66,7 +67,22 @@ if (env.SELF_HOSTED) { // @ts-ignore ACTION_IMPLS["EXECUTE_BASH"] = bash.run // @ts-ignore - ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition + BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition +} + +export async function getActionDefinitions() { + const actionDefinitions = BUILTIN_ACTION_DEFINITIONS + if (env.SELF_HOSTED) { + const plugins = await getPlugins(PluginType.AUTOMATION) + for (let plugin of plugins) { + const schema = plugin.schema.schema as AutomationStep + actionDefinitions[schema.stepId] = { + ...schema, + custom: true, + } + } + } + return actionDefinitions } /* istanbul ignore next */ diff --git a/packages/server/src/automations/index.ts b/packages/server/src/automations/index.ts index e46500d33e..9bbab95a27 100644 --- a/packages/server/src/automations/index.ts +++ b/packages/server/src/automations/index.ts @@ -6,7 +6,7 @@ import BullQueue from "bull" export { automationQueue } from "./bullboard" export { shutdown } from "./bullboard" export { TRIGGER_DEFINITIONS } from "./triggers" -export { ACTION_DEFINITIONS } from "./actions" +export { BUILTIN_ACTION_DEFINITIONS, getActionDefinitions } from "./actions" /** * This module is built purely to kick off the worker farm and manage the inputs/outputs diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts index b653095338..820d0329db 100644 --- a/packages/server/src/automations/steps/bash.ts +++ b/packages/server/src/automations/steps/bash.ts @@ -42,8 +42,8 @@ export const definition: AutomationStepSchema = { description: "Whether the command was successful", }, }, + required: ["stdout"], }, - required: ["stdout"], }, } diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index 5edc300264..4969400552 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -53,8 +53,8 @@ export const definition: AutomationStepSchema = { description: "Whether the action was successful", }, }, + required: ["response", "success"], }, - required: ["response", "success"], }, } diff --git a/packages/server/src/automations/tests/utilities/index.ts b/packages/server/src/automations/tests/utilities/index.ts index 3c990a38d0..9ba4f950f3 100644 --- a/packages/server/src/automations/tests/utilities/index.ts +++ b/packages/server/src/automations/tests/utilities/index.ts @@ -1,6 +1,6 @@ import TestConfig from "../../../tests/utilities/TestConfiguration" import { context } from "@budibase/backend-core" -import { ACTION_DEFINITIONS, getAction } from "../../actions" +import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions" import emitter from "../../../events/index" import env from "../../../environment" @@ -57,4 +57,4 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) { } export const apiKey = "test" -export const actions = ACTION_DEFINITIONS +export const actions = BUILTIN_ACTION_DEFINITIONS diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index 207863a6aa..ed2210270e 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -1,18 +1,18 @@ -import { permissions, roles } from "@budibase/backend-core" +import { permissions, roles, utils } from "@budibase/backend-core" import { createHomeScreen } from "../../constants/screens" import { EMPTY_LAYOUT } from "../../constants/layouts" import { cloneDeep } from "lodash/fp" -import { ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations" +import { BUILTIN_ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations" import { Automation, - AutomationActionStepId, + AutomationActionStepId, AutomationStep, + AutomationStepType, + AutomationTrigger, AutomationTriggerStepId, Datasource, SourceName, } from "@budibase/types" -const { v4: uuidv4 } = require("uuid") - export function basicTable() { return { name: "TestTable", @@ -71,19 +71,19 @@ export function view(tableId: string) { } export function automationStep( - actionDefinition = ACTION_DEFINITIONS.CREATE_ROW -) { + actionDefinition = BUILTIN_ACTION_DEFINITIONS.CREATE_ROW +): AutomationStep { return { - id: uuidv4(), + id: utils.newid(), ...actionDefinition, } } export function automationTrigger( triggerDefinition = TRIGGER_DEFINITIONS.ROW_SAVED -) { +): AutomationTrigger { return { - id: uuidv4(), + id: utils.newid(), ...triggerDefinition, } } @@ -106,7 +106,7 @@ export function newAutomation({ steps, trigger }: any = {}) { return automation } -export function basicAutomation(appId?: string) { +export function basicAutomation(appId?: string): Automation { return { name: "My Automation", screenId: "kasdkfldsafkl", @@ -119,18 +119,22 @@ export function basicAutomation(appId?: string) { tagline: "test", icon: "test", description: "test", - type: "trigger", + type: AutomationStepType.TRIGGER, id: "test", inputs: {}, schema: { - inputs: {}, - outputs: {}, + inputs: { + properties: {}, + }, + outputs: { + properties: {}, + }, }, }, steps: [], }, type: "automation", - appId, + appId: appId!, } } @@ -154,7 +158,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation { inputs: { tableId, }, - schema: ACTION_DEFINITIONS.QUERY_ROWS.schema, + schema: BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS.schema, }, { id: "c", @@ -163,7 +167,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation { internal: true, inputs: loopOpts, blockToLoop: "d", - schema: ACTION_DEFINITIONS.LOOP.schema, + schema: BUILTIN_ACTION_DEFINITIONS.LOOP.schema, }, { id: "d", @@ -173,7 +177,7 @@ export function loopAutomation(tableId: string, loopOpts?: any): Automation { inputs: { text: "log statement", }, - schema: ACTION_DEFINITIONS.SERVER_LOG.schema, + schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema, }, ], trigger: { diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index c19b05a8a7..adc1a83af0 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -27,8 +27,8 @@ import { processObject } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import * as sdkUtils from "../sdk/utils" import env from "../environment" -const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId -const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId +const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId +const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId const CRON_STEP_ID = triggerDefs.CRON.stepId const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED } diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index bf95fdbc1f..aa600c6375 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -73,9 +73,13 @@ export interface Automation extends Document { steps: AutomationStep[] trigger: AutomationTrigger } + screenId?: string + uiTree?: any appId: string live?: boolean name: string + internal?: boolean + type?: string } interface BaseIOStructure { @@ -115,6 +119,7 @@ export interface AutomationStepSchema { inputs: InputOutputBlock outputs: InputOutputBlock } + custom?: boolean } export interface AutomationStep extends AutomationStepSchema { From ac37d00f69ef4d70ac303a535792d36beacf3ff3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 11 Apr 2023 16:37:26 +0100 Subject: [PATCH 04/12] Updating plugins to have a SDK, which can be used for automation action retrieval as well as datasources. --- packages/cli/.gitignore | 3 + .../src/api/controllers/plugin/index.ts | 49 ++------------ packages/server/src/automations/actions.ts | 66 ++++++++++++------- packages/server/src/integrations/index.ts | 6 +- packages/server/src/sdk/index.ts | 2 + packages/server/src/sdk/plugins/index.ts | 5 ++ packages/server/src/sdk/plugins/plugins.ts | 41 ++++++++++++ .../server/src/tests/utilities/structures.ts | 8 ++- .../server/src/utilities/fileSystem/plugin.ts | 17 +++-- 9 files changed, 120 insertions(+), 77 deletions(-) create mode 100644 packages/server/src/sdk/plugins/index.ts create mode 100644 packages/server/src/sdk/plugins/plugins.ts diff --git a/packages/cli/.gitignore b/packages/cli/.gitignore index 8d8de9a0da..b1d9be617c 100644 --- a/packages/cli/.gitignore +++ b/packages/cli/.gitignore @@ -7,3 +7,6 @@ envoy.yaml *.tar.gz prebuilds/ dist/ +budibase-automation/ +budibase-component/ +budibase-datasource/ diff --git a/packages/server/src/api/controllers/plugin/index.ts b/packages/server/src/api/controllers/plugin/index.ts index faecbc1fd8..e8ae847fc6 100644 --- a/packages/server/src/api/controllers/plugin/index.ts +++ b/packages/server/src/api/controllers/plugin/index.ts @@ -1,31 +1,11 @@ -import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders" -import { - plugins as pluginCore, - db as dbCore, - tenancy, - objectStore, -} from "@budibase/backend-core" -import { PluginType, FileType, PluginSource, Plugin } from "@budibase/types" +import { npmUpload, urlUpload, githubUpload } from "./uploaders" +import { plugins as pluginCore } from "@budibase/backend-core" +import { PluginType, FileType, PluginSource } from "@budibase/types" import env from "../../../environment" import { ClientAppSocket } from "../../../websocket" +import sdk from "../../../sdk" import { sdk as pro } from "@budibase/pro" -export async function getPlugins(type?: PluginType) { - const db = tenancy.getGlobalDB() - const response = await db.allDocs( - dbCore.getPluginParams(null, { - include_docs: true, - }) - ) - let plugins = response.rows.map((row: any) => row.doc) as Plugin[] - plugins = objectStore.enrichPluginURLs(plugins) - if (type) { - return plugins.filter((plugin: Plugin) => plugin.schema?.type === type) - } else { - return plugins - } -} - export async function upload(ctx: any) { const plugins: FileType[] = ctx.request.files.file.length > 1 @@ -35,7 +15,7 @@ export async function upload(ctx: any) { let docs = [] // can do single or multiple plugins for (let plugin of plugins) { - const doc = await processUploadedPlugin(plugin, PluginSource.FILE) + const doc = await sdk.plugins.processUploaded(plugin, PluginSource.FILE) docs.push(doc) } ctx.body = { @@ -105,7 +85,7 @@ export async function create(ctx: any) { } export async function fetch(ctx: any) { - ctx.body = await getPlugins() + ctx.body = await sdk.plugins.fetch() } export async function destroy(ctx: any) { @@ -119,20 +99,3 @@ export async function destroy(ctx: any) { ctx.throw(400, err.message) } } - -export async function processUploadedPlugin( - plugin: FileType, - source?: PluginSource -) { - const { metadata, directory } = await fileUpload(plugin) - pluginCore.validate(metadata?.schema) - - // Only allow components in cloud - if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) { - throw new Error("Only component plugins are supported outside of self-host") - } - - const doc = await pro.plugins.storePlugin(metadata, directory, source) - ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash }) - return doc -} diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index a82be686c0..2a6b760725 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -15,8 +15,14 @@ import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" import * as loop from "./steps/loop" import env from "../environment" -import { AutomationStepSchema, AutomationStepInput, PluginType, AutomationStep } from "@budibase/types" -import { getPlugins } from "../api/controllers/plugin" +import { + AutomationStepSchema, + AutomationStepInput, + PluginType, + AutomationStep, +} from "@budibase/types" +import sdk from "../sdk" +import { getAutomationPlugin } from "../utilities/fileSystem" const ACTION_IMPLS: Record< string, @@ -39,25 +45,26 @@ const ACTION_IMPLS: Record< zapier: zapier.run, integromat: integromat.run, } -export const BUILTIN_ACTION_DEFINITIONS: Record = { - SEND_EMAIL_SMTP: sendSmtpEmail.definition, - CREATE_ROW: createRow.definition, - UPDATE_ROW: updateRow.definition, - DELETE_ROW: deleteRow.definition, - OUTGOING_WEBHOOK: outgoingWebhook.definition, - EXECUTE_SCRIPT: executeScript.definition, - EXECUTE_QUERY: executeQuery.definition, - SERVER_LOG: serverLog.definition, - DELAY: delay.definition, - FILTER: filter.definition, - QUERY_ROWS: queryRow.definition, - LOOP: loop.definition, - // these used to be lowercase step IDs, maintain for backwards compat - discord: discord.definition, - slack: slack.definition, - zapier: zapier.definition, - integromat: integromat.definition, -} +export const BUILTIN_ACTION_DEFINITIONS: Record = + { + SEND_EMAIL_SMTP: sendSmtpEmail.definition, + CREATE_ROW: createRow.definition, + UPDATE_ROW: updateRow.definition, + DELETE_ROW: deleteRow.definition, + OUTGOING_WEBHOOK: outgoingWebhook.definition, + EXECUTE_SCRIPT: executeScript.definition, + EXECUTE_QUERY: executeQuery.definition, + SERVER_LOG: serverLog.definition, + DELAY: delay.definition, + FILTER: filter.definition, + QUERY_ROWS: queryRow.definition, + LOOP: loop.definition, + // these used to be lowercase step IDs, maintain for backwards compat + discord: discord.definition, + slack: slack.definition, + zapier: zapier.definition, + integromat: integromat.definition, + } // don't add the bash script/definitions unless in self host // the fact this isn't included in any definitions means it cannot be @@ -73,7 +80,7 @@ if (env.SELF_HOSTED) { export async function getActionDefinitions() { const actionDefinitions = BUILTIN_ACTION_DEFINITIONS if (env.SELF_HOSTED) { - const plugins = await getPlugins(PluginType.AUTOMATION) + const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION) for (let plugin of plugins) { const schema = plugin.schema.schema as AutomationStep actionDefinitions[schema.stepId] = { @@ -86,8 +93,17 @@ export async function getActionDefinitions() { } /* istanbul ignore next */ -export async function getAction(actionName: string) { - if (ACTION_IMPLS[actionName] != null) { - return ACTION_IMPLS[actionName] +export async function getAction(stepId: string) { + if (ACTION_IMPLS[stepId] != null) { + return ACTION_IMPLS[stepId] + } + // must be a plugin + if (env.SELF_HOSTED) { + const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION) + const found = plugins.find(plugin => plugin.schema.schema.stepId === stepId) + if (!found) { + throw new Error(`Unable to find action implementation for "${stepId}"`) + } + return (await getAutomationPlugin(found)).action } } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index edbce6db0a..f3285e441f 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -14,11 +14,11 @@ import firebase from "./firebase" import redis from "./redis" import snowflake from "./snowflake" import oracle from "./oracle" -import { getPlugins } from "../api/controllers/plugin" import { SourceName, Integration, PluginType } from "@budibase/types" import { getDatasourcePlugin } from "../utilities/fileSystem" import env from "../environment" import { cloneDeep } from "lodash" +import sdk from "../sdk" const DEFINITIONS: { [key: string]: Integration } = { [SourceName.POSTGRES]: postgres.schema, @@ -79,7 +79,7 @@ export async function getDefinition(source: SourceName): Promise { export async function getDefinitions() { const pluginSchemas: { [key: string]: Integration } = {} if (env.SELF_HOSTED) { - const plugins = await getPlugins(PluginType.DATASOURCE) + const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE) // extract the actual schema from each custom for (let plugin of plugins) { const sourceId = plugin.name @@ -103,7 +103,7 @@ export async function getIntegration(integration: string) { return INTEGRATIONS[integration] } if (env.SELF_HOSTED) { - const plugins = await getPlugins(PluginType.DATASOURCE) + const plugins = await sdk.plugins.fetch(PluginType.DATASOURCE) for (let plugin of plugins) { if (plugin.name === integration) { // need to use commonJS require due to its dynamic runtime nature diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 294c99a12c..1bf7d89604 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -6,6 +6,7 @@ import { default as datasources } from "./app/datasources" import { default as queries } from "./app/queries" import { default as rows } from "./app/rows" import { default as users } from "./users" +import { default as plugins } from "./plugins" const sdk = { backups, @@ -16,6 +17,7 @@ const sdk = { users, datasources, queries, + plugins, } // default export for TS diff --git a/packages/server/src/sdk/plugins/index.ts b/packages/server/src/sdk/plugins/index.ts new file mode 100644 index 0000000000..863cfa0db6 --- /dev/null +++ b/packages/server/src/sdk/plugins/index.ts @@ -0,0 +1,5 @@ +import * as plugins from "./plugins" + +export default { + ...plugins, +} diff --git a/packages/server/src/sdk/plugins/plugins.ts b/packages/server/src/sdk/plugins/plugins.ts new file mode 100644 index 0000000000..58670146ec --- /dev/null +++ b/packages/server/src/sdk/plugins/plugins.ts @@ -0,0 +1,41 @@ +import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types" +import { + db as dbCore, + objectStore, + plugins as pluginCore, + tenancy, +} from "@budibase/backend-core" +import { fileUpload } from "../../api/controllers/plugin/file" +import env from "../../environment" +import { ClientAppSocket } from "../../websocket" +import { sdk as pro } from "@budibase/pro" + +export async function fetch(type?: PluginType) { + const db = tenancy.getGlobalDB() + const response = await db.allDocs( + dbCore.getPluginParams(null, { + include_docs: true, + }) + ) + let plugins = response.rows.map((row: any) => row.doc) as Plugin[] + plugins = objectStore.enrichPluginURLs(plugins) + if (type) { + return plugins.filter((plugin: Plugin) => plugin.schema?.type === type) + } else { + return plugins + } +} + +export async function processUploaded(plugin: FileType, source?: PluginSource) { + const { metadata, directory } = await fileUpload(plugin) + pluginCore.validate(metadata?.schema) + + // Only allow components in cloud + if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) { + throw new Error("Only component plugins are supported outside of self-host") + } + + const doc = await pro.plugins.storePlugin(metadata, directory, source) + ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash }) + return doc +} diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index ed2210270e..e4e03fb85d 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -2,10 +2,14 @@ import { permissions, roles, utils } from "@budibase/backend-core" import { createHomeScreen } from "../../constants/screens" import { EMPTY_LAYOUT } from "../../constants/layouts" import { cloneDeep } from "lodash/fp" -import { BUILTIN_ACTION_DEFINITIONS, TRIGGER_DEFINITIONS } from "../../automations" +import { + BUILTIN_ACTION_DEFINITIONS, + TRIGGER_DEFINITIONS, +} from "../../automations" import { Automation, - AutomationActionStepId, AutomationStep, + AutomationActionStepId, + AutomationStep, AutomationStepType, AutomationTrigger, AutomationTriggerStepId, diff --git a/packages/server/src/utilities/fileSystem/plugin.ts b/packages/server/src/utilities/fileSystem/plugin.ts index 0bc2b7de44..3e1e9bef4d 100644 --- a/packages/server/src/utilities/fileSystem/plugin.ts +++ b/packages/server/src/utilities/fileSystem/plugin.ts @@ -5,6 +5,7 @@ import { join } from "path" import { objectStore } from "@budibase/backend-core" const DATASOURCE_PATH = join(budibaseTempDir(), "datasource") +const AUTOMATION_PATH = join(budibaseTempDir(), "automation") export const getPluginMetadata = async (path: string) => { let metadata: any = {} @@ -33,12 +34,12 @@ export const getPluginMetadata = async (path: string) => { return { metadata, directory: path } } -export const getDatasourcePlugin = async (plugin: Plugin) => { +async function getPluginImpl(path: string, plugin: Plugin) { const hash = plugin.schema?.hash - if (!fs.existsSync(DATASOURCE_PATH)) { - fs.mkdirSync(DATASOURCE_PATH) + if (!fs.existsSync(path)) { + fs.mkdirSync(path) } - const filename = join(DATASOURCE_PATH, plugin.name) + const filename = join(path, plugin.name) const metadataName = `${filename}.bbmetadata` if (fs.existsSync(filename)) { const currentHash = fs.readFileSync(metadataName, "utf8") @@ -62,3 +63,11 @@ export const getDatasourcePlugin = async (plugin: Plugin) => { return require(filename) } + +export const getDatasourcePlugin = async (plugin: Plugin) => { + return getPluginImpl(DATASOURCE_PATH, plugin) +} + +export const getAutomationPlugin = async (plugin: Plugin) => { + return getPluginImpl(AUTOMATION_PATH, plugin) +} From cb73e3b1c3966eed89289db64795c802383ee00c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 12 Apr 2023 16:22:30 +0100 Subject: [PATCH 05/12] Fixing build issue. --- packages/server/src/watch.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/watch.ts b/packages/server/src/watch.ts index 09c7312e3b..8f3fe22023 100644 --- a/packages/server/src/watch.ts +++ b/packages/server/src/watch.ts @@ -3,7 +3,7 @@ import env from "./environment" import chokidar from "chokidar" import fs from "fs" import { constants, tenancy } from "@budibase/backend-core" -import { processUploadedPlugin } from "./api/controllers/plugin" +import pluginsSdk from "./sdk/plugins" export function watch() { const watchPath = path.join(env.PLUGINS_DIR, "./**/*.tar.gz") @@ -27,7 +27,7 @@ export function watch() { const split = path.split("/") const name = split[split.length - 1] console.log("Importing plugin:", path) - await processUploadedPlugin({ name, path }) + await pluginsSdk.processUploaded({ name, path }) } catch (err: any) { const message = err?.message ? err?.message : err console.error("Failed to import plugin:", message) From d36cd3b1f8a2d2fa7c1f67d28a7507f32ee3dbec Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Tue, 18 Apr 2023 14:37:29 +0100 Subject: [PATCH 06/12] Builder Unit Test PoC (#10173) * wip * wip --- .github/workflows/budibase_ci.yml | 3 +- packages/bbui/src/Label/Label.svelte | 1 + packages/bbui/src/Typography/Body.svelte | 1 + packages/bbui/src/Typography/Heading.svelte | 1 + packages/builder/package.json | 8 +- .../builder/src/builderStore/store/history.js | 4 +- .../src/builderStore/store/history.test.js | 345 ++++++++++ .../common/inputs/EditableLabel.test.js | 37 + .../{dataUtils.spec.js => dataUtils.test.js} | 11 +- .../{duplicate.spec.js => duplicate.test.js} | 4 +- packages/builder/src/stores/portal/admin.js | 32 +- .../builder/src/stores/portal/admin.test.js | 268 ++++++++ .../builder/src/stores/portal/backups.test.js | 173 +++++ packages/builder/vite.config.js | 5 + packages/builder/vitest.setup.js | 28 + yarn.lock | 639 +++++++++++++++++- 16 files changed, 1510 insertions(+), 50 deletions(-) create mode 100644 packages/builder/src/builderStore/store/history.test.js create mode 100644 packages/builder/src/components/common/inputs/EditableLabel.test.js rename packages/builder/src/helpers/tests/{dataUtils.spec.js => dataUtils.test.js} (79%) rename packages/builder/src/helpers/tests/{duplicate.spec.js => duplicate.test.js} (91%) create mode 100644 packages/builder/src/stores/portal/admin.test.js create mode 100644 packages/builder/src/stores/portal/backups.test.js create mode 100644 packages/builder/vitest.setup.js diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index a9de0ba342..65e6529678 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -56,6 +56,7 @@ jobs: run: yarn install:pro $BRANCH $BASE_BRANCH - run: yarn - run: yarn bootstrap + - run: yarn build - run: yarn test - uses: codecov/codecov-action@v3 with: @@ -94,4 +95,4 @@ jobs: yarn test:ci env: BB_ADMIN_USER_EMAIL: admin - BB_ADMIN_USER_PASSWORD: admin \ No newline at end of file + BB_ADMIN_USER_PASSWORD: admin diff --git a/packages/bbui/src/Label/Label.svelte b/packages/bbui/src/Label/Label.svelte index 261ca946ea..71b0967d99 100644 --- a/packages/bbui/src/Label/Label.svelte +++ b/packages/bbui/src/Label/Label.svelte @@ -9,6 +9,7 @@