Some initial work towards webhooks, that generates schema similar to integromat.

packages/builder/src/components/automation/AutomationPanel/BlockList/AutomationBlock.svelte

@@ -1,11 +1,15 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { backendUiStore, automationStore } from "builderStore"
   import analytics from "analytics"
 
   export let blockDefinition
   export let stepId
   export let blockType
 
+  $: blockDefinitions = $automationStore.blockDefinitions
+  $: instanceId = $backendUiStore.selectedDatabase._id
+  $: automation = $automationStore.selectedAutomation?.automation
+
   function addBlockToAutomation() {
     automationStore.actions.addBlockToAutomation({
       ...blockDefinition,
@@ -13,6 +17,12 @@
       stepId,
       type: blockType,
     })
+    if (stepId === blockDefinitions.TRIGGER["WEBHOOK"].stepId) {
+      automationStore.actions.save({
+        instanceId,
+        automation,
+      })
+    }
     analytics.captureEvent("Added Automation Block", {
       name: blockDefinition.name,
     })
@@ -62,6 +72,7 @@
   }
   .automation-text p {
     font-size: 12px;
+    line-height: 1.4;
     color: var(--grey-7);
     margin: 0;
   }

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -1,8 +1,9 @@
 <script>
   import TableSelector from "./ParamInputs/TableSelector.svelte"
   import RowSelector from "./ParamInputs/RowSelector.svelte"
-  import { Input, TextArea, Select, Label } from "@budibase/bbui"
+  import { Button, Input, TextArea, Select, Label } from "@budibase/bbui"
   import { automationStore } from "builderStore"
+  import { notifier } from "builderStore/store/notifications"
   import BindableInput from "../../userInterface/BindableInput.svelte"
 
   export let block
@@ -42,6 +43,20 @@
     }
     return bindings
   }
+
+  function fullWebhookURL(uri) {
+    return `http://localhost:4001/${uri}`
+  }
+
+  function copyToClipboard(input) {
+    const dummy = document.createElement("textarea")
+    document.body.appendChild(dummy)
+    dummy.value = input
+    dummy.select()
+    document.execCommand("copy")
+    document.body.removeChild(dummy)
+    notifier.success(`URL copied to clipboard`)
+  }
 </script>
 
 <div class="container" data-cy="automation-block-setup">
@@ -64,6 +79,13 @@
         <TableSelector bind:value={block.inputs[key]} />
       {:else if value.customType === 'row'}
         <RowSelector bind:value={block.inputs[key]} {bindings} />
+      {:else if value.customType === 'webhookUrl'}
+        <div class="copy-area">
+          <Input disabled="true" thin value={fullWebhookURL(block.inputs[key])} />
+          <span class="copy-btn" on:click={() => copyToClipboard(fullWebhookURL(block.inputs[key]))}>
+            <i class="ri-clipboard-line copy-icon"></i>
+          </span>
+        </div>
       {:else if value.type === 'string' || value.type === 'number'}
         <BindableInput
           type="string"
@@ -92,4 +114,27 @@
     padding: 12px;
     margin-top: 8px;
   }
+
+  .copy-area {
+    position: relative;
+  }
+
+  .copy-btn {
+    position: absolute;
+    border: none;
+    border-radius: 50%;
+    height: 24px;
+    width: 24px;
+    background: white;
+    right: var(--spacing-s);
+    bottom: 9px;
+    display: flex;
+    flex-direction: row;
+    justify-content: center;
+    align-items: center;
+  }
+
+  .copy-btn:hover {
+    background-color: var(--grey-4);
+  }
 </style>

packages/server/package.json

@@ -59,6 +59,7 @@
     "fs-extra": "^8.1.0",
     "jimp": "^0.16.1",
     "joi": "^17.2.1",
+    "jsonschema": "^1.4.0",
     "jsonwebtoken": "^8.5.1",
     "koa": "^2.7.0",
     "koa-body": "^4.2.0",
@@ -77,6 +78,7 @@
     "sanitize-s3-objectkey": "^0.0.1",
     "squirrelly": "^7.5.0",
     "tar-fs": "^2.1.0",
+    "to-json-schema": "^0.2.5",
     "uuid": "^3.3.2",
     "validate.js": "^0.13.1",
     "worker-farm": "^1.7.0",

packages/server/src/api/controllers/automation.js

@@ -2,8 +2,10 @@ const CouchDB = require("../../db")
 const actions = require("../../automations/actions")
 const logic = require("../../automations/logic")
 const triggers = require("../../automations/triggers")
+const webhooks = require("./webhook")
 const { getAutomationParams, generateAutomationID } = require("../../db/utils")
 
+const WH_STEP_ID = triggers.BUILTIN_DEFINITIONS.WEBHOOK.stepId
 /*************************
  *                       *
  *   BUILDER FUNCTIONS   *
@@ -30,6 +32,68 @@ function cleanAutomationInputs(automation) {
   return automation
 }
 
+/**
+ * This function handles checking if any webhooks need to be created or deleted for automations.
+ * @param {object} user The user object, including all auth info
+ * @param {object|undefined} oldAuto The old automation object if updating/deleting
+ * @param {object|undefined} newAuto The new automation object if creating/updating
+ * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
+ * written to DB (this does not write to DB as it would be wasteful to repeat).
+ */
+async function checkForWebhooks({ user, oldAuto, newAuto }) {
+  function isWebhookTrigger(auto) {
+    return (
+      auto &&
+      auto.definition.trigger &&
+      auto.definition.trigger.stepId === WH_STEP_ID
+    )
+  }
+  // need to delete webhook
+  if (
+    isWebhookTrigger(oldAuto) &&
+    !isWebhookTrigger(newAuto) &&
+    oldAuto.definition.trigger.webhook
+  ) {
+    const ctx = {
+      user,
+      params: {
+        id: oldAuto.definition.trigger.webhook.id,
+        rev: oldAuto.definition.trigger.webhook.rev,
+      },
+    }
+    // reset the inputs to remove the URLs
+    if (newAuto && newAuto.definition.trigger) {
+      const trigger = newAuto.definition.trigger
+      delete trigger.webhook
+      delete trigger.inputs.schemaUrl
+      delete trigger.inputs.triggerUrl
+    }
+    await webhooks.destroy(ctx)
+  }
+  // need to create webhook
+  else if (!isWebhookTrigger(oldAuto) && isWebhookTrigger(newAuto)) {
+    const ctx = {
+      user,
+      request: {
+        body: new webhooks.Webhook(
+          "Automation webhook",
+          webhooks.WebhookType.AUTOMATION,
+          newAuto._id
+        ),
+      },
+    }
+    await webhooks.save(ctx)
+    const id = ctx.body.webhook._id,
+      rev = ctx.body.webhook._rev
+    newAuto.definition.trigger.webhook = { id, rev }
+    newAuto.definition.trigger.inputs = {
+      schemaUrl: `api/webhooks/schema/${user.instanceId}/${id}`,
+      triggerUrl: `api/webhooks/trigger/${user.instanceId}/${id}`,
+    }
+  }
+  return newAuto
+}
+
 exports.create = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
@@ -39,7 +103,8 @@ exports.create = async function(ctx) {
 
   automation.type = "automation"
   automation = cleanAutomationInputs(automation)
-  const response = await db.post(automation)
+  automation = await checkForWebhooks({ user: ctx.user, newAuto: automation })
+  const response = await db.put(automation)
   automation._rev = response.rev
 
   ctx.status = 200
@@ -56,8 +121,13 @@ exports.update = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
   automation.appId = ctx.user.appId
-
+  const oldAutomation = await db.get(automation._id)
   automation = cleanAutomationInputs(automation)
+  automation = await checkForWebhooks({
+    user: ctx.user,
+    oldAuto: oldAutomation,
+    newAuto: automation,
+  })
   const response = await db.put(automation)
   automation._rev = response.rev
 
@@ -89,6 +159,8 @@ exports.find = async function(ctx) {
 
 exports.destroy = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
+  const oldAutomation = await db.get(ctx.params.id)
+  await checkForWebhooks({ user: ctx.user, oldAuto: oldAutomation })
   ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
 }
 

packages/server/src/api/controllers/webhook.js

@@ -0,0 +1,94 @@
+const CouchDB = require("../../db")
+const { generateWebhookID, getWebhookParams } = require("../../db/utils")
+const toJsonSchema = require("to-json-schema")
+const validate = require("jsonschema").validate
+const triggers = require("../../automations/triggers")
+
+const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
+
+function Webhook(name, type, target) {
+  this.live = true
+  this.name = name
+  this.action = {
+    type,
+    target,
+  }
+}
+
+exports.Webhook = Webhook
+
+exports.WebhookType = {
+  AUTOMATION: "automation",
+}
+
+exports.fetch = async ctx => {
+  const db = new CouchDB(ctx.user.instanceId)
+  const response = await db.allDocs(
+    getWebhookParams(null, {
+      include_docs: true,
+    })
+  )
+  ctx.body = response.rows.map(row => row.doc)
+}
+
+exports.save = async ctx => {
+  const db = new CouchDB(ctx.user.instanceId)
+  const webhook = ctx.request.body
+  webhook.appId = ctx.user.appId
+
+  // check that the webhook exists
+  if (webhook._id) {
+    await db.get(webhook._id)
+  } else {
+    webhook._id = generateWebhookID()
+  }
+  const response = await db.put(webhook)
+  ctx.body = {
+    message: "Webhook created successfully",
+    webhook: {
+      ...webhook,
+      ...response,
+    },
+  }
+}
+
+exports.destroy = async ctx => {
+  const db = new CouchDB(ctx.user.instanceId)
+  ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
+}
+
+exports.buildSchema = async ctx => {
+  const db = new CouchDB(ctx.params.instance)
+  const webhook = await db.get(ctx.params.id)
+  webhook.bodySchema = toJsonSchema(ctx.request.body)
+  // update the automation outputs
+  if (webhook.action.type === exports.WebhookType.AUTOMATION) {
+    let automation = await db.get(webhook.action.target)
+    const autoOutputs = automation.definition.trigger.schema.outputs
+    let properties = webhook.bodySchema.properties
+    for (let prop of Object.keys(properties)) {
+      autoOutputs.properties[prop] = {
+        type: properties[prop].type,
+        description: AUTOMATION_DESCRIPTION,
+      }
+    }
+    await db.put(automation)
+  }
+  ctx.body = await db.put(webhook)
+}
+
+exports.trigger = async ctx => {
+  const db = new CouchDB(ctx.params.instance)
+  const webhook = await db.get(ctx.params.id)
+  // validate against the schema
+  if (!webhook.bodySchema) {
+    ctx.throw(400, "Webhook has not been fully configured, no schema created")
+  }
+  validate(ctx.request.body, webhook.bodySchema)
+  const target = await db.get(webhook.action.target)
+  if (webhook.action.type === exports.WebhookType.AUTOMATION) {
+    await triggers.externalTrigger(target, ctx.request.body)
+  }
+  ctx.status = 200
+  ctx.body = "Webhook trigger fired successfully"
+}

packages/server/src/api/index.js

@@ -21,6 +21,7 @@ const {
   apiKeysRoutes,
   templatesRoutes,
   analyticsRoutes,
+  webhookRoutes,
 } = require("./routes")
 
 const router = new Router()
@@ -90,6 +91,9 @@ router.use(instanceRoutes.allowedMethods())
 router.use(automationRoutes.routes())
 router.use(automationRoutes.allowedMethods())
 
+router.use(webhookRoutes.routes())
+router.use(webhookRoutes.allowedMethods())
+
 router.use(deployRoutes.routes())
 router.use(deployRoutes.allowedMethods())
 

packages/server/src/api/routes/automation.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/automation")
 const authorized = require("../../middleware/authorized")
 const joiValidator = require("../../middleware/joi-validator")
-const { BUILDER } = require("../../utilities/accessLevels")
+const { BUILDER, EXECUTE_AUTOMATION } = require("../../utilities/accessLevels")
 const Joi = require("joi")
 
 const router = Router()
@@ -33,7 +33,7 @@ function generateValidator(existing = false) {
     type: Joi.string().valid("automation").required(),
     definition: Joi.object({
       steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
-      trigger: generateStepSchema(["TRIGGER"]),
+      trigger: generateStepSchema(["TRIGGER"]).allow(null),
     }).required().unknown(true),
   }).unknown(true))
 }
@@ -73,7 +73,11 @@ router
     generateValidator(false),
     controller.create
   )
-  .post("/api/automations/:id/trigger", controller.trigger)
+  .post(
+    "/api/automations/:id/trigger",
+    authorized(EXECUTE_AUTOMATION),
+    controller.trigger
+  )
   .delete("/api/automations/:id/:rev", authorized(BUILDER), controller.destroy)
 
 module.exports = router

packages/server/src/api/routes/index.js

@@ -10,6 +10,7 @@ const viewRoutes = require("./view")
 const staticRoutes = require("./static")
 const componentRoutes = require("./component")
 const automationRoutes = require("./automation")
+const webhookRoutes = require("./webhook")
 const accesslevelRoutes = require("./accesslevel")
 const deployRoutes = require("./deploy")
 const apiKeysRoutes = require("./apikeys")
@@ -34,4 +35,5 @@ module.exports = {
   apiKeysRoutes,
   templatesRoutes,
   analyticsRoutes,
+  webhookRoutes,
 }

packages/server/src/api/routes/webhook.js

@@ -0,0 +1,45 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/webhook")
+const authorized = require("../../middleware/authorized")
+const joiValidator = require("../../middleware/joi-validator")
+const { BUILDER, EXECUTE_WEBHOOK } = require("../../utilities/accessLevels")
+const Joi = require("joi")
+
+const router = Router()
+
+function generateSaveValidator() {
+  // prettier-ignore
+  return joiValidator.body(Joi.object({
+    live: Joi.bool(),
+    _id: Joi.string().optional(),
+    _rev: Joi.string().optional(),
+    name: Joi.string().required(),
+    bodySchema: Joi.object().optional(),
+    action: Joi.object({
+      type: Joi.string().required().valid(controller.WebhookType.AUTOMATION),
+      target: Joi.string().required(),
+    }).required(),
+  }).unknown(true))
+}
+
+router
+  .get("/api/webhooks", authorized(BUILDER), controller.fetch)
+  .put(
+    "/api/webhooks",
+    authorized(BUILDER),
+    generateSaveValidator(),
+    controller.save
+  )
+  .delete("/api/webhooks/:id/:rev", authorized(BUILDER), controller.destroy)
+  .post(
+    "/api/webhooks/schema/:instance/:id",
+    authorized(BUILDER),
+    controller.buildSchema
+  )
+  .post(
+    "/api/webhooks/trigger/:instance/:id",
+    authorized(EXECUTE_WEBHOOK),
+    controller.trigger
+  )
+
+module.exports = router

packages/server/src/automations/triggers.js

@@ -83,6 +83,37 @@ const BUILTIN_DEFINITIONS = {
     },
     type: "TRIGGER",
   },
+  WEBHOOK: {
+    name: "Webhook",
+    event: "web:trigger",
+    icon: "ri-global-line",
+    tagline: "Webhook endpoint is hit",
+    description: "Trigger an automation when a HTTP POST webhook is hit",
+    stepId: "WEBHOOK",
+    inputs: {},
+    schema: {
+      inputs: {
+        properties: {
+          schemaUrl: {
+            type: "string",
+            customType: "webhookUrl",
+            title: "Schema URL",
+          },
+          triggerUrl: {
+            type: "string",
+            customType: "webhookUrl",
+            title: "Trigger URL",
+          },
+        },
+        required: ["schemaUrl", "triggerUrl"],
+      },
+      outputs: {
+        properties: {},
+        required: [],
+      },
+    },
+    type: "TRIGGER",
+  },
 }
 
 async function queueRelevantRowAutomations(event, eventType) {

packages/server/src/db/utils.js

@@ -11,6 +11,7 @@ const DocumentTypes = {
   LINK: "li",
   APP: "app",
   ACCESS_LEVEL: "ac",
+  WEBHOOK: "wh",
 }
 
 exports.DocumentTypes = DocumentTypes
@@ -164,3 +165,18 @@ exports.generateAccessLevelID = () => {
 exports.getAccessLevelParams = (accessLevelId = null, otherProps = {}) => {
   return getDocParams(DocumentTypes.ACCESS_LEVEL, accessLevelId, otherProps)
 }
+
+/**
+ * Generates a new webhook ID.
+ * @returns {string} The new webhook ID which the webhook doc can be stored under.
+ */
+exports.generateWebhookID = () => {
+  return `${DocumentTypes.WEBHOOK}${SEPARATOR}${newid()}`
+}
+
+/**
+ * Gets parameters for retrieving a webhook, this is a utility function for the getDocParams function.
+ */
+exports.getWebhookParams = (webhookId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.WEBHOOK, webhookId, otherProps)
+}

packages/server/src/middleware/authorized.js

@@ -9,7 +9,13 @@ const environment = require("../environment")
 const { apiKeyTable } = require("../db/dynamoClient")
 const { AuthTypes } = require("../constants")
 
+const LOCAL_PASS = new RegExp(["webhooks/trigger", "webhooks/schema"].join("|"))
+
 module.exports = (permName, getItemId) => async (ctx, next) => {
+  // webhooks can pass locally
+  if (!environment.CLOUD && LOCAL_PASS.test(ctx.request.url)) {
+    return next()
+  }
   if (
     environment.CLOUD &&
     ctx.headers["x-api-key"] &&

packages/server/src/utilities/accessLevels.js

@@ -3,6 +3,7 @@ module.exports.READ_TABLE = "read-table"
 module.exports.WRITE_TABLE = "write-table"
 module.exports.READ_VIEW = "read-view"
 module.exports.EXECUTE_AUTOMATION = "execute-automation"
+module.exports.EXECUTE_WEBHOOK = "execute-webhook"
 module.exports.USER_MANAGEMENT = "user-management"
 module.exports.BUILDER = "builder"
 module.exports.LIST_USERS = "list-users"

packages/server/yarn.lock

@@ -4714,6 +4714,11 @@ jsonfile@^6.0.1:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
+jsonschema@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2"
+  integrity sha512-/YgW6pRMr6M7C+4o8kS+B/2myEpHCrxO4PEWnqJNBFMjn7EWXqlQ4tGwL6xTHeRplwuZmcAncdvfOad1nT2yMw==
+
 jsonwebtoken@^8.5.1:
   version "8.5.1"
   resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
@@ -5160,6 +5165,21 @@ lodash.isstring@^4.0.1:
   resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
   integrity sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=
 
+lodash.keys@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205"
+  integrity sha1-oIYCrBLk+4P5H8H7ejYKTZujUgU=
+
+lodash.merge@^4.6.2:
+  version "4.6.2"
+  resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
+  integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==
+
+lodash.omit@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.omit/-/lodash.omit-4.5.0.tgz#6eb19ae5a1ee1dd9df0b969e66ce0b7fa30b5e60"
+  integrity sha1-brGa5aHuHdnfC5aeZs4Lf6MLXmA=
+
 lodash.once@^4.0.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
@@ -5175,6 +5195,16 @@ lodash.sortby@^4.7.0:
   resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
   integrity sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=
 
+lodash.without@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.without/-/lodash.without-4.4.0.tgz#3cd4574a00b67bae373a94b748772640507b7aac"
+  integrity sha1-PNRXSgC2e643OpS3SHcmQFB7eqw=
+
+lodash.xor@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
+  integrity sha1-TUjtfpgJWwYyWCunFNP/iuj7HbY=
+
 lodash@^4.17.10, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.19, lodash@^4.17.3:
   version "4.17.20"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
@@ -7471,6 +7501,18 @@ to-fast-properties@^2.0.0:
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
   integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=
 
+to-json-schema@^0.2.5:
+  version "0.2.5"
+  resolved "https://registry.yarnpkg.com/to-json-schema/-/to-json-schema-0.2.5.tgz#ef3c3f11ad64460dcfbdbafd0fd525d69d62a98f"
+  integrity sha512-jP1ievOee8pec3tV9ncxLSS48Bnw7DIybgy112rhMCEhf3K4uyVNZZHr03iQQBzbV5v5Hos+dlZRRyk6YSMNDw==
+  dependencies:
+    lodash.isequal "^4.5.0"
+    lodash.keys "^4.2.0"
+    lodash.merge "^4.6.2"
+    lodash.omit "^4.5.0"
+    lodash.without "^4.4.0"
+    lodash.xor "^4.5.0"
+
 to-object-path@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af"