Merge pull request #5145 from Budibase/feature/foreach-block

Foreach Automation Block

charts/budibase/templates/app-service-deployment.yaml

@@ -114,6 +114,9 @@ spec:
           value: {{ .Values.globals.google.clientId | quote }}
         - name: GOOGLE_CLIENT_SECRET
           value: {{ .Values.globals.google.secret | quote }}
+        - name: AUTOMATION_MAX_ITERATIONS
+          value: {{ .Values.globals.automationMaxIterations | quote }}
+
         image: budibase/apps:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
         name: bbapps

charts/budibase/values.yaml

@@ -103,6 +103,7 @@ globals:
   google:
     clientId: "" 
     secret: ""
+  automationMaxIterations: "500"
 
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 

packages/builder/cypress/integration/createAutomation.spec.js

@@ -20,7 +20,6 @@ filterTests(['smoke', 'all'], () => {
       })
 
       // Setup trigger
-      cy.contains("Setup").click()
       cy.get(".spectrum-Picker-label").click()
       cy.wait(500)
       cy.contains("dog").click()
@@ -32,7 +31,6 @@ filterTests(['smoke', 'all'], () => {
         cy.contains("Create Row").trigger('mouseover').click().click()
         cy.get(".spectrum-Button--cta").click()
       })
-      cy.contains("Setup").click()
       cy.get(".spectrum-Picker-label").eq(1).click()
       cy.contains("dog").click()
       cy.get(".spectrum-Textfield-input")

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -39,6 +39,7 @@
     if (v.internal) {
       acc[k] = v
     }
+    delete acc.LOOP
     return acc
   }, {})
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -72,7 +72,9 @@
         animate:flip={{ duration: 500 }}
         in:fly|local={{ x: 500, duration: 1500 }}
       >
+        {#if block.stepId !== "LOOP"}
           <FlowItem {testDataModal} {block} />
+        {/if}
       </div>
     {/each}
   </div>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -9,8 +9,8 @@
     Modal,
     Button,
     StatusLight,
-    ActionButton,
     Select,
+    ActionButton,
     notifications,
   } from "@budibase/bbui"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
@@ -25,8 +25,8 @@
   let webhookModal
   let actionModal
   let resultsModal
-  let setupToggled
   let blockComplete
+  let showLooping = false
 
   $: rowControl = $automationStore.selectedAutomation.automation.rowControl
   $: showBindingPicker =
@@ -52,8 +52,21 @@
     block.schema?.inputs?.properties || {}
   ).every(x => block?.inputs[x])
 
+  $: loopingSelected =
+    $automationStore.selectedAutomation?.automation.definition.steps.find(
+      x => x.blockToLoop === block.id
+    )
+
   async function deleteStep() {
+    let loopBlock =
+      $automationStore.selectedAutomation?.automation.definition.steps.find(
+        x => x.blockToLoop === block.id
+      )
+
     try {
+      if (loopBlock) {
+        automationStore.actions.deleteAutomationBlock(loopBlock)
+      }
       automationStore.actions.deleteAutomationBlock(block)
       await automationStore.actions.save(
         $automationStore.selectedAutomation?.automation
@@ -76,6 +89,23 @@
     )
   }
 
+  async function addLooping() {
+    loopingSelected = true
+    const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
+
+    const loopBlock = $automationStore.selectedAutomation.constructBlock(
+      "ACTION",
+      "LOOP",
+      loopDefinition
+    )
+    loopBlock.blockToLoop = block.id
+    block.loopBlock = loopBlock.id
+    automationStore.actions.addBlockToAutomation(loopBlock, blockIdx)
+    await automationStore.actions.save(
+      $automationStore.selectedAutomation?.automation
+    )
+  }
+
   async function onSelect(block) {
     await automationStore.update(state => {
       state.selectedBlock = block
@@ -84,13 +114,68 @@
   }
 </script>
 
-<div
+<div class={`block ${block.type} hoverable`} class:selected on:click={() => {}}>
-  class={`block ${block.type} hoverable`}
+  {#if loopingSelected}
-  class:selected
+    <div class="blockSection">
+      <div
+        on:click={() => {
+          showLooping = !showLooping
+        }}
+        class="splitHeader"
+      >
+        <div class="center-items">
+          <svg
+            width="28px"
+            height="28px"
+            class="spectrum-Icon"
+            style="color:grey;"
+            focusable="false"
+          >
+            <use xlink:href="#spectrum-icon-18-Reuse" />
+          </svg>
+          <div class="iconAlign">
+            <Detail size="S">Looping</Detail>
+          </div>
+        </div>
+
+        <div class="blockTitle">
+          <div
+            style="margin-left: 10px;"
             on:click={() => {
               onSelect(block)
             }}
->
+          >
+            <Icon name={showLooping ? "ChevronDown" : "ChevronUp"} />
+          </div>
+        </div>
+      </div>
+    </div>
+
+    <Divider noMargin />
+    {#if !showLooping}
+      <div class="blockSection">
+        <div class="block-options">
+          <div class="delete-padding" on:click={() => deleteStep()}>
+            <Icon name="DeleteOutline" />
+          </div>
+        </div>
+        <Layout noPadding gap="S">
+          <AutomationBlockSetup
+            schemaProperties={Object.entries(
+              $automationStore.blockDefinitions.ACTION.LOOP.schema.inputs
+                .properties
+            )}
+            block={$automationStore.selectedAutomation?.automation.definition.steps.find(
+              x => x.blockToLoop === block.id
+            )}
+            {webhookModal}
+          />
+        </Layout>
+      </div>
+      <Divider noMargin />
+    {/if}
+  {/if}
+
   <div class="blockSection">
     <div
       on:click={() => {
@@ -127,55 +212,57 @@
           <Detail size="S">{block?.name?.toUpperCase() || ""}</Detail>
         </div>
       </div>
+      <div class="blockTitle">
         {#if testResult && testResult[0]}
-        <span on:click={() => resultsModal.show()}>
+          <div style="float: right;" on:click={() => resultsModal.show()}>
             <StatusLight
               positive={isTrigger || testResult[0].outputs?.success}
               negative={!testResult[0].outputs?.success}
               ><Body size="XS">View response</Body></StatusLight
             >
-        </span>
+          </div>
         {/if}
+        <div
+          style="margin-left: 10px;"
+          on:click={() => {
+            onSelect(block)
+          }}
+        >
+          <Icon name={blockComplete ? "ChevronDown" : "ChevronUp"} />
+        </div>
+      </div>
     </div>
   </div>
   {#if !blockComplete}
     <Divider noMargin />
     <div class="blockSection">
       <Layout noPadding gap="S">
-        <div class="splitHeader">
-          <ActionButton
-            on:click={() => {
-              onSelect(block)
-              setupToggled = !setupToggled
-            }}
-            quiet
-            icon={setupToggled ? "ChevronDown" : "ChevronRight"}
-          >
-            <Detail size="S">Setup</Detail>
-          </ActionButton>
         {#if !isTrigger}
-            <div class="block-options">
-              {#if showBindingPicker}
           <div>
+            <div class="block-options">
+              {#if !loopingSelected}
+                <ActionButton on:click={() => addLooping()} icon="Reuse"
+                  >Add Looping</ActionButton
+                >
+              {/if}
+              {#if showBindingPicker}
                 <Select
                   on:change={toggleFieldControl}
-                    quiet
                   defaultValue="Use values"
                   autoWidth
                   value={rowControl ? "Use bindings" : "Use values"}
                   options={["Use values", "Use bindings"]}
                   placeholder={null}
                 />
-                </div>
               {/if}
-              <div class="delete-padding" on:click={() => deleteStep()}>
+              <ActionButton
-                <Icon name="DeleteOutline" />
+                on:click={() => deleteStep()}
+                icon="DeleteOutline"
+              />
             </div>
           </div>
         {/if}
-        </div>
 
-        {#if setupToggled}
         <AutomationBlockSetup
           schemaProperties={Object.entries(block.schema.inputs.properties)}
           {block}
@@ -186,7 +273,6 @@
             >Finish and test automation</Button
           >
         {/if}
-        {/if}
       </Layout>
     </div>
   {/if}
@@ -220,8 +306,10 @@
     padding-left: 30px;
   }
   .block-options {
-    display: flex;
+    justify-content: flex-end;
     align-items: center;
+    display: flex;
+    gap: var(--spacing-m);
   }
   .center-items {
     display: flex;
@@ -256,4 +344,9 @@
     /* center horizontally */
     align-self: center;
   }
+
+  .blockTitle {
+    display: flex;
+    align-items: center;
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ResultsModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { ModalContent, Icon, Detail, TextArea } from "@budibase/bbui"
+  import { ModalContent, Icon, Detail, TextArea, Label } from "@budibase/bbui"
 
   export let testResult
   export let isTrigger
@@ -10,7 +10,7 @@
 <ModalContent
   showCloseIcon={false}
   showConfirmButton={false}
-  title="Test Automation"
+  title="Test Results"
   cancelText="Close"
 >
   <div slot="header">
@@ -26,7 +26,18 @@
       {/if}
     </div>
   </div>
-
+  <span>
+    {#if testResult[0].outputs.iterations}
+      <div style="display: flex;">
+        <Icon name="Reuse" />
+        <div style="margin-left: 10px;">
+          <Label>
+            This loop ran {testResult[0].outputs.iterations} times.</Label
+          >
+        </div>
+      </div>
+    {/if}
+  </span>
   <div
     on:click={() => {
       inputToggled = !inputToggled

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -88,36 +88,65 @@
     if (!block || !automation) {
       return []
     }
-
     // Find previous steps to the selected one
     let allSteps = [...automation.steps]
+
     if (automation.trigger) {
       allSteps = [automation.trigger, ...allSteps]
     }
-    const blockIdx = allSteps.findIndex(step => step.id === block.id)
+    let blockIdx = allSteps.findIndex(step => step.id === block.id)
 
-    // Extract all outputs from all previous steps as available bindings
+    // Extract all outputs from all previous steps as available bindins
     let bindings = []
     for (let idx = 0; idx < blockIdx; idx++) {
-      const outputs = Object.entries(
+      let wasLoopBlock = allSteps[idx]?.stepId === "LOOP"
-        allSteps[idx].schema?.outputs?.properties ?? {}
+      let isLoopBlock =
-      )
+        allSteps[idx]?.stepId === "LOOP" &&
+        allSteps.find(x => x.blockToLoop === block.id)
+
+      // If the previous block was a loop block, decerement the index so the following
+      // steps are in the correct order
+      if (wasLoopBlock) {
+        blockIdx--
+      }
+
+      let schema = allSteps[idx]?.schema?.outputs?.properties ?? {}
+
+      // If its a Loop Block, we need to add this custom schema
+      if (isLoopBlock) {
+        schema = {
+          currentItem: {
+            type: "string",
+            description: "the item currently being executed",
+          },
+        }
+      }
+      const outputs = Object.entries(schema)
+
       bindings = bindings.concat(
         outputs.map(([name, value]) => {
-          const stepsLabel = block.name.startsWith("JS")
+          let runtimeName = isLoopBlock
+            ? `loop.${name}`
+            : block.name.startsWith("JS")
             ? `steps[${idx}].${name}`
             : `steps.${idx}.${name}`
-          const runtime = idx === 0 ? `trigger.${name}` : stepsLabel
+          const runtime = idx === 0 ? `trigger.${name}` : runtimeName
           return {
             label: runtime,
             type: value.type,
             description: value.description,
-            category: idx === 0 ? "Trigger outputs" : `Step ${idx} outputs`,
+            category:
+              idx === 0
+                ? "Trigger outputs"
+                : isLoopBlock
+                ? "Loop Outputs"
+                : `Step ${idx} outputs`,
             path: runtime,
           }
         })
       )
     }
+
     return bindings
   }
 
@@ -264,6 +293,14 @@
             value={inputData[key]}
           />
         </CodeEditorModal>
+      {:else if value.customType === "loopOption"}
+        <Select
+          on:change={e => onChange(e, key)}
+          autoWidth
+          value={inputData[key]}
+          options={["Array", "String"]}
+          defaultValue={"Array"}
+        />
       {:else if value.type === "string" || value.type === "number" || value.type === "integer"}
         {#if isTestModal}
           <ModalBindableInput

packages/server/src/automations/actions.js

@@ -13,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+let loop = require("./steps/loop")
 const env = require("../environment")
 
 const ACTION_IMPLS = {
@@ -27,6 +28,7 @@ const ACTION_IMPLS = {
   DELAY: delay.run,
   FILTER: filter.run,
   QUERY_ROWS: queryRow.run,
+  LOOP: loop.run,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.run,
   slack: slack.run,
@@ -45,6 +47,7 @@ const ACTION_DEFINITIONS = {
   DELAY: delay.definition,
   FILTER: filter.definition,
   QUERY_ROWS: queryRow.definition,
+  LOOP: loop.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.definition,
   slack: slack.definition,

packages/server/src/automations/automationUtils.js

@@ -1,4 +1,5 @@
 const { getTable } = require("../api/controllers/table/utils")
+const { findHBSBlocks } = require("@budibase/string-templates")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -74,3 +75,14 @@ exports.getError = err => {
   }
   return typeof err !== "string" ? err.toString() : err
 }
+
+exports.substituteLoopStep = (hbsString, substitute) => {
+  let blocks = findHBSBlocks(hbsString)
+  for (let block of blocks) {
+    let oldBlock = block
+    block = block.replace(/loop/, substitute)
+    hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
+  }
+
+  return hbsString
+}

packages/server/src/automations/steps/loop.js

@@ -0,0 +1,50 @@
+exports.definition = {
+  name: "Looping",
+  icon: "Reuse",
+  tagline: "Loop the block",
+  description: "Loop",
+  stepId: "LOOP",
+  internal: true,
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        option: {
+          customType: "loopOption",
+          title: "Input type",
+        },
+        binding: {
+          type: "string",
+          title: "Binding / Value",
+        },
+        iterations: {
+          type: "number",
+          title: "Max loop iterations",
+        },
+        failure: {
+          type: "string",
+          title: "Failure Condition",
+        },
+      },
+      required: ["type", "value", "iterations", "failure"],
+    },
+    outputs: {
+      properties: {
+        items: {
+          customType: "item",
+          description: "The item currently being executed",
+        },
+        success: {
+          type: "boolean",
+          description: "Whether the message loop was successfully",
+        },
+        iterations: {
+          type: "number",
+          descriptions: "The amount of times the block ran",
+        },
+      },
+      required: ["success", "items", "iterations"],
+    },
+  },
+  type: "LOGIC",
+}

packages/server/src/constants/index.js

@@ -190,5 +190,11 @@ exports.WebhookType = {
   AUTOMATION: "automation",
 }
 
+exports.AutomationErrors = {
+  INCORRECT_TYPE: "INCORRECT_TYPE",
+  MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",
+  FAILURE_CONDITION: "FAILURE_CONDITION_MET",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

packages/server/src/environment.js

@@ -59,6 +59,7 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL,
   AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
+  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,

packages/server/src/threads/automation.js

@@ -8,10 +8,14 @@ const { DocumentTypes } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-
+const { AutomationErrors } = require("../constants")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
+
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
+const { cloneDeep } = require("lodash/fp")
+const env = require("../environment")
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -74,34 +78,156 @@ class Orchestrator {
     this.executionOutput.steps.push(stepObj)
   }
 
+  updateContextAndOutput(loopStepNumber, step, output, result) {
+    this.executionOutput.steps.splice(loopStepNumber, 0, {
+      id: step.id,
+      stepId: step.stepId,
+      outputs: {
+        ...output,
+        success: result.success,
+        status: result.status,
+      },
+      inputs: step.inputs,
+    })
+    this._context.steps.splice(loopStepNumber, 0, {
+      ...output,
+      success: result.success,
+      status: result.status,
+    })
+  }
+
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
     let stopped = false
+    let loopStep
+
+    let stepCount = 0
+    let loopStepNumber
+    let loopSteps = []
     for (let step of automation.definition.steps) {
+      stepCount++
+      let input
+      if (step.stepId === LOOP_STEP_ID) {
+        loopStep = step
+        loopStepNumber = stepCount
+        continue
+      }
+
+      if (loopStep) {
+        input = await processObject(loopStep.inputs, this._context)
+      }
+      let iterations = loopStep ? input.binding.length : 1
+      let iterationCount = 0
+      for (let index = 0; index < iterations; index++) {
+        let originalStepInput = cloneDeep(step.inputs)
+
+        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
+        if (loopStep) {
+          // lets first of all handle the input
+          // if the input is array then use it, if it is a string then split it on every new line
+          let newInput = await processObject(
+            loopStep.inputs,
+            cloneDeep(this._context)
+          )
+          newInput = automationUtils.cleanInputValues(
+            newInput,
+            loopStep.schema.inputs
+          )
+          this._context.steps[loopStepNumber] = {
+            currentItem: newInput.binding[index],
+          }
+
+          let tempOutput = { items: loopSteps, iterations: iterationCount }
+          if (
+            (loopStep.inputs.option === "Array" &&
+              !Array.isArray(newInput.binding)) ||
+            (loopStep.inputs.option === "String" &&
+              typeof newInput.binding !== "string")
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.INCORRECT_TYPE,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
+          // Pretty hacky because we need to account for the row object
+          for (let [key, value] of Object.entries(originalStepInput)) {
+            if (typeof value === "object") {
+              for (let [innerKey, innerValue] of Object.entries(
+                originalStepInput[key]
+              )) {
+                if (typeof innerValue === "string") {
+                  originalStepInput[key][innerKey] =
+                    automationUtils.substituteLoopStep(
+                      innerValue,
+                      `steps.${loopStepNumber}`
+                    )
+                }
+              }
+            } else {
+              if (typeof value === "string") {
+                originalStepInput[key] = automationUtils.substituteLoopStep(
+                  value,
+                  `steps.${loopStepNumber}`
+                )
+              }
+            }
+          }
+
+          if (
+            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
+            index === loopStep.inputs.iterations
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.MAX_ITERATIONS,
+              success: true,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+
+          if (
+            this._context.steps[loopStepNumber]?.currentItem ===
+            loopStep.inputs.failure
+          ) {
+            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
+              status: AutomationErrors.FAILURE_CONDITION,
+              success: false,
+            })
+            loopSteps = null
+            loopStep = null
+            break
+          }
+        }
+
         // execution stopped, record state for that
         if (stopped) {
           this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
           continue
         }
+
+        // If it's a loop step, we need to manually add the bindings to the context
         let stepFn = await this.getStepFunctionality(step.stepId)
-      step.inputs = await processObject(step.inputs, this._context)
+        let inputs = await processObject(originalStepInput, this._context)
-      step.inputs = automationUtils.cleanInputValues(
+        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
-        step.inputs,
-        step.schema.inputs
-      )
-      // appId is always passed
         try {
+          // appId is always passed
           let tenantId = app.tenantId || DEFAULT_TENANT_ID
           const outputs = await doInTenant(tenantId, () => {
             return stepFn({
-            inputs: step.inputs,
+              inputs: inputs,
               appId: this._appId,
               emitter: this._emitter,
               context: this._context,
             })
           })
-        this._context.steps.push(outputs)
+          this._context.steps[stepCount] = outputs
           // if filter causes us to stop execution don't break the loop, set a var
           // so that we can finish iterating through the steps and record that it stopped
           if (step.stepId === FILTER_STEP_ID && !outputs.success) {
@@ -112,12 +238,48 @@ class Orchestrator {
             })
             continue
           }
-        this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
+          if (loopStep && loopSteps) {
+            loopSteps.push(outputs)
+          } else {
+            this.updateExecutionOutput(
+              step.id,
+              step.stepId,
+              step.inputs,
+              outputs
+            )
+          }
         } catch (err) {
           console.error(`Automation error - ${step.stepId} - ${err}`)
           return err
         }
+        if (loopStep) {
+          iterationCount++
+          if (index === iterations - 1) {
+            loopStep = null
+            this._context.steps.splice(loopStepNumber, 1)
+            break
           }
+        }
+      }
+
+      if (loopSteps && loopSteps.length) {
+        let tempOutput = {
+          success: true,
+          items: loopSteps,
+          iterations: iterationCount,
+        }
+        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
+          id: step.id,
+          stepId: step.stepId,
+          outputs: tempOutput,
+          inputs: step.inputs,
+        })
+
+        this._context.steps.splice(loopStepNumber, 0, tempOutput)
+        loopSteps = null
+      }
+    }
+
     return this.executionOutput
   }
 }