Merge branch 'master' of github.com:Budibase/budibase into feature/sql-query-aliasing

packages/server/scripts/test.sh

@@ -5,10 +5,10 @@ if [[ -n $CI ]]
 then
   # Running in ci, where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
-  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
+  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
+  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --maxWorkers=2 --forceExit"
-  jest --coverage --maxWorkers=2 --forceExit
+  echo "jest --coverage --maxWorkers=2 --forceExit $@"
+  jest --coverage --maxWorkers=2 --forceExit $@
 fi

packages/server/src/automations/automationUtils.ts

@@ -5,7 +5,7 @@ import {
 } from "@budibase/string-templates"
 import sdk from "../sdk"
 import { Row } from "@budibase/types"
-import { LoopStep, LoopStepType, LoopInput } from "../definitions/automations"
+import { LoopInput, LoopStep, LoopStepType } from "../definitions/automations"
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -128,23 +128,28 @@ export function substituteLoopStep(hbsString: string, substitute: string) {
 }
 
 export function stringSplit(value: string | string[]) {
-  if (value == null || Array.isArray(value)) {
-    return value || []
+  if (value == null) {
+    return []
   }
-  if (value.split("\n").length > 1) {
-    value = value.split("\n")
-  } else {
-    value = value.split(",")
+  if (Array.isArray(value)) {
+    return value
   }
-  return value
+  if (typeof value !== "string") {
+    throw new Error(`Unable to split value of type ${typeof value}: ${value}`)
+  }
+  const splitOnNewLine = value.split("\n")
+  if (splitOnNewLine.length > 1) {
+    return splitOnNewLine
+  }
+  return value.split(",")
 }
 
-export function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
+export function typecastForLooping(input: LoopInput) {
   if (!input || !input.binding) {
     return null
   }
   try {
-    switch (loopStep.inputs.option) {
+    switch (input.option) {
       case LoopStepType.ARRAY:
         if (typeof input.binding === "string") {
           return JSON.parse(input.binding)

packages/server/src/automations/tests/loop.spec.ts

@@ -3,11 +3,13 @@ import * as triggers from "../triggers"
 import { loopAutomation } from "../../tests/utilities/structures"
 import { context } from "@budibase/backend-core"
 import * as setup from "./utilities"
+import { Row, Table } from "@budibase/types"
+import { LoopInput, LoopStepType } from "../../definitions/automations"
 
 describe("Attempt to run a basic loop automation", () => {
   let config = setup.getConfig(),
-    table: any,
-    row: any
+    table: Table,
+    row: Row
 
   beforeEach(async () => {
     await automation.init()
@@ -18,12 +20,12 @@ describe("Attempt to run a basic loop automation", () => {
 
   afterAll(setup.afterAll)
 
-  async function runLoop(loopOpts?: any) {
+  async function runLoop(loopOpts?: LoopInput) {
     const appId = config.getAppId()
     return await context.doInAppContext(appId, async () => {
       const params = { fields: { appId } }
       return await triggers.externalTrigger(
-        loopAutomation(table._id, loopOpts),
+        loopAutomation(table._id!, loopOpts),
         params,
         { getResponses: true }
       )
@@ -37,9 +39,17 @@ describe("Attempt to run a basic loop automation", () => {
 
   it("test a loop with a string", async () => {
     const resp = await runLoop({
-      type: "String",
+      option: LoopStepType.STRING,
       binding: "a,b,c",
     })
     expect(resp.steps[2].outputs.iterations).toBe(3)
   })
+
+  it("test a loop with a binding that returns an integer", async () => {
+    const resp = await runLoop({
+      option: LoopStepType.ARRAY,
+      binding: "{{ 1 }}",
+    })
+    expect(resp.steps[2].outputs.iterations).toBe(1)
+  })
 })

packages/server/src/automations/triggers.ts

@@ -9,7 +9,7 @@ import * as utils from "./utils"
 import env from "../environment"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types"
-import { executeSynchronously } from "../threads/automation"
+import { executeInThread } from "../threads/automation"
 
 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -117,8 +117,7 @@ export async function externalTrigger(
       appId: context.getAppId(),
       automation,
     }
-    const job = { data } as AutomationJob
-    return executeSynchronously(job)
+    return executeInThread({ data } as AutomationJob)
   } else {
     return automationQueue.add(data, JOB_OPTS)
   }

packages/server/src/automations/unitTests/automationUtils.spec.ts

@@ -1,10 +1,15 @@
-const automationUtils = require("../automationUtils")
+import { LoopStep, LoopStepType } from "../../definitions/automations"
+import {
+  typecastForLooping,
+  cleanInputValues,
+  substituteLoopStep,
+} from "../automationUtils"
 
 describe("automationUtils", () => {
   describe("substituteLoopStep", () => {
     it("should allow multiple loop binding substitutes", () => {
       expect(
-        automationUtils.substituteLoopStep(
+        substituteLoopStep(
           `{{ loop.currentItem._id }} {{ loop.currentItem._id }} {{ loop.currentItem._id }}`,
           "step.2"
         )
@@ -15,7 +20,7 @@ describe("automationUtils", () => {
 
     it("should handle not subsituting outside of curly braces", () => {
       expect(
-        automationUtils.substituteLoopStep(
+        substituteLoopStep(
           `loop {{ loop.currentItem._id }}loop loop{{ loop.currentItem._id }}loop`,
           "step.2"
         )
@@ -28,37 +33,20 @@ describe("automationUtils", () => {
   describe("typeCastForLooping", () => {
     it("should parse to correct type", () => {
       expect(
-        automationUtils.typecastForLooping(
-          { inputs: { option: "Array" } },
-          { binding: [1, 2, 3] }
-        )
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: [1, 2, 3] })
       ).toEqual([1, 2, 3])
       expect(
-        automationUtils.typecastForLooping(
-          { inputs: { option: "Array" } },
-          { binding: "[1, 2, 3]" }
-        )
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: "[1,2,3]" })
       ).toEqual([1, 2, 3])
       expect(
-        automationUtils.typecastForLooping(
-          { inputs: { option: "String" } },
-          { binding: [1, 2, 3] }
-        )
+        typecastForLooping({ option: LoopStepType.STRING, binding: [1, 2, 3] })
       ).toEqual("1,2,3")
     })
     it("should handle null values", () => {
       // expect it to handle where the binding is null
-      expect(
-        automationUtils.typecastForLooping(
-          { inputs: { option: "Array" } },
-          { binding: null }
-        )
-      ).toEqual(null)
+      expect(typecastForLooping({ option: LoopStepType.ARRAY })).toEqual(null)
       expect(() =>
-        automationUtils.typecastForLooping(
-          { inputs: { option: "Array" } },
-          { binding: "test" }
-        )
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: "test" })
       ).toThrow()
     })
   })
@@ -80,7 +68,7 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        automationUtils.cleanInputValues(
+        cleanInputValues(
           {
             row: {
               relationship: `[{"_id": "ro_ta_users_us_3"}]`,
@@ -113,7 +101,7 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        automationUtils.cleanInputValues(
+        cleanInputValues(
           {
             row: {
               relationship: `ro_ta_users_us_3`,

packages/server/src/definitions/automations.ts

@@ -6,14 +6,14 @@ export enum LoopStepType {
 }
 
 export interface LoopStep extends AutomationStep {
-  inputs: {
-    option: LoopStepType
-    [key: string]: any
-  }
+  inputs: LoopInput
 }
 
 export interface LoopInput {
-  binding: string[] | string
+  option: LoopStepType
+  binding?: string[] | string | number[]
+  iterations?: string
+  failure?: any
 }
 
 export interface TriggerOutput {

packages/server/src/tests/utilities/structures.ts

@@ -23,6 +23,7 @@ import {
   TableSourceType,
   Query,
 } from "@budibase/types"
+import { LoopInput, LoopStepType } from "../../definitions/automations"
 
 const { BUILTIN_ROLE_IDS } = roles
 
@@ -204,10 +205,13 @@ export function serverLogAutomation(appId?: string): Automation {
   }
 }
 
-export function loopAutomation(tableId: string, loopOpts?: any): Automation {
+export function loopAutomation(
+  tableId: string,
+  loopOpts?: LoopInput
+): Automation {
   if (!loopOpts) {
     loopOpts = {
-      option: "Array",
+      option: LoopStepType.ARRAY,
       binding: "{{ steps.1.rows }}",
     }
   }

packages/server/src/threads/automation.ts

@@ -43,22 +43,19 @@ const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
 function getLoopIterations(loopStep: LoopStep) {
-  let binding = loopStep.inputs.binding
+  const binding = loopStep.inputs.binding
   if (!binding) {
     return 0
   }
-  const isString = typeof binding === "string"
   try {
-    if (isString) {
-      binding = JSON.parse(binding)
+    const json = typeof binding === "string" ? JSON.parse(binding) : binding
+    if (Array.isArray(json)) {
+      return json.length
     }
   } catch (err) {
     // ignore error - wasn't able to parse
   }
-  if (Array.isArray(binding)) {
-    return binding.length
-  }
-  if (isString) {
+  if (typeof binding === "string") {
     return automationUtils.stringSplit(binding).length
   }
   return 0
@@ -256,7 +253,7 @@ class Orchestrator {
         this._context.env = await sdkUtils.getEnvironmentVariables()
         let automation = this._automation
         let stopped = false
-        let loopStep: AutomationStep | undefined = undefined
+        let loopStep: LoopStep | undefined = undefined
 
         let stepCount = 0
         let loopStepNumber: any = undefined
@@ -311,7 +308,7 @@ class Orchestrator {
 
             stepCount++
             if (step.stepId === LOOP_STEP_ID) {
-              loopStep = step
+              loopStep = step as LoopStep
               loopStepNumber = stepCount
               continue
             }
@@ -331,7 +328,6 @@ class Orchestrator {
                 }
                 try {
                   loopStep.inputs.binding = automationUtils.typecastForLooping(
-                    loopStep as LoopStep,
                     loopStep.inputs as LoopInput
                   )
                 } catch (err) {
@@ -348,7 +344,7 @@ class Orchestrator {
                   loopStep = undefined
                   break
                 }
-                let item = []
+                let item: any[] = []
                 if (
                   typeof loopStep.inputs.binding === "string" &&
                   loopStep.inputs.option === "String"
@@ -399,7 +395,8 @@ class Orchestrator {
 
                 if (
                   index === env.AUTOMATION_MAX_ITERATIONS ||
-                  index === parseInt(loopStep.inputs.iterations)
+                  (loopStep.inputs.iterations &&
+                    index === parseInt(loopStep.inputs.iterations))
                 ) {
                   this.updateContextAndOutput(
                     loopStepNumber,
@@ -615,7 +612,7 @@ export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
   })
 }
 
-export function executeSynchronously(job: Job) {
+export async function executeInThread(job: Job<AutomationData>) {
   const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
@@ -627,10 +624,10 @@ export function executeSynchronously(job: Job) {
     }, job.data.event.timeout || 12000)
   })
 
-  return context.doInAppContext(appId, async () => {
+  return await context.doInAppContext(appId, async () => {
     const envVars = await sdkUtils.getEnvironmentVariables()
     // put into automation thread for whole context
-    return context.doInEnvironmentContext(envVars, async () => {
+    return await context.doInEnvironmentContext(envVars, async () => {
       const automationOrchestrator = new Orchestrator(job)
       return await Promise.race([
         automationOrchestrator.execute(),