Main bulk of logic around detecting cron recurring failure, still need to disable.

packages/backend-core/src/logging.ts

@@ -15,6 +15,17 @@ export function logAlert(message: string, e?: any) {
   console.error(`bb-alert: ${message} ${errorJson}`)
 }
 
+export function logAlertWithInfo(
+  message: string,
+  db: string,
+  id: string,
+  error: any
+) {
+  message = `${message} - db: ${db} - doc: ${id} - error: `
+  logAlert(message, error)
+}
+
 export default {
   logAlert,
+  logAlertWithInfo,
 }

packages/server/src/automations/utils.ts

@@ -10,6 +10,7 @@ import { cloneDeep } from "lodash/fp"
 import { getAppDB, getAppId } from "@budibase/backend-core/context"
 import { tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
+import { Automation } from "@budibase/types"
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
@@ -204,3 +205,12 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
 export async function cleanupAutomations(appId: any) {
   await disableAllCrons(appId)
 }
+
+/**
+ * Checks if the supplied automation is of a recurring type.
+ * @param automation The automation to check.
+ * @return {boolean} if it is recurring (cron).
+ */
+export function isRecurring(automation: Automation) {
+  return automation.definition.trigger.stepId === definitions.CRON.stepId
+}

packages/server/src/constants/index.js

@@ -210,3 +210,5 @@ exports.AutomationErrors = {
 
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets
+
+exports.MAX_AUTOMATION_RECURRING_ERRORS = 5

packages/server/src/definitions/automations.ts

@@ -1,4 +1,9 @@
-import { Automation, AutomationResults, AutomationStep } from "@budibase/types"
+import {
+  Automation,
+  AutomationResults,
+  AutomationStep,
+  Document,
+} from "@budibase/types"
 
 export enum LoopStepTypes {
   ARRAY = "Array",
@@ -33,3 +38,7 @@ export interface AutomationContext extends AutomationResults {
   steps: any[]
   trigger: any
 }
+
+export interface AutomationMetadata extends Document {
+  errorCount?: number
+}

packages/server/src/threads/automation.ts

@@ -1,13 +1,14 @@
 import { threadSetup } from "./utils"
 threadSetup()
+import { isRecurring } from "../automations/utils"
 import { default as actions } from "../automations/actions"
 import { default as automationUtils } from "../automations/automationUtils"
 import { default as AutomationEmitter } from "../events/AutomationEmitter"
 import { generateAutomationMetadataID } from "../db/utils"
 import { definitions as triggerDefs } from "../automations/triggerInfo"
-import { AutomationErrors } from "../constants"
+import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants"
 import { storeLog } from "../automations/logging"
-import { Automation, AutomationStep } from "@budibase/types"
+import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
 import {
   LoopStep,
   LoopStepTypes,
@@ -15,13 +16,15 @@ import {
   AutomationEvent,
   TriggerOutput,
   AutomationContext,
+  AutomationMetadata,
 } from "../definitions/automations"
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
+const { logAlertWithInfo } = require("@budibase/backend-core/logging")
 const { processObject } = require("@budibase/string-templates")
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
-const STOPPED_STATUS = { success: true, status: "STOPPED" }
+const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 const { cloneDeep } = require("lodash/fp")
 const env = require("../environment")
 
@@ -64,7 +67,6 @@ function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
  * inputs and handles any outputs.
  */
 class Orchestrator {
-  _metadata: any
   _chainCount: number
   _appId: string
   _automation: Automation
@@ -73,8 +75,8 @@ class Orchestrator {
   executionOutput: AutomationContext
 
   constructor(automation: Automation, triggerOutput: TriggerOutput) {
-    this._metadata = triggerOutput.metadata
-    this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
+    const metadata = triggerOutput.metadata
+    this._chainCount = metadata ? metadata.automationChainCount : 0
     this._appId = triggerOutput.appId as string
     const triggerStepId = automation.definition.trigger.stepId
     triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
@@ -108,14 +110,63 @@ class Orchestrator {
     return step
   }
 
-  async getMetadata() {
+  async getMetadata(): Promise<AutomationMetadata> {
     const metadataId = generateAutomationMetadataID(this._automation._id)
     const db = getAppDB()
-    let metadata: any
+    let metadata: AutomationMetadata
     try {
       metadata = await db.get(metadataId)
     } catch (err) {
-      metadata = {}
+      metadata = {
+        _id: metadataId,
+        errorCount: 0,
+      }
+    }
+    return metadata
+  }
+
+  async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
+    if (!metadata.errorCount) {
+      return false
+    }
+    const automation = this._automation
+    const trigger = automation.definition.trigger
+    if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
+      // TODO: need to disable the recurring here
+      this.updateExecutionOutput(trigger.id, trigger.stepId, {}, STOPPED_STATUS)
+      await storeLog(automation, this.executionOutput)
+      return true
+    }
+    return false
+  }
+
+  async updateMetadata(metadata: AutomationMetadata) {
+    const output = this.executionOutput,
+      automation = this._automation
+    if (!output || !isRecurring(automation)) {
+      return
+    }
+    const count = metadata.errorCount
+    const isError = output.status === AutomationStatus.ERROR
+    // nothing to do in this scenario, escape
+    if (!count && !isError) {
+      return
+    }
+    if (isError) {
+      metadata.errorCount = count ? count + 1 : 1
+    } else {
+      metadata.errorCount = 0
+    }
+    const db = getAppDB()
+    try {
+      await db.put(metadata)
+    } catch (err) {
+      logAlertWithInfo(
+        "Failed to write automation metadata",
+        db.name,
+        automation._id,
+        err
+      )
     }
   }
 
@@ -162,6 +213,17 @@ class Orchestrator {
     let stepCount = 0
     let loopStepNumber: any = undefined
     let loopSteps: LoopStep[] | undefined = []
+    let metadata
+
+    // check if this is a recurring automation,
+    if (isRecurring(automation)) {
+      metadata = await this.getMetadata()
+      const shouldStop = await this.checkIfShouldStop(metadata)
+      if (shouldStop) {
+        return
+      }
+    }
+
     for (let step of automation.definition.steps) {
       stepCount++
       let input,
@@ -349,11 +411,14 @@ class Orchestrator {
 
     // store the logs for the automation run
     await storeLog(this._automation, this.executionOutput)
+    if (isRecurring(automation) && metadata) {
+      await this.updateMetadata(metadata)
+    }
     return this.executionOutput
   }
 }
 
-module.exports = (
+export default (
   input: AutomationEvent,
   callback: (error: any, response?: any) => void
 ) => {

packages/types/src/documents/app/automation.ts

@@ -35,7 +35,7 @@ export enum AutomationStatus {
 
 export interface AutomationResults {
   automationId?: string
-  status?: string
+  status?: AutomationStatus
   trigger?: any
   steps: {
     stepId: string

packages/worker/yarn.lock

@@ -291,12 +291,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.1.17":
-  version "1.1.17"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.17.tgz#5102a6b457823735e4da71eb9bca8218a667f57b"
-  integrity sha512-SQQA3IvlD+ARMIXnAErFzOAzC67e9hM72qgZvOc3cqfBpvObSg1VnmNAuuG4GB0y3dm21ldLpCBQFzPH2Ylkkg==
+"@budibase/backend-core@1.1.18-alpha.2":
+  version "1.1.18-alpha.2"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.18-alpha.2.tgz#c3fe04afa11fcbba288c7ba2113469d203213a23"
+  integrity sha512-T6Cv2k/F59eYCHLOr76gDOPcRAhGIEemoqmKt22JwctrArdFQkICdcT0LDFt375oR+svuK6cGY9bWZsC2J9dqg==
   dependencies:
-    "@budibase/types" "^1.1.17"
+    "@budibase/types" "^1.1.18-alpha.2"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
     bcrypt "5.0.1"
@@ -324,19 +324,24 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
-"@budibase/pro@1.1.17":
-  version "1.1.17"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.17.tgz#6d6493b4df1c796469c0925c9d7b83e5f12b7b92"
-  integrity sha512-k0tfmeTf1Hh6m9U17wpUqfC0U2XvwhIf1Bbh7cbk8c41esm8blO0LxRpQGCS49lM85bfMFJvAGDr3+BHkiN7HA==
+"@budibase/pro@1.1.18-alpha.2":
+  version "1.1.18-alpha.2"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.18-alpha.2.tgz#d399fbc11d6d7ebee95e05d68c617581189830ed"
+  integrity sha512-z7kJ7PQehPbMjaUJk5xIvlGql0UxiYH1M/wrDhucpQa4FXgnjrY9eL17c3BiHNQe2+E2IbtL+pgCnXNgHxGTDQ==
   dependencies:
-    "@budibase/backend-core" "1.1.17"
-    "@budibase/types" "1.1.17"
+    "@budibase/backend-core" "1.1.18-alpha.2"
+    "@budibase/types" "1.1.18-alpha.2"
     node-fetch "^2.6.1"
 
-"@budibase/types@1.1.17", "@budibase/types@^1.1.17":
-  version "1.1.17"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.17.tgz#fa58e86f3858c04a8b79094193ff87ac57a499fe"
-  integrity sha512-3sTH3tjPd+NEk5CIN23bgwyGXXNYq/hwaxKMbYLKGr45K6m9WDZs6saWxJ2JwguNtKGB9RggzCVR/DFJH2zI1A==
+"@budibase/types@1.1.18-alpha.2":
+  version "1.1.18-alpha.2"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.18-alpha.2.tgz#55c585c0efa983a006b2b981dc5a7a6018a768fd"
+  integrity sha512-1PfwibyzpuCjeSzhbjb+KgO+pIzkntbhxYKUZY5u7HxrYxvfzwAiemD5ywmGY2/hgJwShLsNpZOFMucSkfTXhg==
+
+"@budibase/types@^1.1.18-alpha.2":
+  version "1.1.18"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.18.tgz#06b4fc5029ea9a73f086d2d0fc3ed27543175797"
+  integrity sha512-10j8zbPsdE6k/DIJyDDNGR1yJ8WL+bSvId+Ev66hleSr2kc1ntP0hji+uFNHH/k709w8rzA0D4Ugx5bO9zA5Qw==
 
 "@cspotcode/source-map-consumer@0.8.0":
   version "0.8.0"