Main bulk of logic around detecting cron recurring failure, still need to disable.
This commit is contained in:
parent
21a120ed87
commit
07de5faa2d
10 changed files with 187 additions and 3017 deletions
|
@ -15,6 +15,17 @@ export function logAlert(message: string, e?: any) {
|
||||||
console.error(`bb-alert: ${message} ${errorJson}`)
|
console.error(`bb-alert: ${message} ${errorJson}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function logAlertWithInfo(
|
||||||
|
message: string,
|
||||||
|
db: string,
|
||||||
|
id: string,
|
||||||
|
error: any
|
||||||
|
) {
|
||||||
|
message = `${message} - db: ${db} - doc: ${id} - error: `
|
||||||
|
logAlert(message, error)
|
||||||
|
}
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
logAlert,
|
logAlert,
|
||||||
|
logAlertWithInfo,
|
||||||
}
|
}
|
||||||
|
|
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
@ -10,6 +10,7 @@ import { cloneDeep } from "lodash/fp"
|
||||||
import { getAppDB, getAppId } from "@budibase/backend-core/context"
|
import { getAppDB, getAppId } from "@budibase/backend-core/context"
|
||||||
import { tenancy } from "@budibase/backend-core"
|
import { tenancy } from "@budibase/backend-core"
|
||||||
import { quotas } from "@budibase/pro"
|
import { quotas } from "@budibase/pro"
|
||||||
|
import { Automation } from "@budibase/types"
|
||||||
|
|
||||||
const WH_STEP_ID = definitions.WEBHOOK.stepId
|
const WH_STEP_ID = definitions.WEBHOOK.stepId
|
||||||
const CRON_STEP_ID = definitions.CRON.stepId
|
const CRON_STEP_ID = definitions.CRON.stepId
|
||||||
|
@ -204,3 +205,12 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
|
||||||
export async function cleanupAutomations(appId: any) {
|
export async function cleanupAutomations(appId: any) {
|
||||||
await disableAllCrons(appId)
|
await disableAllCrons(appId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if the supplied automation is of a recurring type.
|
||||||
|
* @param automation The automation to check.
|
||||||
|
* @return {boolean} if it is recurring (cron).
|
||||||
|
*/
|
||||||
|
export function isRecurring(automation: Automation) {
|
||||||
|
return automation.definition.trigger.stepId === definitions.CRON.stepId
|
||||||
|
}
|
||||||
|
|
|
@ -210,3 +210,5 @@ exports.AutomationErrors = {
|
||||||
|
|
||||||
// pass through the list from the auth/core lib
|
// pass through the list from the auth/core lib
|
||||||
exports.ObjectStoreBuckets = ObjectStoreBuckets
|
exports.ObjectStoreBuckets = ObjectStoreBuckets
|
||||||
|
|
||||||
|
exports.MAX_AUTOMATION_RECURRING_ERRORS = 5
|
||||||
|
|
|
@ -1,4 +1,9 @@
|
||||||
import { Automation, AutomationResults, AutomationStep } from "@budibase/types"
|
import {
|
||||||
|
Automation,
|
||||||
|
AutomationResults,
|
||||||
|
AutomationStep,
|
||||||
|
Document,
|
||||||
|
} from "@budibase/types"
|
||||||
|
|
||||||
export enum LoopStepTypes {
|
export enum LoopStepTypes {
|
||||||
ARRAY = "Array",
|
ARRAY = "Array",
|
||||||
|
@ -33,3 +38,7 @@ export interface AutomationContext extends AutomationResults {
|
||||||
steps: any[]
|
steps: any[]
|
||||||
trigger: any
|
trigger: any
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface AutomationMetadata extends Document {
|
||||||
|
errorCount?: number
|
||||||
|
}
|
||||||
|
|
|
@ -1,13 +1,14 @@
|
||||||
import { threadSetup } from "./utils"
|
import { threadSetup } from "./utils"
|
||||||
threadSetup()
|
threadSetup()
|
||||||
|
import { isRecurring } from "../automations/utils"
|
||||||
import { default as actions } from "../automations/actions"
|
import { default as actions } from "../automations/actions"
|
||||||
import { default as automationUtils } from "../automations/automationUtils"
|
import { default as automationUtils } from "../automations/automationUtils"
|
||||||
import { default as AutomationEmitter } from "../events/AutomationEmitter"
|
import { default as AutomationEmitter } from "../events/AutomationEmitter"
|
||||||
import { generateAutomationMetadataID } from "../db/utils"
|
import { generateAutomationMetadataID } from "../db/utils"
|
||||||
import { definitions as triggerDefs } from "../automations/triggerInfo"
|
import { definitions as triggerDefs } from "../automations/triggerInfo"
|
||||||
import { AutomationErrors } from "../constants"
|
import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants"
|
||||||
import { storeLog } from "../automations/logging"
|
import { storeLog } from "../automations/logging"
|
||||||
import { Automation, AutomationStep } from "@budibase/types"
|
import { Automation, AutomationStep, AutomationStatus } from "@budibase/types"
|
||||||
import {
|
import {
|
||||||
LoopStep,
|
LoopStep,
|
||||||
LoopStepTypes,
|
LoopStepTypes,
|
||||||
|
@ -15,13 +16,15 @@ import {
|
||||||
AutomationEvent,
|
AutomationEvent,
|
||||||
TriggerOutput,
|
TriggerOutput,
|
||||||
AutomationContext,
|
AutomationContext,
|
||||||
|
AutomationMetadata,
|
||||||
} from "../definitions/automations"
|
} from "../definitions/automations"
|
||||||
const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
|
const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
|
||||||
|
const { logAlertWithInfo } = require("@budibase/backend-core/logging")
|
||||||
const { processObject } = require("@budibase/string-templates")
|
const { processObject } = require("@budibase/string-templates")
|
||||||
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
|
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
|
||||||
const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
|
const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
|
||||||
const CRON_STEP_ID = triggerDefs.CRON.stepId
|
const CRON_STEP_ID = triggerDefs.CRON.stepId
|
||||||
const STOPPED_STATUS = { success: true, status: "STOPPED" }
|
const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
|
||||||
const { cloneDeep } = require("lodash/fp")
|
const { cloneDeep } = require("lodash/fp")
|
||||||
const env = require("../environment")
|
const env = require("../environment")
|
||||||
|
|
||||||
|
@ -64,7 +67,6 @@ function getLoopIterations(loopStep: LoopStep, input: LoopInput) {
|
||||||
* inputs and handles any outputs.
|
* inputs and handles any outputs.
|
||||||
*/
|
*/
|
||||||
class Orchestrator {
|
class Orchestrator {
|
||||||
_metadata: any
|
|
||||||
_chainCount: number
|
_chainCount: number
|
||||||
_appId: string
|
_appId: string
|
||||||
_automation: Automation
|
_automation: Automation
|
||||||
|
@ -73,8 +75,8 @@ class Orchestrator {
|
||||||
executionOutput: AutomationContext
|
executionOutput: AutomationContext
|
||||||
|
|
||||||
constructor(automation: Automation, triggerOutput: TriggerOutput) {
|
constructor(automation: Automation, triggerOutput: TriggerOutput) {
|
||||||
this._metadata = triggerOutput.metadata
|
const metadata = triggerOutput.metadata
|
||||||
this._chainCount = this._metadata ? this._metadata.automationChainCount : 0
|
this._chainCount = metadata ? metadata.automationChainCount : 0
|
||||||
this._appId = triggerOutput.appId as string
|
this._appId = triggerOutput.appId as string
|
||||||
const triggerStepId = automation.definition.trigger.stepId
|
const triggerStepId = automation.definition.trigger.stepId
|
||||||
triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
|
triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
|
||||||
|
@ -108,14 +110,63 @@ class Orchestrator {
|
||||||
return step
|
return step
|
||||||
}
|
}
|
||||||
|
|
||||||
async getMetadata() {
|
async getMetadata(): Promise<AutomationMetadata> {
|
||||||
const metadataId = generateAutomationMetadataID(this._automation._id)
|
const metadataId = generateAutomationMetadataID(this._automation._id)
|
||||||
const db = getAppDB()
|
const db = getAppDB()
|
||||||
let metadata: any
|
let metadata: AutomationMetadata
|
||||||
try {
|
try {
|
||||||
metadata = await db.get(metadataId)
|
metadata = await db.get(metadataId)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
metadata = {}
|
metadata = {
|
||||||
|
_id: metadataId,
|
||||||
|
errorCount: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
|
}
|
||||||
|
|
||||||
|
async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
|
||||||
|
if (!metadata.errorCount) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
const automation = this._automation
|
||||||
|
const trigger = automation.definition.trigger
|
||||||
|
if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
|
||||||
|
// TODO: need to disable the recurring here
|
||||||
|
this.updateExecutionOutput(trigger.id, trigger.stepId, {}, STOPPED_STATUS)
|
||||||
|
await storeLog(automation, this.executionOutput)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
async updateMetadata(metadata: AutomationMetadata) {
|
||||||
|
const output = this.executionOutput,
|
||||||
|
automation = this._automation
|
||||||
|
if (!output || !isRecurring(automation)) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const count = metadata.errorCount
|
||||||
|
const isError = output.status === AutomationStatus.ERROR
|
||||||
|
// nothing to do in this scenario, escape
|
||||||
|
if (!count && !isError) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (isError) {
|
||||||
|
metadata.errorCount = count ? count + 1 : 1
|
||||||
|
} else {
|
||||||
|
metadata.errorCount = 0
|
||||||
|
}
|
||||||
|
const db = getAppDB()
|
||||||
|
try {
|
||||||
|
await db.put(metadata)
|
||||||
|
} catch (err) {
|
||||||
|
logAlertWithInfo(
|
||||||
|
"Failed to write automation metadata",
|
||||||
|
db.name,
|
||||||
|
automation._id,
|
||||||
|
err
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -162,6 +213,17 @@ class Orchestrator {
|
||||||
let stepCount = 0
|
let stepCount = 0
|
||||||
let loopStepNumber: any = undefined
|
let loopStepNumber: any = undefined
|
||||||
let loopSteps: LoopStep[] | undefined = []
|
let loopSteps: LoopStep[] | undefined = []
|
||||||
|
let metadata
|
||||||
|
|
||||||
|
// check if this is a recurring automation,
|
||||||
|
if (isRecurring(automation)) {
|
||||||
|
metadata = await this.getMetadata()
|
||||||
|
const shouldStop = await this.checkIfShouldStop(metadata)
|
||||||
|
if (shouldStop) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (let step of automation.definition.steps) {
|
for (let step of automation.definition.steps) {
|
||||||
stepCount++
|
stepCount++
|
||||||
let input,
|
let input,
|
||||||
|
@ -349,11 +411,14 @@ class Orchestrator {
|
||||||
|
|
||||||
// store the logs for the automation run
|
// store the logs for the automation run
|
||||||
await storeLog(this._automation, this.executionOutput)
|
await storeLog(this._automation, this.executionOutput)
|
||||||
|
if (isRecurring(automation) && metadata) {
|
||||||
|
await this.updateMetadata(metadata)
|
||||||
|
}
|
||||||
return this.executionOutput
|
return this.executionOutput
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = (
|
export default (
|
||||||
input: AutomationEvent,
|
input: AutomationEvent,
|
||||||
callback: (error: any, response?: any) => void
|
callback: (error: any, response?: any) => void
|
||||||
) => {
|
) => {
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -35,7 +35,7 @@ export enum AutomationStatus {
|
||||||
|
|
||||||
export interface AutomationResults {
|
export interface AutomationResults {
|
||||||
automationId?: string
|
automationId?: string
|
||||||
status?: string
|
status?: AutomationStatus
|
||||||
trigger?: any
|
trigger?: any
|
||||||
steps: {
|
steps: {
|
||||||
stepId: string
|
stepId: string
|
||||||
|
|
|
@ -291,12 +291,12 @@
|
||||||
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
|
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
|
||||||
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
|
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
|
||||||
|
|
||||||
"@budibase/backend-core@1.1.17":
|
"@budibase/backend-core@1.1.18-alpha.2":
|
||||||
version "1.1.17"
|
version "1.1.18-alpha.2"
|
||||||
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.17.tgz#5102a6b457823735e4da71eb9bca8218a667f57b"
|
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.1.18-alpha.2.tgz#c3fe04afa11fcbba288c7ba2113469d203213a23"
|
||||||
integrity sha512-SQQA3IvlD+ARMIXnAErFzOAzC67e9hM72qgZvOc3cqfBpvObSg1VnmNAuuG4GB0y3dm21ldLpCBQFzPH2Ylkkg==
|
integrity sha512-T6Cv2k/F59eYCHLOr76gDOPcRAhGIEemoqmKt22JwctrArdFQkICdcT0LDFt375oR+svuK6cGY9bWZsC2J9dqg==
|
||||||
dependencies:
|
dependencies:
|
||||||
"@budibase/types" "^1.1.17"
|
"@budibase/types" "^1.1.18-alpha.2"
|
||||||
"@techpass/passport-openidconnect" "0.3.2"
|
"@techpass/passport-openidconnect" "0.3.2"
|
||||||
aws-sdk "2.1030.0"
|
aws-sdk "2.1030.0"
|
||||||
bcrypt "5.0.1"
|
bcrypt "5.0.1"
|
||||||
|
@ -324,19 +324,24 @@
|
||||||
uuid "8.3.2"
|
uuid "8.3.2"
|
||||||
zlib "1.0.5"
|
zlib "1.0.5"
|
||||||
|
|
||||||
"@budibase/pro@1.1.17":
|
"@budibase/pro@1.1.18-alpha.2":
|
||||||
version "1.1.17"
|
version "1.1.18-alpha.2"
|
||||||
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.17.tgz#6d6493b4df1c796469c0925c9d7b83e5f12b7b92"
|
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.1.18-alpha.2.tgz#d399fbc11d6d7ebee95e05d68c617581189830ed"
|
||||||
integrity sha512-k0tfmeTf1Hh6m9U17wpUqfC0U2XvwhIf1Bbh7cbk8c41esm8blO0LxRpQGCS49lM85bfMFJvAGDr3+BHkiN7HA==
|
integrity sha512-z7kJ7PQehPbMjaUJk5xIvlGql0UxiYH1M/wrDhucpQa4FXgnjrY9eL17c3BiHNQe2+E2IbtL+pgCnXNgHxGTDQ==
|
||||||
dependencies:
|
dependencies:
|
||||||
"@budibase/backend-core" "1.1.17"
|
"@budibase/backend-core" "1.1.18-alpha.2"
|
||||||
"@budibase/types" "1.1.17"
|
"@budibase/types" "1.1.18-alpha.2"
|
||||||
node-fetch "^2.6.1"
|
node-fetch "^2.6.1"
|
||||||
|
|
||||||
"@budibase/types@1.1.17", "@budibase/types@^1.1.17":
|
"@budibase/types@1.1.18-alpha.2":
|
||||||
version "1.1.17"
|
version "1.1.18-alpha.2"
|
||||||
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.17.tgz#fa58e86f3858c04a8b79094193ff87ac57a499fe"
|
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.18-alpha.2.tgz#55c585c0efa983a006b2b981dc5a7a6018a768fd"
|
||||||
integrity sha512-3sTH3tjPd+NEk5CIN23bgwyGXXNYq/hwaxKMbYLKGr45K6m9WDZs6saWxJ2JwguNtKGB9RggzCVR/DFJH2zI1A==
|
integrity sha512-1PfwibyzpuCjeSzhbjb+KgO+pIzkntbhxYKUZY5u7HxrYxvfzwAiemD5ywmGY2/hgJwShLsNpZOFMucSkfTXhg==
|
||||||
|
|
||||||
|
"@budibase/types@^1.1.18-alpha.2":
|
||||||
|
version "1.1.18"
|
||||||
|
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.1.18.tgz#06b4fc5029ea9a73f086d2d0fc3ed27543175797"
|
||||||
|
integrity sha512-10j8zbPsdE6k/DIJyDDNGR1yJ8WL+bSvId+Ev66hleSr2kc1ntP0hji+uFNHH/k709w8rzA0D4Ugx5bO9zA5Qw==
|
||||||
|
|
||||||
"@cspotcode/source-map-consumer@0.8.0":
|
"@cspotcode/source-map-consumer@0.8.0":
|
||||||
version "0.8.0"
|
version "0.8.0"
|
||||||
|
|
Loading…
Reference in a new issue