CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-03 05:20:32 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into security/yarn-audit

Browse source
This commit is contained in:
Adria Navarro 2023-12-20 12:01:39 +01:00 committed by GitHub
parent f0a789ca5b 895c9a9ac3
commit 45edddb32d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 681 additions and 300 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
.github/workflows/budibase_ci.yml vendored
View file

@ -76,6 +76,18 @@ jobs:
yarn check:types
fi
helm-lint:
runs-on: ubuntu-latest
steps:
- name: Checkout repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Use Node.js 18.x
uses: azure/setup-helm@v3
- run: cd charts/budibase && helm lint .
test-libraries:
runs-on: ubuntu-latest
steps:

8
charts/budibase/templates/app-service-deployment.yaml
View file

@ -227,6 +227,14 @@ spec:
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
{{ if .Values.services.apps.command }}
command:
{{- toYaml .Values.services.apps.command | nindent 10 }}
{{ end }}
{{ if .Values.services.apps.args }}
args:
{{- toYaml .Values.services.apps.args | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}

8
charts/budibase/templates/automation-worker-service-deployment.yaml
View file

@ -227,6 +227,13 @@ spec:
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
command:
{{- toYaml .Values.services.automationWorkers.command | nindent 10 }}
{{ end }}
{{ if .Values.services.automationWorkers.args }}
args:
{{- toYaml .Values.services.automationWorkers.args | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
@ -244,5 +251,6 @@ spec:
{{ end }}
restartPolicy: Always
serviceAccountName: ""
{{ if .Values.services.automationWorkers.command }}}
status: {}
{{- end }}

8
charts/budibase/templates/proxy-service-deployment.yaml
View file

@ -100,5 +100,13 @@ spec:
{{ end }}
restartPolicy: Always
serviceAccountName: ""
{{ if .Values.services.proxy.command }}
command:
{{- toYaml .Values.services.proxy.command | nindent 8 }}
{{ end }}
{{ if .Values.services.proxy.args }}
args:
{{- toYaml .Values.services.proxy.args | nindent 8 }}
{{ end }}
volumes:
status: {}

8
charts/budibase/templates/worker-service-deployment.yaml
View file

@ -213,6 +213,14 @@ spec:
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
{{ if .Values.services.worker.command }}
command:
{{- toYaml .Values.services.worker.command | nindent 10 }}
{{ end }}
{{ if .Values.services.worker.args }}
args:
{{- toYaml .Values.services.worker.args | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}

9
hosting/single/runner.sh
View file

@ -7,7 +7,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000
[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000
[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
@ -77,7 +77,12 @@ mkdir -p ${DATA_DIR}/minio
chown -R couchdb:couchdb ${DATA_DIR}/couch
redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
/bbcouch-runner.sh &
/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
# only start minio if use s3 isn't passed
if [[ -z "${USE_S3}"]]; then
/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
fi
/etc/init.d/nginx restart
if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
# Add monthly cron job to renew certbot certificate

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.13.46",
"version": "2.13.48",
"npmClient": "yarn",
"packages": [
"packages/*",

2
packages/account-portal

@ -1 +1 @@
Subproject commit 09dae295e3ba6149c4e1d7fe567870c3a38bd277
Subproject commit abf2d7804c940b011328bb0979ebc6261420fe85

8
packages/backend-core/src/context/mainContext.ts
View file

@ -335,3 +335,11 @@ export function isScim(): boolean {
const scimCall = context?.isScim
return !!scimCall
}
export function getCurrentContext(): ContextMap | undefined {
try {
return Context.get()
} catch (e) {
return undefined
}
}

2
packages/backend-core/src/context/types.ts
View file

@ -1,4 +1,5 @@
import { IdentityContext } from "@budibase/types"
import { ExecutionTimeTracker } from "../timers"
// keep this out of Budibase types, don't want to expose context info
export type ContextMap = {
@ -9,4 +10,5 @@ export type ContextMap = {
isScim?: boolean
automationId?: string
isMigrating?: boolean
jsExecutionTracker?: ExecutionTimeTracker
}

2
packages/backend-core/src/db/couch/DatabaseImpl.ts
View file

@ -37,7 +37,7 @@ export function DatabaseWithConnection(
opts?: DatabaseOpts
) {
const db = new DatabaseImpl(dbName, opts, connection)
return new DDInstrumentedDatabase(db, "couchdb")
return new DDInstrumentedDatabase(db)
}
export class DatabaseImpl implements Database {

2
packages/backend-core/src/db/db.ts
View file

@ -3,7 +3,7 @@ import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types"
import { DDInstrumentedDatabase } from "./instrumentation"
export function getDB(dbName: string, opts?: DatabaseOpts): Database {
return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts), "couchdb")
return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts))
}
// we have to use a callback for this so that we can close

37
packages/backend-core/src/db/instrumentation.ts
View file

@ -18,31 +18,28 @@ import tracer from "dd-trace"
import { Writable } from "stream"
export class DDInstrumentedDatabase implements Database {
constructor(
private readonly db: Database,
private readonly resource: string
) {}
constructor(private readonly db: Database) {}
get name(): string {
return this.db.name
}
exists(): Promise<boolean> {
return tracer.trace("exists", { resource: this.resource }, span => {
return tracer.trace("db.exists", span => {
span?.addTags({ db_name: this.name })
return this.db.exists()
})
}
checkSetup(): Promise<DocumentScope<any>> {
return tracer.trace("checkSetup", { resource: this.resource }, span => {
return tracer.trace("db.checkSetup", span => {
span?.addTags({ db_name: this.name })
return this.db.checkSetup()
})
}
get<T extends Document>(id?: string | undefined): Promise<T> {
return tracer.trace("get", { resource: this.resource }, span => {
return tracer.trace("db.get", span => {
span?.addTags({ db_name: this.name, doc_id: id })
return this.db.get(id)
})
@ -52,7 +49,7 @@ export class DDInstrumentedDatabase implements Database {
ids: string[],
opts?: { allowMissing?: boolean | undefined } | undefined
): Promise<T[]> {
return tracer.trace("getMultiple", { resource: this.resource }, span => {
return tracer.trace("db.getMultiple", span => {
span?.addTags({
db_name: this.name,
num_docs: ids.length,
@ -66,7 +63,7 @@ export class DDInstrumentedDatabase implements Database {
id: string | Document,
rev?: string | undefined
): Promise<DocumentDestroyResponse> {
return tracer.trace("remove", { resource: this.resource }, span => {
return tracer.trace("db.remove", span => {
span?.addTags({ db_name: this.name, doc_id: id })
return this.db.remove(id, rev)
})
@ -76,14 +73,14 @@ export class DDInstrumentedDatabase implements Database {
document: AnyDocument,
opts?: DatabasePutOpts | undefined
): Promise<DocumentInsertResponse> {
return tracer.trace("put", { resource: this.resource }, span => {
return tracer.trace("db.put", span => {
span?.addTags({ db_name: this.name, doc_id: document._id })
return this.db.put(document, opts)
})
}
bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
return tracer.trace("bulkDocs", { resource: this.resource }, span => {
return tracer.trace("db.bulkDocs", span => {
span?.addTags({ db_name: this.name, num_docs: documents.length })
return this.db.bulkDocs(documents)
})
@ -92,7 +89,7 @@ export class DDInstrumentedDatabase implements Database {
allDocs<T extends Document>(
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {
return tracer.trace("allDocs", { resource: this.resource }, span => {
return tracer.trace("db.allDocs", span => {
span?.addTags({ db_name: this.name })
return this.db.allDocs(params)
})
@ -102,56 +99,56 @@ export class DDInstrumentedDatabase implements Database {
viewName: string,
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {
return tracer.trace("query", { resource: this.resource }, span => {
return tracer.trace("db.query", span => {
span?.addTags({ db_name: this.name, view_name: viewName })
return this.db.query(viewName, params)
})
}
destroy(): Promise<void | OkResponse> {
return tracer.trace("destroy", { resource: this.resource }, span => {
return tracer.trace("db.destroy", span => {
span?.addTags({ db_name: this.name })
return this.db.destroy()
})
}
compact(): Promise<void | OkResponse> {
return tracer.trace("compact", { resource: this.resource }, span => {
return tracer.trace("db.compact", span => {
span?.addTags({ db_name: this.name })
return this.db.compact()
})
}
dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
return tracer.trace("dump", { resource: this.resource }, span => {
return tracer.trace("db.dump", span => {
span?.addTags({ db_name: this.name })
return this.db.dump(stream, opts)
})
}
load(...args: any[]): Promise<any> {
return tracer.trace("load", { resource: this.resource }, span => {
return tracer.trace("db.load", span => {
span?.addTags({ db_name: this.name })
return this.db.load(...args)
})
}
createIndex(...args: any[]): Promise<any> {
return tracer.trace("createIndex", { resource: this.resource }, span => {
return tracer.trace("db.createIndex", span => {
span?.addTags({ db_name: this.name })
return this.db.createIndex(...args)
})
}
deleteIndex(...args: any[]): Promise<any> {
return tracer.trace("deleteIndex", { resource: this.resource }, span => {
return tracer.trace("db.deleteIndex", span => {
span?.addTags({ db_name: this.name })
return this.db.deleteIndex(...args)
})
}
getIndexes(...args: any[]): Promise<any> {
return tracer.trace("getIndexes", { resource: this.resource }, span => {
return tracer.trace("db.getIndexes", span => {
span?.addTags({ db_name: this.name })
return this.db.getIndexes(...args)
})

1
packages/backend-core/src/queue/inMemoryQueue.ts
View file

@ -15,6 +15,7 @@ function newJob(queue: string, message: any) {
timestamp: Date.now(),
queue: queue,
data: message,
opts: {},
}
}

38
packages/backend-core/src/timers/timers.ts
View file

@ -20,3 +20,41 @@ export function cleanup() {
}
intervals = []
}
export class ExecutionTimeoutError extends Error {
public readonly name = "ExecutionTimeoutError"
}
export class ExecutionTimeTracker {
static withLimit(limitMs: number) {
return new ExecutionTimeTracker(limitMs)
}
constructor(readonly limitMs: number) {}
private totalTimeMs = 0
track<T>(f: () => T): T {
this.checkLimit()
const start = process.hrtime.bigint()
try {
return f()
} finally {
const end = process.hrtime.bigint()
this.totalTimeMs += Number(end - start) / 1e6
this.checkLimit()
}
}
get elapsedMS() {
return this.totalTimeMs
}
private checkLimit() {
if (this.totalTimeMs > this.limitMs) {
throw new ExecutionTimeoutError(
`Execution time limit of ${this.limitMs}ms exceeded: ${this.totalTimeMs}ms`
)
}
}
}

4
packages/backend-core/src/utils/Duration.ts
View file

@ -49,4 +49,8 @@ export class Duration {
static fromDays(duration: number) {
return Duration.from(DurationType.DAYS, duration)
}
static fromMilliseconds(duration: number) {
return Duration.from(DurationType.MILLISECONDS, duration)
}
}

108
packages/server/src/api/routes/tests/row.spec.ts
View file

@ -2086,4 +2086,112 @@ describe.each([
expect(row.formula).toBe(relatedRow.name)
})
})
describe("Formula JS protection", () => {
it("should time out JS execution if a single cell takes too long", async () => {
await config.withEnv({ JS_PER_EXECUTION_TIME_LIMIT_MS: 20 }, async () => {
const js = Buffer.from(
`
let i = 0;
while (true) {
i++;
}
return i;
`
).toString("base64")
const table = await config.createTable({
name: "table",
type: "table",
schema: {
text: {
name: "text",
type: FieldType.STRING,
},
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formulaType: FormulaTypes.DYNAMIC,
},
},
})
await config.api.row.save(table._id!, { text: "foo" })
const { rows } = await config.api.row.search(table._id!)
expect(rows).toHaveLength(1)
const row = rows[0]
expect(row.text).toBe("foo")
expect(row.formula).toBe("Timed out while executing JS")
})
})
it("should time out JS execution if a multiple cells take too long", async () => {
await config.withEnv(
{
JS_PER_EXECUTION_TIME_LIMIT_MS: 20,
JS_PER_REQUEST_TIME_LIMIT_MS: 40,
},
async () => {
const js = Buffer.from(
`
let i = 0;
while (true) {
i++;
}
return i;
`
).toString("base64")
const table = await config.createTable({
name: "table",
type: "table",
schema: {
text: {
name: "text",
type: FieldType.STRING,
},
formula: {
name: "formula",
type: FieldType.FORMULA,
formula: `{{ js "${js}"}}`,
formulaType: FormulaTypes.DYNAMIC,
},
},
})
for (let i = 0; i < 10; i++) {
await config.api.row.save(table._id!, { text: "foo" })
}
// Run this test 3 times to make sure that there's no cross-request
// pollution of the execution time tracking.
for (let reqs = 0; reqs < 3; reqs++) {
const { rows } = await config.api.row.search(table._id!)
expect(rows).toHaveLength(10)
let i = 0
for (; i < 10; i++) {
const row = rows[i]
if (row.formula !== "Timed out while executing JS") {
break
}
}
// Given the execution times are not deterministic, we can't be sure
// of the exact number of rows that were executed before the timeout
// but it should absolutely be at least 1.
expect(i).toBeGreaterThan(0)
expect(i).toBeLessThan(5)
for (; i < 10; i++) {
const row = rows[i]
expect(row.text).toBe("foo")
expect(row.formula).toBe("Request JS execution limit hit")
}
}
}
)
})
})
})

69
packages/server/src/automations/utils.ts
View file

@ -16,6 +16,7 @@ import {
} from "@budibase/types"
import sdk from "../sdk"
import { automationsEnabled } from "../features"
import tracer from "dd-trace"
const REBOOT_CRON = "@reboot"
const WH_STEP_ID = definitions.WEBHOOK.stepId
@ -39,26 +40,62 @@ function loggingArgs(job: AutomationJob) {
}
export async function processEvent(job: AutomationJob) {
const appId = job.data.event.appId!
const automationId = job.data.automation._id!
const task = async () => {
try {
// need to actually await these so that an error can be captured properly
console.log("automation running", ...loggingArgs(job))
return tracer.trace(
"processEvent",
{ resource: "automation" },
async span => {
const appId = job.data.event.appId!
const automationId = job.data.automation._id!
const runFn = () => Runner.run(job)
const result = await quotas.addAutomation(runFn, {
span?.addTags({
appId,
automationId,
job: {
id: job.id,
name: job.name,
attemptsMade: job.attemptsMade,
opts: {
attempts: job.opts.attempts,
priority: job.opts.priority,
delay: job.opts.delay,
repeat: job.opts.repeat,
backoff: job.opts.backoff,
lifo: job.opts.lifo,
timeout: job.opts.timeout,
jobId: job.opts.jobId,
removeOnComplete: job.opts.removeOnComplete,
removeOnFail: job.opts.removeOnFail,
stackTraceLimit: job.opts.stackTraceLimit,
preventParsingData: job.opts.preventParsingData,
},
},
})
console.log("automation completed", ...loggingArgs(job))
return result
} catch (err) {
console.error(`automation was unable to run`, err, ...loggingArgs(job))
return { err }
}
}
return await context.doInAutomationContext({ appId, automationId, task })
const task = async () => {
try {
// need to actually await these so that an error can be captured properly
console.log("automation running", ...loggingArgs(job))
const runFn = () => Runner.run(job)
const result = await quotas.addAutomation(runFn, {
automationId,
})
console.log("automation completed", ...loggingArgs(job))
return result
} catch (err) {
span?.addTags({ error: true })
console.error(
`automation was unable to run`,
err,
...loggingArgs(job)
)
return { err }
}
}
return await context.doInAutomationContext({ appId, automationId, task })
}
)
}
export async function updateTestHistory(

5
packages/server/src/environment.ts
View file

@ -70,6 +70,11 @@ const environment = {
SELF_HOSTED: process.env.SELF_HOSTED,
HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
JS_PER_EXECUTION_TIME_LIMIT_MS:
parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
JS_PER_REQUEST_TIME_LIMIT_MS: parseIntSafe(
process.env.JS_PER_REQUEST_TIME_LIMIT_MS
),
// old
CLIENT_ID: process.env.CLIENT_ID,
_set(key: string, value: any) {

45
packages/server/src/jsRunner.ts Normal file
View file

@ -0,0 +1,45 @@
import vm from "vm"
import env from "./environment"
import { setJSRunner } from "@budibase/string-templates"
import { context, timers } from "@budibase/backend-core"
import tracer from "dd-trace"
type TrackerFn = <T>(f: () => T) => T
export function init() {
setJSRunner((js: string, ctx: vm.Context) => {
return tracer.trace("runJS", {}, span => {
const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
let track: TrackerFn = f => f()
if (perRequestLimit) {
const bbCtx = context.getCurrentContext()
if (bbCtx) {
if (!bbCtx.jsExecutionTracker) {
bbCtx.jsExecutionTracker =
timers.ExecutionTimeTracker.withLimit(perRequestLimit)
}
track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
span?.addTags({
js: {
limitMS: bbCtx.jsExecutionTracker.limitMs,
elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
},
})
}
}
ctx = {
...ctx,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
vm.createContext(ctx)
return track(() =>
vm.runInNewContext(js, ctx, {
timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
})
)
})
})
}

10
packages/server/src/middleware/currentapp.ts
View file

@ -23,7 +23,7 @@ export default async (ctx: UserCtx, next: any) => {
if (requestAppId) {
const span = tracer.scope().active()
span?.addTags({ app_id: requestAppId })
span?.setTag("appId", requestAppId)
}
// deny access to application preview
@ -76,6 +76,14 @@ export default async (ctx: UserCtx, next: any) => {
return next()
}
if (ctx.user) {
const span = tracer.scope().active()
if (ctx.user._id) {
span?.setTag("userId", ctx.user._id)
}
span?.setTag("tenantId", ctx.user.tenantId)
}
const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined
// if the user is not in the right tenant then make sure to wipe their cookie

3
packages/server/src/startup.ts
View file

@ -23,6 +23,7 @@ import { automationsEnabled, printFeatures } from "./features"
import Koa from "koa"
import { Server } from "http"
import { AddressInfo } from "net"
import * as jsRunner from "./jsRunner"
let STARTUP_RAN = false
@ -152,4 +153,6 @@ export async function startup(app?: Koa, server?: Server) {
}
})
}
jsRunner.init()
}

580
packages/server/src/threads/automation.ts
View file

@ -34,6 +34,7 @@ import { cloneDeep } from "lodash/fp"
import { performance } from "perf_hooks"
import * as sdkUtils from "../sdk/utils"
import env from "../environment"
import tracer from "dd-trace"
threadUtils.threadSetup()
const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
@ -242,278 +243,347 @@ class Orchestrator {
}
async execute(): Promise<any> {
// this will retrieve from context created at start of thread
this._context.env = await sdkUtils.getEnvironmentVariables()
let automation = this._automation
let stopped = false
let loopStep: AutomationStep | undefined = undefined
return tracer.trace(
"Orchestrator.execute",
{ resource: "automation" },
async span => {
span?.addTags({
appId: this._appId,
automationId: this._automation._id,
})
let stepCount = 0
let loopStepNumber: any = undefined
let loopSteps: LoopStep[] | undefined = []
let metadata
let timeoutFlag = false
let wasLoopStep = false
let timeout = this._job.data.event.timeout
// check if this is a recurring automation,
if (isProdAppID(this._appId) && isRecurring(automation)) {
metadata = await this.getMetadata()
const shouldStop = await this.checkIfShouldStop(metadata)
if (shouldStop) {
return
}
}
const start = performance.now()
for (let step of automation.definition.steps) {
if (timeoutFlag) {
break
}
// this will retrieve from context created at start of thread
this._context.env = await sdkUtils.getEnvironmentVariables()
let automation = this._automation
let stopped = false
let loopStep: AutomationStep | undefined = undefined
if (timeout) {
setTimeout(() => {
timeoutFlag = true
}, timeout || 12000)
}
stepCount++
let input: any,
iterations = 1,
iterationCount = 0
if (step.stepId === LOOP_STEP_ID) {
loopStep = step
loopStepNumber = stepCount
continue
}
if (loopStep) {
input = await processObject(loopStep.inputs, this._context)
iterations = getLoopIterations(loopStep as LoopStep)
}
for (let index = 0; index < iterations; index++) {
let originalStepInput = cloneDeep(step.inputs)
// Handle if the user has set a max iteration count or if it reaches the max limit set by us
if (loopStep && input.binding) {
let tempOutput = { items: loopSteps, iterations: iterationCount }
try {
loopStep.inputs.binding = automationUtils.typecastForLooping(
loopStep as LoopStep,
loopStep.inputs as LoopInput
)
} catch (err) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
status: AutomationErrors.INCORRECT_TYPE,
success: false,
})
loopSteps = undefined
loopStep = undefined
break
}
let item = []
if (
typeof loopStep.inputs.binding === "string" &&
loopStep.inputs.option === "String"
) {
item = automationUtils.stringSplit(loopStep.inputs.binding)
} else if (Array.isArray(loopStep.inputs.binding)) {
item = loopStep.inputs.binding
}
this._context.steps[loopStepNumber] = {
currentItem: item[index],
}
// The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
// Pretty hacky because we need to account for the row object
for (let [key, value] of Object.entries(originalStepInput)) {
if (typeof value === "object") {
for (let [innerKey, innerValue] of Object.entries(
originalStepInput[key]
)) {
if (typeof innerValue === "string") {
originalStepInput[key][innerKey] =
automationUtils.substituteLoopStep(
innerValue,
`steps.${loopStepNumber}`
)
} else if (typeof value === "object") {
for (let [innerObject, innerValue] of Object.entries(
originalStepInput[key][innerKey]
)) {
originalStepInput[key][innerKey][innerObject] =
automationUtils.substituteLoopStep(
innerValue as string,
`steps.${loopStepNumber}`
)
}
}
}
} else {
if (typeof value === "string") {
originalStepInput[key] = automationUtils.substituteLoopStep(
value,
`steps.${loopStepNumber}`
)
}
}
}
if (
index === env.AUTOMATION_MAX_ITERATIONS ||
index === parseInt(loopStep.inputs.iterations)
) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
status: AutomationErrors.MAX_ITERATIONS,
success: true,
})
loopSteps = undefined
loopStep = undefined
break
}
let isFailure = false
const currentItem = this._context.steps[loopStepNumber]?.currentItem
if (currentItem && typeof currentItem === "object") {
isFailure = Object.keys(currentItem).some(value => {
return currentItem[value] === loopStep?.inputs.failure
})
} else {
isFailure = currentItem && currentItem === loopStep.inputs.failure
}
if (isFailure) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
status: AutomationErrors.FAILURE_CONDITION,
success: false,
})
loopSteps = undefined
loopStep = undefined
break
let stepCount = 0
let loopStepNumber: any = undefined
let loopSteps: LoopStep[] | undefined = []
let metadata
let timeoutFlag = false
let wasLoopStep = false
let timeout = this._job.data.event.timeout
// check if this is a recurring automation,
if (isProdAppID(this._appId) && isRecurring(automation)) {
span?.addTags({ recurring: true })
metadata = await this.getMetadata()
const shouldStop = await this.checkIfShouldStop(metadata)
if (shouldStop) {
span?.addTags({ shouldStop: true })
return
}
}
// execution stopped, record state for that
if (stopped) {
this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
continue
}
// If it's a loop step, we need to manually add the bindings to the context
let stepFn = await this.getStepFunctionality(step.stepId)
let inputs = await processObject(originalStepInput, this._context)
inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
try {
// appId is always passed
const outputs = await stepFn({
inputs: inputs,
appId: this._appId,
emitter: this._emitter,
context: this._context,
const start = performance.now()
for (let step of automation.definition.steps) {
const stepSpan = tracer.startSpan("Orchestrator.execute.step", {
childOf: span,
})
stepSpan.addTags({
resource: "automation",
step: {
stepId: step.stepId,
id: step.id,
name: step.name,
type: step.type,
title: step.stepTitle,
internal: step.internal,
deprecated: step.deprecated,
},
})
this._context.steps[stepCount] = outputs
// if filter causes us to stop execution don't break the loop, set a var
// so that we can finish iterating through the steps and record that it stopped
if (step.stepId === FILTER_STEP_ID && !outputs.result) {
stopped = true
this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
...outputs,
...STOPPED_STATUS,
})
continue
}
if (loopStep && loopSteps) {
loopSteps.push(outputs)
} else {
this.updateExecutionOutput(
step.id,
step.stepId,
step.inputs,
outputs
)
}
} catch (err) {
console.error(`Automation error - ${step.stepId} - ${err}`)
return err
}
let input: any,
iterations = 1,
iterationCount = 0
if (loopStep) {
iterationCount++
if (index === iterations - 1) {
try {
if (timeoutFlag) {
span?.addTags({ timedOut: true })
break
}
if (timeout) {
setTimeout(() => {
timeoutFlag = true
}, timeout || 12000)
}
stepCount++
if (step.stepId === LOOP_STEP_ID) {
loopStep = step
loopStepNumber = stepCount
continue
}
if (loopStep) {
input = await processObject(loopStep.inputs, this._context)
iterations = getLoopIterations(loopStep as LoopStep)
stepSpan?.addTags({ step: { iterations } })
}
for (let index = 0; index < iterations; index++) {
let originalStepInput = cloneDeep(step.inputs)
// Handle if the user has set a max iteration count or if it reaches the max limit set by us
if (loopStep && input.binding) {
let tempOutput = {
items: loopSteps,
iterations: iterationCount,
}
try {
loopStep.inputs.binding = automationUtils.typecastForLooping(
loopStep as LoopStep,
loopStep.inputs as LoopInput
)
} catch (err) {
this.updateContextAndOutput(
loopStepNumber,
step,
tempOutput,
{
status: AutomationErrors.INCORRECT_TYPE,
success: false,
}
)
loopSteps = undefined
loopStep = undefined
break
}
let item = []
if (
typeof loopStep.inputs.binding === "string" &&
loopStep.inputs.option === "String"
) {
item = automationUtils.stringSplit(loopStep.inputs.binding)
} else if (Array.isArray(loopStep.inputs.binding)) {
item = loopStep.inputs.binding
}
this._context.steps[loopStepNumber] = {
currentItem: item[index],
}
// The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
// Pretty hacky because we need to account for the row object
for (let [key, value] of Object.entries(originalStepInput)) {
if (typeof value === "object") {
for (let [innerKey, innerValue] of Object.entries(
originalStepInput[key]
)) {
if (typeof innerValue === "string") {
originalStepInput[key][innerKey] =
automationUtils.substituteLoopStep(
innerValue,
`steps.${loopStepNumber}`
)
} else if (typeof value === "object") {
for (let [innerObject, innerValue] of Object.entries(
originalStepInput[key][innerKey]
)) {
originalStepInput[key][innerKey][innerObject] =
automationUtils.substituteLoopStep(
innerValue as string,
`steps.${loopStepNumber}`
)
}
}
}
} else {
if (typeof value === "string") {
originalStepInput[key] =
automationUtils.substituteLoopStep(
value,
`steps.${loopStepNumber}`
)
}
}
}
if (
index === env.AUTOMATION_MAX_ITERATIONS ||
index === parseInt(loopStep.inputs.iterations)
) {
this.updateContextAndOutput(
loopStepNumber,
step,
tempOutput,
{
status: AutomationErrors.MAX_ITERATIONS,
success: true,
}
)
loopSteps = undefined
loopStep = undefined
break
}
let isFailure = false
const currentItem =
this._context.steps[loopStepNumber]?.currentItem
if (currentItem && typeof currentItem === "object") {
isFailure = Object.keys(currentItem).some(value => {
return currentItem[value] === loopStep?.inputs.failure
})
} else {
isFailure =
currentItem && currentItem === loopStep.inputs.failure
}
if (isFailure) {
this.updateContextAndOutput(
loopStepNumber,
step,
tempOutput,
{
status: AutomationErrors.FAILURE_CONDITION,
success: false,
}
)
loopSteps = undefined
loopStep = undefined
break
}
}
// execution stopped, record state for that
if (stopped) {
this.updateExecutionOutput(
step.id,
step.stepId,
{},
STOPPED_STATUS
)
continue
}
// If it's a loop step, we need to manually add the bindings to the context
let stepFn = await this.getStepFunctionality(step.stepId)
let inputs = await processObject(originalStepInput, this._context)
inputs = automationUtils.cleanInputValues(
inputs,
step.schema.inputs
)
try {
// appId is always passed
const outputs = await stepFn({
inputs: inputs,
appId: this._appId,
emitter: this._emitter,
context: this._context,
})
this._context.steps[stepCount] = outputs
// if filter causes us to stop execution don't break the loop, set a var
// so that we can finish iterating through the steps and record that it stopped
if (step.stepId === FILTER_STEP_ID && !outputs.result) {
stopped = true
this.updateExecutionOutput(
step.id,
step.stepId,
step.inputs,
{
...outputs,
...STOPPED_STATUS,
}
)
continue
}
if (loopStep && loopSteps) {
loopSteps.push(outputs)
} else {
this.updateExecutionOutput(
step.id,
step.stepId,
step.inputs,
outputs
)
}
} catch (err) {
console.error(`Automation error - ${step.stepId} - ${err}`)
return err
}
if (loopStep) {
iterationCount++
if (index === iterations - 1) {
loopStep = undefined
this._context.steps.splice(loopStepNumber, 1)
break
}
}
}
} finally {
stepSpan?.finish()
}
if (loopStep && iterations === 0) {
loopStep = undefined
this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
id: step.id,
stepId: step.stepId,
outputs: {
status: AutomationStepStatus.NO_ITERATIONS,
success: true,
},
inputs: {},
})
this._context.steps.splice(loopStepNumber, 1)
break
iterations = 1
}
// Delete the step after the loop step as it's irrelevant, since information is included
// in the loop step
if (wasLoopStep && !loopStep) {
this._context.steps.splice(loopStepNumber + 1, 1)
wasLoopStep = false
}
if (loopSteps && loopSteps.length) {
let tempOutput = {
success: true,
items: loopSteps,
iterations: iterationCount,
}
this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
id: step.id,
stepId: step.stepId,
outputs: tempOutput,
inputs: step.inputs,
})
this._context.steps[loopStepNumber] = tempOutput
wasLoopStep = true
loopSteps = []
}
}
}
if (loopStep && iterations === 0) {
loopStep = undefined
this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
id: step.id,
stepId: step.stepId,
outputs: {
status: AutomationStepStatus.NO_ITERATIONS,
success: true,
},
inputs: {},
})
const end = performance.now()
const executionTime = end - start
this._context.steps.splice(loopStepNumber, 1)
iterations = 1
}
console.info(
`Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
{
_logKey: "automation",
executionTime,
}
)
// Delete the step after the loop step as it's irrelevant, since information is included
// in the loop step
if (wasLoopStep && !loopStep) {
this._context.steps.splice(loopStepNumber + 1, 1)
wasLoopStep = false
}
if (loopSteps && loopSteps.length) {
let tempOutput = {
success: true,
items: loopSteps,
iterations: iterationCount,
// store the logs for the automation run
try {
await storeLog(this._automation, this.executionOutput)
} catch (e: any) {
if (e.status === 413 && e.request?.data) {
// if content is too large we shouldn't log it
delete e.request.data
e.request.data = { message: "removed due to large size" }
}
logging.logAlert("Error writing automation log", e)
}
this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
id: step.id,
stepId: step.stepId,
outputs: tempOutput,
inputs: step.inputs,
})
this._context.steps[loopStepNumber] = tempOutput
wasLoopStep = true
loopSteps = []
}
}
const end = performance.now()
const executionTime = end - start
console.info(
`Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
{
_logKey: "automation",
executionTime,
if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
await this.updateMetadata(metadata)
}
return this.executionOutput
}
)
// store the logs for the automation run
try {
await storeLog(this._automation, this.executionOutput)
} catch (e: any) {
if (e.status === 413 && e.request?.data) {
// if content is too large we shouldn't log it
delete e.request.data
e.request.data = { message: "removed due to large size" }
}
logging.logAlert("Error writing automation log", e)
}
if (isProdAppID(this._appId) && isRecurring(automation) && metadata) {
await this.updateMetadata(metadata)
}
return this.executionOutput
}
}

4
packages/string-templates/src/helpers/javascript.js
View file

@ -56,10 +56,12 @@ module.exports.processJS = (handlebars, context) => {
const res = { data: runJS(js, sandboxContext) }
return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}`
} catch (error) {
console.log(`JS error: ${typeof error} ${JSON.stringify(error)}`)
if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
return "Timed out while executing JS"
}
if (error.name === "ExecutionTimeoutError") {
return "Request JS execution limit hit"
}
return "Error while executing JS"
}
}

1
packages/string-templates/src/index.cjs
View file

@ -18,6 +18,7 @@ module.exports.doesContainString = templates.doesContainString
module.exports.disableEscaping = templates.disableEscaping
module.exports.findHBSBlocks = templates.findHBSBlocks
module.exports.convertToJS = templates.convertToJS
module.exports.setJSRunner = templates.setJSRunner
module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
if (!process.env.NO_JS) {

3
packages/string-templates/src/index.js
View file

@ -9,6 +9,7 @@ const {
findDoubleHbsInstances,
} = require("./utilities")
const { convertHBSBlock } = require("./conversion")
const javascript = require("./helpers/javascript")
const hbsInstance = handlebars.create()
registerAll(hbsInstance)
@ -362,6 +363,8 @@ module.exports.doesContainString = (template, string) => {
return exports.doesContainStrings(template, [string])
}
module.exports.setJSRunner = javascript.setJSRunner
module.exports.convertToJS = hbs => {
const blocks = exports.findHBSBlocks(hbs)
let js = "return `",

2
packages/string-templates/src/index.mjs
View file

@ -1,6 +1,5 @@
import vm from "vm"
import templates from "./index.js"
import { setJSRunner } from "./helpers/javascript"
/**
* ES6 entrypoint for rollup
@ -20,6 +19,7 @@ export const doesContainString = templates.doesContainString
export const disableEscaping = templates.disableEscaping
export const findHBSBlocks = templates.findHBSBlocks
export const convertToJS = templates.convertToJS
export const setJSRunner = templates.setJSRunner
export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
if (process && !process.env.NO_JS) {