diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 14cc110214..39a79193d3 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -76,6 +76,18 @@ jobs: yarn check:types fi + helm-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Use Node.js 18.x + uses: azure/setup-helm@v3 + - run: cd charts/budibase && helm lint . + test-libraries: runs-on: ubuntu-latest steps: diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 9fb435c2a3..c6ded3cee2 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -227,6 +227,14 @@ spec: resources: {{- toYaml . | nindent 10 }} {{ end }} + {{ if .Values.services.apps.command }} + command: + {{- toYaml .Values.services.apps.command | nindent 10 }} + {{ end }} + {{ if .Values.services.apps.args }} + args: + {{- toYaml .Values.services.apps.args | nindent 10 }} + {{ end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 46be6a4435..b7eece6b85 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -227,6 +227,13 @@ spec: resources: {{- toYaml . | nindent 10 }} {{ end }} + command: + {{- toYaml .Values.services.automationWorkers.command | nindent 10 }} + {{ end }} + {{ if .Values.services.automationWorkers.args }} + args: + {{- toYaml .Values.services.automationWorkers.args | nindent 10 }} + {{ end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -244,5 +251,6 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" + {{ if .Values.services.automationWorkers.command }}} status: {} {{- end }} \ No newline at end of file diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index 706e9b4b73..2e6217008b 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -100,5 +100,13 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" + {{ if .Values.services.proxy.command }} + command: + {{- toYaml .Values.services.proxy.command | nindent 8 }} + {{ end }} + {{ if .Values.services.proxy.args }} + args: + {{- toYaml .Values.services.proxy.args | nindent 8 }} + {{ end }} volumes: status: {} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 1d90aaf954..04791df869 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -213,6 +213,14 @@ spec: resources: {{- toYaml . | nindent 10 }} {{ end }} + {{ if .Values.services.worker.command }} + command: + {{- toYaml .Values.services.worker.command | nindent 10 }} + {{ end }} + {{ if .Values.services.worker.args }} + args: + {{- toYaml .Values.services.worker.args | nindent 10 }} + {{ end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index f4b2b5b127..eb9ec07464 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -7,7 +7,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME [[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION [[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80 [[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker -[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000 +[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU [[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR" @@ -77,7 +77,12 @@ mkdir -p ${DATA_DIR}/minio chown -R couchdb:couchdb ${DATA_DIR}/couch redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & /bbcouch-runner.sh & -/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & + +# only start minio if use s3 isn't passed +if [[ -z "${USE_S3}"]]; then + /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & +fi + /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate diff --git a/lerna.json b/lerna.json index b47f9e4aac..d2a623fcd2 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.46", + "version": "2.13.48", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index 09dae295e3..abf2d7804c 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 09dae295e3ba6149c4e1d7fe567870c3a38bd277 +Subproject commit abf2d7804c940b011328bb0979ebc6261420fe85 diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 983a4d20e1..e439eb5fd0 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -335,3 +335,11 @@ export function isScim(): boolean { const scimCall = context?.isScim return !!scimCall } + +export function getCurrentContext(): ContextMap | undefined { + try { + return Context.get() + } catch (e) { + return undefined + } +} diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index a1606a17b9..f73dc9f5c7 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,5 @@ import { IdentityContext } from "@budibase/types" +import { ExecutionTimeTracker } from "../timers" // keep this out of Budibase types, don't want to expose context info export type ContextMap = { @@ -9,4 +10,5 @@ export type ContextMap = { isScim?: boolean automationId?: string isMigrating?: boolean + jsExecutionTracker?: ExecutionTimeTracker } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 45aefc36f7..3fec573bb9 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -37,7 +37,7 @@ export function DatabaseWithConnection( opts?: DatabaseOpts ) { const db = new DatabaseImpl(dbName, opts, connection) - return new DDInstrumentedDatabase(db, "couchdb") + return new DDInstrumentedDatabase(db) } export class DatabaseImpl implements Database { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 3d0f522139..197770298e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -3,7 +3,7 @@ import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" import { DDInstrumentedDatabase } from "./instrumentation" export function getDB(dbName: string, opts?: DatabaseOpts): Database { - return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts), "couchdb") + return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts)) } // we have to use a callback for this so that we can close diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index 6d34478952..ba5febcba6 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -18,31 +18,28 @@ import tracer from "dd-trace" import { Writable } from "stream" export class DDInstrumentedDatabase implements Database { - constructor( - private readonly db: Database, - private readonly resource: string - ) {} + constructor(private readonly db: Database) {} get name(): string { return this.db.name } exists(): Promise { - return tracer.trace("exists", { resource: this.resource }, span => { + return tracer.trace("db.exists", span => { span?.addTags({ db_name: this.name }) return this.db.exists() }) } checkSetup(): Promise> { - return tracer.trace("checkSetup", { resource: this.resource }, span => { + return tracer.trace("db.checkSetup", span => { span?.addTags({ db_name: this.name }) return this.db.checkSetup() }) } get(id?: string | undefined): Promise { - return tracer.trace("get", { resource: this.resource }, span => { + return tracer.trace("db.get", span => { span?.addTags({ db_name: this.name, doc_id: id }) return this.db.get(id) }) @@ -52,7 +49,7 @@ export class DDInstrumentedDatabase implements Database { ids: string[], opts?: { allowMissing?: boolean | undefined } | undefined ): Promise { - return tracer.trace("getMultiple", { resource: this.resource }, span => { + return tracer.trace("db.getMultiple", span => { span?.addTags({ db_name: this.name, num_docs: ids.length, @@ -66,7 +63,7 @@ export class DDInstrumentedDatabase implements Database { id: string | Document, rev?: string | undefined ): Promise { - return tracer.trace("remove", { resource: this.resource }, span => { + return tracer.trace("db.remove", span => { span?.addTags({ db_name: this.name, doc_id: id }) return this.db.remove(id, rev) }) @@ -76,14 +73,14 @@ export class DDInstrumentedDatabase implements Database { document: AnyDocument, opts?: DatabasePutOpts | undefined ): Promise { - return tracer.trace("put", { resource: this.resource }, span => { + return tracer.trace("db.put", span => { span?.addTags({ db_name: this.name, doc_id: document._id }) return this.db.put(document, opts) }) } bulkDocs(documents: AnyDocument[]): Promise { - return tracer.trace("bulkDocs", { resource: this.resource }, span => { + return tracer.trace("db.bulkDocs", span => { span?.addTags({ db_name: this.name, num_docs: documents.length }) return this.db.bulkDocs(documents) }) @@ -92,7 +89,7 @@ export class DDInstrumentedDatabase implements Database { allDocs( params: DatabaseQueryOpts ): Promise> { - return tracer.trace("allDocs", { resource: this.resource }, span => { + return tracer.trace("db.allDocs", span => { span?.addTags({ db_name: this.name }) return this.db.allDocs(params) }) @@ -102,56 +99,56 @@ export class DDInstrumentedDatabase implements Database { viewName: string, params: DatabaseQueryOpts ): Promise> { - return tracer.trace("query", { resource: this.resource }, span => { + return tracer.trace("db.query", span => { span?.addTags({ db_name: this.name, view_name: viewName }) return this.db.query(viewName, params) }) } destroy(): Promise { - return tracer.trace("destroy", { resource: this.resource }, span => { + return tracer.trace("db.destroy", span => { span?.addTags({ db_name: this.name }) return this.db.destroy() }) } compact(): Promise { - return tracer.trace("compact", { resource: this.resource }, span => { + return tracer.trace("db.compact", span => { span?.addTags({ db_name: this.name }) return this.db.compact() }) } dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise { - return tracer.trace("dump", { resource: this.resource }, span => { + return tracer.trace("db.dump", span => { span?.addTags({ db_name: this.name }) return this.db.dump(stream, opts) }) } load(...args: any[]): Promise { - return tracer.trace("load", { resource: this.resource }, span => { + return tracer.trace("db.load", span => { span?.addTags({ db_name: this.name }) return this.db.load(...args) }) } createIndex(...args: any[]): Promise { - return tracer.trace("createIndex", { resource: this.resource }, span => { + return tracer.trace("db.createIndex", span => { span?.addTags({ db_name: this.name }) return this.db.createIndex(...args) }) } deleteIndex(...args: any[]): Promise { - return tracer.trace("deleteIndex", { resource: this.resource }, span => { + return tracer.trace("db.deleteIndex", span => { span?.addTags({ db_name: this.name }) return this.db.deleteIndex(...args) }) } getIndexes(...args: any[]): Promise { - return tracer.trace("getIndexes", { resource: this.resource }, span => { + return tracer.trace("db.getIndexes", span => { span?.addTags({ db_name: this.name }) return this.db.getIndexes(...args) }) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index ac7cdf550b..c05bbffbe9 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -15,6 +15,7 @@ function newJob(queue: string, message: any) { timestamp: Date.now(), queue: queue, data: message, + opts: {}, } } diff --git a/packages/backend-core/src/timers/timers.ts b/packages/backend-core/src/timers/timers.ts index 000be74821..9de57af7f1 100644 --- a/packages/backend-core/src/timers/timers.ts +++ b/packages/backend-core/src/timers/timers.ts @@ -20,3 +20,41 @@ export function cleanup() { } intervals = [] } + +export class ExecutionTimeoutError extends Error { + public readonly name = "ExecutionTimeoutError" +} + +export class ExecutionTimeTracker { + static withLimit(limitMs: number) { + return new ExecutionTimeTracker(limitMs) + } + + constructor(readonly limitMs: number) {} + + private totalTimeMs = 0 + + track(f: () => T): T { + this.checkLimit() + const start = process.hrtime.bigint() + try { + return f() + } finally { + const end = process.hrtime.bigint() + this.totalTimeMs += Number(end - start) / 1e6 + this.checkLimit() + } + } + + get elapsedMS() { + return this.totalTimeMs + } + + private checkLimit() { + if (this.totalTimeMs > this.limitMs) { + throw new ExecutionTimeoutError( + `Execution time limit of ${this.limitMs}ms exceeded: ${this.totalTimeMs}ms` + ) + } + } +} diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts index 3c7ef23b11..730b59d1dc 100644 --- a/packages/backend-core/src/utils/Duration.ts +++ b/packages/backend-core/src/utils/Duration.ts @@ -49,4 +49,8 @@ export class Duration { static fromDays(duration: number) { return Duration.from(DurationType.DAYS, duration) } + + static fromMilliseconds(duration: number) { + return Duration.from(DurationType.MILLISECONDS, duration) + } } diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 5b3c69b87a..2d47e5f54a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -2086,4 +2086,112 @@ describe.each([ expect(row.formula).toBe(relatedRow.name) }) }) + + describe("Formula JS protection", () => { + it("should time out JS execution if a single cell takes too long", async () => { + await config.withEnv({ JS_PER_EXECUTION_TIME_LIMIT_MS: 20 }, async () => { + const js = Buffer.from( + ` + let i = 0; + while (true) { + i++; + } + return i; + ` + ).toString("base64") + + const table = await config.createTable({ + name: "table", + type: "table", + schema: { + text: { + name: "text", + type: FieldType.STRING, + }, + formula: { + name: "formula", + type: FieldType.FORMULA, + formula: `{{ js "${js}"}}`, + formulaType: FormulaTypes.DYNAMIC, + }, + }, + }) + + await config.api.row.save(table._id!, { text: "foo" }) + const { rows } = await config.api.row.search(table._id!) + expect(rows).toHaveLength(1) + const row = rows[0] + expect(row.text).toBe("foo") + expect(row.formula).toBe("Timed out while executing JS") + }) + }) + + it("should time out JS execution if a multiple cells take too long", async () => { + await config.withEnv( + { + JS_PER_EXECUTION_TIME_LIMIT_MS: 20, + JS_PER_REQUEST_TIME_LIMIT_MS: 40, + }, + async () => { + const js = Buffer.from( + ` + let i = 0; + while (true) { + i++; + } + return i; + ` + ).toString("base64") + + const table = await config.createTable({ + name: "table", + type: "table", + schema: { + text: { + name: "text", + type: FieldType.STRING, + }, + formula: { + name: "formula", + type: FieldType.FORMULA, + formula: `{{ js "${js}"}}`, + formulaType: FormulaTypes.DYNAMIC, + }, + }, + }) + + for (let i = 0; i < 10; i++) { + await config.api.row.save(table._id!, { text: "foo" }) + } + + // Run this test 3 times to make sure that there's no cross-request + // pollution of the execution time tracking. + for (let reqs = 0; reqs < 3; reqs++) { + const { rows } = await config.api.row.search(table._id!) + expect(rows).toHaveLength(10) + + let i = 0 + for (; i < 10; i++) { + const row = rows[i] + if (row.formula !== "Timed out while executing JS") { + break + } + } + + // Given the execution times are not deterministic, we can't be sure + // of the exact number of rows that were executed before the timeout + // but it should absolutely be at least 1. + expect(i).toBeGreaterThan(0) + expect(i).toBeLessThan(5) + + for (; i < 10; i++) { + const row = rows[i] + expect(row.text).toBe("foo") + expect(row.formula).toBe("Request JS execution limit hit") + } + } + } + ) + }) + }) }) diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 53c4d9d3b7..0c28787f67 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -16,6 +16,7 @@ import { } from "@budibase/types" import sdk from "../sdk" import { automationsEnabled } from "../features" +import tracer from "dd-trace" const REBOOT_CRON = "@reboot" const WH_STEP_ID = definitions.WEBHOOK.stepId @@ -39,26 +40,62 @@ function loggingArgs(job: AutomationJob) { } export async function processEvent(job: AutomationJob) { - const appId = job.data.event.appId! - const automationId = job.data.automation._id! - const task = async () => { - try { - // need to actually await these so that an error can be captured properly - console.log("automation running", ...loggingArgs(job)) + return tracer.trace( + "processEvent", + { resource: "automation" }, + async span => { + const appId = job.data.event.appId! + const automationId = job.data.automation._id! - const runFn = () => Runner.run(job) - const result = await quotas.addAutomation(runFn, { + span?.addTags({ + appId, automationId, + job: { + id: job.id, + name: job.name, + attemptsMade: job.attemptsMade, + opts: { + attempts: job.opts.attempts, + priority: job.opts.priority, + delay: job.opts.delay, + repeat: job.opts.repeat, + backoff: job.opts.backoff, + lifo: job.opts.lifo, + timeout: job.opts.timeout, + jobId: job.opts.jobId, + removeOnComplete: job.opts.removeOnComplete, + removeOnFail: job.opts.removeOnFail, + stackTraceLimit: job.opts.stackTraceLimit, + preventParsingData: job.opts.preventParsingData, + }, + }, }) - console.log("automation completed", ...loggingArgs(job)) - return result - } catch (err) { - console.error(`automation was unable to run`, err, ...loggingArgs(job)) - return { err } - } - } - return await context.doInAutomationContext({ appId, automationId, task }) + const task = async () => { + try { + // need to actually await these so that an error can be captured properly + console.log("automation running", ...loggingArgs(job)) + + const runFn = () => Runner.run(job) + const result = await quotas.addAutomation(runFn, { + automationId, + }) + console.log("automation completed", ...loggingArgs(job)) + return result + } catch (err) { + span?.addTags({ error: true }) + console.error( + `automation was unable to run`, + err, + ...loggingArgs(job) + ) + return { err } + } + } + + return await context.doInAutomationContext({ appId, automationId, task }) + } + ) } export async function updateTestHistory( diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 98a90fbddc..f692a8b6cf 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -70,6 +70,11 @@ const environment = { SELF_HOSTED: process.env.SELF_HOSTED, HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT, FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main", + JS_PER_EXECUTION_TIME_LIMIT_MS: + parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000, + JS_PER_REQUEST_TIME_LIMIT_MS: parseIntSafe( + process.env.JS_PER_REQUEST_TIME_LIMIT_MS + ), // old CLIENT_ID: process.env.CLIENT_ID, _set(key: string, value: any) { diff --git a/packages/server/src/jsRunner.ts b/packages/server/src/jsRunner.ts new file mode 100644 index 0000000000..ab0381a399 --- /dev/null +++ b/packages/server/src/jsRunner.ts @@ -0,0 +1,45 @@ +import vm from "vm" +import env from "./environment" +import { setJSRunner } from "@budibase/string-templates" +import { context, timers } from "@budibase/backend-core" +import tracer from "dd-trace" + +type TrackerFn = (f: () => T) => T + +export function init() { + setJSRunner((js: string, ctx: vm.Context) => { + return tracer.trace("runJS", {}, span => { + const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS + let track: TrackerFn = f => f() + if (perRequestLimit) { + const bbCtx = context.getCurrentContext() + if (bbCtx) { + if (!bbCtx.jsExecutionTracker) { + bbCtx.jsExecutionTracker = + timers.ExecutionTimeTracker.withLimit(perRequestLimit) + } + track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker) + span?.addTags({ + js: { + limitMS: bbCtx.jsExecutionTracker.limitMs, + elapsedMS: bbCtx.jsExecutionTracker.elapsedMS, + }, + }) + } + } + + ctx = { + ...ctx, + alert: undefined, + setInterval: undefined, + setTimeout: undefined, + } + vm.createContext(ctx) + return track(() => + vm.runInNewContext(js, ctx, { + timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, + }) + ) + }) + }) +} diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts index 4b11b933af..ad6f2afa18 100644 --- a/packages/server/src/middleware/currentapp.ts +++ b/packages/server/src/middleware/currentapp.ts @@ -23,7 +23,7 @@ export default async (ctx: UserCtx, next: any) => { if (requestAppId) { const span = tracer.scope().active() - span?.addTags({ app_id: requestAppId }) + span?.setTag("appId", requestAppId) } // deny access to application preview @@ -76,6 +76,14 @@ export default async (ctx: UserCtx, next: any) => { return next() } + if (ctx.user) { + const span = tracer.scope().active() + if (ctx.user._id) { + span?.setTag("userId", ctx.user._id) + } + span?.setTag("tenantId", ctx.user.tenantId) + } + const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined // if the user is not in the right tenant then make sure to wipe their cookie diff --git a/packages/server/src/startup.ts b/packages/server/src/startup.ts index 2db6e5ae6a..6860fe5f9b 100644 --- a/packages/server/src/startup.ts +++ b/packages/server/src/startup.ts @@ -23,6 +23,7 @@ import { automationsEnabled, printFeatures } from "./features" import Koa from "koa" import { Server } from "http" import { AddressInfo } from "net" +import * as jsRunner from "./jsRunner" let STARTUP_RAN = false @@ -152,4 +153,6 @@ export async function startup(app?: Koa, server?: Server) { } }) } + + jsRunner.init() } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index d1fcc2be72..4447899f96 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -34,6 +34,7 @@ import { cloneDeep } from "lodash/fp" import { performance } from "perf_hooks" import * as sdkUtils from "../sdk/utils" import env from "../environment" +import tracer from "dd-trace" threadUtils.threadSetup() const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId @@ -242,278 +243,347 @@ class Orchestrator { } async execute(): Promise { - // this will retrieve from context created at start of thread - this._context.env = await sdkUtils.getEnvironmentVariables() - let automation = this._automation - let stopped = false - let loopStep: AutomationStep | undefined = undefined + return tracer.trace( + "Orchestrator.execute", + { resource: "automation" }, + async span => { + span?.addTags({ + appId: this._appId, + automationId: this._automation._id, + }) - let stepCount = 0 - let loopStepNumber: any = undefined - let loopSteps: LoopStep[] | undefined = [] - let metadata - let timeoutFlag = false - let wasLoopStep = false - let timeout = this._job.data.event.timeout - // check if this is a recurring automation, - if (isProdAppID(this._appId) && isRecurring(automation)) { - metadata = await this.getMetadata() - const shouldStop = await this.checkIfShouldStop(metadata) - if (shouldStop) { - return - } - } - const start = performance.now() - for (let step of automation.definition.steps) { - if (timeoutFlag) { - break - } + // this will retrieve from context created at start of thread + this._context.env = await sdkUtils.getEnvironmentVariables() + let automation = this._automation + let stopped = false + let loopStep: AutomationStep | undefined = undefined - if (timeout) { - setTimeout(() => { - timeoutFlag = true - }, timeout || 12000) - } - - stepCount++ - let input: any, - iterations = 1, - iterationCount = 0 - - if (step.stepId === LOOP_STEP_ID) { - loopStep = step - loopStepNumber = stepCount - continue - } - - if (loopStep) { - input = await processObject(loopStep.inputs, this._context) - iterations = getLoopIterations(loopStep as LoopStep) - } - for (let index = 0; index < iterations; index++) { - let originalStepInput = cloneDeep(step.inputs) - // Handle if the user has set a max iteration count or if it reaches the max limit set by us - if (loopStep && input.binding) { - let tempOutput = { items: loopSteps, iterations: iterationCount } - try { - loopStep.inputs.binding = automationUtils.typecastForLooping( - loopStep as LoopStep, - loopStep.inputs as LoopInput - ) - } catch (err) { - this.updateContextAndOutput(loopStepNumber, step, tempOutput, { - status: AutomationErrors.INCORRECT_TYPE, - success: false, - }) - loopSteps = undefined - loopStep = undefined - break - } - let item = [] - if ( - typeof loopStep.inputs.binding === "string" && - loopStep.inputs.option === "String" - ) { - item = automationUtils.stringSplit(loopStep.inputs.binding) - } else if (Array.isArray(loopStep.inputs.binding)) { - item = loopStep.inputs.binding - } - this._context.steps[loopStepNumber] = { - currentItem: item[index], - } - - // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it - // Pretty hacky because we need to account for the row object - for (let [key, value] of Object.entries(originalStepInput)) { - if (typeof value === "object") { - for (let [innerKey, innerValue] of Object.entries( - originalStepInput[key] - )) { - if (typeof innerValue === "string") { - originalStepInput[key][innerKey] = - automationUtils.substituteLoopStep( - innerValue, - `steps.${loopStepNumber}` - ) - } else if (typeof value === "object") { - for (let [innerObject, innerValue] of Object.entries( - originalStepInput[key][innerKey] - )) { - originalStepInput[key][innerKey][innerObject] = - automationUtils.substituteLoopStep( - innerValue as string, - `steps.${loopStepNumber}` - ) - } - } - } - } else { - if (typeof value === "string") { - originalStepInput[key] = automationUtils.substituteLoopStep( - value, - `steps.${loopStepNumber}` - ) - } - } - } - - if ( - index === env.AUTOMATION_MAX_ITERATIONS || - index === parseInt(loopStep.inputs.iterations) - ) { - this.updateContextAndOutput(loopStepNumber, step, tempOutput, { - status: AutomationErrors.MAX_ITERATIONS, - success: true, - }) - loopSteps = undefined - loopStep = undefined - break - } - - let isFailure = false - const currentItem = this._context.steps[loopStepNumber]?.currentItem - if (currentItem && typeof currentItem === "object") { - isFailure = Object.keys(currentItem).some(value => { - return currentItem[value] === loopStep?.inputs.failure - }) - } else { - isFailure = currentItem && currentItem === loopStep.inputs.failure - } - - if (isFailure) { - this.updateContextAndOutput(loopStepNumber, step, tempOutput, { - status: AutomationErrors.FAILURE_CONDITION, - success: false, - }) - loopSteps = undefined - loopStep = undefined - break + let stepCount = 0 + let loopStepNumber: any = undefined + let loopSteps: LoopStep[] | undefined = [] + let metadata + let timeoutFlag = false + let wasLoopStep = false + let timeout = this._job.data.event.timeout + // check if this is a recurring automation, + if (isProdAppID(this._appId) && isRecurring(automation)) { + span?.addTags({ recurring: true }) + metadata = await this.getMetadata() + const shouldStop = await this.checkIfShouldStop(metadata) + if (shouldStop) { + span?.addTags({ shouldStop: true }) + return } } - - // execution stopped, record state for that - if (stopped) { - this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS) - continue - } - - // If it's a loop step, we need to manually add the bindings to the context - let stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await processObject(originalStepInput, this._context) - inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) - - try { - // appId is always passed - const outputs = await stepFn({ - inputs: inputs, - appId: this._appId, - emitter: this._emitter, - context: this._context, + const start = performance.now() + for (let step of automation.definition.steps) { + const stepSpan = tracer.startSpan("Orchestrator.execute.step", { + childOf: span, + }) + stepSpan.addTags({ + resource: "automation", + step: { + stepId: step.stepId, + id: step.id, + name: step.name, + type: step.type, + title: step.stepTitle, + internal: step.internal, + deprecated: step.deprecated, + }, }) - this._context.steps[stepCount] = outputs - // if filter causes us to stop execution don't break the loop, set a var - // so that we can finish iterating through the steps and record that it stopped - if (step.stepId === FILTER_STEP_ID && !outputs.result) { - stopped = true - this.updateExecutionOutput(step.id, step.stepId, step.inputs, { - ...outputs, - ...STOPPED_STATUS, - }) - continue - } - if (loopStep && loopSteps) { - loopSteps.push(outputs) - } else { - this.updateExecutionOutput( - step.id, - step.stepId, - step.inputs, - outputs - ) - } - } catch (err) { - console.error(`Automation error - ${step.stepId} - ${err}`) - return err - } + let input: any, + iterations = 1, + iterationCount = 0 - if (loopStep) { - iterationCount++ - if (index === iterations - 1) { + try { + if (timeoutFlag) { + span?.addTags({ timedOut: true }) + break + } + + if (timeout) { + setTimeout(() => { + timeoutFlag = true + }, timeout || 12000) + } + + stepCount++ + if (step.stepId === LOOP_STEP_ID) { + loopStep = step + loopStepNumber = stepCount + continue + } + + if (loopStep) { + input = await processObject(loopStep.inputs, this._context) + iterations = getLoopIterations(loopStep as LoopStep) + stepSpan?.addTags({ step: { iterations } }) + } + for (let index = 0; index < iterations; index++) { + let originalStepInput = cloneDeep(step.inputs) + // Handle if the user has set a max iteration count or if it reaches the max limit set by us + if (loopStep && input.binding) { + let tempOutput = { + items: loopSteps, + iterations: iterationCount, + } + try { + loopStep.inputs.binding = automationUtils.typecastForLooping( + loopStep as LoopStep, + loopStep.inputs as LoopInput + ) + } catch (err) { + this.updateContextAndOutput( + loopStepNumber, + step, + tempOutput, + { + status: AutomationErrors.INCORRECT_TYPE, + success: false, + } + ) + loopSteps = undefined + loopStep = undefined + break + } + let item = [] + if ( + typeof loopStep.inputs.binding === "string" && + loopStep.inputs.option === "String" + ) { + item = automationUtils.stringSplit(loopStep.inputs.binding) + } else if (Array.isArray(loopStep.inputs.binding)) { + item = loopStep.inputs.binding + } + this._context.steps[loopStepNumber] = { + currentItem: item[index], + } + + // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it + // Pretty hacky because we need to account for the row object + for (let [key, value] of Object.entries(originalStepInput)) { + if (typeof value === "object") { + for (let [innerKey, innerValue] of Object.entries( + originalStepInput[key] + )) { + if (typeof innerValue === "string") { + originalStepInput[key][innerKey] = + automationUtils.substituteLoopStep( + innerValue, + `steps.${loopStepNumber}` + ) + } else if (typeof value === "object") { + for (let [innerObject, innerValue] of Object.entries( + originalStepInput[key][innerKey] + )) { + originalStepInput[key][innerKey][innerObject] = + automationUtils.substituteLoopStep( + innerValue as string, + `steps.${loopStepNumber}` + ) + } + } + } + } else { + if (typeof value === "string") { + originalStepInput[key] = + automationUtils.substituteLoopStep( + value, + `steps.${loopStepNumber}` + ) + } + } + } + + if ( + index === env.AUTOMATION_MAX_ITERATIONS || + index === parseInt(loopStep.inputs.iterations) + ) { + this.updateContextAndOutput( + loopStepNumber, + step, + tempOutput, + { + status: AutomationErrors.MAX_ITERATIONS, + success: true, + } + ) + loopSteps = undefined + loopStep = undefined + break + } + + let isFailure = false + const currentItem = + this._context.steps[loopStepNumber]?.currentItem + if (currentItem && typeof currentItem === "object") { + isFailure = Object.keys(currentItem).some(value => { + return currentItem[value] === loopStep?.inputs.failure + }) + } else { + isFailure = + currentItem && currentItem === loopStep.inputs.failure + } + + if (isFailure) { + this.updateContextAndOutput( + loopStepNumber, + step, + tempOutput, + { + status: AutomationErrors.FAILURE_CONDITION, + success: false, + } + ) + loopSteps = undefined + loopStep = undefined + break + } + } + + // execution stopped, record state for that + if (stopped) { + this.updateExecutionOutput( + step.id, + step.stepId, + {}, + STOPPED_STATUS + ) + continue + } + + // If it's a loop step, we need to manually add the bindings to the context + let stepFn = await this.getStepFunctionality(step.stepId) + let inputs = await processObject(originalStepInput, this._context) + inputs = automationUtils.cleanInputValues( + inputs, + step.schema.inputs + ) + + try { + // appId is always passed + const outputs = await stepFn({ + inputs: inputs, + appId: this._appId, + emitter: this._emitter, + context: this._context, + }) + + this._context.steps[stepCount] = outputs + // if filter causes us to stop execution don't break the loop, set a var + // so that we can finish iterating through the steps and record that it stopped + if (step.stepId === FILTER_STEP_ID && !outputs.result) { + stopped = true + this.updateExecutionOutput( + step.id, + step.stepId, + step.inputs, + { + ...outputs, + ...STOPPED_STATUS, + } + ) + continue + } + if (loopStep && loopSteps) { + loopSteps.push(outputs) + } else { + this.updateExecutionOutput( + step.id, + step.stepId, + step.inputs, + outputs + ) + } + } catch (err) { + console.error(`Automation error - ${step.stepId} - ${err}`) + return err + } + + if (loopStep) { + iterationCount++ + if (index === iterations - 1) { + loopStep = undefined + this._context.steps.splice(loopStepNumber, 1) + break + } + } + } + } finally { + stepSpan?.finish() + } + + if (loopStep && iterations === 0) { loopStep = undefined + this.executionOutput.steps.splice(loopStepNumber + 1, 0, { + id: step.id, + stepId: step.stepId, + outputs: { + status: AutomationStepStatus.NO_ITERATIONS, + success: true, + }, + inputs: {}, + }) + this._context.steps.splice(loopStepNumber, 1) - break + iterations = 1 + } + + // Delete the step after the loop step as it's irrelevant, since information is included + // in the loop step + if (wasLoopStep && !loopStep) { + this._context.steps.splice(loopStepNumber + 1, 1) + wasLoopStep = false + } + if (loopSteps && loopSteps.length) { + let tempOutput = { + success: true, + items: loopSteps, + iterations: iterationCount, + } + this.executionOutput.steps.splice(loopStepNumber + 1, 0, { + id: step.id, + stepId: step.stepId, + outputs: tempOutput, + inputs: step.inputs, + }) + this._context.steps[loopStepNumber] = tempOutput + + wasLoopStep = true + loopSteps = [] } } - } - if (loopStep && iterations === 0) { - loopStep = undefined - this.executionOutput.steps.splice(loopStepNumber + 1, 0, { - id: step.id, - stepId: step.stepId, - outputs: { - status: AutomationStepStatus.NO_ITERATIONS, - success: true, - }, - inputs: {}, - }) + const end = performance.now() + const executionTime = end - start - this._context.steps.splice(loopStepNumber, 1) - iterations = 1 - } + console.info( + `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`, + { + _logKey: "automation", + executionTime, + } + ) - // Delete the step after the loop step as it's irrelevant, since information is included - // in the loop step - if (wasLoopStep && !loopStep) { - this._context.steps.splice(loopStepNumber + 1, 1) - wasLoopStep = false - } - if (loopSteps && loopSteps.length) { - let tempOutput = { - success: true, - items: loopSteps, - iterations: iterationCount, + // store the logs for the automation run + try { + await storeLog(this._automation, this.executionOutput) + } catch (e: any) { + if (e.status === 413 && e.request?.data) { + // if content is too large we shouldn't log it + delete e.request.data + e.request.data = { message: "removed due to large size" } + } + logging.logAlert("Error writing automation log", e) } - this.executionOutput.steps.splice(loopStepNumber + 1, 0, { - id: step.id, - stepId: step.stepId, - outputs: tempOutput, - inputs: step.inputs, - }) - this._context.steps[loopStepNumber] = tempOutput - - wasLoopStep = true - loopSteps = [] - } - } - - const end = performance.now() - const executionTime = end - start - - console.info( - `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`, - { - _logKey: "automation", - executionTime, + if (isProdAppID(this._appId) && isRecurring(automation) && metadata) { + await this.updateMetadata(metadata) + } + return this.executionOutput } ) - - // store the logs for the automation run - try { - await storeLog(this._automation, this.executionOutput) - } catch (e: any) { - if (e.status === 413 && e.request?.data) { - // if content is too large we shouldn't log it - delete e.request.data - e.request.data = { message: "removed due to large size" } - } - logging.logAlert("Error writing automation log", e) - } - if (isProdAppID(this._appId) && isRecurring(automation) && metadata) { - await this.updateMetadata(metadata) - } - return this.executionOutput } } diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.js index 53baec8613..585a2ba6b7 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.js @@ -56,10 +56,12 @@ module.exports.processJS = (handlebars, context) => { const res = { data: runJS(js, sandboxContext) } return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` } catch (error) { - console.log(`JS error: ${typeof error} ${JSON.stringify(error)}`) if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") { return "Timed out while executing JS" } + if (error.name === "ExecutionTimeoutError") { + return "Request JS execution limit hit" + } return "Error while executing JS" } } diff --git a/packages/string-templates/src/index.cjs b/packages/string-templates/src/index.cjs index 1cf3c8d0c6..aedb7fc052 100644 --- a/packages/string-templates/src/index.cjs +++ b/packages/string-templates/src/index.cjs @@ -18,6 +18,7 @@ module.exports.doesContainString = templates.doesContainString module.exports.disableEscaping = templates.disableEscaping module.exports.findHBSBlocks = templates.findHBSBlocks module.exports.convertToJS = templates.convertToJS +module.exports.setJSRunner = templates.setJSRunner module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX if (!process.env.NO_JS) { diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 4844bf5fb6..63da7fde4d 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -9,6 +9,7 @@ const { findDoubleHbsInstances, } = require("./utilities") const { convertHBSBlock } = require("./conversion") +const javascript = require("./helpers/javascript") const hbsInstance = handlebars.create() registerAll(hbsInstance) @@ -362,6 +363,8 @@ module.exports.doesContainString = (template, string) => { return exports.doesContainStrings(template, [string]) } +module.exports.setJSRunner = javascript.setJSRunner + module.exports.convertToJS = hbs => { const blocks = exports.findHBSBlocks(hbs) let js = "return `", diff --git a/packages/string-templates/src/index.mjs b/packages/string-templates/src/index.mjs index ae640624b2..43cda8183f 100644 --- a/packages/string-templates/src/index.mjs +++ b/packages/string-templates/src/index.mjs @@ -1,6 +1,5 @@ import vm from "vm" import templates from "./index.js" -import { setJSRunner } from "./helpers/javascript" /** * ES6 entrypoint for rollup @@ -20,6 +19,7 @@ export const doesContainString = templates.doesContainString export const disableEscaping = templates.disableEscaping export const findHBSBlocks = templates.findHBSBlocks export const convertToJS = templates.convertToJS +export const setJSRunner = templates.setJSRunner export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX if (process && !process.env.NO_JS) {