diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte index 7ff0bb7f4c..33946d16dc 100644 --- a/packages/builder/src/components/integration/RestQueryViewer.svelte +++ b/packages/builder/src/components/integration/RestQueryViewer.svelte @@ -159,7 +159,7 @@ newQuery.fields.queryString = queryString newQuery.fields.authConfigId = authConfigId newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders) - newQuery.schema = schema + newQuery.schema = schema || {} return newQuery } diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 6feba9fb2d..186b68f3b7 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -1,4 +1,3 @@ -import * as actions from "../../automations/actions" import * as triggers from "../../automations/triggers" import { getAutomationParams, @@ -20,11 +19,12 @@ import { Automation, AutomationActionStepId, AutomationResults, - BBContext, + UserCtx, } from "@budibase/types" import { getActionDefinitions as actionDefs } from "../../automations/actions" import sdk from "../../sdk" import { builderSocket } from "../../websockets" +import env from "../../environment" async function getActionDefinitions() { return removeDeprecated(await actionDefs()) @@ -72,7 +72,7 @@ function cleanAutomationInputs(automation: Automation) { return automation } -export async function create(ctx: BBContext) { +export async function create(ctx: UserCtx) { const db = context.getAppDB() let automation = ctx.request.body automation.appId = ctx.appId @@ -141,7 +141,7 @@ export async function handleStepEvents( } } -export async function update(ctx: BBContext) { +export async function update(ctx: UserCtx) { const db = context.getAppDB() let automation = ctx.request.body automation.appId = ctx.appId @@ -192,7 +192,7 @@ export async function update(ctx: BBContext) { builderSocket?.emitAutomationUpdate(ctx, automation) } -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: UserCtx) { const db = context.getAppDB() const response = await db.allDocs( getAutomationParams(null, { @@ -202,12 +202,12 @@ export async function fetch(ctx: BBContext) { ctx.body = response.rows.map(row => row.doc) } -export async function find(ctx: BBContext) { +export async function find(ctx: UserCtx) { const db = context.getAppDB() ctx.body = await db.get(ctx.params.id) } -export async function destroy(ctx: BBContext) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const automationId = ctx.params.id const oldAutomation = await db.get(automationId) @@ -221,11 +221,11 @@ export async function destroy(ctx: BBContext) { builderSocket?.emitAutomationDeletion(ctx, automationId) } -export async function logSearch(ctx: BBContext) { +export async function logSearch(ctx: UserCtx) { ctx.body = await automations.logs.logSearch(ctx.request.body) } -export async function clearLogError(ctx: BBContext) { +export async function clearLogError(ctx: UserCtx) { const { automationId, appId } = ctx.request.body await context.doInAppContext(appId, async () => { const db = context.getProdAppDB() @@ -244,15 +244,15 @@ export async function clearLogError(ctx: BBContext) { }) } -export async function getActionList(ctx: BBContext) { +export async function getActionList(ctx: UserCtx) { ctx.body = await getActionDefinitions() } -export async function getTriggerList(ctx: BBContext) { +export async function getTriggerList(ctx: UserCtx) { ctx.body = getTriggerDefinitions() } -export async function getDefinitionList(ctx: BBContext) { +export async function getDefinitionList(ctx: UserCtx) { ctx.body = { trigger: getTriggerDefinitions(), action: await getActionDefinitions(), @@ -265,7 +265,7 @@ export async function getDefinitionList(ctx: BBContext) { * * *********************/ -export async function trigger(ctx: BBContext) { +export async function trigger(ctx: UserCtx) { const db = context.getAppDB() let automation = await db.get(ctx.params.id) @@ -275,7 +275,9 @@ export async function trigger(ctx: BBContext) { automation, { fields: ctx.request.body.fields, - timeout: ctx.request.body.timeout * 1000 || 120000, + timeout: + ctx.request.body.timeout * 1000 || + env.getDefaults().AUTOMATION_SYNC_TIMEOUT, }, { getResponses: true } ) @@ -310,7 +312,7 @@ function prepareTestInput(input: any) { return input } -export async function test(ctx: BBContext) { +export async function test(ctx: UserCtx) { const db = context.getAppDB() let automation = await db.get(ctx.params.id) await setTestFlag(automation._id!) diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 1be836b169..8dabe5b3cc 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -22,7 +22,7 @@ import { import { ValidQueryNameRegex } from "@budibase/shared-core" const Runner = new Thread(ThreadType.QUERY, { - timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000, + timeoutMs: env.QUERY_THREAD_TIMEOUT, }) // simple function to append "readable" to all read queries diff --git a/packages/server/src/automations/bullboard.ts b/packages/server/src/automations/bullboard.ts index df784eacff..34f18754a2 100644 --- a/packages/server/src/automations/bullboard.ts +++ b/packages/server/src/automations/bullboard.ts @@ -15,7 +15,7 @@ const PATH_PREFIX = "/bulladmin" export async function init() { // Set up queues for bull board admin - const backupQueue = await backups.getBackupQueue() + const backupQueue = backups.getBackupQueue() const queues = [automationQueue] if (backupQueue) { queues.push(backupQueue) diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts index 61d446f12c..1a13f651ec 100644 --- a/packages/server/src/automations/steps/bash.ts +++ b/packages/server/src/automations/steps/bash.ts @@ -65,7 +65,7 @@ export async function run({ inputs, context }: AutomationStepInput) { success = true try { stdout = execSync(command, { - timeout: environment.QUERY_THREAD_TIMEOUT || 500, + timeout: environment.QUERY_THREAD_TIMEOUT, }).toString() } catch (err: any) { stdout = err.message diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index a9517b01a0..ea0737c86a 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -10,6 +10,8 @@ import { AutomationStepSchema, AutomationStepType, } from "@budibase/types" +import { utils } from "@budibase/backend-core" +import env from "../../environment" export const definition: AutomationStepSchema = { name: "External Data Connector", diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index cb6126ca01..83e1722877 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -9,8 +9,9 @@ import { AutomationCustomIOType, } from "@budibase/types" import * as triggers from "../triggers" -import { db as dbCore, context } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { features } from "@budibase/pro" +import env from "../../environment" export const definition: AutomationStepSchema = { name: "Trigger an automation", @@ -76,7 +77,8 @@ export async function run({ inputs }: AutomationStepInput) { automation, { fields: { ...fieldParams }, - timeout: inputs.timeout * 1000 || 120000, + timeout: + inputs.timeout * 1000 || env.getDefaults().AUTOMATION_SYNC_TIMEOUT, }, { getResponses: true } ) diff --git a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts index f8cf647e79..9d699e15fa 100644 --- a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts +++ b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts @@ -3,6 +3,7 @@ jest.spyOn(global.console, "error") import * as setup from "./utilities" import * as automation from "../index" import { serverLogAutomation } from "../../tests/utilities/structures" +import env from "../../environment" describe("Test triggering an automation from another automation", () => { let config = setup.getConfig() @@ -22,7 +23,10 @@ describe("Test triggering an automation from another automation", () => { let newAutomation = await config.createAutomation(automation) const inputs: any = { - automation: { automationId: newAutomation._id, timeout: 12000 }, + automation: { + automationId: newAutomation._id, + timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT, + }, } const res = await setup.runStep( setup.actions.TRIGGER_AUTOMATION_RUN.stepId, @@ -33,7 +37,12 @@ describe("Test triggering an automation from another automation", () => { }) it("should fail gracefully if the automation id is incorrect", async () => { - const inputs: any = { automation: { automationId: null, timeout: 12000 } } + const inputs: any = { + automation: { + automationId: null, + timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT, + }, + } const res = await setup.runStep( setup.actions.TRIGGER_AUTOMATION_RUN.stepId, inputs diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 8e6866d5e4..20142776b8 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -18,6 +18,21 @@ function parseIntSafe(number?: string) { } } +const DEFAULTS = { + QUERY_THREAD_TIMEOUT: 10000, + AUTOMATION_THREAD_TIMEOUT: 12000, + AUTOMATION_SYNC_TIMEOUT: 120000, + AUTOMATION_MAX_ITERATIONS: 200, + JS_PER_EXECUTION_TIME_LIMIT_MS: 1000, + TEMPLATE_REPOSITORY: "app", + PLUGINS_DIR: "/plugins", + FORKED_PROCESS_NAME: "main", + JS_RUNNER_MEMORY_LIMIT: 64, +} + +const QUERY_THREAD_TIMEOUT = + parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) || + DEFAULTS.QUERY_THREAD_TIMEOUT const environment = { // features APP_FEATURES: process.env.APP_FEATURES, @@ -42,7 +57,8 @@ const environment = { JEST_WORKER_ID: process.env.JEST_WORKER_ID, BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, - TEMPLATE_REPOSITORY: process.env.TEMPLATE_REPOSITORY || "app", + TEMPLATE_REPOSITORY: + process.env.TEMPLATE_REPOSITORY || DEFAULTS.TEMPLATE_REPOSITORY, DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC, SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD, // minor @@ -50,14 +66,20 @@ const environment = { LOGGER: process.env.LOGGER, ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL, AUTOMATION_MAX_ITERATIONS: - parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200, + parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || + DEFAULTS.AUTOMATION_MAX_ITERATIONS, SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, - QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT), + QUERY_THREAD_TIMEOUT: QUERY_THREAD_TIMEOUT, + AUTOMATION_THREAD_TIMEOUT: + parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) || + DEFAULTS.AUTOMATION_THREAD_TIMEOUT > QUERY_THREAD_TIMEOUT + ? DEFAULTS.AUTOMATION_THREAD_TIMEOUT + : QUERY_THREAD_TIMEOUT, SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, - PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins", + PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR, OPENAI_API_KEY: process.env.OPENAI_API_KEY, MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB, SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS, @@ -70,12 +92,21 @@ const environment = { ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT, - FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main", + FORKED_PROCESS_NAME: + process.env.FORKED_PROCESS_NAME || DEFAULTS.FORKED_PROCESS_NAME, JS_PER_INVOCATION_TIMEOUT_MS: - parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000, + parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || + DEFAULTS.JS_PER_EXECUTION_TIME_LIMIT_MS, JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe( process.env.JS_PER_REQUEST_TIME_LIMIT_MS ), + TOP_LEVEL_PATH: + process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH, + APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT), + JS_RUNNER_MEMORY_LIMIT: + parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || + DEFAULTS.JS_RUNNER_MEMORY_LIMIT, + LOG_JS_ERRORS: process.env.LOG_JS_ERRORS, // old CLIENT_ID: process.env.CLIENT_ID, _set(key: string, value: any) { @@ -92,12 +123,9 @@ const environment = { isInThread: () => { return process.env.FORKED_PROCESS }, - TOP_LEVEL_PATH: - process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH, - APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT), - JS_RUNNER_MEMORY_LIMIT: - parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64, - LOG_JS_ERRORS: process.env.LOG_JS_ERRORS, + getDefaults: () => { + return DEFAULTS + }, } // clean up any environment variable edge cases diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index a828af5d19..a4938bb138 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -303,7 +303,7 @@ class Orchestrator { if (timeout) { setTimeout(() => { timeoutFlag = true - }, timeout || 12000) + }, timeout || env.AUTOMATION_THREAD_TIMEOUT) } stepCount++ @@ -621,7 +621,7 @@ export async function executeInThread(job: Job) { const timeoutPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error("Timeout exceeded")) - }, job.data.event.timeout || 12000) + }, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT) }) return await context.doInAppContext(appId, async () => {