Merge remote-tracking branch 'origin/master' into feature/form-screen-template

.eslintignore

@@ -10,4 +10,5 @@ packages/builder/.routify
 packages/sdk/sdk
 packages/account-portal/packages/server/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/ui/build
+packages/account-portal/packages/ui/build
+**/*.ivm.bundle.js

.eslintrc.json

@@ -43,7 +43,8 @@
         "no-useless-escape": "off",
         "no-undef": "off",
         "no-prototype-builtins": "off",
-        "local-rules/no-budibase-imports": "error"
+        "local-rules/no-budibase-imports": "error",
+        "local-rules/no-test-com": "error"
       }
     },
     {
@@ -53,7 +54,7 @@
         "packages/frontend-core/**/*"
       ],
       "rules": {
-        "no-console": ["error", { "allow": ["warn", "error", "debug"] } ]
+        "no-console": ["error", { "allow": ["warn", "error", "debug"] }]
       }
     }
   ],

.prettierignore

@@ -11,4 +11,5 @@ packages/sdk/sdk
 packages/pro/coverage
 packages/account-portal/packages/ui/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/server/build
+packages/account-portal/packages/server/build
+**/*.ivm.bundle.js

eslint-local-rules/index.js

@@ -18,4 +18,37 @@ module.exports = {
       }
     },
   },
+  "no-test-com": {
+    meta: {
+      type: "problem",
+      docs: {
+        description:
+          "disallow the use of 'test.com' in strings and replace it with 'example.com'",
+        category: "Possible Errors",
+        recommended: false,
+      },
+      schema: [], // no options
+      fixable: "code", // Indicates that this rule supports automatic fixing
+    },
+    create: function (context) {
+      return {
+        Literal(node) {
+          if (
+            typeof node.value === "string" &&
+            node.value.includes("test.com")
+          ) {
+            context.report({
+              node,
+              message:
+                "test.com is a privately owned domain and could point anywhere, use example.com instead.",
+              fix: function (fixer) {
+                const newText = node.raw.replace(/test\.com/g, "example.com")
+                return fixer.replaceText(node, newText)
+              },
+            })
+          }
+        },
+      }
+    },
+  },
 }

hosting/single/Dockerfile

@@ -124,6 +124,8 @@ HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh
 
 # must set this just before running
 ENV NODE_ENV=production
+# this is required for isolated-vm to work on Node 20+
+ENV NODE_OPTIONS="--no-node-snapshot"
 WORKDIR /
 
 CMD ["./runner.sh"]

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.17.8",
+  "version": "2.19.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -97,7 +97,17 @@
     "@budibase/backend-core": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
-    "@budibase/types": "0.0.0"
+    "@budibase/types": "0.0.0",
+    "tough-cookie": "4.1.3",
+    "node-fetch": "2.6.7",
+    "semver": "7.5.3",
+    "http-cache-semantics": "4.1.1",
+    "msgpackr": "1.10.1",
+    "axios": "1.6.3",
+    "xml2js": "0.6.2",
+    "unset-value": "2.0.1",
+    "got": "13.0.0",
+    "passport": "0.6.0"
   },
   "engines": {
     "node": ">=20.0.0 <21.0.0"

packages/backend-core/package.json

@@ -25,19 +25,19 @@
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@techpass/passport-openidconnect": "0.3.2",
+    "@govtechsg/passport-openidconnect": "^1.0.2",
     "aws-cloudfront-sign": "3.0.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
-    "dd-trace": "5.0.0",
+    "dd-trace": "5.2.0",
     "dotenv": "16.0.1",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.2",
-    "koa-passport": "4.1.4",
+    "koa-passport": "^6.0.0",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
     "node-fetch": "2.6.7",
@@ -52,9 +52,9 @@
     "redlock": "4.2.0",
     "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
-    "semver": "7.3.7",
+    "semver": "^7.5.4",
     "tar-fs": "2.1.1",
-    "uuid": "8.3.2"
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
     "@shopify/jest-koa-mocks": "5.1.1",

packages/backend-core/src/context/types.ts

@@ -1,4 +1,4 @@
-import { IdentityContext } from "@budibase/types"
+import { IdentityContext, VM } from "@budibase/types"
 import { ExecutionTimeTracker } from "../timers"
 
 // keep this out of Budibase types, don't want to expose context info
@@ -11,4 +11,5 @@ export type ContextMap = {
   automationId?: string
   isMigrating?: boolean
   jsExecutionTracker?: ExecutionTimeTracker
+  vm?: VM
 }

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -44,11 +44,11 @@ describe("utils", () => {
 
     it("gets appId from url", async () => {
       await config.doInTenant(async () => {
-        const url = "http://test.com"
+        const url = "http://example.com"
         env._set("PLATFORM_URL", url)
 
         const ctx = structures.koa.newContext()
-        ctx.host = `${config.tenantId}.test.com`
+        ctx.host = `${config.tenantId}.example.com`
 
         const expected = db.generateAppID(config.tenantId)
         const app = structures.apps.app(expected)
@@ -89,7 +89,7 @@ describe("utils", () => {
       const ctx = structures.koa.newContext()
       const expected = db.generateAppID()
       ctx.request.headers = {
-        referer: `http://test.com/builder/app/${expected}/design/screen_123/screens`,
+        referer: `http://example.com/builder/app/${expected}/design/screen_123/screens`,
       }
 
       const actual = await utils.getAppIdFromCtx(ctx)
@@ -100,7 +100,7 @@ describe("utils", () => {
       const ctx = structures.koa.newContext()
       const appId = db.generateAppID()
       ctx.request.headers = {
-        referer: `http://test.com/foo/app/${appId}/bar`,
+        referer: `http://example.com/foo/app/${appId}/bar`,
       }
 
       const actual = await utils.getAppIdFromCtx(ctx)

packages/backend-core/tests/core/utilities/structures/common.ts

@@ -3,5 +3,5 @@ import { v4 as uuid } from "uuid"
 export { v4 as uuid } from "uuid"
 
 export const email = () => {
-  return `${uuid()}@test.com`
+  return `${uuid()}@example.com`
 }

packages/backend-core/tests/core/utilities/structures/sso.ts

@@ -61,7 +61,7 @@ export function ssoProfile(user?: User): SSOProfile {
     },
     _json: {
       email: user.email,
-      picture: "http://test.com",
+      picture: "http://example.com",
     },
     provider: generator.string(),
   }

packages/backend-core/tests/core/utilities/structures/users.ts

@@ -25,7 +25,7 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
     roles: { app_test: "admin" },
     firstName: generator.first(),
     lastName: generator.last(),
-    pictureUrl: "http://test.com",
+    pictureUrl: "http://example.com",
     tenantId: tenant.id(),
     ...userProps,
   }

packages/bbui/src/Form/Core/EnvDropdown.svelte

@@ -184,7 +184,7 @@
       {#if environmentVariablesEnabled}
         <div on:click={() => showModal()} class="add-variable">
           <svg
-            class="spectrum-Icon spectrum-Icon--sizeS "
+            class="spectrum-Icon spectrum-Icon--sizeS"
             focusable="false"
             aria-hidden="true"
           >
@@ -195,7 +195,7 @@
       {:else}
         <div on:click={() => handleUpgradePanel()} class="add-variable">
           <svg
-            class="spectrum-Icon spectrum-Icon--sizeS "
+            class="spectrum-Icon spectrum-Icon--sizeS"
             focusable="false"
             aria-hidden="true"
           >

packages/builder/src/components/integration/QueryViewer.svelte

@@ -41,6 +41,7 @@
 
   let autoSchema = {}
   let rows = []
+  let keys = {}
 
   const parseQuery = query => {
     modified = false
@@ -143,8 +144,20 @@
   const handleScroll = e => {
     scrolling = e.target.scrollTop !== 0
   }
+
+  async function handleKeyDown(evt) {
+    keys[evt.key] = true
+    if ((keys["Meta"] || keys["Control"]) && keys["Enter"]) {
+      await runQuery({ suppressErrors: false })
+    }
+  }
+
+  function handleKeyUp(evt) {
+    delete keys[evt.key]
+  }
 </script>
 
+<svelte:window on:keydown={handleKeyDown} on:keyup={handleKeyUp} />
 <QueryViewerSavePromptModal
   checkIsModified={() => checkIsModified(newQuery)}
   attemptSave={() => runQuery({ suppressErrors: false }).then(saveQuery)}

packages/builder/src/components/integration/QueryViewerSidePanel/JSONPanel.svelte

@@ -1,7 +1,20 @@
 <script>
   export let data
+  export let maxRowsToDisplay = 5
 
-  $: string = JSON.stringify(data || {}, null, 2)
+  let string
+  $: {
+    string = JSON.stringify(data || {}, null, 2)
+    if (Array.isArray(data) && data.length > maxRowsToDisplay) {
+      string = JSON.stringify(data.slice(0, maxRowsToDisplay) || {}, null, 2)
+
+      // Display '...' at the end of the array
+      string = string.replace(
+        /(}\n])/,
+        `},\n  ...${data.length - maxRowsToDisplay} further items\n]`
+      )
+    }
+  }
 </script>
 
 <textarea class="json" disabled value={string} />

packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte

@@ -4,13 +4,17 @@
 
   export let schema = {}
   export let rows = []
+  export let maxRowsToDisplay = 5
 
-  $: rowsCopy = cloneDeep(rows)
+  let rowsToDisplay
+  $: rowsToDisplay = [...cloneDeep(rows).slice(0, maxRowsToDisplay)]
+
+  $: additionalRows = rows.length - maxRowsToDisplay
 
   // Cast field in query preview response to number if specified by schema
   $: {
-    for (let i = 0; i < rowsCopy.length; i++) {
+    for (let i = 0; i < rowsToDisplay.length; i++) {
-      let row = rowsCopy[i]
+      let row = rowsToDisplay[i]
       for (let fieldName of Object.keys(schema)) {
         if (schema[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
           row[fieldName] = Number(row[fieldName])
@@ -23,11 +27,27 @@
 </script>
 
 <div class="table">
-  <Table {schema} data={rowsCopy} allowEditing={false} />
+  <Table {schema} data={rowsToDisplay} allowEditing={false} />
+  {#if additionalRows > 0}
+    <div class="show-more">
+      ...{additionalRows} further items
+    </div>
+  {/if}
 </div>
 
 <style>
-  .table :global(.spectrum-Table-cell) {
+  .table :global(.spectrum-Table-cell),
+  .show-more {
     min-width: 100px;
   }
+
+  .show-more {
+    display: flex;
+    padding: 16px;
+    justify-content: center;
+
+    background-color: var(--spectrum-global-color-gray-50);
+    border: 1px solid var(--spectrum-alias-border-color-mid);
+    border-top: 0;
+  }
 </style>

packages/builder/src/components/integration/QueryViewerSidePanel/index.svelte

@@ -41,7 +41,7 @@
   </div>
   <div class="content">
     {#if activeTab === "JSON"}
-      <JSONPanel data={rows[0] || {}} />
+      <JSONPanel data={rows?.length === 1 ? rows[0] : rows || {}} />
     {:else if activeTab === "Schema"}
       <SchemaPanel {onSchemaChange} {schema} />
     {:else}

packages/pro

@@ -1 +1 @@
-Subproject commit 9b14e5d5182bf5e5ee98f717997e7352e5904799
+Subproject commit 336bf2184cf632fdc2bffbad5628e8b15dd381bd

packages/server/Dockerfile

@@ -82,6 +82,8 @@ EXPOSE 4001
 # due to this causing yarn to stop installing dev dependencies
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
+# this is required for isolated-vm to work on Node 20+
+ENV NODE_OPTIONS="--no-node-snapshot"
 ENV CLUSTER_MODE=${CLUSTER_MODE}
 ENV TOP_LEVEL_PATH=/app
 

packages/server/__mocks__/@sendgrid/mail.ts

@@ -11,7 +11,7 @@ module SendgridMock {
     }
 
     async send(msg: any) {
-      if (msg.to === "invalid@test.com") {
+      if (msg.to === "invalid@example.com") {
         throw "Invalid"
       }
       return msg

packages/server/__mocks__/aws-sdk.ts

@@ -60,7 +60,7 @@ module AwsMock {
 
     // @ts-ignore
     this.getSignedUrl = (operation, params) => {
-      return `http://test.com/${params.Bucket}/${params.Key}`
+      return `http://example.com/${params.Bucket}/${params.Key}`
     }
 
     // @ts-ignore

packages/server/__mocks__/node-fetch.ts

@@ -36,8 +36,8 @@ module FetchMock {
 
     if (url.includes("/api/global")) {
       const user = {
-        email: "test@test.com",
+        email: "test@example.com",
-        _id: "us_test@test.com",
+        _id: "us_test@example.com",
         status: "active",
         roles: {},
         builder: {
@@ -58,7 +58,7 @@ module FetchMock {
           url: "/app1",
         },
       })
-    } else if (url.includes("test.com")) {
+    } else if (url.includes("example.com")) {
       return json({
         body: opts.body,
         url,

packages/server/nodemon.json

@@ -8,6 +8,6 @@
     "../string-templates"
   ],
   "ext": "js,ts,json,svelte",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
+  "ignore": ["**/*.spec.ts", "**/*.spec.js", "../*/dist/**/*"],
-  "exec": "yarn build && node ./dist/index.js"
+  "exec": "yarn build && node --no-node-snapshot ./dist/index.js"
 }

packages/server/package.json

@@ -13,8 +13,12 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
+    "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=esm --external:handlebars",
+    "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=esm",
+    "build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates && yarn build:isolated-vm-lib:bson",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
+    "jest": "NODE_OPTIONS=\"--no-node-snapshot $NODE_OPTIONS\" jest",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
@@ -49,8 +53,8 @@
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@bull-board/api": "3.7.0",
+    "@bull-board/api": "5.10.2",
-    "@bull-board/koa": "3.9.4",
+    "@bull-board/koa": "5.10.2",
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "6.8.0",
     "@koa/router": "8.0.8",
@@ -65,14 +69,15 @@
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
-    "dd-trace": "5.0.0",
+    "dd-trace": "5.2.0",
     "dotenv": "8.2.0",
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
     "ioredis": "5.3.2",
-    "jimp": "0.16.1",
+    "isolated-vm": "^4.7.2",
+    "jimp": "0.22.10",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
@@ -85,7 +90,7 @@
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
-    "mongodb": "5.7",
+    "mongodb": "^6.3.0",
     "mssql": "10.0.1",
     "mysql2": "3.5.2",
     "node-fetch": "2.6.7",
@@ -104,7 +109,9 @@
     "svelte": "^3.49.0",
     "tar": "6.1.15",
     "to-json-schema": "0.2.5",
-    "uuid": "3.3.2",
+    "undici": "^6.0.1",
+    "undici-types": "^6.0.1",
+    "uuid": "^8.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.19",
     "worker-farm": "1.7.0",
@@ -129,6 +136,7 @@
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/tar": "6.1.5",
+    "@types/uuid": "8.3.4",
     "apidoc": "0.50.4",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",

packages/server/scripts/integrations/mongo/docker-compose.yaml

@@ -0,0 +1,24 @@
+# Use root/example as user/password credentials
+version: "3.1"
+
+services:
+  mongo:
+    image: mongo
+    restart: always
+    ports:
+      - 27017:27017
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: example
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: root
+      ME_CONFIG_MONGODB_ADMINPASSWORD: example
+      ME_CONFIG_MONGODB_AUTH_USERNAME: admin
+      ME_CONFIG_MONGODB_AUTH_PASSWORD: pass
+      ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/

packages/server/scripts/test.sh

@@ -3,12 +3,12 @@ set -e
 
 if [[ -n $CI ]]
 then
-  # Running in ci, where resources are limited
+  export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot"
-  export NODE_OPTIONS="--max-old-space-size=4096"
   echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
   jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
 else
   # --maxWorkers performs better in development
+  export NODE_OPTIONS="--no-node-snapshot"
   echo "jest --coverage --maxWorkers=2 --forceExit $@"
   jest --coverage --maxWorkers=2 --forceExit $@
 fi

packages/server/specs/openapi.json

@@ -367,7 +367,7 @@
         "value": {
           "data": {
             "_id": "us_693a73206518477283a8d5ae31103252",
-            "email": "test@test.com",
+            "email": "test@example.com",
             "roles": {
               "app_957b12f943d348faa61db7e18e088d0f": "BASIC"
             },
@@ -397,7 +397,7 @@
           "data": [
             {
               "_id": "us_693a73206518477283a8d5ae31103252",
-              "email": "test@test.com",
+              "email": "test@example.com",
               "roles": {
                 "app_957b12f943d348faa61db7e18e088d0f": "BASIC"
               },

packages/server/specs/openapi.yaml

@@ -256,7 +256,7 @@ components:
       value:
         data:
           _id: us_693a73206518477283a8d5ae31103252
-          email: test@test.com
+          email: test@example.com
           roles:
             app_957b12f943d348faa61db7e18e088d0f: BASIC
           builder:
@@ -278,7 +278,7 @@ components:
       value:
         data:
           - _id: us_693a73206518477283a8d5ae31103252
-            email: test@test.com
+            email: test@example.com
             roles:
               app_957b12f943d348faa61db7e18e088d0f: BASIC
             builder:

packages/server/specs/resources/user.ts

@@ -3,7 +3,7 @@ import Resource from "./utils/Resource"
 
 const user = {
   _id: "us_693a73206518477283a8d5ae31103252",
-  email: "test@test.com",
+  email: "test@example.com",
   roles: {
     app_957b12f943d348faa61db7e18e088d0f: "BASIC",
   },

packages/server/src/api/controllers/script.ts

@@ -1,12 +1,12 @@
 import ScriptRunner from "../../utilities/scriptRunner"
-import { BBContext } from "@budibase/types"
+import { Ctx } from "@budibase/types"
 
-export async function execute(ctx: BBContext) {
+export async function execute(ctx: Ctx) {
   const { script, context } = ctx.request.body
   const runner = new ScriptRunner(script, context)
   ctx.body = runner.execute()
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: Ctx) {
   ctx.throw(501, "Not currently implemented")
 }

packages/server/src/api/controllers/static/index.ts

@@ -3,7 +3,7 @@ import { InvalidFileExtensions } from "@budibase/shared-core"
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
-import uuid from "uuid"
+import * as uuid from "uuid"
 import { ObjectStoreBuckets } from "../../../constants"
 import { processString } from "@budibase/string-templates"
 import {

packages/server/src/api/routes/tests/appSync.spec.ts

@@ -12,7 +12,7 @@ describe("/api/applications/:appId/sync", () => {
     app = await config.init()
     // create some users which we will use throughout the tests
     await config.createUser({
-      email: "sync1@test.com",
+      email: "sync1@example.com",
       roles: {
         [app._id!]: roles.BUILTIN_ROLE_IDS.BASIC,
       },

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -77,7 +77,7 @@ describe("/datasources", () => {
         const { datasource, query } = await config.dynamicVariableDatasource()
         // preview once to cache variables
         await preview(datasource, {
-          path: "www.test.com",
+          path: "www.example.com",
           queryString: "test={{ variable3 }}",
         })
         // check variables in cache

packages/server/src/api/routes/tests/row.spec.ts

@@ -2031,7 +2031,7 @@ describe.each([
 
   describe("Formula JS protection", () => {
     it("should time out JS execution if a single cell takes too long", async () => {
-      await config.withEnv({ JS_PER_EXECUTION_TIME_LIMIT_MS: 20 }, async () => {
+      await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => {
         const js = Buffer.from(
           `
               let i = 0;
@@ -2071,8 +2071,8 @@ describe.each([
     it("should time out JS execution if a multiple cells take too long", async () => {
       await config.withEnv(
         {
-          JS_PER_EXECUTION_TIME_LIMIT_MS: 20,
+          JS_PER_INVOCATION_TIMEOUT_MS: 20,
-          JS_PER_REQUEST_TIME_LIMIT_MS: 40,
+          JS_PER_REQUEST_TIMEOUT_MS: 40,
         },
         async () => {
           const js = Buffer.from(

packages/server/src/api/routes/tests/static.spec.js

@@ -80,7 +80,7 @@ describe("/static", () => {
           .set(config.defaultHeaders())
           .expect("Content-Type", /json/)
           .expect(200)
-        expect(res.body.signedUrl).toEqual("http://test.com/foo/bar")
+        expect(res.body.signedUrl).toEqual("http://example.com/foo/bar")
         expect(res.body.publicUrl).toEqual(
           `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
         )

packages/server/src/api/routes/tests/table.spec.ts

@@ -16,7 +16,7 @@ import {
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import sdk from "../../../sdk"
-import uuid from "uuid"
+import * as uuid from "uuid"
 
 const { basicTable } = setup.structures
 

packages/server/src/api/routes/tests/utilities/index.ts

@@ -9,7 +9,7 @@ function user() {
     _id: "user",
     _rev: "rev",
     createdAt: Date.now(),
-    email: "test@test.com",
+    email: "test@example.com",
     roles: {},
     tenantId: "default",
     status: "active",

packages/server/src/automations/tests/discord.spec.js

@@ -11,7 +11,7 @@ describe("test the outgoing webhook action", () => {
     await config.init()
     inputs = {
       username: "joe_bloggs",
-      url: "http://www.test.com",
+      url: "http://www.example.com",
     }
   })
 
@@ -19,7 +19,7 @@ describe("test the outgoing webhook action", () => {
 
   it("should be able to run the action", async () => {
     const res = await setup.runStep(setup.actions.discord.stepId, inputs)
-    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.url).toEqual("http://www.example.com")
     expect(res.response.method).toEqual("post")
     expect(res.success).toEqual(true)
   })

packages/server/src/automations/tests/executeScript.spec.js

@@ -9,34 +9,40 @@ describe("test the execute script action", () => {
   afterAll(setup.afterAll)
 
   it("should be able to execute a script", async () => {
-    let res = await setup.runStep(
+    const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
-      setup.actions.EXECUTE_SCRIPT.stepId,
+      code: "return 1 + 1",
-      (inputs = {
+    })
-        code: "return 1 + 1",
-      })
-    )
     expect(res.value).toEqual(2)
     expect(res.success).toEqual(true)
   })
 
   it("should handle a null value", async () => {
-    let res = await setup.runStep(
+    const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
-      setup.actions.EXECUTE_SCRIPT.stepId,
+      code: null,
-      (inputs = {
+    })
-        code: null,
-      })
-    )
     expect(res.response.message).toEqual("Invalid inputs")
     expect(res.success).toEqual(false)
   })
 
-  it("should be able to handle an error gracefully", async () => {
+  it("should be able to get a value from context", async () => {
-    let res = await setup.runStep(
+    const res = await setup.runStep(
       setup.actions.EXECUTE_SCRIPT.stepId,
-      (inputs = {
+      {
-        code: "return something.map(x => x.name)",
+        code: "return steps.map(d => d.value)",
-      })
+      },
+      {
+        steps: [{ value: 0 }, { value: 1 }],
+      }
     )
+    expect(res.value).toEqual([0, 1])
+    expect(res.response).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+
+  it("should be able to handle an error gracefully", async () => {
+    const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
+      code: "return something.map(x => x.name)",
+    })
     expect(res.response).toEqual("ReferenceError: something is not defined")
     expect(res.success).toEqual(false)
   })

packages/server/src/automations/tests/make.spec.ts

@@ -12,9 +12,9 @@ describe("test the outgoing webhook action", () => {
   it("should be able to run the action", async () => {
     const res = await runStep(actions.integromat.stepId, {
       value1: "test",
-      url: "http://www.test.com",
+      url: "http://www.example.com",
     })
-    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.url).toEqual("http://www.example.com")
     expect(res.response.method).toEqual("post")
     expect(res.success).toEqual(true)
   })
@@ -30,9 +30,9 @@ describe("test the outgoing webhook action", () => {
       body: {
         value: payload,
       },
-      url: "http://www.test.com",
+      url: "http://www.example.com",
     })
-    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.url).toEqual("http://www.example.com")
     expect(res.response.method).toEqual("post")
     expect(res.response.body).toEqual(payload)
     expect(res.success).toEqual(true)
@@ -45,7 +45,7 @@ describe("test the outgoing webhook action", () => {
       body: {
         value: payload,
       },
-      url: "http://www.test.com",
+      url: "http://www.example.com",
     })
     expect(res.httpStatus).toEqual(400)
     expect(res.response).toEqual("Invalid payload JSON")

packages/server/src/automations/tests/outgoingWebhook.spec.js

@@ -11,7 +11,7 @@ describe("test the outgoing webhook action", () => {
     await config.init()
     inputs = {
       requestMethod: "POST",
-      url: "www.test.com",
+      url: "www.example.com",
       requestBody: JSON.stringify({
         a: 1,
       }),
@@ -26,7 +26,7 @@ describe("test the outgoing webhook action", () => {
       inputs
     )
     expect(res.success).toEqual(true)
-    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.url).toEqual("http://www.example.com")
     expect(res.response.method).toEqual("POST")
     expect(JSON.parse(res.response.body).a).toEqual(1)
   })

packages/server/src/automations/tests/sendSmtpEmail.spec.ts

@@ -33,7 +33,7 @@ describe("test the outgoing webhook action", () => {
     jest
       .spyOn(workerRequests, "sendSmtpEmail")
       .mockImplementationOnce(async () =>
-        generateResponse("user1@test.com", "admin@test.com")
+        generateResponse("user1@example.com", "admin@example.com")
       )
     const invite = {
       startTime: new Date(),
@@ -43,8 +43,8 @@ describe("test the outgoing webhook action", () => {
       url: "url",
     }
     inputs = {
-      to: "user1@test.com",
+      to: "user1@example.com",
-      from: "admin@test.com",
+      from: "admin@example.com",
       subject: "hello",
       contents: "testing",
       cc: "cc",
@@ -61,8 +61,8 @@ describe("test the outgoing webhook action", () => {
     expect(res.success).toEqual(true)
     expect(workerRequests.sendSmtpEmail).toHaveBeenCalledTimes(1)
     expect(workerRequests.sendSmtpEmail).toHaveBeenCalledWith({
-      to: "user1@test.com",
+      to: "user1@example.com",
-      from: "admin@test.com",
+      from: "admin@example.com",
       subject: "hello",
       contents: "testing",
       cc: "cc",

packages/server/src/automations/tests/zapier.spec.ts

@@ -12,9 +12,9 @@ describe("test the outgoing webhook action", () => {
   it("should be able to run the action", async () => {
     const res = await runStep(actions.zapier.stepId, {
       value1: "test",
-      url: "http://www.test.com",
+      url: "http://www.example.com",
     })
-    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.url).toEqual("http://www.example.com")
     expect(res.response.method).toEqual("post")
     expect(res.success).toEqual(true)
   })
@@ -30,9 +30,9 @@ describe("test the outgoing webhook action", () => {
       body: {
         value: payload,
       },
-      url: "http://www.test.com",
+      url: "http://www.example.com",
     })
-    expect(res.response.url).toEqual("http://www.test.com")
+    expect(res.response.url).toEqual("http://www.example.com")
     expect(res.response.method).toEqual("post")
     expect(res.response.body).toEqual(
       `{"platform":"budibase","value1":1,"value2":2,"value3":3,"value4":4,"value5":5,"name":"Adam","age":9}`
@@ -47,7 +47,7 @@ describe("test the outgoing webhook action", () => {
       body: {
         value: payload,
       },
-      url: "http://www.test.com",
+      url: "http://www.example.com",
     })
     expect(res.httpStatus).toEqual(400)
     expect(res.response).toEqual("Invalid payload JSON")

packages/server/src/db/newid.ts

@@ -1,4 +1,4 @@
-const { v4 } = require("uuid")
+import { v4 } from "uuid"
 
 export default function (): string {
   return v4().replace(/-/g, "")

packages/server/src/environment.ts

@@ -71,9 +71,9 @@ const environment = {
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
   FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
-  JS_PER_EXECUTION_TIME_LIMIT_MS:
+  JS_PER_INVOCATION_TIMEOUT_MS:
     parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
-  JS_PER_REQUEST_TIME_LIMIT_MS: parseIntSafe(
+  JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe(
     process.env.JS_PER_REQUEST_TIME_LIMIT_MS
   ),
   // old
@@ -95,6 +95,8 @@ const environment = {
   TOP_LEVEL_PATH:
     process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
   APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
+  JS_RUNNER_MEMORY_LIMIT:
+    parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64,
 }
 
 // threading can cause memory issues with node-ts in development

packages/server/src/integrations/mongodb.ts

@@ -21,7 +21,6 @@ import environment from "../environment"
 interface MongoDBConfig {
   connectionString: string
   db: string
-  tlsCertificateFile: string
   tlsCertificateKeyFile: string
   tlsCAFile: string
 }
@@ -320,16 +319,11 @@ const getSchema = () => {
   if (environment.SELF_HOSTED) {
     schema.datasource = {
       ...schema.datasource,
-      //@ts-ignore
+      // @ts-ignore
       tls: {
         type: DatasourceFieldType.FIELD_GROUP,
         display: "Configure SSL",
         fields: {
-          tlsCertificateFile: {
-            type: DatasourceFieldType.STRING,
-            required: false,
-            display: "Certificate file path",
-          },
           tlsCertificateKeyFile: {
             type: DatasourceFieldType.STRING,
             required: false,
@@ -356,7 +350,6 @@ class MongoIntegration implements IntegrationBase {
   constructor(config: MongoDBConfig) {
     this.config = config
     const options: MongoClientOptions = {
-      tlsCertificateFile: config.tlsCertificateFile || undefined,
       tlsCertificateKeyFile: config.tlsCertificateKeyFile || undefined,
       tlsCAFile: config.tlsCAFile || undefined,
     }
@@ -525,7 +518,10 @@ class MongoIntegration implements IntegrationBase {
           return await collection.findOneAndUpdate(
             findAndUpdateJson.filter,
             findAndUpdateJson.update,
-            findAndUpdateJson.options
+            {
+              ...findAndUpdateJson.options,
+              includeResultMetadata: true,
+            }
           )
         }
         case "count": {

packages/server/src/integrations/tests/mongo.spec.ts

@@ -221,6 +221,7 @@ describe("MongoDB Integration", () => {
     })
     expect(args[2]).toEqual({
       upsert: false,
+      includeResultMetadata: true,
     })
   })
 

packages/server/src/jsRunner/bundles/README.md

@@ -0,0 +1,15 @@
+# Bundles for isolated-vm
+
+[Isolated-vm](https://github.com/laverdet/isolated-vm) requires for us to have some libraries, such as string-templates helpers, built in a single file without external dependencies. These libraries are pretty much static. To avoid building this in every dev command, in every test command and in every pipeline, these libraries are already compiled and commited into the repo.
+
+## How are they consumed?
+
+These libaries are compiled with a special extension: .ivm.bundle.js. This extension is configured in [esbuild](/scripts/build.js) in order to not be bundled as javascript, and to be treated as a `string` instead. This will allow us to read it's context on runtime and inject it to `isolated-vm`.
+
+## How to update it?
+
+These libraries are pretty much static, but they might require some updates from time to time when something changes on the source code. In order to do this, we just need to run the following command and commit the updated bundles:
+
+```
+yarn build:isolated-vm-libs
+```

packages/server/src/jsRunner/bundles/bsonPackage.ts

@@ -0,0 +1,4 @@
+import { EJSON } from "bson"
+
+export { deserialize } from "bson"
+export const toJson = EJSON.deserialize

packages/server/src/jsRunner/bundles/index-helpers.ts

@@ -0,0 +1,12 @@
+const {
+  getJsHelperList,
+} = require("../../../../string-templates/src/helpers/list.js")
+
+const helpers = getJsHelperList()
+export default {
+  ...helpers,
+  // pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
+  // @ts-ignore
+  // eslint-disable-next-line no-undef
+  stripProtocol: helpersStripProtocol,
+}

packages/server/src/jsRunner/bundles/index.ts

@@ -0,0 +1,28 @@
+import { utils } from "@budibase/shared-core"
+import environment from "../../environment"
+import fs from "fs"
+
+export const enum BundleType {
+  HELPERS = "helpers",
+  BSON = "bson",
+}
+
+const bundleSourceCode = {
+  [BundleType.HELPERS]: "../bundles/index-helpers.ivm.bundle.js",
+  [BundleType.BSON]: "../bundles/bson.ivm.bundle.js",
+}
+
+export function loadBundle(type: BundleType) {
+  if (environment.isJest()) {
+    return fs.readFileSync(require.resolve(bundleSourceCode[type]), "utf-8")
+  }
+
+  switch (type) {
+    case BundleType.HELPERS:
+      return require("../bundles/index-helpers.ivm.bundle.js")
+    case BundleType.BSON:
+      return require("../bundles/bson.ivm.bundle.js")
+    default:
+      utils.unreachable(type)
+  }
+}

packages/server/src/jsRunner/index.ts

@@ -1,5 +1,5 @@
 import vm from "vm"
-import env from "./environment"
+import env from "../environment"
 import { setJSRunner } from "@budibase/string-templates"
 import { context, timers } from "@budibase/backend-core"
 import tracer from "dd-trace"
@@ -9,7 +9,7 @@ type TrackerFn = <T>(f: () => T) => T
 export function init() {
   setJSRunner((js: string, ctx: vm.Context) => {
     return tracer.trace("runJS", {}, span => {
-      const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
+      const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
       let track: TrackerFn = f => f()
       if (perRequestLimit) {
         const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
@@ -53,7 +53,7 @@ export function init() {
       vm.createContext(ctx)
       return track(() =>
         vm.runInNewContext(js, ctx, {
-          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+          timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
         })
       )
     })

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -0,0 +1,83 @@
+// import { validate as isValidUUID } from "uuid"
+
+jest.mock("@budibase/handlebars-helpers/lib/math", () => {
+  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
+
+  return {
+    ...actual,
+    random: () => 10,
+  }
+})
+jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
+  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/uuid")
+
+  return {
+    ...actual,
+    uuid: () => "f34ebc66-93bd-4f7c-b79b-92b5569138bc",
+  }
+})
+
+import { processStringSync, encodeJSBinding } from "@budibase/string-templates"
+
+const { runJsHelpersTests } = require("@budibase/string-templates/test/utils")
+
+import tk from "timekeeper"
+import { init } from ".."
+import TestConfiguration from "../../tests/utilities/TestConfiguration"
+
+tk.freeze("2021-01-21T12:00:00")
+
+describe("jsRunner", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    // Register js runner
+    init()
+    await config.init()
+  })
+
+  const processJS = (js: string, context?: object) => {
+    return config.doInContext(config.getAppId(), async () =>
+      processStringSync(encodeJSBinding(js), context || {})
+    )
+  }
+
+  it("it can run a basic javascript", async () => {
+    const output = await processJS(`return 1 + 2`)
+    expect(output).toBe(3)
+  })
+
+  // TODO This should be reenabled when running on isolated-vm
+  it.skip("should prevent sandbox escape", async () => {
+    const output = await processJS(
+      `return this.constructor.constructor("return process")()`
+    )
+    expect(output).toBe("Error while executing JS")
+  })
+
+  describe("helpers", () => {
+    runJsHelpersTests({
+      funcWrap: (func: any) => config.doInContext(config.getAppId(), func),
+      // testsToSkip: ["random", "uuid"],
+    })
+
+    // describe("uuid", () => {
+    //   it("uuid helper returns a valid uuid", async () => {
+    //     const result = await processJS("return helpers.uuid()")
+    //     expect(result).toBeDefined()
+    //     expect(isValidUUID(result)).toBe(true)
+    //   })
+    // })
+
+    // describe("random", () => {
+    //   it("random helper returns a valid number", async () => {
+    //     const min = 1
+    //     const max = 8
+    //     const result = await processJS(`return helpers.random(${min}, ${max})`)
+    //     expect(result).toBeDefined()
+    //     expect(result).toBeGreaterThanOrEqual(min)
+    //     expect(result).toBeLessThanOrEqual(max)
+    //   })
+    // })
+  })
+})

packages/server/src/jsRunner/vm/index.ts

@@ -0,0 +1,270 @@
+import ivm from "isolated-vm"
+import bson from "bson"
+
+import url from "url"
+import crypto from "crypto"
+import querystring from "querystring"
+
+import { BundleType, loadBundle } from "../bundles"
+import { VM } from "@budibase/types"
+
+class ExecutionTimeoutError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "ExecutionTimeoutError"
+  }
+}
+
+class ModuleHandler {
+  private modules: {
+    import: string
+    moduleKey: string
+    module: ivm.Module
+  }[] = []
+
+  private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}`
+
+  registerModule(module: ivm.Module, imports: string) {
+    this.modules.push({
+      moduleKey: this.generateRandomKey(),
+      import: imports,
+      module: module,
+    })
+  }
+
+  generateImports() {
+    return this.modules
+      .map(m => `import ${m.import} from "${m.moduleKey}"`)
+      .join(";")
+  }
+
+  getModule(key: string) {
+    const module = this.modules.find(m => m.moduleKey === key)
+    return module?.module
+  }
+}
+
+export class IsolatedVM implements VM {
+  private isolate: ivm.Isolate
+  private vm: ivm.Context
+  private jail: ivm.Reference
+  private invocationTimeout: number
+  private isolateAccumulatedTimeout?: number
+
+  // By default the wrapper returns itself
+  private codeWrapper: (code: string) => string = code => code
+
+  private moduleHandler = new ModuleHandler()
+
+  private readonly resultKey = "results"
+
+  constructor({
+    memoryLimit,
+    invocationTimeout,
+    isolateAccumulatedTimeout,
+  }: {
+    memoryLimit: number
+    invocationTimeout: number
+    isolateAccumulatedTimeout?: number
+  }) {
+    this.isolate = new ivm.Isolate({ memoryLimit })
+    this.vm = this.isolate.createContextSync()
+    this.jail = this.vm.global
+    this.jail.setSync("global", this.jail.derefInto())
+
+    this.addToContext({
+      [this.resultKey]: { out: "" },
+    })
+
+    this.invocationTimeout = invocationTimeout
+    this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
+  }
+
+  withHelpers() {
+    const urlModule = this.registerCallbacks({
+      resolve: url.resolve,
+      parse: url.parse,
+    })
+
+    const querystringModule = this.registerCallbacks({
+      escape: querystring.escape,
+    })
+
+    this.addToContext({
+      helpersStripProtocol: new ivm.Callback((str: string) => {
+        var parsed = url.parse(str) as any
+        parsed.protocol = ""
+        return parsed.format()
+      }),
+    })
+
+    const injectedRequire = `const require=function req(val) {
+        switch (val) {
+            case "url": return ${urlModule};
+            case "querystring": return ${querystringModule};
+        }
+      }`
+    const helpersSource = loadBundle(BundleType.HELPERS)
+    const helpersModule = this.isolate.compileModuleSync(
+      `${injectedRequire};${helpersSource}`
+    )
+
+    helpersModule.instantiateSync(this.vm, specifier => {
+      if (specifier === "crypto") {
+        const cryptoModule = this.registerCallbacks({
+          randomUUID: crypto.randomUUID,
+        })
+        const module = this.isolate.compileModuleSync(
+          `export default ${cryptoModule}`
+        )
+        module.instantiateSync(this.vm, specifier => {
+          throw new Error(`No imports allowed. Required: ${specifier}`)
+        })
+        return module
+      }
+      throw new Error(`No imports allowed. Required: ${specifier}`)
+    })
+
+    this.moduleHandler.registerModule(helpersModule, "helpers")
+    return this
+  }
+
+  withContext(context: Record<string, any>) {
+    this.addToContext(context)
+
+    return this
+  }
+
+  withParsingBson(data: any) {
+    this.addToContext({
+      bsonData: bson.BSON.serialize({ data }),
+    })
+
+    // If we need to parse bson, we follow the next steps:
+    // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
+    // 2. Deserialise the data within the isolate, to get the original data
+    // 3. Process script
+    // 4. Stringify the result in order to convert the result from BSON to json
+    this.codeWrapper = code =>
+      `(function(){
+            const data = deserialize(bsonData, { validation: { utf8: false } }).data;
+            const result = ${code}
+            return toJson(result);
+        })();`
+
+    const bsonSource = loadBundle(BundleType.BSON)
+
+    this.addToContext({
+      textDecoderCb: new ivm.Callback(
+        (args: {
+          constructorArgs: any
+          functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
+        }) => {
+          const result = new TextDecoder(...args.constructorArgs).decode(
+            ...args.functionArgs
+          )
+          return result
+        }
+      ),
+    })
+
+    // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
+    const textDecoderPolyfill = class TextDecoder {
+      constructorArgs
+
+      constructor(...constructorArgs: any) {
+        this.constructorArgs = constructorArgs
+      }
+
+      decode(...input: any) {
+        // @ts-ignore
+        return textDecoderCb({
+          constructorArgs: this.constructorArgs,
+          functionArgs: input,
+        })
+      }
+    }.toString()
+    const bsonModule = this.isolate.compileModuleSync(
+      `${textDecoderPolyfill};${bsonSource}`
+    )
+    bsonModule.instantiateSync(this.vm, specifier => {
+      throw new Error(`No imports allowed. Required: ${specifier}`)
+    })
+
+    this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}")
+
+    return this
+  }
+
+  execute(code: string): any {
+    if (this.isolateAccumulatedTimeout) {
+      const cpuMs = Number(this.isolate.cpuTime) / 1e6
+      if (cpuMs > this.isolateAccumulatedTimeout) {
+        throw new ExecutionTimeoutError(
+          `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
+        )
+      }
+    }
+
+    code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper(
+      code
+    )};`
+
+    const script = this.isolate.compileModuleSync(code)
+
+    script.instantiateSync(this.vm, specifier => {
+      const module = this.moduleHandler.getModule(specifier)
+      if (module) {
+        return module
+      }
+
+      throw new Error(`"${specifier}" import not allowed`)
+    })
+
+    script.evaluateSync({ timeout: this.invocationTimeout })
+
+    const result = this.getFromContext(this.resultKey)
+    return result.out
+  }
+
+  private registerCallbacks(functions: Record<string, any>) {
+    const libId = crypto.randomUUID().replace(/-/g, "")
+
+    const x: Record<string, string> = {}
+    for (const [funcName, func] of Object.entries(functions)) {
+      const key = `f${libId}${funcName}cb`
+      x[funcName] = key
+
+      this.addToContext({
+        [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
+      })
+    }
+
+    const mod =
+      `{` +
+      Object.entries(x)
+        .map(([key, func]) => `${key}: ${func}`)
+        .join() +
+      "}"
+    return mod
+  }
+
+  private addToContext(context: Record<string, any>) {
+    for (let key in context) {
+      const value = context[key]
+      this.jail.setSync(
+        key,
+        typeof value === "function"
+          ? value
+          : new ivm.ExternalCopy(value).copyInto({ release: true })
+      )
+    }
+  }
+
+  private getFromContext(key: string) {
+    const ref = this.vm.global.getSync(key, { reference: true })
+    const result = ref.copySync()
+    ref.release()
+    return result
+  }
+}

packages/server/src/migrations/tests/structures.ts

@@ -46,7 +46,7 @@ export const smtp = (conf?: SMTPConfig): SMTPConfig => {
     config: {
       port: 12345,
       host: "smtptesthost.com",
-      from: "testfrom@test.com",
+      from: "testfrom@example.com",
       subject: "Hello!",
       secure: false,
       ...conf,

packages/server/src/sdk/app/applications/tests/sync.spec.ts

@@ -94,8 +94,8 @@ function buildRoles() {
 }
 
 describe("app user/group sync", () => {
-  const groupEmail = "test2@test.com",
+  const groupEmail = "test2@example.com",
-    normalEmail = "test@test.com"
+    normalEmail = "test@example.com"
   async function checkEmail(
     email: string,
     opts?: { group?: boolean; notFound?: boolean }
@@ -131,7 +131,7 @@ describe("app user/group sync", () => {
   })
 
   it("should be able to handle builder users", async () => {
-    await createUser("test3@test.com", {}, true)
+    await createUser("test3@example.com", {}, true)
-    await checkEmail("test3@test.com")
+    await checkEmail("test3@example.com")
   })
 })

packages/server/src/sdk/app/backups/exports.ts

@@ -14,6 +14,7 @@ import {
   ATTACHMENT_DIRECTORY,
 } from "./constants"
 import fs from "fs"
+import fsp from "fs/promises"
 import { join } from "path"
 import env from "../../../environment"
 import { v4 as uuid } from "uuid"
@@ -117,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
           ObjectStoreBuckets.APPS,
           join(appPath, path)
         )
-        fs.writeFileSync(join(tmpPath, path), contents)
+        await fsp.writeFile(join(tmpPath, path), contents)
       }
     }
     // get all the files
@@ -131,14 +132,14 @@ export async function exportApp(appId: string, config?: ExportOpts) {
 
   const downloadedPath = join(tmpPath, appPath)
   if (fs.existsSync(downloadedPath)) {
-    const allFiles = fs.readdirSync(downloadedPath)
+    const allFiles = await fsp.readdir(downloadedPath)
     for (let file of allFiles) {
       const path = join(downloadedPath, file)
       // move out of app directory, simplify structure
-      fs.renameSync(path, join(downloadedPath, "..", file))
+      await fsp.rename(path, join(downloadedPath, "..", file))
     }
     // remove the old app directory created by object export
-    fs.rmdirSync(downloadedPath)
+    await fsp.rmdir(downloadedPath)
   }
   // enforce an export of app DB to the tmp path
   const dbPath = join(tmpPath, DB_EXPORT_FILE)
@@ -148,7 +149,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   })
 
   if (config?.encryptPassword) {
-    for (let file of fs.readdirSync(tmpPath)) {
+    for (let file of await fsp.readdir(tmpPath)) {
       const path = join(tmpPath, file)
 
       // skip the attachments - too big to encrypt
@@ -157,7 +158,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
           { dir: tmpPath, filename: file },
           config.encryptPassword
         )
-        fs.rmSync(path)
+        await fsp.rm(path)
       }
     }
   }
@@ -165,9 +166,9 @@ export async function exportApp(appId: string, config?: ExportOpts) {
   // if tar requested, return where the tarball is
   if (config?.tar) {
     // now the tmpPath contains both the DB export and attachments, tar this
-    const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
+    const tarPath = await tarFilesToTmp(tmpPath, await fsp.readdir(tmpPath))
     // cleanup the tmp export files as tarball returned
-    fs.rmSync(tmpPath, { recursive: true, force: true })
+    await fsp.rm(tmpPath, { recursive: true, force: true })
 
     return tarPath
   }

packages/server/src/sdk/app/backups/imports.ts

@@ -17,6 +17,7 @@ import { downloadTemplate } from "../../../utilities/fileSystem"
 import { ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
+import fsp from "fs/promises"
 import sdk from "../../"
 import { v4 as uuid } from "uuid"
 import tar from "tar"
@@ -119,7 +120,7 @@ async function getTemplateStream(template: TemplateType) {
 
 export async function untarFile(file: { path: string }) {
   const tmpPath = join(budibaseTempDir(), uuid())
-  fs.mkdirSync(tmpPath)
+  await fsp.mkdir(tmpPath)
   // extract the tarball
   await tar.extract({
     cwd: tmpPath,
@@ -130,12 +131,12 @@ export async function untarFile(file: { path: string }) {
 
 async function decryptFiles(path: string, password: string) {
   try {
-    for (let file of fs.readdirSync(path)) {
+    for (let file of await fsp.readdir(path)) {
       const inputPath = join(path, file)
       if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) {
         const outputPath = inputPath.replace(/\.enc$/, "")
         await encryption.decryptFile(inputPath, outputPath, password)
-        fs.rmSync(inputPath)
+        await fsp.rm(inputPath)
       }
     }
   } catch (err: any) {
@@ -164,14 +165,14 @@ export async function importApp(
   let dbStream: any
   const isTar = template.file && template?.file?.type?.endsWith("gzip")
   const isDirectory =
-    template.file && fs.lstatSync(template.file.path).isDirectory()
+    template.file && (await fsp.lstat(template.file.path)).isDirectory()
   let tmpPath: string | undefined = undefined
   if (template.file && (isTar || isDirectory)) {
     tmpPath = isTar ? await untarFile(template.file) : template.file.path
     if (isTar && template.file.password) {
       await decryptFiles(tmpPath, template.file.password)
     }
-    const contents = fs.readdirSync(tmpPath)
+    const contents = await fsp.readdir(tmpPath)
     // have to handle object import
     if (contents.length && opts.importObjStoreContents) {
       let promises = []
@@ -182,7 +183,7 @@ export async function importApp(
           continue
         }
         filename = join(prodAppId, filename)
-        if (fs.lstatSync(path).isDirectory()) {
+        if ((await fsp.lstat(path)).isDirectory()) {
           promises.push(
             objectStore.uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
           )
@@ -211,7 +212,7 @@ export async function importApp(
   await updateAutomations(prodAppId, db)
   // clear up afterward
   if (tmpPath) {
-    fs.rmSync(tmpPath, { recursive: true, force: true })
+    await fsp.rm(tmpPath, { recursive: true, force: true })
   }
   return ok
 }

packages/server/src/sdk/app/rows/search/tests/utils.spec.ts

@@ -76,7 +76,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
     })
 
     it("shouldn't change any other input", () => {
-      const email = "test@test.com"
+      const email = "test@example.com"
       const params: SearchParams = {
         tableId,
         query: {

packages/server/src/threads/definitions.ts

@@ -1,10 +1,10 @@
-import { QuerySchema, Row } from "@budibase/types"
+import { Datasource, QuerySchema, Row } from "@budibase/types"
 
 export type WorkerCallback = (error: any, response?: any) => void
 
 export interface QueryEvent {
   appId?: string
-  datasource: any
+  datasource: Datasource
   queryVerb: string
   fields: { [key: string]: any }
   parameters: { [key: string]: any }

packages/server/src/threads/query.ts

@@ -14,13 +14,13 @@ import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
-import { Query } from "@budibase/types"
+import { Datasource, Query, SourceName } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
 
 class QueryRunner {
-  datasource: any
+  datasource: Datasource
   queryVerb: string
   queryId: string
   fields: any
@@ -68,7 +68,7 @@ class QueryRunner {
       throw "Integration type does not exist."
     }
 
-    if (datasourceClone.config.authConfigs) {
+    if (datasourceClone.config?.authConfigs) {
       const updatedConfigs = []
       for (let config of datasourceClone.config.authConfigs) {
         updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
@@ -93,7 +93,7 @@ class QueryRunner {
     const enrichedContext = { ...enrichedParameters, ...this.ctx }
 
     // Parse global headers
-    if (datasourceClone.config.defaultHeaders) {
+    if (datasourceClone.config?.defaultHeaders) {
       datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
         datasourceClone.config.defaultHeaders,
         enrichedContext
@@ -152,11 +152,6 @@ class QueryRunner {
       return this.execute()
     }
 
-    // check for undefined response
-    if (!rows) {
-      rows = []
-    }
-
     // needs to an array for next step
     if (!Array.isArray(rows)) {
       rows = [rows]

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -4,7 +4,7 @@ import { resolve, join } from "path"
 import env from "../../environment"
 import tar from "tar"
 
-const uuid = require("uuid/v4")
+import { v4 as uuid } from "uuid"
 
 export const TOP_LEVEL_PATH =
   env.TOP_LEVEL_PATH || resolve(join(__dirname, "..", "..", ".."))

packages/string-templates/package.json

@@ -2,16 +2,17 @@
   "name": "@budibase/string-templates",
   "version": "0.0.0",
   "description": "Handlebars wrapper for Budibase templating.",
-  "main": "src/index.cjs",
+  "main": "src/index.js",
   "module": "dist/bundle.mjs",
   "license": "MPL-2.0",
   "types": "dist/index.d.ts",
   "exports": {
     ".": {
-      "require": "./src/index.cjs",
+      "require": "./src/index.js",
       "import": "./dist/bundle.mjs"
     },
-    "./package.json": "./package.json"
+    "./package.json": "./package.json",
+    "./test/utils": "./test/utils.js"
   },
   "files": [
     "dist",
@@ -20,7 +21,7 @@
   ],
   "scripts": {
     "build": "tsc && rollup -c",
-    "dev": "tsc && rollup -cw",
+    "dev": "concurrently \"tsc --watch\" \"rollup -cw\"",
     "test": "jest",
     "manifest": "node ./scripts/gen-collection-info.js"
   },
@@ -28,12 +29,12 @@
     "@budibase/handlebars-helpers": "^0.13.1",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
-    "lodash.clonedeep": "^4.5.0",
+    "lodash.clonedeep": "^4.5.0"
-    "vm2": "^3.9.19"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-json": "^4.1.0",
+    "concurrently": "^8.2.2",
     "doctrine": "^3.0.0",
     "jest": "29.7.0",
     "marked": "^4.0.10",

packages/string-templates/src/errors.js

@@ -0,0 +1,11 @@
+class JsErrorTimeout extends Error {
+  code = "ERR_SCRIPT_EXECUTION_TIMEOUT"
+
+  constructor() {
+    super()
+  }
+}
+
+module.exports = {
+  JsErrorTimeout,
+}

packages/string-templates/src/helpers/javascript.js

@@ -4,7 +4,7 @@ const { LITERAL_MARKER } = require("../helpers/constants")
 const { getJsHelperList } = require("./list")
 
 // The method of executing JS scripts depends on the bundle being built.
-// This setter is used in the entrypoint (either index.cjs or index.mjs).
+// This setter is used in the entrypoint (either index.js or index.mjs).
 let runJS
 module.exports.setJSRunner = runner => (runJS = runner)
 
@@ -42,7 +42,7 @@ module.exports.processJS = (handlebars, context) => {
   try {
     // Wrap JS in a function and immediately invoke it.
     // This is required to allow the final `return` statement to be valid.
-    const js = `function run(){${atob(handlebars)}};run();`
+    const js = `(function(){${atob(handlebars)}})();`
 
     // Our $ context function gets a value from context.
     // We clone the context to avoid mutation in the binding affecting real

packages/string-templates/src/helpers/list.js

@@ -1,29 +1,42 @@
-const externalHandlebars = require("./external")
+const { date, duration } = require("./date")
-const helperList = require("@budibase/handlebars-helpers")
 
-let helpers = undefined
+// https://github.com/evanw/esbuild/issues/56
+const externalCollections = {
+  math: require("@budibase/handlebars-helpers/lib/math"),
+  array: require("@budibase/handlebars-helpers/lib/array"),
+  number: require("@budibase/handlebars-helpers/lib/number"),
+  url: require("@budibase/handlebars-helpers/lib/url"),
+  string: require("@budibase/handlebars-helpers/lib/string"),
+  comparison: require("@budibase/handlebars-helpers/lib/comparison"),
+  object: require("@budibase/handlebars-helpers/lib/object"),
+  regex: require("@budibase/handlebars-helpers/lib/regex"),
+  uuid: require("@budibase/handlebars-helpers/lib/uuid"),
+}
 
 const helpersToRemoveForJs = ["sortBy"]
 module.exports.helpersToRemoveForJs = helpersToRemoveForJs
 
+const addedHelpers = {
+  date: date,
+  duration: duration,
+}
+
+let helpers = undefined
+
 module.exports.getJsHelperList = () => {
   if (helpers) {
     return helpers
   }
 
   helpers = {}
-  let constructed = []
+  for (let collection of Object.values(externalCollections)) {
-  for (let collection of externalHandlebars.externalCollections) {
-    constructed.push(helperList[collection]())
-  }
-  for (let collection of constructed) {
     for (let [key, func] of Object.entries(collection)) {
       // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
       helpers[key] = (...props) => func(...props, {})
     }
   }
-  for (let key of Object.keys(externalHandlebars.addedHelpers)) {
+  for (let key of Object.keys(addedHelpers)) {
-    helpers[key] = externalHandlebars.addedHelpers[key]
+    helpers[key] = addedHelpers[key]
   }
 
   for (const toRemove of helpersToRemoveForJs) {

packages/string-templates/src/index.cjs

@@ -1,38 +0,0 @@
-const templates = require("./index.js")
-
-/**
- * CJS entrypoint for rollup
- */
-module.exports.isValid = templates.isValid
-module.exports.makePropSafe = templates.makePropSafe
-module.exports.getManifest = templates.getManifest
-module.exports.isJSBinding = templates.isJSBinding
-module.exports.encodeJSBinding = templates.encodeJSBinding
-module.exports.decodeJSBinding = templates.decodeJSBinding
-module.exports.processStringSync = templates.processStringSync
-module.exports.processObjectSync = templates.processObjectSync
-module.exports.processString = templates.processString
-module.exports.processObject = templates.processObject
-module.exports.doesContainStrings = templates.doesContainStrings
-module.exports.doesContainString = templates.doesContainString
-module.exports.disableEscaping = templates.disableEscaping
-module.exports.findHBSBlocks = templates.findHBSBlocks
-module.exports.convertToJS = templates.convertToJS
-module.exports.setJSRunner = templates.setJSRunner
-module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
-module.exports.helpersToRemoveForJs = templates.helpersToRemoveForJs
-
-if (!process.env.NO_JS) {
-  const { VM } = require("vm2")
-  const { setJSRunner } = require("./helpers/javascript")
-  /**
-   * Use vm2 to run JS scripts in a node env
-   */
-  setJSRunner((js, context) => {
-    const vm = new VM({
-      sandbox: context,
-      timeout: 1000,
-    })
-    return vm.run(js)
-  })
-}

packages/string-templates/src/index.js

@@ -395,4 +395,9 @@ module.exports.convertToJS = hbs => {
 }
 
 module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX
+
+const errors = require("./errors")
+// We cannot use dynamic exports, otherwise the typescript file will not be generating it
+module.exports.JsErrorTimeout = errors.JsErrorTimeout
+
 module.exports.helpersToRemoveForJs = helpersToRemoveForJs

packages/string-templates/src/index.mjs

@@ -38,3 +38,5 @@ if (process && !process.env.NO_JS) {
     return vm.runInNewContext(js, context, { timeout: 1000 })
   })
 }
+
+export * from "./errors.js"

packages/string-templates/test/basic.spec.js

@@ -8,7 +8,7 @@ const {
   doesContainString,
   disableEscaping,
   findHBSBlocks,
-} = require("../src/index.cjs")
+} = require("../src/index.js")
 
 describe("Test that the string processing works correctly", () => {
   it("should process a basic template string", async () => {

packages/string-templates/test/escapes.spec.js

@@ -1,4 +1,4 @@
-const { processString } = require("../src/index.cjs")
+const { processString } = require("../src/index.js")
 
 describe("Handling context properties with spaces in their name", () => {
   it("should allow through literal specifiers", async () => {

packages/string-templates/test/examples/table.json

@@ -2,7 +2,7 @@
   "user":{
     "_id":"ro_ta_users_us_b0bc7ba0ce304294accc1ced8165dd23",
     "_rev":"1-e9199d92e7286005a9c11c614fdbcc51",
-    "email":"test2@test.com",
+    "email":"test2@example.com",
     "status":"active",
     "roleId":"PUBLIC",
     "test-Created By_text":"",

packages/string-templates/test/hbsToJs.spec.js

@@ -1,4 +1,4 @@
-const { convertToJS } = require("../src/index.cjs")
+const { convertToJS } = require("../src/index.js")
 
 function checkLines(response, lines) {
   const toCheck = response.split("\n")

packages/string-templates/test/helpers.spec.js

@@ -1,4 +1,4 @@
-const { processString, processObject, isValid } = require("../src/index.cjs")
+const { processString, processObject, isValid } = require("../src/index.js")
 const tableJson = require("./examples/table.json")
 const dayjs = require("dayjs")
 const { UUID_REGEX } = require("./constants")
@@ -448,7 +448,7 @@ describe("Cover a few complex use cases", () => {
   it("getting a nice date from the user", async () => {
     const input = { text: `{{ date user.subscriptionDue "DD-MM" }}` }
     const context = JSON.parse(
-      `{"user":{"email":"test@test.com","roleId":"ADMIN","type":"user","tableId":"ta_users","subscriptionDue":"2021-01-12T12:00:00.000Z","_id":"ro_ta_users_us_test@test.com","_rev":"2-24cc794985eb54183ecb93e148563f3d"}}`
+      `{"user":{"email":"test@example.com","roleId":"ADMIN","type":"user","tableId":"ta_users","subscriptionDue":"2021-01-12T12:00:00.000Z","_id":"ro_ta_users_us_test@example.com","_rev":"2-24cc794985eb54183ecb93e148563f3d"}}`
     )
     const output = await processObject(input, context)
     expect(output.text).toBe("12-01")

packages/string-templates/test/javascript.spec.js

@@ -1,149 +1,156 @@
-const { processStringSync, encodeJSBinding } = require("../src/index.cjs")
+const vm = require("vm")
+
+const {
+  processStringSync,
+  encodeJSBinding,
+  setJSRunner,
+} = require("../src/index.js")
 const { UUID_REGEX } = require("./constants")
 
 const processJS = (js, context) => {
   return processStringSync(encodeJSBinding(js), context)
 }
 
-describe("Test the JavaScript helper", () => {
+describe("Javascript", () => {
-  it("should execute a simple expression", () => {
+  beforeAll(() => {
-    const output = processJS(`return 1 + 2`)
+    setJSRunner((js, context) => {
-    expect(output).toBe(3)
+      return vm.runInNewContext(js, context, { timeout: 1000 })
-  })
-
-  it("should be able to use primitive bindings", () => {
-    const output = processJS(`return $("foo")`, {
-      foo: "bar",
     })
-    expect(output).toBe("bar")
   })
 
-  it("should be able to use an object binding", () => {
+  describe("Test the JavaScript helper", () => {
-    const output = processJS(`return $("foo").bar`, {
+    it("should execute a simple expression", () => {
-      foo: {
+      const output = processJS(`return 1 + 2`)
-        bar: "baz",
+      expect(output).toBe(3)
-      },
     })
-    expect(output).toBe("baz")
-  })
 
-  it("should be able to use a complex object binding", () => {
+    it("should be able to use primitive bindings", () => {
-    const output = processJS(`return $("foo").bar[0].baz`, {
+      const output = processJS(`return $("foo")`, {
-      foo: {
+        foo: "bar",
-        bar: [
+      })
-          {
+      expect(output).toBe("bar")
-            baz: "shazbat",
-          },
-        ],
-      },
     })
-    expect(output).toBe("shazbat")
-  })
 
-  it("should be able to use a deep binding", () => {
+    it("should be able to use an object binding", () => {
-    const output = processJS(`return $("foo.bar.baz")`, {
+      const output = processJS(`return $("foo").bar`, {
-      foo: {
+        foo: {
-        bar: {
-          baz: "shazbat",
-        },
-      },
-    })
-    expect(output).toBe("shazbat")
-  })
-
-  it("should be able to return an object", () => {
-    const output = processJS(`return $("foo")`, {
-      foo: {
-        bar: {
-          baz: "shazbat",
-        },
-      },
-    })
-    expect(output.bar.baz).toBe("shazbat")
-  })
-
-  it("should be able to return an array", () => {
-    const output = processJS(`return $("foo")`, {
-      foo: ["a", "b", "c"],
-    })
-    expect(output[2]).toBe("c")
-  })
-
-  it("should be able to return null", () => {
-    const output = processJS(`return $("foo")`, {
-      foo: null,
-    })
-    expect(output).toBe(null)
-  })
-
-  it("should be able to return undefined", () => {
-    const output = processJS(`return $("foo")`, {
-      foo: undefined,
-    })
-    expect(output).toBe(undefined)
-  })
-
-  it("should be able to return 0", () => {
-    const output = processJS(`return $("foo")`, {
-      foo: 0,
-    })
-    expect(output).toBe(0)
-  })
-
-  it("should be able to return an empty string", () => {
-    const output = processJS(`return $("foo")`, {
-      foo: "",
-    })
-    expect(output).toBe("")
-  })
-
-  it("should be able to use a deep array binding", () => {
-    const output = processJS(`return $("foo.0.bar")`, {
-      foo: [
-        {
           bar: "baz",
         },
-      ],
+      })
+      expect(output).toBe("baz")
+    })
+
+    it("should be able to use a complex object binding", () => {
+      const output = processJS(`return $("foo").bar[0].baz`, {
+        foo: {
+          bar: [
+            {
+              baz: "shazbat",
+            },
+          ],
+        },
+      })
+      expect(output).toBe("shazbat")
+    })
+
+    it("should be able to use a deep binding", () => {
+      const output = processJS(`return $("foo.bar.baz")`, {
+        foo: {
+          bar: {
+            baz: "shazbat",
+          },
+        },
+      })
+      expect(output).toBe("shazbat")
+    })
+
+    it("should be able to return an object", () => {
+      const output = processJS(`return $("foo")`, {
+        foo: {
+          bar: {
+            baz: "shazbat",
+          },
+        },
+      })
+      expect(output.bar.baz).toBe("shazbat")
+    })
+
+    it("should be able to return an array", () => {
+      const output = processJS(`return $("foo")`, {
+        foo: ["a", "b", "c"],
+      })
+      expect(output[2]).toBe("c")
+    })
+
+    it("should be able to return null", () => {
+      const output = processJS(`return $("foo")`, {
+        foo: null,
+      })
+      expect(output).toBe(null)
+    })
+
+    it("should be able to return undefined", () => {
+      const output = processJS(`return $("foo")`, {
+        foo: undefined,
+      })
+      expect(output).toBe(undefined)
+    })
+
+    it("should be able to return 0", () => {
+      const output = processJS(`return $("foo")`, {
+        foo: 0,
+      })
+      expect(output).toBe(0)
+    })
+
+    it("should be able to return an empty string", () => {
+      const output = processJS(`return $("foo")`, {
+        foo: "",
+      })
+      expect(output).toBe("")
+    })
+
+    it("should be able to use a deep array binding", () => {
+      const output = processJS(`return $("foo.0.bar")`, {
+        foo: [
+          {
+            bar: "baz",
+          },
+        ],
+      })
+      expect(output).toBe("baz")
+    })
+
+    it("should handle errors", () => {
+      const output = processJS(`throw "Error"`)
+      expect(output).toBe("Error while executing JS")
+    })
+
+    it("should timeout after one second", () => {
+      const output = processJS(`while (true) {}`)
+      expect(output).toBe("Timed out while executing JS")
+    })
+
+    it("should prevent access to the process global", () => {
+      const output = processJS(`return process`)
+      expect(output).toBe("Error while executing JS")
     })
-    expect(output).toBe("baz")
   })
 
-  it("should handle errors", () => {
+  describe("check JS helpers", () => {
-    const output = processJS(`throw "Error"`)
+    it("should error if using the format helper. not helpers.", () => {
-    expect(output).toBe("Error while executing JS")
+      const output = processJS(`return helper.toInt(4.3)`)
-  })
+      expect(output).toBe("Error while executing JS")
+    })
 
-  it("should timeout after one second", () => {
+    it("should be able to use toInt", () => {
-    const output = processJS(`while (true) {}`)
+      const output = processJS(`return helpers.toInt(4.3)`)
-    expect(output).toBe("Timed out while executing JS")
+      expect(output).toBe(4)
-  })
+    })
 
-  it("should prevent access to the process global", () => {
+    it("should be able to use uuid", () => {
-    const output = processJS(`return process`)
+      const output = processJS(`return helpers.uuid()`)
-    expect(output).toBe("Error while executing JS")
+      expect(output).toMatch(UUID_REGEX)
-  })
+    })
-
-  it("should prevent sandbox escape", () => {
-    const output = processJS(
-      `return this.constructor.constructor("return process")()`
-    )
-    expect(output).toBe("Error while executing JS")
-  })
-})
-
-describe("check JS helpers", () => {
-  it("should error if using the format helper. not helpers.", () => {
-    const output = processJS(`return helper.toInt(4.3)`)
-    expect(output).toBe("Error while executing JS")
-  })
-
-  it("should be able to use toInt", () => {
-    const output = processJS(`return helpers.toInt(4.3)`)
-    expect(output).toBe(4)
-  })
-
-  it("should be able to use uuid", () => {
-    const output = processJS(`return helpers.uuid()`)
-    expect(output).toMatch(UUID_REGEX)
   })
 })

packages/string-templates/test/manifest.spec.js

@@ -1,3 +1,5 @@
+const vm = require("vm")
+
 jest.mock("@budibase/handlebars-helpers/lib/math", () => {
   const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
 
@@ -15,81 +17,29 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
   }
 })
 
-const fs = require("fs")
+const { processString, setJSRunner } = require("../src/index.js")
-const {
-  processString,
-  convertToJS,
-  processStringSync,
-  encodeJSBinding,
-} = require("../src/index.cjs")
 
 const tk = require("timekeeper")
-const { getJsHelperList } = require("../src/helpers")
+const { getParsedManifest, runJsHelpersTests } = require("./utils")
 
 tk.freeze("2021-01-21T12:00:00")
 
-const processJS = (js, context) => {
-  return processStringSync(encodeJSBinding(js), context)
-}
-
-const manifest = JSON.parse(
-  fs.readFileSync(require.resolve("../manifest.json"), "utf8")
-)
-
-const collections = Object.keys(manifest)
-const examples = collections.reduce((acc, collection) => {
-  const functions = Object.entries(manifest[collection])
-    .filter(([_, details]) => details.example)
-    .map(([name, details]) => {
-      const example = details.example
-      let [hbs, js] = example.split("->").map(x => x.trim())
-      if (!js) {
-        // The function has no return value
-        return
-      }
-
-      // Trim 's
-      js = js.replace(/^\'|\'$/g, "")
-      if ((parsedExpected = tryParseJson(js))) {
-        if (Array.isArray(parsedExpected)) {
-          if (typeof parsedExpected[0] === "object") {
-            js = JSON.stringify(parsedExpected)
-          } else {
-            js = parsedExpected.join(",")
-          }
-        }
-      }
-      const requiresHbsBody = details.requiresBlock
-      return [name, { hbs, js, requiresHbsBody }]
-    })
-    .filter(x => !!x)
-
-  if (Object.keys(functions).length) {
-    acc[collection] = functions
-  }
-  return acc
-}, {})
-
 function escapeRegExp(string) {
   return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
 }
 
-function tryParseJson(str) {
-  if (typeof str !== "string") {
-    return
-  }
-
-  try {
-    return JSON.parse(str.replace(/\'/g, '"'))
-  } catch (e) {
-    return
-  }
-}
-
 describe("manifest", () => {
+  const manifest = getParsedManifest()
+
+  beforeAll(() => {
+    setJSRunner((js, context) => {
+      return vm.runInNewContext(js, context, { timeout: 1000 })
+    })
+  })
+
   describe("examples are valid", () => {
-    describe.each(Object.keys(examples))("%s", collection => {
+    describe.each(Object.keys(manifest))("%s", collection => {
-      it.each(examples[collection])("%s", async (_, { hbs, js }) => {
+      it.each(manifest[collection])("%s", async (_, { hbs, js }) => {
         const context = {
           double: i => i * 2,
           isString: x => typeof x === "string",
@@ -108,36 +58,5 @@ describe("manifest", () => {
     })
   })
 
-  describe("can be parsed and run as js", () => {
+  runJsHelpersTests()
-    const jsHelpers = getJsHelperList()
-    const jsExamples = Object.keys(examples).reduce((acc, v) => {
-      acc[v] = examples[v].filter(([key]) => jsHelpers[key])
-      return acc
-    }, {})
-
-    describe.each(Object.keys(jsExamples))("%s", collection => {
-      it.each(
-        jsExamples[collection].filter(
-          ([_, { requiresHbsBody }]) => !requiresHbsBody
-        )
-      )("%s", async (_, { hbs, js }) => {
-        const context = {
-          double: i => i * 2,
-          isString: x => typeof x === "string",
-        }
-
-        const arrays = hbs.match(/\[[^/\]]+\]/)
-        arrays?.forEach((arrayString, i) => {
-          hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
-          context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
-        })
-
-        let convertedJs = convertToJS(hbs)
-
-        let result = processJS(convertedJs, context)
-        result = result.replace(/ /g, " ")
-        expect(result).toEqual(js)
-      })
-    })
-  })
 })

packages/string-templates/test/renderApp.spec.js

@@ -1,4 +1,4 @@
-const { processString } = require("../src/index.cjs")
+const { processString } = require("../src/index.js")
 
 describe("specific test case for whether or not full app template can still be rendered", () => {
   it("should be able to render the app template", async () => {

packages/string-templates/test/utils.js

@@ -0,0 +1,111 @@
+const { getManifest } = require("../src")
+const { getJsHelperList } = require("../src/helpers")
+
+const {
+  convertToJS,
+  processStringSync,
+  encodeJSBinding,
+} = require("../src/index.js")
+
+function tryParseJson(str) {
+  if (typeof str !== "string") {
+    return
+  }
+
+  try {
+    return JSON.parse(str.replace(/'/g, '"'))
+  } catch (e) {
+    return
+  }
+}
+
+const getParsedManifest = () => {
+  const manifest = getManifest()
+  const collections = Object.keys(manifest)
+  const examples = collections.reduce((acc, collection) => {
+    const functions = Object.entries(manifest[collection])
+      .filter(([_, details]) => details.example)
+      .map(([name, details]) => {
+        const example = details.example
+        let [hbs, js] = example.split("->").map(x => x.trim())
+        if (!js) {
+          // The function has no return value
+          return
+        }
+
+        // Trim 's
+        js = js.replace(/^'|'$/g, "")
+        let parsedExpected
+        if ((parsedExpected = tryParseJson(js))) {
+          if (Array.isArray(parsedExpected)) {
+            if (typeof parsedExpected[0] === "object") {
+              js = JSON.stringify(parsedExpected)
+            } else {
+              js = parsedExpected.join(",")
+            }
+          }
+        }
+        const requiresHbsBody = details.requiresBlock
+        return [name, { hbs, js, requiresHbsBody }]
+      })
+      .filter(x => !!x)
+
+    if (Object.keys(functions).length) {
+      acc[collection] = functions
+    }
+    return acc
+  }, {})
+
+  return examples
+}
+module.exports.getParsedManifest = getParsedManifest
+
+module.exports.runJsHelpersTests = ({ funcWrap, testsToSkip } = {}) => {
+  funcWrap = funcWrap || (delegate => delegate())
+  const manifest = getParsedManifest()
+
+  const processJS = (js, context) => {
+    return funcWrap(() => processStringSync(encodeJSBinding(js), context))
+  }
+
+  function escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
+  }
+
+  describe("can be parsed and run as js", () => {
+    const jsHelpers = getJsHelperList()
+    const jsExamples = Object.keys(manifest).reduce((acc, v) => {
+      acc[v] = manifest[v].filter(([key]) => jsHelpers[key])
+      return acc
+    }, {})
+
+    describe.each(Object.keys(jsExamples))("%s", collection => {
+      const examplesToRun = jsExamples[collection]
+        .filter(([_, { requiresHbsBody }]) => !requiresHbsBody)
+        .filter(([key]) => !testsToSkip?.includes(key))
+
+      examplesToRun.length &&
+        it.each(examplesToRun)("%s", async (_, { hbs, js }) => {
+          const context = {
+            double: i => i * 2,
+            isString: x => typeof x === "string",
+          }
+
+          const arrays = hbs.match(/\[[^/\]]+\]/)
+          arrays?.forEach((arrayString, i) => {
+            hbs = hbs.replace(
+              new RegExp(escapeRegExp(arrayString)),
+              `array${i}`
+            )
+            context[`array${i}`] = JSON.parse(arrayString.replace(/'/g, '"'))
+          })
+
+          let convertedJs = convertToJS(hbs)
+
+          let result = await processJS(convertedJs, context)
+          result = result.replace(/ /g, " ")
+          expect(result).toEqual(js)
+        })
+    })
+  })
+}

packages/types/src/sdk/index.ts

@@ -20,3 +20,4 @@ export * from "./cli"
 export * from "./websocket"
 export * from "./permissions"
 export * from "./row"
+export * from "./vm"

packages/types/src/sdk/vm.ts

@@ -0,0 +1,3 @@
+export interface VM {
+  execute(code: string): any
+}

packages/worker/Dockerfile

@@ -44,6 +44,8 @@ EXPOSE 4001
 # due to this causing yarn to stop installing dev dependencies
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
+# this is required for isolated-vm to work on Node 20+
+ENV NODE_OPTIONS="--no-node-snapshot"
 ENV CLUSTER_MODE=${CLUSTER_MODE}
 ENV SERVICE=worker-service
 ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU

packages/worker/nodemon.json

@@ -8,6 +8,6 @@
     "../string-templates"
   ],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
+  "ignore": ["**/*.spec.ts", "**/*.spec.js", "../*/dist/**/*"],
-  "exec": "yarn build && node dist/index.js"
+  "exec": "yarn build && node --no-node-snapshot dist/index.js"
 }

packages/worker/package.json

@@ -48,7 +48,7 @@
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
-    "dd-trace": "5.0.0",
+    "dd-trace": "5.2.0",
     "dotenv": "8.6.0",
     "global-agent": "3.0.0",
     "ical-generator": "4.1.0",
@@ -68,7 +68,9 @@
     "passport-local": "1.0.0",
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
-    "server-destroy": "1.0.1"
+    "server-destroy": "1.0.1",
+    "undici": "^6.0.1",
+    "undici-types": "^6.0.1"
   },
   "devDependencies": {
     "@swc/core": "1.3.71",

packages/worker/scripts/load/heavyUse.js

@@ -23,27 +23,27 @@ const USERS = [
     password: "test",
   },
   {
-    email: "loadtest2@test.com",
+    email: "loadtest2@example.com",
     password: "test",
   },
   {
-    email: "loadtest3@test.com",
+    email: "loadtest3@example.com",
     password: "test",
   },
   {
-    email: "loadtest4@test.com",
+    email: "loadtest4@example.com",
     password: "test",
   },
   {
-    email: "loadtest5@test.com",
+    email: "loadtest5@example.com",
     password: "test",
   },
   {
-    email: "loadtest6@test.com",
+    email: "loadtest6@example.com",
     password: "test",
   },
   {
-    email: "loadtest7@test.com",
+    email: "loadtest7@example.com",
     password: "test",
   },
 ]

packages/worker/src/api/routes/global/tests/auth.spec.ts

@@ -80,7 +80,7 @@ describe("/api/global/auth", () => {
 
       it("should return 403 when user doesn't exist", async () => {
         const tenantId = config.tenantId!
-        const email = "invaliduser@test.com"
+        const email = "invaliduser@example.com"
         const password = "password"
 
         const response = await config.api.auth.login(

packages/worker/src/api/routes/global/tests/users.spec.ts

@@ -490,7 +490,7 @@ describe("/api/global/users", () => {
     it("should not be able to update email address", async () => {
       const email = structures.email()
       const user = await config.createUser(structures.users.user({ email }))
-      user.email = "new@test.com"
+      user.email = "new@example.com"
 
       const response = await config.api.users.saveUser(user, 400)
 

packages/worker/src/tests/api/email.ts

@@ -10,7 +10,7 @@ export class EmailAPI extends TestAPI {
     return this.request
       .post(`/api/global/email/send`)
       .send({
-        email: "test@test.com",
+        email: "test@example.com",
         purpose,
         tenantId: this.config.getTenantId(),
         userId: this.config.user?._id!,

packages/worker/src/tests/structures/configs.ts

@@ -45,7 +45,7 @@ export function smtp(conf?: any): SMTPConfig {
     config: {
       port: 12345,
       host: "smtptesthost.com",
-      from: "testfrom@test.com",
+      from: "testfrom@example.com",
       subject: "Hello!",
       secure: false,
       ...conf,
@@ -59,7 +59,7 @@ export function smtpEthereal(): SMTPConfig {
     config: {
       port: 587,
       host: "smtp.ethereal.email",
-      from: "testfrom@test.com",
+      from: "testfrom@example.com",
       secure: false,
       auth: {
         user: "wyatt.zulauf29@ethereal.email",

scripts/build.js

@@ -49,6 +49,7 @@ function runBuild(entry, outfile) {
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",
+      ".ivm.bundle.js": "text",
     },
     metafile: true,
     external: [
@@ -60,6 +61,7 @@ function runBuild(entry, outfile) {
       "bcrypt",
       "bcryptjs",
       "graphql/*",
+      "bson",
     ],
   }