Merge pull request #12963 from Budibase/tidy-esbuilt-libs

Move bundle responsibility from string-templates to server

.eslintignore

@@ -10,4 +10,5 @@ packages/builder/.routify
 packages/sdk/sdk
 packages/account-portal/packages/server/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/ui/build
+packages/account-portal/packages/ui/build
+**/*.ivm.bundle.js

.prettierignore

@@ -11,4 +11,5 @@ packages/sdk/sdk
 packages/pro/coverage
 packages/account-portal/packages/ui/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/server/build
+packages/account-portal/packages/server/build
+**/*.ivm.bundle.js

packages/server/package.json

@@ -13,6 +13,8 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
+    "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=esm --external:handlebars",
+    "build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
     "jest": "NODE_OPTIONS=\"--no-node-snapshot $NODE_OPTIONS\" jest",
@@ -165,12 +167,6 @@
               "@budibase/backend-core"
             ],
             "target": "build"
-          },
-          {
-            "projects": [
-              "@budibase/string-templates"
-            ],
-            "target": "build:esbuild"
           }
         ]
       },
@@ -189,16 +185,6 @@
             "target": "build"
           }
         ]
-      },
-      "test": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/string-templates"
-            ],
-            "target": "build:esbuild"
-          }
-        ]
       }
     }
   }

packages/server/src/jsRunner/bundles/README.md

@@ -0,0 +1,15 @@
+# Bundles for isolated-vm
+
+[Isolated-vm](https://github.com/laverdet/isolated-vm) requires for us to have some libraries, such as string-templates helpers, built in a single file without external dependencies. These libraries are pretty much static. To avoid building this in every dev command, in every test command and in every pipeline, these libraries are already compiled and commited into the repo.
+
+## How are they consumed?
+
+These libaries are compiled with a special extension: .ivm.bundle.js. This extension is configured in [esbuild](/scripts/build.js) in order to not be bundled as javascript, and to be treated as a `string` instead. This will allow us to read it's context on runtime and inject it to `isolated-vm`.
+
+## How to update it?
+
+These libraries are pretty much static, but they might require some updates from time to time when something changes on the source code. In order to do this, we just need to run the following command and commit the updated bundles:
+
+```
+yarn build:isolated-vm-libs
+```

packages/server/src/jsRunner/bundles/index-helpers.ts

@@ -1,9 +1,12 @@
-const { getJsHelperList } = require("./helpers/list")
+const {
+  getJsHelperList,
+} = require("../../../../string-templates/src/helpers/list.js")
 
 const helpers = getJsHelperList()
-module.exports = {
+export default {
   ...helpers,
   // pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
+  // @ts-ignore
   // eslint-disable-next-line no-undef
   stripProtocol: helpersStripProtocol,
 }

packages/server/src/jsRunner/bundles/index.ts

@@ -0,0 +1,24 @@
+import { utils } from "@budibase/shared-core"
+import environment from "../../environment"
+import fs from "fs"
+
+export enum BundleType {
+  HELPERS = "helpers",
+}
+
+const bundleSourceCode = {
+  [BundleType.HELPERS]: "../bundles/index-helpers.ivm.bundle.js",
+}
+
+export function loadBundle(type: BundleType) {
+  if (environment.isJest()) {
+    return fs.readFileSync(require.resolve(bundleSourceCode[type]), "utf-8")
+  }
+
+  switch (type) {
+    case BundleType.HELPERS:
+      return require("../bundles/index-helpers.ivm.bundle.js")
+    default:
+      utils.unreachable(type)
+  }
+}

packages/server/src/jsRunner/index.ts

@@ -3,15 +3,10 @@ import env from "../environment"
 import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
 import { context } from "@budibase/backend-core"
 import tracer from "dd-trace"
-import fs from "fs"
 import url from "url"
 import crypto from "crypto"
 import querystring from "querystring"
-
-const helpersSource = fs.readFileSync(
-  `${require.resolve("@budibase/string-templates/index-helpers")}`,
-  "utf8"
-)
+import { BundleType, loadBundle } from "./bundles"
 
 class ExecutionTimeoutError extends Error {
   constructor(message: string) {
@@ -21,6 +16,7 @@ class ExecutionTimeoutError extends Error {
 }
 
 export function init() {
+  const helpersSource = loadBundle(BundleType.HELPERS)
   setJSRunner((js: string, ctx: Record<string, any>) => {
     return tracer.trace("runJS", {}, span => {
       try {
@@ -33,19 +29,20 @@ export function init() {
           })
           const jsContext = jsIsolate.createContextSync()
 
-          const injectedRequire = `const require = function(val){
-        switch (val) {
-          case "url": 
-            return {
-              resolve: (...params) => urlResolveCb(...params),
-              parse: (...params) => urlParseCb(...params),
-            }
-          case "querystring":
-            return {
-              escape: (...params) => querystringEscapeCb(...params),
-            }
-        }
-      };`
+          const injectedRequire = `
+            const require = function(val){
+              switch (val) {
+                case "url": 
+                  return {
+                    resolve: (...params) => urlResolveCb(...params),
+                    parse: (...params) => urlParseCb(...params),
+                  }
+                case "querystring":
+                  return {
+                    escape: (...params) => querystringEscapeCb(...params),
+                  }
+              }
+            };`
 
           const global = jsContext.global
           global.setSync(
@@ -83,9 +80,9 @@ export function init() {
             `${injectedRequire};${helpersSource}`
           )
 
-          const cryptoModule = jsIsolate.compileModuleSync(`export default {
-        randomUUID: cryptoRandomUUIDCb,
-      }`)
+          const cryptoModule = jsIsolate.compileModuleSync(
+            `export default { randomUUID: cryptoRandomUUIDCb }`
+          )
           cryptoModule.instantiateSync(jsContext, specifier => {
             throw new Error(`No imports allowed. Required: ${specifier}`)
           })

packages/server/src/utilities/scriptRunner.ts

@@ -25,14 +25,14 @@ class ScriptRunner {
 class IsolatedVM {
   isolate: ivm.Isolate
   vm: ivm.Context
-  jail: ivm.Reference
+  #jail: ivm.Reference
   script: any
 
   constructor({ memoryLimit }: { memoryLimit: number }) {
     this.isolate = new ivm.Isolate({ memoryLimit })
     this.vm = this.isolate.createContextSync()
-    this.jail = this.vm.global
-    this.jail.setSync("global", this.jail.derefInto())
+    this.#jail = this.vm.global
+    this.#jail.setSync("global", this.#jail.derefInto())
   }
 
   getValue(key: string) {
@@ -44,7 +44,7 @@ class IsolatedVM {
 
   set context(context: Record<string, any>) {
     for (let key in context) {
-      this.jail.setSync(key, this.copyRefToVm(context[key]))
+      this.#jail.setSync(key, this.copyRefToVm(context[key]))
     }
   }
 

packages/string-templates/package.json

@@ -12,7 +12,6 @@
       "import": "./dist/bundle.mjs"
     },
     "./package.json": "./package.json",
-    "./index-helpers": "./dist/index-helpers.bundled.js",
     "./test/utils": "./test/utils.js"
   },
   "files": [
@@ -21,9 +20,8 @@
     "manifest.json"
   ],
   "scripts": {
-    "build:esbuild": "esbuild --minify --bundle src/index-helpers.js --outfile=dist/index-helpers.bundled.js --platform=node --format=esm --external:handlebars",
-    "build": "yarn build:esbuild && tsc && rollup -c",
-    "dev": "concurrently \"yarn build:esbuild --watch\" \"tsc && rollup -cw\"",
+    "build": "tsc && rollup -c",
+    "dev": "concurrently \"tsc --watch\" \"rollup -cw\"",
     "test": "jest",
     "manifest": "node ./scripts/gen-collection-info.js"
   },

scripts/build.js

@@ -49,6 +49,7 @@ function runBuild(entry, outfile) {
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",
+      ".ivm.bundle.js": "text",
     },
     metafile: true,
     external: [
@@ -60,6 +61,7 @@ function runBuild(entry, outfile) {
       "bcrypt",
       "bcryptjs",
       "graphql/*",
+      "bson",
     ],
   }