Merge pull request #12963 from Budibase/tidy-esbuilt-libs
Move bundle responsibility from string-templates to server
This commit is contained in:
commit
9dddff47cc
11 changed files with 136 additions and 50 deletions
|
@ -10,4 +10,5 @@ packages/builder/.routify
|
||||||
packages/sdk/sdk
|
packages/sdk/sdk
|
||||||
packages/account-portal/packages/server/build
|
packages/account-portal/packages/server/build
|
||||||
packages/account-portal/packages/ui/.routify
|
packages/account-portal/packages/ui/.routify
|
||||||
packages/account-portal/packages/ui/build
|
packages/account-portal/packages/ui/build
|
||||||
|
**/*.ivm.bundle.js
|
|
@ -11,4 +11,5 @@ packages/sdk/sdk
|
||||||
packages/pro/coverage
|
packages/pro/coverage
|
||||||
packages/account-portal/packages/ui/build
|
packages/account-portal/packages/ui/build
|
||||||
packages/account-portal/packages/ui/.routify
|
packages/account-portal/packages/ui/.routify
|
||||||
packages/account-portal/packages/server/build
|
packages/account-portal/packages/server/build
|
||||||
|
**/*.ivm.bundle.js
|
|
@ -13,6 +13,8 @@
|
||||||
"build": "node ./scripts/build.js",
|
"build": "node ./scripts/build.js",
|
||||||
"postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
|
"postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
|
||||||
"check:types": "tsc -p tsconfig.json --noEmit --paths null",
|
"check:types": "tsc -p tsconfig.json --noEmit --paths null",
|
||||||
|
"build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=esm --external:handlebars",
|
||||||
|
"build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates",
|
||||||
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
|
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
|
||||||
"debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
|
"debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
|
||||||
"jest": "NODE_OPTIONS=\"--no-node-snapshot $NODE_OPTIONS\" jest",
|
"jest": "NODE_OPTIONS=\"--no-node-snapshot $NODE_OPTIONS\" jest",
|
||||||
|
@ -165,12 +167,6 @@
|
||||||
"@budibase/backend-core"
|
"@budibase/backend-core"
|
||||||
],
|
],
|
||||||
"target": "build"
|
"target": "build"
|
||||||
},
|
|
||||||
{
|
|
||||||
"projects": [
|
|
||||||
"@budibase/string-templates"
|
|
||||||
],
|
|
||||||
"target": "build:esbuild"
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
@ -189,16 +185,6 @@
|
||||||
"target": "build"
|
"target": "build"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
|
||||||
"test": {
|
|
||||||
"dependsOn": [
|
|
||||||
{
|
|
||||||
"projects": [
|
|
||||||
"@budibase/string-templates"
|
|
||||||
],
|
|
||||||
"target": "build:esbuild"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
15
packages/server/src/jsRunner/bundles/README.md
Normal file
15
packages/server/src/jsRunner/bundles/README.md
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
# Bundles for isolated-vm
|
||||||
|
|
||||||
|
[Isolated-vm](https://github.com/laverdet/isolated-vm) requires for us to have some libraries, such as string-templates helpers, built in a single file without external dependencies. These libraries are pretty much static. To avoid building this in every dev command, in every test command and in every pipeline, these libraries are already compiled and commited into the repo.
|
||||||
|
|
||||||
|
## How are they consumed?
|
||||||
|
|
||||||
|
These libaries are compiled with a special extension: .ivm.bundle.js. This extension is configured in [esbuild](/scripts/build.js) in order to not be bundled as javascript, and to be treated as a `string` instead. This will allow us to read it's context on runtime and inject it to `isolated-vm`.
|
||||||
|
|
||||||
|
## How to update it?
|
||||||
|
|
||||||
|
These libraries are pretty much static, but they might require some updates from time to time when something changes on the source code. In order to do this, we just need to run the following command and commit the updated bundles:
|
||||||
|
|
||||||
|
```
|
||||||
|
yarn build:isolated-vm-libs
|
||||||
|
```
|
File diff suppressed because one or more lines are too long
|
@ -1,9 +1,12 @@
|
||||||
const { getJsHelperList } = require("./helpers/list")
|
const {
|
||||||
|
getJsHelperList,
|
||||||
|
} = require("../../../../string-templates/src/helpers/list.js")
|
||||||
|
|
||||||
const helpers = getJsHelperList()
|
const helpers = getJsHelperList()
|
||||||
module.exports = {
|
export default {
|
||||||
...helpers,
|
...helpers,
|
||||||
// pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
|
// pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
|
||||||
|
// @ts-ignore
|
||||||
// eslint-disable-next-line no-undef
|
// eslint-disable-next-line no-undef
|
||||||
stripProtocol: helpersStripProtocol,
|
stripProtocol: helpersStripProtocol,
|
||||||
}
|
}
|
24
packages/server/src/jsRunner/bundles/index.ts
Normal file
24
packages/server/src/jsRunner/bundles/index.ts
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
import { utils } from "@budibase/shared-core"
|
||||||
|
import environment from "../../environment"
|
||||||
|
import fs from "fs"
|
||||||
|
|
||||||
|
export enum BundleType {
|
||||||
|
HELPERS = "helpers",
|
||||||
|
}
|
||||||
|
|
||||||
|
const bundleSourceCode = {
|
||||||
|
[BundleType.HELPERS]: "../bundles/index-helpers.ivm.bundle.js",
|
||||||
|
}
|
||||||
|
|
||||||
|
export function loadBundle(type: BundleType) {
|
||||||
|
if (environment.isJest()) {
|
||||||
|
return fs.readFileSync(require.resolve(bundleSourceCode[type]), "utf-8")
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (type) {
|
||||||
|
case BundleType.HELPERS:
|
||||||
|
return require("../bundles/index-helpers.ivm.bundle.js")
|
||||||
|
default:
|
||||||
|
utils.unreachable(type)
|
||||||
|
}
|
||||||
|
}
|
|
@ -3,15 +3,10 @@ import env from "../environment"
|
||||||
import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
|
import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
|
||||||
import { context } from "@budibase/backend-core"
|
import { context } from "@budibase/backend-core"
|
||||||
import tracer from "dd-trace"
|
import tracer from "dd-trace"
|
||||||
import fs from "fs"
|
|
||||||
import url from "url"
|
import url from "url"
|
||||||
import crypto from "crypto"
|
import crypto from "crypto"
|
||||||
import querystring from "querystring"
|
import querystring from "querystring"
|
||||||
|
import { BundleType, loadBundle } from "./bundles"
|
||||||
const helpersSource = fs.readFileSync(
|
|
||||||
`${require.resolve("@budibase/string-templates/index-helpers")}`,
|
|
||||||
"utf8"
|
|
||||||
)
|
|
||||||
|
|
||||||
class ExecutionTimeoutError extends Error {
|
class ExecutionTimeoutError extends Error {
|
||||||
constructor(message: string) {
|
constructor(message: string) {
|
||||||
|
@ -21,6 +16,7 @@ class ExecutionTimeoutError extends Error {
|
||||||
}
|
}
|
||||||
|
|
||||||
export function init() {
|
export function init() {
|
||||||
|
const helpersSource = loadBundle(BundleType.HELPERS)
|
||||||
setJSRunner((js: string, ctx: Record<string, any>) => {
|
setJSRunner((js: string, ctx: Record<string, any>) => {
|
||||||
return tracer.trace("runJS", {}, span => {
|
return tracer.trace("runJS", {}, span => {
|
||||||
try {
|
try {
|
||||||
|
@ -33,19 +29,20 @@ export function init() {
|
||||||
})
|
})
|
||||||
const jsContext = jsIsolate.createContextSync()
|
const jsContext = jsIsolate.createContextSync()
|
||||||
|
|
||||||
const injectedRequire = `const require = function(val){
|
const injectedRequire = `
|
||||||
switch (val) {
|
const require = function(val){
|
||||||
case "url":
|
switch (val) {
|
||||||
return {
|
case "url":
|
||||||
resolve: (...params) => urlResolveCb(...params),
|
return {
|
||||||
parse: (...params) => urlParseCb(...params),
|
resolve: (...params) => urlResolveCb(...params),
|
||||||
}
|
parse: (...params) => urlParseCb(...params),
|
||||||
case "querystring":
|
}
|
||||||
return {
|
case "querystring":
|
||||||
escape: (...params) => querystringEscapeCb(...params),
|
return {
|
||||||
}
|
escape: (...params) => querystringEscapeCb(...params),
|
||||||
}
|
}
|
||||||
};`
|
}
|
||||||
|
};`
|
||||||
|
|
||||||
const global = jsContext.global
|
const global = jsContext.global
|
||||||
global.setSync(
|
global.setSync(
|
||||||
|
@ -83,9 +80,9 @@ export function init() {
|
||||||
`${injectedRequire};${helpersSource}`
|
`${injectedRequire};${helpersSource}`
|
||||||
)
|
)
|
||||||
|
|
||||||
const cryptoModule = jsIsolate.compileModuleSync(`export default {
|
const cryptoModule = jsIsolate.compileModuleSync(
|
||||||
randomUUID: cryptoRandomUUIDCb,
|
`export default { randomUUID: cryptoRandomUUIDCb }`
|
||||||
}`)
|
)
|
||||||
cryptoModule.instantiateSync(jsContext, specifier => {
|
cryptoModule.instantiateSync(jsContext, specifier => {
|
||||||
throw new Error(`No imports allowed. Required: ${specifier}`)
|
throw new Error(`No imports allowed. Required: ${specifier}`)
|
||||||
})
|
})
|
||||||
|
|
|
@ -25,14 +25,14 @@ class ScriptRunner {
|
||||||
class IsolatedVM {
|
class IsolatedVM {
|
||||||
isolate: ivm.Isolate
|
isolate: ivm.Isolate
|
||||||
vm: ivm.Context
|
vm: ivm.Context
|
||||||
jail: ivm.Reference
|
#jail: ivm.Reference
|
||||||
script: any
|
script: any
|
||||||
|
|
||||||
constructor({ memoryLimit }: { memoryLimit: number }) {
|
constructor({ memoryLimit }: { memoryLimit: number }) {
|
||||||
this.isolate = new ivm.Isolate({ memoryLimit })
|
this.isolate = new ivm.Isolate({ memoryLimit })
|
||||||
this.vm = this.isolate.createContextSync()
|
this.vm = this.isolate.createContextSync()
|
||||||
this.jail = this.vm.global
|
this.#jail = this.vm.global
|
||||||
this.jail.setSync("global", this.jail.derefInto())
|
this.#jail.setSync("global", this.#jail.derefInto())
|
||||||
}
|
}
|
||||||
|
|
||||||
getValue(key: string) {
|
getValue(key: string) {
|
||||||
|
@ -44,7 +44,7 @@ class IsolatedVM {
|
||||||
|
|
||||||
set context(context: Record<string, any>) {
|
set context(context: Record<string, any>) {
|
||||||
for (let key in context) {
|
for (let key in context) {
|
||||||
this.jail.setSync(key, this.copyRefToVm(context[key]))
|
this.#jail.setSync(key, this.copyRefToVm(context[key]))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
"import": "./dist/bundle.mjs"
|
"import": "./dist/bundle.mjs"
|
||||||
},
|
},
|
||||||
"./package.json": "./package.json",
|
"./package.json": "./package.json",
|
||||||
"./index-helpers": "./dist/index-helpers.bundled.js",
|
|
||||||
"./test/utils": "./test/utils.js"
|
"./test/utils": "./test/utils.js"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
|
@ -21,9 +20,8 @@
|
||||||
"manifest.json"
|
"manifest.json"
|
||||||
],
|
],
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build:esbuild": "esbuild --minify --bundle src/index-helpers.js --outfile=dist/index-helpers.bundled.js --platform=node --format=esm --external:handlebars",
|
"build": "tsc && rollup -c",
|
||||||
"build": "yarn build:esbuild && tsc && rollup -c",
|
"dev": "concurrently \"tsc --watch\" \"rollup -cw\"",
|
||||||
"dev": "concurrently \"yarn build:esbuild --watch\" \"tsc && rollup -cw\"",
|
|
||||||
"test": "jest",
|
"test": "jest",
|
||||||
"manifest": "node ./scripts/gen-collection-info.js"
|
"manifest": "node ./scripts/gen-collection-info.js"
|
||||||
},
|
},
|
||||||
|
|
|
@ -49,6 +49,7 @@ function runBuild(entry, outfile) {
|
||||||
preserveSymlinks: true,
|
preserveSymlinks: true,
|
||||||
loader: {
|
loader: {
|
||||||
".svelte": "copy",
|
".svelte": "copy",
|
||||||
|
".ivm.bundle.js": "text",
|
||||||
},
|
},
|
||||||
metafile: true,
|
metafile: true,
|
||||||
external: [
|
external: [
|
||||||
|
@ -60,6 +61,7 @@ function runBuild(entry, outfile) {
|
||||||
"bcrypt",
|
"bcrypt",
|
||||||
"bcryptjs",
|
"bcryptjs",
|
||||||
"graphql/*",
|
"graphql/*",
|
||||||
|
"bson",
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue