Merge branch 'develop' of github.com:Budibase/budibase into views-v2-frontend

.github/workflows/budibase_ci.yml

@@ -25,20 +25,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn lint
 
   build:
@@ -46,20 +46,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       # Run build all the projects
       - run: yarn build
       # Check the types of the projects built via esbuild
@@ -70,20 +70,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
       - uses: codecov/codecov-action@v3
         with:
@@ -96,21 +96,22 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
-      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
+      - run: yarn --frozen-lockfile
+      - name: Test worker and server
+        run: yarn test --scope=@budibase/worker --scope=@budibase/server
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -119,7 +120,7 @@ jobs:
 
   test-pro:
     runs-on: ubuntu-latest
-    if: github.repository == 'Budibase/budibase'
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
@@ -127,12 +128,12 @@ jobs:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn test --scope=@budibase/pro
 
   integration-test:
@@ -140,20 +141,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == 'Budibase/budibase'
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != 'Budibase/budibase'
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client
       - name: Run tests
         run: |
@@ -166,7 +167,7 @@ jobs:
 
   check-pro-submodule:
     runs-on: ubuntu-latest
-    if: github.repository == 'Budibase/budibase'
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3

.github/workflows/release-develop.yml

@@ -44,7 +44,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - run: yarn install --frozen-lockfile
       - name: Update versions

.github/workflows/release-master.yml

@@ -60,9 +60,9 @@ jobs:
       - name: "Get Current tag"
         id: currenttag
         run: |
-          version=v$(./scripts/getCurrentVersion.sh)
-          echo 'Using tag $version'
-          echo "::set-output name=tag::$resversionult"
+          version=$(./scripts/getCurrentVersion.sh)
+          echo "Using tag $version"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
 
       - name: Build/release Docker images
         run: |
@@ -71,7 +71,7 @@ jobs:
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.tag }}
+          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

.github/workflows/release-singleimage.yml

@@ -8,7 +8,7 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   REGISTRY_URL: registry.hub.docker.com
 jobs:
-  build-amd64:
+  build-amd64-arm64:
     name: "build-amd64"
     runs-on: ubuntu-latest
     strategy:
@@ -68,81 +68,7 @@ jobs:
         with:
           context: .
           push: true
-          platforms: linux/amd64
-          tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
-
-      - name: Tag and release Budibase Azure App Service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64
-          build-args: TARGETBUILD=aas
-          tags: budibase/budibase-aas,budibase/budibase-aas:v${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
-
-  build-arm64:
-    name: "build-arm64"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Runt Yarn Lint
-        run: yarn lint
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build:docker:pre
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-      - name: Tag and release Budibase service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/arm64
+          platforms: linux/amd64,linux/arm64
           tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile
 

.nvmrc

@@ -1 +1 @@
-v14.20.1
+v18.17.0

.tool-versions

@@ -1,3 +1,3 @@
-nodejs 14.21.3
+nodejs 18.17.0
 python 3.10.0
-yarn 1.22.19
+yarn 1.22.19

charts/budibase/templates/app-service-deployment.yaml

@@ -120,6 +120,8 @@ spec:
         {{ end }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.apps.logLevel | quote }}
         - name: REDIS_PASSWORD

charts/budibase/templates/worker-service-deployment.yaml

@@ -116,6 +116,8 @@ spec:
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.worker.logLevel | quote }}
         - name: REDIS_PASSWORD

charts/budibase/values.yaml

@@ -82,6 +82,7 @@ globals:
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  offlineMode: "0" # set to 1 to enable offline mode
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""

docs/CONTRIBUTING.md

@@ -90,7 +90,7 @@ Component libraries are collections of components as well as the definition of t
 
 #### 1. Prerequisites
 
-- NodeJS version `14.x.x`
+- NodeJS version `18.x.x`
 - Python version `3.x`
 
 ### Using asdf (recommended)

hosting/single/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-slim as build
+FROM node:18-slim as build
 
 # install node-gyp dependencies
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.32-alpha.6",
+  "version": "2.9.8-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -109,7 +109,7 @@
     "@budibase/types": "0.0.0"
   },
   "engines": {
-    "node": ">=14.0.0 <15.0.0"
+    "node": ">=18.0.0 <19.0.0"
   },
   "dependencies": {}
 }

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -80,7 +80,7 @@ function getRedisConfig() {
 
 export function setupEnv(...envs: any[]) {
   const couch = getCouchConfig(),
-    minio = getCouchConfig(),
+    minio = getMinioConfig(),
     redis = getRedisConfig()
   const configs = [
     { key: "COUCH_DB_PORT", value: couch.port },

packages/cli/package.json

@@ -2,16 +2,16 @@
   "name": "@budibase/cli",
   "version": "0.0.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
-  "main": "dist/src/index.js",
+  "main": "dist/index.js",
   "bin": {
-    "budi": "dist/src/index.js"
+    "budi": "dist/index.js"
   },
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {
     "prebuild": "rm -rf prebuilds 2> /dev/null && cp -r ../../node_modules/leveldown/prebuilds prebuilds",
     "rename": "renamer --find .node --replace .fake 'prebuilds/**'",
-    "tsc": "tsc -p tsconfig.build.json",
+    "tsc": "node ../../scripts/build.js",
     "pkg": "pkg . --out-path build --no-bytecode --public --public-packages \"*\" -C GZip",
     "build": "yarn prebuild && yarn rename && yarn tsc && yarn pkg && yarn postbuild",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
@@ -19,12 +19,11 @@
   },
   "pkg": {
     "targets": [
-      "node16-linux",
-      "node16-win",
-      "node16-macos"
+      "node18-linux",
+      "node18-win",
+      "node18-macos"
     ],
     "assets": [
-      "node_modules/@budibase/backend-core/dist/**/*",
       "prebuilds/**/*"
     ],
     "outputPath": "build"

packages/cli/src/exec.ts

@@ -1,5 +1,6 @@
 import util from "util"
-const runCommand = util.promisify(require("child_process").exec)
+import childProcess from "child_process"
+const runCommand = util.promisify(childProcess.exec)
 
 export async function exec(command: string, dir = "./") {
   const { stdout } = await runCommand(command, { cwd: dir })
@@ -16,12 +17,12 @@ export async function utilityInstalled(utilName: string) {
 }
 
 export async function runPkgCommand(command: string, dir = "./") {
-  const yarn = await exports.utilityInstalled("yarn")
-  const npm = await exports.utilityInstalled("npm")
+  const yarn = await utilityInstalled("yarn")
+  const npm = await utilityInstalled("npm")
   if (!yarn && !npm) {
     throw new Error("Must have yarn or npm installed to run build.")
   }
   const npmCmd = command === "install" ? `npm ${command}` : `npm run ${command}`
   const cmd = yarn ? `yarn ${command} --ignore-engines` : npmCmd
-  await exports.exec(cmd, dir)
+  await exec(cmd, dir)
 }

packages/cli/src/prebuilds.ts

@@ -5,7 +5,7 @@ import { error } from "./utils"
 
 const PREBUILDS = "prebuilds"
 const ARCH = `${os.platform()}-${os.arch()}`
-const PREBUILD_DIR = join(process.execPath, "..", PREBUILDS, ARCH)
+const PREBUILD_DIR = join(process.execPath, "..", "cli", PREBUILDS, ARCH)
 
 // running as built CLI pkg bundle
 if (!process.argv[0].includes("node")) {
@@ -13,17 +13,19 @@ if (!process.argv[0].includes("node")) {
 }
 
 function checkForBinaries() {
-  const readDir = join(__filename, "..", "..", "..", PREBUILDS, ARCH)
+  const readDir = join(__filename, "..", "..", "..", "cli", PREBUILDS, ARCH)
   if (fs.existsSync(PREBUILD_DIR) || !fs.existsSync(readDir)) {
     return
   }
   const natives = fs.readdirSync(readDir)
   if (fs.existsSync(readDir)) {
-    fs.mkdirSync(PREBUILD_DIR, { recursive: true })
+    const writePath = join(process.execPath, PREBUILDS, ARCH)
+    fs.mkdirSync(writePath, { recursive: true })
     for (let native of natives) {
       const filename = `${native.split(".fake")[0]}.node`
-      fs.cpSync(join(readDir, native), join(PREBUILD_DIR, filename))
+      fs.cpSync(join(readDir, native), join(writePath, filename))
     }
+    console.log("copied something")
   }
 }
 
@@ -39,8 +41,9 @@ function cleanup(evt?: number) {
     )
     console.error(error(evt))
   }
-  if (fs.existsSync(PREBUILD_DIR)) {
-    fs.rmSync(PREBUILD_DIR, { recursive: true })
+  const path = join(process.execPath, PREBUILDS)
+  if (fs.existsSync(path)) {
+    fs.rmSync(path, { recursive: true })
   }
 }
 

packages/cli/tsconfig.build.json

@@ -10,7 +10,12 @@
     "incremental": true,
     "types": [ "node", "jest" ],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"]
+    }
   },
   "include": [
     "src/**/*"

packages/cli/tsconfig.json

@@ -5,12 +5,7 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": ".",
-    "resolveJsonModule": true,
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"]
-    }
+    "resolveJsonModule": true
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],

packages/server/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-slim
+FROM node:18-slim
 
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"

packages/server/scripts/test.sh

@@ -5,8 +5,9 @@ if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  echo "jest --coverage --runInBand --forceExit --bail"
-  jest --coverage --runInBand --forceExit --bail
+  node ../../node_modules/jest/bin/jest.js --version
+  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
+  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2 --forceExit"

packages/server/src/api/controllers/row/external.ts

@@ -44,7 +44,7 @@ export async function handleRequest(
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = ctx.params.tableId
-  const { id, ...rowData } = ctx.request.body
+  const { _id, ...rowData } = ctx.request.body
 
   const validateResult = await sdk.rows.utils.validate({
     row: rowData,
@@ -54,10 +54,10 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     throw { validation: validateResult.errors }
   }
   const response = await handleRequest(Operation.UPDATE, tableId, {
-    id: breakRowIdField(id),
+    id: breakRowIdField(_id),
     row: rowData,
   })
-  const row = await sdk.rows.external.getRow(tableId, id, {
+  const row = await sdk.rows.external.getRow(tableId, _id, {
     relationships: true,
   })
   const table = await sdk.tables.getTable(tableId)
@@ -104,9 +104,9 @@ export async function find(ctx: UserCtx) {
 
 export async function destroy(ctx: UserCtx) {
   const tableId = ctx.params.tableId
-  const id = ctx.request.body._id
+  const _id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
-    id: breakRowIdField(id),
+    id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as { row: Row }
   return { response: { ok: true }, row }

packages/server/src/threads/automation.ts

@@ -486,10 +486,13 @@ class Orchestrator {
     const end = performance.now()
     const executionTime = end - start
 
-    console.info(`Execution time: ${executionTime} milliseconds`, {
-      _logKey: "automation",
-      executionTime,
-    })
+    console.info(
+      `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
+      {
+        _logKey: "automation",
+        executionTime,
+      }
+    )
 
     // store the logs for the automation run
     try {

packages/worker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-alpine
+FROM node:18-alpine
 
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"

scripts/build.js

@@ -43,7 +43,6 @@ function runBuild(entry, outfile) {
       TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }),
       nodeExternalsPlugin(),
     ],
-    target: "node14",
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",