Merge branch 'develop' of github.com:Budibase/budibase into feature/BUDI-7108

.github/workflows/budibase_ci.yml

@@ -25,20 +25,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn lint
 
   build:
@@ -46,20 +46,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       # Run build all the projects
       - run: yarn build
       # Check the types of the projects built via esbuild
@@ -70,20 +70,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
       - uses: codecov/codecov-action@v3
         with:
@@ -96,21 +96,22 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
-      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
+      - run: yarn --frozen-lockfile
+      - name: Test worker and server
+        run: yarn test --scope=@budibase/worker --scope=@budibase/server
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos
@@ -119,7 +120,7 @@ jobs:
 
   test-pro:
     runs-on: ubuntu-latest
-    if: github.repository == github.event.pull_request.head.repo.full_name
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
@@ -127,12 +128,12 @@ jobs:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn test --scope=@budibase/pro
 
   integration-test:
@@ -140,20 +141,20 @@ jobs:
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
-        if: github.repository == github.event.pull_request.head.repo.full_name
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
         with:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
       - name: Checkout repo only
         uses: actions/checkout@v3
-        if: github.repository != github.event.pull_request.head.repo.full_name
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
 
-      - name: Use Node.js 14.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 14.x
+          node-version: 18.x
           cache: "yarn"
-      - run: yarn
+      - run: yarn --frozen-lockfile
       - run: yarn build --projects=@budibase/server,@budibase/worker,@budibase/client
       - name: Run tests
         run: |
@@ -166,7 +167,7 @@ jobs:
 
   check-pro-submodule:
     runs-on: ubuntu-latest
-    if: github.repository == github.event.pull_request.head.repo.full_name
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
     steps:
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
@@ -190,6 +191,8 @@ jobs:
             base_commit=$(git rev-parse origin/develop)
           fi
 
+          echo "target_branch=$branch"
+          echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
           echo "pro_commit=$pro_commit"
           echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
           echo "base_commit=$base_commit"
@@ -204,7 +207,7 @@ jobs:
             const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}';
 
             if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the develop branch.');
+              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.');
               console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md')
               process.exit(1);
             } else {

.github/workflows/check_unreleased_changes.yml

@@ -0,0 +1,29 @@
+name: check_unreleased_changes
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check_unreleased:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for unreleased changes
+        env:
+          REPO: "Budibase/budibase"
+          TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+        run: |
+          RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
+              "https://api.github.com/repos/$REPO/releases/latest" | \
+              jq -r .published_at)
+          COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
+              "https://api.github.com/repos/$REPO/commits/master" | \
+              jq -r .commit.committer.date)
+          RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s")
+          COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s")
+          if (( COMMIT_SECONDS > RELEASE_SECONDS )); then
+              echo "There are unreleased changes. Please release these changes before merging."
+              exit 1
+          fi
+          echo "No unreleased changes detected."

.github/workflows/release-develop.yml

@@ -44,7 +44,7 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 18.x
 
       - run: yarn install --frozen-lockfile
       - name: Update versions

.github/workflows/release-master.yml

@@ -60,9 +60,9 @@ jobs:
       - name: "Get Current tag"
         id: currenttag
         run: |
-          version=v$(./scripts/getCurrentVersion.sh)
-          echo 'Using tag $version'
-          echo "::set-output name=tag::$resversionult"
+          version=$(./scripts/getCurrentVersion.sh)
+          echo "Using tag $version"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
 
       - name: Build/release Docker images
         run: |
@@ -71,7 +71,7 @@ jobs:
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.tag }}
+          BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
 
   release-helm-chart:
     needs: [release-images]

.github/workflows/release-singleimage.yml

@@ -1,4 +1,4 @@
-name: release-singleimage
+name: Deploy Budibase Single Container Image to DockerHub
 
 on:
   workflow_dispatch:
@@ -8,13 +8,20 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   REGISTRY_URL: registry.hub.docker.com
 jobs:
-  build-amd64:
-    name: "build-amd64"
+  build:
+    name: "build"
     runs-on: ubuntu-latest
     strategy:
       matrix:
         node-version: [14.x]
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Fail if not a tag
         run: |
           if [[ $GITHUB_REF != refs/tags/* ]]; then 
@@ -27,12 +34,14 @@ jobs:
           submodules: true
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
           fetch-depth: 0
+
       - name: Fail if tag is not in master
         run: |
           if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
             echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
             exit 1
           fi
+
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
@@ -68,139 +77,9 @@ jobs:
         with:
           context: .
           push: true
-          platforms: linux/amd64
-          tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
+          platforms: linux/amd64,linux/arm64
+          tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
           file: ./hosting/single/Dockerfile
-
-      - name: Tag and release Budibase Azure App Service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64
-          build-args: TARGETBUILD=aas
-          tags: budibase/budibase-aas,budibase/budibase-aas:v${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
-
-  build-arm64:
-    name: "build-arm64"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Runt Yarn Lint
-        run: yarn lint
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build:docker:pre
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-      - name: Tag and release Budibase service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/arm64
-          tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile
-
-  build-aas:
-    name: "build-aas"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [14.x]
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Runt Yarn Lint
-        run: yarn lint
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build:docker:pre
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
       - name: Tag and release Budibase Azure App Service docker image
         uses: docker/build-push-action@v2
         with:

.nvmrc

@@ -1 +1 @@
-v14.20.1
+v18.17.0

.tool-versions

@@ -1,3 +1,3 @@
-nodejs 14.21.3
+nodejs 18.17.0
 python 3.10.0
-yarn 1.22.19
+yarn 1.22.19

charts/budibase/templates/app-service-deployment.yaml

@@ -120,6 +120,8 @@ spec:
         {{ end }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.apps.logLevel | quote }}
         - name: REDIS_PASSWORD

charts/budibase/templates/worker-service-deployment.yaml

@@ -116,6 +116,8 @@ spec:
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY
           value: {{ .Values.globals.multiTenancy | quote }}
+        - name: OFFLINE_MODE
+          value: {{ .Values.globals.offlineMode | quote }}
         - name: LOG_LEVEL
           value: {{ .Values.services.worker.logLevel | quote }}
         - name: REDIS_PASSWORD

charts/budibase/values.yaml

@@ -82,6 +82,7 @@ globals:
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  offlineMode: "0" # set to 1 to enable offline mode
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""

docs/CONTRIBUTING.md

@@ -90,7 +90,7 @@ Component libraries are collections of components as well as the definition of t
 
 #### 1. Prerequisites
 
-- NodeJS version `14.x.x`
+- NodeJS version `18.x.x`
 - Python version `3.x`
 
 ### Using asdf (recommended)

hosting/single/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-slim as build
+FROM node:18-slim as build
 
 # install node-gyp dependencies
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.29-alpha.19",
+  "version": "2.9.21-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -109,7 +109,7 @@
     "@budibase/types": "0.0.0"
   },
   "engines": {
-    "node": ">=14.0.0 <15.0.0"
+    "node": ">=18.0.0 <19.0.0"
   },
   "dependencies": {}
 }

packages/backend-core/.npmignore

@@ -0,0 +1,4 @@
+*
+!dist/**/*
+dist/tsconfig.build.tsbuildinfo
+!package.json

packages/backend-core/package.json

@@ -14,7 +14,7 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node ./scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
     "test": "bash scripts/test.sh",
@@ -88,5 +88,20 @@
     "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }

packages/backend-core/plugins.ts

@@ -1 +0,0 @@
-export * from "./src/plugin"

packages/backend-core/scripts/build.js

@@ -0,0 +1,6 @@
+#!/usr/bin/node
+const coreBuild = require("../../../scripts/build")
+
+coreBuild("./src/plugin/index.ts", "./dist/plugins.js")
+coreBuild("./src/index.ts", "./dist/index.js")
+coreBuild("./tests/index.ts", "./dist/tests.js")

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -80,7 +80,7 @@ function getRedisConfig() {
 
 export function setupEnv(...envs: any[]) {
   const couch = getCouchConfig(),
-    minio = getCouchConfig(),
+    minio = getMinioConfig(),
     redis = getRedisConfig()
   const configs = [
     { key: "COUCH_DB_PORT", value: couch.port },

packages/backend-core/tsconfig.build.json

@@ -12,7 +12,11 @@
     "declaration": true,
     "types": ["node", "jest"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/shared-core": ["../shared-core/src"]
+    }
   },
   "include": ["**/*.js", "**/*.ts"],
   "exclude": [

packages/backend-core/tsconfig.json

@@ -1,12 +1,4 @@
 {
   "extends": "./tsconfig.build.json",
-  "compilerOptions": {
-    "composite": true,
-    "baseUrl": ".",
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/shared-core": ["../shared-core/src"]
-    }
-  },
   "exclude": ["node_modules", "dist"]
 }

packages/bbui/package.json

@@ -98,8 +98,7 @@
           {
             "projects": [
               "@budibase/string-templates",
-              "@budibase/shared-core",
-              "@budibase/types"
+              "@budibase/shared-core"
             ],
             "target": "build"
           }

packages/builder/package.json

@@ -133,9 +133,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
-              "@budibase/string-templates",
-              "@budibase/types"
+              "@budibase/string-templates"
             ],
             "target": "build"
           }
@@ -145,9 +143,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
-              "@budibase/string-templates",
-              "@budibase/types"
+              "@budibase/string-templates"
             ],
             "target": "build"
           }
@@ -157,9 +153,7 @@
         "dependsOn": [
           {
             "projects": [
-              "@budibase/shared-core",
-              "@budibase/string-templates",
-              "@budibase/types"
+              "@budibase/string-templates"
             ],
             "target": "build"
           }

packages/builder/src/pages/builder/app/[application]/data/_layout.svelte

@@ -2,14 +2,15 @@
   import { Button, Layout } from "@budibase/bbui"
   import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
   import Panel from "components/design/Panel.svelte"
-  import { isActive, goto, redirect } from "@roxi/routify"
+  import { isActive, redirect, goto, params } from "@roxi/routify"
   import BetaButton from "./_components/BetaButton.svelte"
   import { datasources } from "stores/backend"
 
   $: {
     // If we ever don't have any data other than the users table, prompt the
     // user to add some
-    if (!$datasources.hasData) {
+    // Don't redirect if setting up google sheets, or we lose the query parameter
+    if (!$datasources.hasData && !$params["?continue_google_setup"]) {
       $redirect("./new")
     }
   }

packages/builder/vite.config.js

@@ -127,6 +127,14 @@ export default defineConfig(({ mode }) => {
           find: "helpers",
           replacement: path.resolve("./src/helpers"),
         },
+        {
+          find: "@budibase/types",
+          replacement: path.resolve("../types/src"),
+        },
+        {
+          find: "@budibase/shared-core",
+          replacement: path.resolve("../shared-core/src"),
+        },
       ],
     },
   }

packages/cli/package.json

@@ -2,16 +2,16 @@
   "name": "@budibase/cli",
   "version": "0.0.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
-  "main": "dist/src/index.js",
+  "main": "dist/index.js",
   "bin": {
-    "budi": "dist/src/index.js"
+    "budi": "dist/index.js"
   },
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {
     "prebuild": "rm -rf prebuilds 2> /dev/null && cp -r ../../node_modules/leveldown/prebuilds prebuilds",
     "rename": "renamer --find .node --replace .fake 'prebuilds/**'",
-    "tsc": "tsc -p tsconfig.build.json",
+    "tsc": "node ../../scripts/build.js",
     "pkg": "pkg . --out-path build --no-bytecode --public --public-packages \"*\" -C GZip",
     "build": "yarn prebuild && yarn rename && yarn tsc && yarn pkg && yarn postbuild",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
@@ -19,12 +19,11 @@
   },
   "pkg": {
     "targets": [
-      "node16-linux",
-      "node16-win",
-      "node16-macos"
+      "node18-linux",
+      "node18-win",
+      "node18-macos"
     ],
     "assets": [
-      "node_modules/@budibase/backend-core/dist/**/*",
       "prebuilds/**/*"
     ],
     "outputPath": "build"
@@ -64,20 +63,5 @@
     "renamer": "^4.0.0",
     "ts-node": "^10.9.1",
     "typescript": "4.7.3"
-  },
-  "nx": {
-    "targets": {
-      "build": {
-        "dependsOn": [
-          {
-            "projects": [
-              "@budibase/backend-core",
-              "@budibase/string-templates"
-            ],
-            "target": "build"
-          }
-        ]
-      }
-    }
   }
 }

packages/cli/src/exec.ts

@@ -1,5 +1,6 @@
 import util from "util"
-const runCommand = util.promisify(require("child_process").exec)
+import childProcess from "child_process"
+const runCommand = util.promisify(childProcess.exec)
 
 export async function exec(command: string, dir = "./") {
   const { stdout } = await runCommand(command, { cwd: dir })
@@ -16,12 +17,12 @@ export async function utilityInstalled(utilName: string) {
 }
 
 export async function runPkgCommand(command: string, dir = "./") {
-  const yarn = await exports.utilityInstalled("yarn")
-  const npm = await exports.utilityInstalled("npm")
+  const yarn = await utilityInstalled("yarn")
+  const npm = await utilityInstalled("npm")
   if (!yarn && !npm) {
     throw new Error("Must have yarn or npm installed to run build.")
   }
   const npmCmd = command === "install" ? `npm ${command}` : `npm run ${command}`
   const cmd = yarn ? `yarn ${command} --ignore-engines` : npmCmd
-  await exports.exec(cmd, dir)
+  await exec(cmd, dir)
 }

packages/cli/src/prebuilds.ts

@@ -5,7 +5,7 @@ import { error } from "./utils"
 
 const PREBUILDS = "prebuilds"
 const ARCH = `${os.platform()}-${os.arch()}`
-const PREBUILD_DIR = join(process.execPath, "..", PREBUILDS, ARCH)
+const PREBUILD_DIR = join(process.execPath, "..", "cli", PREBUILDS, ARCH)
 
 // running as built CLI pkg bundle
 if (!process.argv[0].includes("node")) {
@@ -13,17 +13,19 @@ if (!process.argv[0].includes("node")) {
 }
 
 function checkForBinaries() {
-  const readDir = join(__filename, "..", "..", "..", PREBUILDS, ARCH)
+  const readDir = join(__filename, "..", "..", "..", "cli", PREBUILDS, ARCH)
   if (fs.existsSync(PREBUILD_DIR) || !fs.existsSync(readDir)) {
     return
   }
   const natives = fs.readdirSync(readDir)
   if (fs.existsSync(readDir)) {
-    fs.mkdirSync(PREBUILD_DIR, { recursive: true })
+    const writePath = join(process.execPath, PREBUILDS, ARCH)
+    fs.mkdirSync(writePath, { recursive: true })
     for (let native of natives) {
       const filename = `${native.split(".fake")[0]}.node`
-      fs.cpSync(join(readDir, native), join(PREBUILD_DIR, filename))
+      fs.cpSync(join(readDir, native), join(writePath, filename))
     }
+    console.log("copied something")
   }
 }
 
@@ -39,8 +41,9 @@ function cleanup(evt?: number) {
     )
     console.error(error(evt))
   }
-  if (fs.existsSync(PREBUILD_DIR)) {
-    fs.rmSync(PREBUILD_DIR, { recursive: true })
+  const path = join(process.execPath, PREBUILDS)
+  if (fs.existsSync(path)) {
+    fs.rmSync(path, { recursive: true })
   }
 }
 

packages/cli/tsconfig.build.json

@@ -10,7 +10,12 @@
     "incremental": true,
     "types": [ "node", "jest" ],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"]
+    }
   },
   "include": [
     "src/**/*"

packages/cli/tsconfig.json

@@ -5,12 +5,7 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": ".",
-    "resolveJsonModule": true,
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"]
-    }
+    "resolveJsonModule": true
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],

packages/pro

@@ -1 +1 @@
-Subproject commit cf3bef2aad9c739111b306fd0712397adc363f81
+Subproject commit ecee8071ebe0f98a5bb19646954e373264be210d

packages/server/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-slim
+FROM node:18-slim
 
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"

packages/server/scripts/test.sh

@@ -5,8 +5,9 @@ if [[ -n $CI ]]
 then
   # --runInBand performs better in ci where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  echo "jest --coverage --runInBand --forceExit --bail"
-  jest --coverage --runInBand --forceExit --bail
+  node ../../node_modules/jest/bin/jest.js --version
+  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
+  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
 else
   # --maxWorkers performs better in development
   echo "jest --coverage --maxWorkers=2 --forceExit"

packages/server/src/api/controllers/application.ts

@@ -317,6 +317,11 @@ async function performAppCreate(ctx: UserCtx) {
         }
       })
 
+      // Keep existing validation setting
+      if (!existing.features?.componentValidation) {
+        newApplication.features!.componentValidation = false
+      }
+
       // Migrate navigation settings and screens if required
       if (existing) {
         const navigation = await migrateAppNavigation()

packages/server/src/api/controllers/row/external.ts

@@ -44,7 +44,7 @@ export async function handleRequest(
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = ctx.params.tableId
-  const { id, ...rowData } = ctx.request.body
+  const { _id, ...rowData } = ctx.request.body
 
   const validateResult = await sdk.rows.utils.validate({
     row: rowData,
@@ -54,10 +54,10 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     throw { validation: validateResult.errors }
   }
   const response = await handleRequest(Operation.UPDATE, tableId, {
-    id: breakRowIdField(id),
+    id: breakRowIdField(_id),
     row: rowData,
   })
-  const row = await sdk.rows.external.getRow(tableId, id, {
+  const row = await sdk.rows.external.getRow(tableId, _id, {
     relationships: true,
   })
   const table = await sdk.tables.getTable(tableId)
@@ -104,9 +104,9 @@ export async function find(ctx: UserCtx) {
 
 export async function destroy(ctx: UserCtx) {
   const tableId = ctx.params.tableId
-  const id = ctx.request.body._id
+  const _id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
-    id: breakRowIdField(id),
+    id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as { row: Row }
   return { response: { ok: true }, row }

packages/server/src/api/controllers/row/index.ts

@@ -11,6 +11,9 @@ import {
   Row,
   PatchRowRequest,
   PatchRowResponse,
+  SearchRowResponse,
+  SearchRowRequest,
+  SearchParams,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -197,10 +200,10 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
   ctx.body = response
 }
 
-export async function search(ctx: any) {
+export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   const tableId = utils.getTableId(ctx)
 
-  const searchParams = {
+  const searchParams: SearchParams = {
     ...ctx.request.body,
     tableId,
   }

packages/server/src/api/controllers/row/views.ts

@@ -1,14 +1,18 @@
 import { quotas } from "@budibase/pro"
 import {
   UserCtx,
-  SearchResponse,
-  SortOrder,
-  SortType,
   ViewV2,
+  SearchRowResponse,
+  SearchViewRowRequest,
+  RequiredKeys,
+  SearchParams,
 } from "@budibase/types"
+import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
 
-export async function searchView(ctx: UserCtx<void, SearchResponse>) {
+export async function searchView(
+  ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
+) {
   const { viewId } = ctx.params
 
   const view = await sdk.views.get(viewId)
@@ -29,49 +33,35 @@ export async function searchView(ctx: UserCtx<void, SearchResponse>) {
     undefined
 
   ctx.status = 200
-  const result = await quotas.addQuery(
-    () =>
-      sdk.rows.search({
-        tableId: view.tableId,
-        query: view.query || {},
-        fields: viewFields,
-        ...getSortOptions(ctx, view),
-      }),
-    {
-      datasourceId: view.tableId,
-    }
-  )
+
+  const { body } = ctx.request
+  const query = dataFilters.buildLuceneQuery(view.query || [])
+
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
+    RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
+    tableId: view.tableId,
+    query,
+    fields: viewFields,
+    ...getSortOptions(body, view),
+    limit: body.limit,
+    bookmark: body.bookmark,
+    paginate: body.paginate,
+  }
+
+  const result = await quotas.addQuery(() => sdk.rows.search(searchOptions), {
+    datasourceId: view.tableId,
+  })
 
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
 
-function getSortOptions(
-  ctx: UserCtx,
-  view: ViewV2
-):
-  | {
-      sort: string
-      sortOrder?: SortOrder
-      sortType?: SortType
-    }
-  | undefined {
-  const { sort_column, sort_order, sort_type } = ctx.query
-  if (Array.isArray(sort_column)) {
-    ctx.throw(400, "sort_column cannot be an array")
-  }
-  if (Array.isArray(sort_order)) {
-    ctx.throw(400, "sort_order cannot be an array")
-  }
-  if (Array.isArray(sort_type)) {
-    ctx.throw(400, "sort_type cannot be an array")
-  }
-
-  if (sort_column) {
+function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
+  if (request.sort) {
     return {
-      sort: sort_column,
-      sortOrder: sort_order as SortOrder,
-      sortType: sort_type as SortType,
+      sort: request.sort,
+      sortOrder: request.sortOrder,
+      sortType: request.sortType,
     }
   }
   if (view.sort) {
@@ -82,5 +72,9 @@ function getSortOptions(
     }
   }
 
-  return
+  return {
+    sort: undefined,
+    sortOrder: undefined,
+    sortType: undefined,
+  }
 }

packages/server/src/api/routes/row.ts

@@ -269,7 +269,7 @@ router
   )
 
 router
-  .get(
+  .post(
     "/api/v2/views/:viewId/search",
     authorized(PermissionType.VIEW, PermissionLevel.READ),
     rowController.views.searchView

packages/server/src/api/routes/tests/row.spec.ts

@@ -813,252 +813,6 @@ describe("/rows", () => {
     })
   })
 
-  describe("view search", () => {
-    function userTable(): Table {
-      return {
-        name: "user",
-        type: "user",
-        schema: {
-          name: {
-            type: FieldType.STRING,
-            name: "name",
-            constraints: { type: "string" },
-          },
-          age: {
-            type: FieldType.NUMBER,
-            name: "age",
-            constraints: {},
-          },
-        },
-      }
-    }
-
-    it("returns table rows from view", async () => {
-      const table = await config.createTable(userTable())
-      const rows = []
-      for (let i = 0; i < 10; i++) {
-        rows.push(await config.createRow({ tableId: table._id }))
-      }
-
-      const createViewResponse = await config.api.viewV2.create()
-      const response = await config.api.viewV2.search(createViewResponse.id)
-
-      expect(response.body.rows).toHaveLength(10)
-      expect(response.body).toEqual({
-        rows: expect.arrayContaining(rows.map(expect.objectContaining)),
-      })
-    })
-
-    it("searching respects the view filters", async () => {
-      const table = await config.createTable(userTable())
-      const expectedRows = []
-      for (let i = 0; i < 10; i++)
-        await config.createRow({
-          tableId: table._id,
-          name: generator.name(),
-          age: generator.integer({ min: 10, max: 30 }),
-        })
-
-      for (let i = 0; i < 5; i++)
-        expectedRows.push(
-          await config.createRow({
-            tableId: table._id,
-            name: generator.name(),
-            age: 40,
-          })
-        )
-
-      const createViewResponse = await config.api.viewV2.create({
-        query: { equal: { age: 40 } },
-      })
-
-      const response = await config.api.viewV2.search(createViewResponse.id)
-
-      expect(response.body.rows).toHaveLength(5)
-      expect(response.body).toEqual({
-        rows: expect.arrayContaining(expectedRows.map(expect.objectContaining)),
-      })
-    })
-
-    const sortTestOptions: [
-      {
-        field: string
-        order?: SortOrder
-        type?: SortType
-      },
-      string[]
-    ][] = [
-      [
-        {
-          field: "name",
-          order: SortOrder.ASCENDING,
-          type: SortType.STRING,
-        },
-        ["Alice", "Bob", "Charly", "Danny"],
-      ],
-      [
-        {
-          field: "name",
-        },
-        ["Alice", "Bob", "Charly", "Danny"],
-      ],
-      [
-        {
-          field: "name",
-          order: SortOrder.DESCENDING,
-        },
-        ["Danny", "Charly", "Bob", "Alice"],
-      ],
-      [
-        {
-          field: "name",
-          order: SortOrder.DESCENDING,
-          type: SortType.STRING,
-        },
-        ["Danny", "Charly", "Bob", "Alice"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.ASCENDING,
-          type: SortType.number,
-        },
-        ["Danny", "Alice", "Charly", "Bob"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.ASCENDING,
-        },
-        ["Danny", "Alice", "Charly", "Bob"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.DESCENDING,
-        },
-        ["Bob", "Charly", "Alice", "Danny"],
-      ],
-      [
-        {
-          field: "age",
-          order: SortOrder.DESCENDING,
-          type: SortType.number,
-        },
-        ["Bob", "Charly", "Alice", "Danny"],
-      ],
-    ]
-
-    it.each(sortTestOptions)(
-      "allow sorting (%s)",
-      async (sortParams, expected) => {
-        await config.createTable(userTable())
-        const users = [
-          { name: "Alice", age: 25 },
-          { name: "Bob", age: 30 },
-          { name: "Charly", age: 27 },
-          { name: "Danny", age: 15 },
-        ]
-        for (const user of users) {
-          await config.createRow({
-            tableId: config.table!._id,
-            ...user,
-          })
-        }
-
-        const createViewResponse = await config.api.viewV2.create({
-          sort: sortParams,
-        })
-
-        const response = await config.api.viewV2.search(createViewResponse.id)
-
-        expect(response.body.rows).toHaveLength(4)
-        expect(response.body).toEqual({
-          rows: expected.map(name => expect.objectContaining({ name })),
-        })
-      }
-    )
-
-    it.each(sortTestOptions)(
-      "allow override the default view sorting (%s)",
-      async (sortParams, expected) => {
-        await config.createTable(userTable())
-        const users = [
-          { name: "Alice", age: 25 },
-          { name: "Bob", age: 30 },
-          { name: "Charly", age: 27 },
-          { name: "Danny", age: 15 },
-        ]
-        for (const user of users) {
-          await config.createRow({
-            tableId: config.table!._id,
-            ...user,
-          })
-        }
-
-        const createViewResponse = await config.api.viewV2.create({
-          sort: {
-            field: "name",
-            order: SortOrder.ASCENDING,
-            type: SortType.STRING,
-          },
-        })
-
-        const response = await config.api.viewV2.search(createViewResponse.id, {
-          sort: {
-            column: sortParams.field,
-            order: sortParams.order,
-            type: sortParams.type,
-          },
-        })
-
-        expect(response.body.rows).toHaveLength(4)
-        expect(response.body).toEqual({
-          rows: expected.map(name => expect.objectContaining({ name })),
-        })
-      }
-    )
-
-    it("when schema is defined, defined columns and row attributes are returned", async () => {
-      const table = await config.createTable(userTable())
-      const rows = []
-      for (let i = 0; i < 10; i++) {
-        rows.push(
-          await config.createRow({
-            tableId: table._id,
-            name: generator.name(),
-            age: generator.age(),
-          })
-        )
-      }
-
-      const view = await config.api.viewV2.create({
-        schema: { name: {} },
-      })
-      const response = await config.api.viewV2.search(view.id)
-
-      expect(response.body.rows).toHaveLength(10)
-      expect(response.body.rows).toEqual(
-        expect.arrayContaining(
-          rows.map(r => ({
-            ...expectAnyInternalColsAttributes,
-            _viewId: view.id,
-            name: r.name,
-          }))
-        )
-      )
-    })
-
-    it("views without data can be returned", async () => {
-      const table = await config.createTable(userTable())
-
-      const createViewResponse = await config.api.viewV2.create()
-      const response = await config.api.viewV2.search(createViewResponse.id)
-
-      expect(response.body.rows).toHaveLength(0)
-    })
-  })
-
   describe("view 2.0", () => {
     function userTable(): Table {
       return {
@@ -1261,5 +1015,327 @@ describe("/rows", () => {
         await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
       })
     })
+
+    describe("view search", () => {
+      function userTable(): Table {
+        return {
+          name: "user",
+          type: "user",
+          schema: {
+            name: {
+              type: FieldType.STRING,
+              name: "name",
+              constraints: { type: "string" },
+            },
+            age: {
+              type: FieldType.NUMBER,
+              name: "age",
+              constraints: {},
+            },
+          },
+        }
+      }
+
+      it("returns table rows from view", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(await config.createRow({ tableId: table._id }))
+        }
+
+        const createViewResponse = await config.api.viewV2.create()
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(10)
+        expect(response.body).toEqual({
+          rows: expect.arrayContaining(rows.map(expect.objectContaining)),
+        })
+      })
+
+      it("searching respects the view filters", async () => {
+        const table = await config.createTable(userTable())
+        const expectedRows = []
+        for (let i = 0; i < 10; i++)
+          await config.createRow({
+            tableId: table._id,
+            name: generator.name(),
+            age: generator.integer({ min: 10, max: 30 }),
+          })
+
+        for (let i = 0; i < 5; i++)
+          expectedRows.push(
+            await config.createRow({
+              tableId: table._id,
+              name: generator.name(),
+              age: 40,
+            })
+          )
+
+        const createViewResponse = await config.api.viewV2.create({
+          query: [{ operator: "equal", field: "age", value: 40 }],
+        })
+
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(5)
+        expect(response.body).toEqual({
+          rows: expect.arrayContaining(
+            expectedRows.map(expect.objectContaining)
+          ),
+        })
+      })
+
+      const sortTestOptions: [
+        {
+          field: string
+          order?: SortOrder
+          type?: SortType
+        },
+        string[]
+      ][] = [
+        [
+          {
+            field: "name",
+            order: SortOrder.ASCENDING,
+            type: SortType.STRING,
+          },
+          ["Alice", "Bob", "Charly", "Danny"],
+        ],
+        [
+          {
+            field: "name",
+          },
+          ["Alice", "Bob", "Charly", "Danny"],
+        ],
+        [
+          {
+            field: "name",
+            order: SortOrder.DESCENDING,
+          },
+          ["Danny", "Charly", "Bob", "Alice"],
+        ],
+        [
+          {
+            field: "name",
+            order: SortOrder.DESCENDING,
+            type: SortType.STRING,
+          },
+          ["Danny", "Charly", "Bob", "Alice"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.ASCENDING,
+            type: SortType.number,
+          },
+          ["Danny", "Alice", "Charly", "Bob"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.ASCENDING,
+          },
+          ["Danny", "Alice", "Charly", "Bob"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.DESCENDING,
+          },
+          ["Bob", "Charly", "Alice", "Danny"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.DESCENDING,
+            type: SortType.number,
+          },
+          ["Bob", "Charly", "Alice", "Danny"],
+        ],
+      ]
+
+      it.each(sortTestOptions)(
+        "allow sorting (%s)",
+        async (sortParams, expected) => {
+          await config.createTable(userTable())
+          const users = [
+            { name: "Alice", age: 25 },
+            { name: "Bob", age: 30 },
+            { name: "Charly", age: 27 },
+            { name: "Danny", age: 15 },
+          ]
+          for (const user of users) {
+            await config.createRow({
+              tableId: config.table!._id,
+              ...user,
+            })
+          }
+
+          const createViewResponse = await config.api.viewV2.create({
+            sort: sortParams,
+          })
+
+          const response = await config.api.viewV2.search(createViewResponse.id)
+
+          expect(response.body.rows).toHaveLength(4)
+          expect(response.body).toEqual({
+            rows: expected.map(name => expect.objectContaining({ name })),
+          })
+        }
+      )
+
+      it.each(sortTestOptions)(
+        "allow override the default view sorting (%s)",
+        async (sortParams, expected) => {
+          await config.createTable(userTable())
+          const users = [
+            { name: "Alice", age: 25 },
+            { name: "Bob", age: 30 },
+            { name: "Charly", age: 27 },
+            { name: "Danny", age: 15 },
+          ]
+          for (const user of users) {
+            await config.createRow({
+              tableId: config.table!._id,
+              ...user,
+            })
+          }
+
+          const createViewResponse = await config.api.viewV2.create({
+            sort: {
+              field: "name",
+              order: SortOrder.ASCENDING,
+              type: SortType.STRING,
+            },
+          })
+
+          const response = await config.api.viewV2.search(
+            createViewResponse.id,
+            {
+              sort: sortParams.field,
+              sortOrder: sortParams.order,
+              sortType: sortParams.type,
+            }
+          )
+
+          expect(response.body.rows).toHaveLength(4)
+          expect(response.body).toEqual({
+            rows: expected.map(name => expect.objectContaining({ name })),
+          })
+        }
+      )
+
+      it("when schema is defined, defined columns and row attributes are returned", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(
+            await config.createRow({
+              tableId: table._id,
+              name: generator.name(),
+              age: generator.age(),
+            })
+          )
+        }
+
+        const view = await config.api.viewV2.create({
+          schema: { name: {} },
+        })
+        const response = await config.api.viewV2.search(view.id)
+
+        expect(response.body.rows).toHaveLength(10)
+        expect(response.body.rows).toEqual(
+          expect.arrayContaining(
+            rows.map(r => ({
+              ...expectAnyInternalColsAttributes,
+              _viewId: view.id,
+              name: r.name,
+            }))
+          )
+        )
+      })
+
+      it("views without data can be returned", async () => {
+        const table = await config.createTable(userTable())
+
+        const createViewResponse = await config.api.viewV2.create()
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(0)
+      })
+
+      it("respects the limit parameter", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(await config.createRow({ tableId: table._id }))
+        }
+        const limit = generator.integer({ min: 1, max: 8 })
+
+        const createViewResponse = await config.api.viewV2.create()
+        const response = await config.api.viewV2.search(createViewResponse.id, {
+          limit,
+        })
+
+        expect(response.body.rows).toHaveLength(limit)
+      })
+
+      it("can handle pagination", async () => {
+        const table = await config.createTable(userTable())
+        const rows = []
+        for (let i = 0; i < 10; i++) {
+          rows.push(await config.createRow({ tableId: table._id }))
+        }
+        // rows.sort((a, b) => (a._id! > b._id! ? 1 : -1))
+
+        const createViewResponse = await config.api.viewV2.create()
+        const allRows = (await config.api.viewV2.search(createViewResponse.id))
+          .body.rows
+
+        const firstPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+          }
+        )
+        expect(firstPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(0, 4)),
+          totalRows: 10,
+          hasNextPage: true,
+          bookmark: expect.any(String),
+        })
+
+        const secondPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+            bookmark: firstPageResponse.body.bookmark,
+          }
+        )
+        expect(secondPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(4, 8)),
+          totalRows: 10,
+          hasNextPage: true,
+          bookmark: expect.any(String),
+        })
+
+        const lastPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+            bookmark: secondPageResponse.body.bookmark,
+          }
+        )
+        expect(lastPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(8)),
+          totalRows: 10,
+          hasNextPage: false,
+          bookmark: expect.any(String),
+        })
+      })
+    })
   })
 })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -62,7 +62,7 @@ describe("/v2/views", () => {
         name: generator.name(),
         tableId: config.table!._id!,
         primaryDisplay: generator.word(),
-        query: { allOr: false, equal: { field: "value" } },
+        query: [{ operator: "equal", field: "field", value: "value" }],
         sort: {
           field: "fieldToSort",
           order: SortOrder.DESCENDING,
@@ -190,7 +190,7 @@ describe("/v2/views", () => {
       const tableId = config.table!._id!
       await config.api.viewV2.update({
         ...view,
-        query: { equal: { newField: "thatValue" } },
+        query: [{ operator: "equal", field: "newField", value: "thatValue" }],
       })
 
       expect(await config.api.table.get(tableId)).toEqual({
@@ -198,7 +198,9 @@ describe("/v2/views", () => {
         views: {
           [view.name]: {
             ...view,
-            query: { equal: { newField: "thatValue" } },
+            query: [
+              { operator: "equal", field: "newField", value: "thatValue" },
+            ],
             schema: expect.anything(),
           },
         },
@@ -216,7 +218,13 @@ describe("/v2/views", () => {
         tableId,
         name: view.name,
         primaryDisplay: generator.word(),
-        query: { equal: { [generator.word()]: generator.word() } },
+        query: [
+          {
+            operator: "equal",
+            field: generator.word(),
+            value: generator.word(),
+          },
+        ],
         sort: {
           field: generator.word(),
           order: SortOrder.DESCENDING,
@@ -285,7 +293,7 @@ describe("/v2/views", () => {
         {
           ...view,
           tableId: generator.guid(),
-          query: { equal: { newField: "thatValue" } },
+          query: [{ operator: "equal", field: "newField", value: "thatValue" }],
         },
         { expectStatus: 404 }
       )

packages/server/src/sdk/app/rows/search.ts

@@ -1,23 +1,9 @@
-import { SearchFilters, SortOrder, SortType } from "@budibase/types"
+import { SearchFilters, SearchParams } from "@budibase/types"
 import { isExternalTable } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
 import { Format } from "../../../api/controllers/view/exporters"
 
-export interface SearchParams {
-  tableId: string
-  paginate?: boolean
-  query: SearchFilters
-  bookmark?: string
-  limit?: number
-  sort?: string
-  sortOrder?: SortOrder
-  sortType?: SortType
-  version?: string
-  disableEscaping?: boolean
-  fields?: string[]
-}
-
 export interface ViewParams {
   calculation: string
   group: string

packages/server/src/sdk/app/rows/search/external.ts

@@ -6,6 +6,7 @@ import {
   IncludeRelationship,
   Row,
   SearchFilters,
+  SearchParams,
 } from "@budibase/types"
 import * as exporters from "../../../../api/controllers/view/exporters"
 import sdk from "../../../../sdk"
@@ -13,7 +14,7 @@ import { handleRequest } from "../../../../api/controllers/row/external"
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { cleanExportRows } from "../utils"
 import { utils } from "@budibase/shared-core"
-import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import { ExportRowsParams, ExportRowsResult } from "../search"
 import { HTTPError, db } from "@budibase/backend-core"
 import pick from "lodash/pick"
 

packages/server/src/sdk/app/rows/search/internal.ts

@@ -12,7 +12,7 @@ import {
 } from "../../../../db/utils"
 import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
-import { Database, Row, Table } from "@budibase/types"
+import { Database, Row, Table, SearchParams } from "@budibase/types"
 import { cleanExportRows } from "../utils"
 import {
   Format,
@@ -28,7 +28,7 @@ import {
   getFromMemoryDoc,
 } from "../../../../api/controllers/view/utils"
 import sdk from "../../../../sdk"
-import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import { ExportRowsParams, ExportRowsResult } from "../search"
 import pick from "lodash/pick"
 
 export async function search(options: SearchParams) {

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -1,8 +1,15 @@
 import { GenericContainer } from "testcontainers"
 
-import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types"
+import {
+  Datasource,
+  FieldType,
+  Row,
+  SourceName,
+  Table,
+  SearchParams,
+} from "@budibase/types"
+
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
-import { SearchParams } from "../../search"
 import { search } from "../external"
 import {
   expectAnyExternalColsAttributes,

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -1,6 +1,5 @@
-import { FieldType, Row, Table } from "@budibase/types"
+import { FieldType, Row, Table, SearchParams } from "@budibase/types"
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
-import { SearchParams } from "../../search"
 import { search } from "../internal"
 import {
   expectAnyInternalColsAttributes,

packages/server/src/tests/utilities/api/viewV2.ts

@@ -1,13 +1,12 @@
 import {
   CreateViewRequest,
-  SortOrder,
-  SortType,
   UpdateViewRequest,
   DeleteRowRequest,
   PatchRowRequest,
   PatchRowResponse,
   Row,
   ViewV2,
+  SearchViewRowRequest,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -81,31 +80,12 @@ export class ViewV2API extends TestAPI {
 
   search = async (
     viewId: string,
-    options?: {
-      sort: {
-        column: string
-        order?: SortOrder
-        type?: SortType
-      }
-    },
+    params?: SearchViewRowRequest,
     { expectStatus } = { expectStatus: 200 }
   ) => {
-    const qs: [string, any][] = []
-    if (options?.sort.column) {
-      qs.push(["sort_column", options.sort.column])
-    }
-    if (options?.sort.order) {
-      qs.push(["sort_order", options.sort.order])
-    }
-    if (options?.sort.type) {
-      qs.push(["sort_type", options.sort.type])
-    }
-    let url = `/api/v2/views/${viewId}/search`
-    if (qs.length) {
-      url += "?" + qs.map(q => q.join("=")).join("&")
-    }
     return this.request
-      .get(url)
+      .post(`/api/v2/views/${viewId}/search`)
+      .send(params)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)

packages/server/src/threads/automation.ts

@@ -486,10 +486,13 @@ class Orchestrator {
     const end = performance.now()
     const executionTime = end - start
 
-    console.info(`Execution time: ${executionTime} milliseconds`, {
-      _logKey: "automation",
-      executionTime,
-    })
+    console.info(
+      `Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
+      {
+        _logKey: "automation",
+        executionTime,
+      }
+    )
 
     // store the logs for the automation run
     try {

packages/shared-core/.npmignore

@@ -0,0 +1,4 @@
+*
+!dist/**/*
+dist/tsconfig.build.tsbuildinfo
+!package.json

packages/shared-core/package.json

@@ -2,19 +2,13 @@
   "name": "@budibase/shared-core",
   "version": "0.0.0",
   "description": "Shared data utils",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
-  "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "require": "./src/index.ts"
-    }
-  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null"
@@ -26,5 +20,19 @@
     "concurrently": "^7.6.0",
     "rimraf": "3.0.2",
     "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
   }
 }

packages/shared-core/src/filters.ts

@@ -1,4 +1,12 @@
-import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types"
+import {
+  Datasource,
+  FieldType,
+  SearchFilter,
+  SearchQuery,
+  SearchQueryFields,
+  SortDirection,
+  SortType,
+} from "@budibase/types"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet } from "./helpers"
 
@@ -73,13 +81,13 @@ export const NoEmptyFilterStrings = [
   OperatorOptions.NotEquals.value,
   OperatorOptions.Contains.value,
   OperatorOptions.NotContains.value,
-] as (keyof QueryFields)[]
+] as (keyof SearchQueryFields)[]
 
 /**
  * Removes any fields that contain empty strings that would cause inconsistent
  * behaviour with how backend tables are filtered (no value means no filter).
  */
-const cleanupQuery = (query: Query) => {
+const cleanupQuery = (query: SearchQuery) => {
   if (!query) {
     return query
   }
@@ -110,66 +118,12 @@ const removeKeyNumbering = (key: string) => {
   }
 }
 
-type Filter = {
-  operator: keyof Query
-  field: string
-  type: any
-  value: any
-  externalType: keyof typeof SqlNumberTypeRangeMap
-}
-
-type Query = QueryFields & QueryConfig
-type QueryFields = {
-  string?: {
-    [key: string]: string
-  }
-  fuzzy?: {
-    [key: string]: string
-  }
-  range?: {
-    [key: string]: {
-      high: number | string
-      low: number | string
-    }
-  }
-  equal?: {
-    [key: string]: any
-  }
-  notEqual?: {
-    [key: string]: any
-  }
-  empty?: {
-    [key: string]: any
-  }
-  notEmpty?: {
-    [key: string]: any
-  }
-  oneOf?: {
-    [key: string]: any[]
-  }
-  contains?: {
-    [key: string]: any[]
-  }
-  notContains?: {
-    [key: string]: any[]
-  }
-  containsAny?: {
-    [key: string]: any[]
-  }
-}
-
-type QueryConfig = {
-  allOr?: boolean
-}
-
-type QueryFieldsType = keyof QueryFields
-
 /**
  * Builds a lucene JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
  */
-export const buildLuceneQuery = (filter: Filter[]) => {
-  let query: Query = {
+export const buildLuceneQuery = (filter: SearchFilter[]) => {
+  let query: SearchQuery = {
     string: {},
     fuzzy: {},
     range: {},
@@ -227,9 +181,13 @@ export const buildLuceneQuery = (filter: Filter[]) => {
       }
       if (operator.startsWith("range") && query.range) {
         const minint =
-          SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
+          SqlNumberTypeRangeMap[
+            externalType as keyof typeof SqlNumberTypeRangeMap
+          ]?.min || Number.MIN_SAFE_INTEGER
         const maxint =
-          SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
+          SqlNumberTypeRangeMap[
+            externalType as keyof typeof SqlNumberTypeRangeMap
+          ]?.max || Number.MAX_SAFE_INTEGER
         if (!query.range[field]) {
           query.range[field] = {
             low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
@@ -275,7 +233,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
  * @param docs the data
  * @param query the JSON lucene query
  */
-export const runLuceneQuery = (docs: any[], query?: Query) => {
+export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
   if (!docs || !Array.isArray(docs)) {
     return []
   }
@@ -289,7 +247,7 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
   // Iterates over a set of filters and evaluates a fail function against a doc
   const match =
     (
-      type: QueryFieldsType,
+      type: keyof SearchQueryFields,
       failFn: (docValue: any, testValue: any) => boolean
     ) =>
     (doc: any) => {
@@ -456,7 +414,7 @@ export const luceneLimit = (docs: any[], limit: string) => {
   return docs.slice(0, numLimit)
 }
 
-export const hasFilters = (query?: Query) => {
+export const hasFilters = (query?: SearchQuery) => {
   if (!query) {
     return false
   }

packages/shared-core/tsconfig.build.json

@@ -12,7 +12,10 @@
     "declaration": true,
     "types": ["node"],
     "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"]
+    }
   },
   "include": ["**/*.js", "**/*.ts"],
   "exclude": [

packages/shared-core/tsconfig.json

@@ -1,13 +1,4 @@
 {
   "extends": "./tsconfig.build.json",
-  "compilerOptions": {
-    "baseUrl": ".",
-    "rootDir": "./src",
-    "composite": true,
-    "tsBuildInfoFile": "dist/tsconfig.tsbuildinfo",
-    "paths": {
-      "@budibase/types": ["../../types/src"]
-    }
-  },
   "exclude": ["node_modules", "dist"]
 }

packages/types/.npmignore

@@ -0,0 +1,4 @@
+*
+!dist/**/*
+dist/tsconfig.build.tsbuildinfo
+!package.json

packages/types/package.json

@@ -2,19 +2,13 @@
   "name": "@budibase/types",
   "version": "0.0.0",
   "description": "Budibase types",
-  "main": "src/index.ts",
-  "types": "src/index.ts",
-  "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "require": "./src/index.ts"
-    }
-  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {
     "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node ../../scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "dev:builder": "yarn prebuild && tsc -p tsconfig.json --watch --preserveWatchOutput",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null"

packages/types/src/api/web/app/rows.ts

@@ -1,3 +1,4 @@
+import { SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
 
 export interface PatchRowRequest extends Row {
@@ -8,6 +9,14 @@ export interface PatchRowRequest extends Row {
 
 export interface PatchRowResponse extends Row {}
 
-export interface SearchResponse {
+export interface SearchRowRequest extends Omit<SearchParams, "tableId"> {}
+
+export interface SearchViewRowRequest
+  extends Pick<
+    SearchRowRequest,
+    "sort" | "sortOrder" | "sortType" | "limit" | "bookmark" | "paginate"
+  > {}
+
+export interface SearchRowResponse {
   rows: any[]
 }

packages/types/src/api/web/index.ts

@@ -8,3 +8,4 @@ export * from "./system"
 export * from "./app"
 export * from "./global"
 export * from "./pagination"
+export * from "./searchFilter"

packages/types/src/api/web/searchFilter.ts

@@ -0,0 +1,51 @@
+import { FieldType } from "../../documents"
+
+export type SearchFilter = {
+  operator: keyof SearchQuery
+  field: string
+  type?: FieldType
+  value: any
+  externalType?: string
+}
+
+export type SearchQuery = {
+  allOr?: boolean
+  string?: {
+    [key: string]: string
+  }
+  fuzzy?: {
+    [key: string]: string
+  }
+  range?: {
+    [key: string]: {
+      high: number | string
+      low: number | string
+    }
+  }
+  equal?: {
+    [key: string]: any
+  }
+  notEqual?: {
+    [key: string]: any
+  }
+  empty?: {
+    [key: string]: any
+  }
+  notEmpty?: {
+    [key: string]: any
+  }
+  oneOf?: {
+    [key: string]: any[]
+  }
+  contains?: {
+    [key: string]: any[]
+  }
+  notContains?: {
+    [key: string]: any[]
+  }
+  containsAny?: {
+    [key: string]: any[]
+  }
+}
+
+export type SearchQueryFields = Omit<SearchQuery, "allOr">

packages/types/src/documents/app/view.ts

@@ -1,6 +1,5 @@
-import { SortOrder, SortType } from "../../api"
-import { SearchFilters } from "../../sdk"
-import { TableSchema, UIFieldMetadata } from "./table"
+import { SearchFilter, SortOrder, SortType } from "../../api"
+import { UIFieldMetadata } from "./table"
 
 export interface View {
   name: string
@@ -20,7 +19,7 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilters
+  query?: SearchFilter[]
   sort?: {
     field: string
     order?: SortOrder

packages/types/src/sdk/index.ts

@@ -19,3 +19,4 @@ export * from "./user"
 export * from "./cli"
 export * from "./websocket"
 export * from "./permissions"
+export * from "./row"

packages/types/src/sdk/row.ts

@@ -0,0 +1,16 @@
+import { SortOrder, SortType } from "../api"
+import { SearchFilters } from "./search"
+
+export interface SearchParams {
+  tableId: string
+  paginate?: boolean
+  query: SearchFilters
+  bookmark?: string
+  limit?: number
+  sort?: string
+  sortOrder?: SortOrder
+  sortType?: SortType
+  version?: string
+  disableEscaping?: boolean
+  fields?: string[]
+}

packages/worker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-alpine
+FROM node:18-alpine
 
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh"
 LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh"

packages/worker/src/api/routes/validation/users.ts

@@ -1,12 +1,14 @@
 import { auth } from "@budibase/backend-core"
 import Joi from "joi"
 
+const OPTIONAL_STRING = Joi.string().allow(null, "")
+
 let schema: any = {
-  email: Joi.string().allow(null, ""),
-  password: Joi.string().allow(null, ""),
+  email: OPTIONAL_STRING,
+  password: OPTIONAL_STRING,
   forceResetPassword: Joi.boolean().optional(),
-  firstName: Joi.string().allow(null, ""),
-  lastName: Joi.string().allow(null, ""),
+  firstName: OPTIONAL_STRING,
+  lastName: OPTIONAL_STRING,
   builder: Joi.object({
     global: Joi.boolean().optional(),
     apps: Joi.array().optional(),
@@ -21,8 +23,8 @@ export const buildSelfSaveValidation = () => {
   schema = {
     password: Joi.string().optional(),
     forceResetPassword: Joi.boolean().optional(),
-    firstName: Joi.string().allow("").optional(),
-    lastName: Joi.string().allow("").optional(),
+    firstName: OPTIONAL_STRING,
+    lastName: OPTIONAL_STRING,
     onboardedAt: Joi.string().optional(),
   }
   return auth.joiValidator.body(Joi.object(schema).required().unknown(false))

scripts/build.js

@@ -43,7 +43,6 @@ function runBuild(entry, outfile) {
       TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }),
       nodeExternalsPlugin(),
     ],
-    target: "node14",
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",