Merge branch 'dnd-improvements' of github.com:Budibase/budibase into cheeks-lab-day-grid

docs/DEV-SETUP-DEBIAN.md

@@ -1,12 +1,15 @@
 ## Dev Environment on Debian 11
 
-### Install Node
+### Install NVM & Node 14
+NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
 
-Budibase requires a recent version of node (14+):
+Install NVM
 ```
-curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
-apt -y install nodejs
+```
-node -v
+Install Node 14
+```
+nvm install 14
 ```
 
 ### Install npm requirements
@@ -31,7 +34,7 @@ This setup process was tested on Debian 11 (bullseye) with version numbers show
 
 - Docker: 20.10.5
 - Docker-Compose: 1.29.2
-- Node: v16.15.1
+- Node: v14.20.1
 - Yarn: 1.22.19
 - Lerna: 5.1.4
 

docs/DEV-SETUP-MACOSX.md

@@ -11,7 +11,7 @@ through brew.
 
 ### Install Node
 
-Budibase requires a recent version of node (14+):
+Budibase requires a recent version of node 14:
 ```
 brew install node npm
 node -v
@@ -38,7 +38,7 @@ This setup process was tested on Mac OSX 12 (Monterey) with version numbers show
 
 - Docker: 20.10.14
 - Docker-Compose: 2.6.0
-- Node: 18.3.0
+- Node: 14.20.1
 - Yarn: 1.22.19
 - Lerna: 5.1.4
 
@@ -59,4 +59,7 @@ The dev version will be available on port 10000 i.e.
 http://127.0.0.1:10000/builder/admin
 
 | **NOTE**: If you are working on a M1 Apple Silicon, you will need to uncomment `# platform: linux/amd64` line in
-[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
+[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
+
+### Troubleshooting
+If there are errors with the `yarn setup` command, you can try installing nvm and node 14. This is the same as the instructions for Debian 11. 

docs/DEV-SETUP-WINDOWS.md

@@ -0,0 +1,81 @@
+## Dev Environment on Windows 10/11 (WSL2)
+
+
+### Install WSL with Ubuntu LTS
+
+Enable WSL 2 on Windows 10/11 for docker support.
+```
+wsl --set-default-version 2
+```
+Install Ubuntu LTS.
+```
+wsl --install Ubuntu
+```
+
+Or follow the instruction here:
+https://learn.microsoft.com/en-us/windows/wsl/install
+
+### Install Docker in windows
+Download the installer from docker and install it.
+
+Check this url for more detailed instructions:
+https://docs.docker.com/desktop/install/windows-install/
+
+You should follow the next steps from within the Ubuntu terminal.
+
+### Install NVM & Node 14
+NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
+
+Install NVM
+```
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+```
+Install Node 14
+```
+nvm install 14
+```
+
+
+### Install npm requirements
+
+```
+npm install -g yarn jest lerna
+```
+
+### Clone the repo
+```
+git clone https://github.com/Budibase/budibase.git
+```
+
+### Check Versions
+
+This setup process was tested on Windows 11 with version numbers show below. Your mileage may vary using anything else.
+
+- Docker: 20.10.7
+- Docker-Compose: 2.10.2
+- Node: v14.20.1
+- Yarn: 1.22.19
+- Lerna: 5.5.4
+
+### Build
+
+```
+cd budibase
+yarn setup
+```
+The yarn setup command runs several build steps i.e.
+```
+node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
+```
+So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
+
+The dev version will be available on port 10000 i.e.
+
+http://127.0.0.1:10000/builder/admin
+
+### Working with the code
+Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine.
+
+https://code.visualstudio.com/docs/remote/wsl
+
+Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows.

hosting/proxy/10-listen-on-ipv6-by-default.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# vim:sw=4:ts=4:et
+
+set -e
+
+ME=$(basename $0)
+NGINX_CONF_FILE="/etc/nginx/nginx.conf"
+DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf"
+
+# check if we have ipv6 available
+if [ ! -f "/proc/net/if_inet6" ]; then
+    # ipv6 not available so delete lines from nginx conf
+    if [ -f "$NGINX_CONF_FILE" ]; then
+        sed -i '/listen  \[::\]/d' $NGINX_CONF_FILE
+    fi
+    if [ -f "$DEFAULT_CONF_FILE" ]; then
+        sed -i '/listen  \[::\]/d' $DEFAULT_CONF_FILE
+    fi
+    echo "$ME: info: ipv6 not available so delete lines from nginx conf"
+else
+    echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf"
+fi
+
+exit 0

hosting/proxy/Dockerfile

@@ -5,7 +5,7 @@ FROM nginx:latest
 # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
 ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
 COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
-
+COPY 10-listen-on-ipv6-by-default.sh /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
 # Error handling
 COPY error.html /usr/share/nginx/html/error.html
 

hosting/scripts/build-target-paths.sh

@@ -4,6 +4,7 @@ echo ${TARGETBUILD} > /buildtarget.txt
 if [[ "${TARGETBUILD}" = "aas" ]]; then
     # Azure AppService uses /home for persisent data & SSH on port 2222
     DATA_DIR=/home
+    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     mkdir -p $DATA_DIR/{search,minio,couch}
     mkdir -p $DATA_DIR/couch/{dbs,views}
     chown -R couchdb:couchdb $DATA_DIR/couch/

hosting/single/Dockerfile

@@ -19,8 +19,8 @@ ADD packages/worker .
 RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
 
 FROM couchdb:3.2.1
-# TARGETARCH can be amd64 or arm e.g. docker build --build-arg TARGETARCH=amd64
+ARG TARGETARCH
-ARG TARGETARCH=amd64
+ENV TARGETARCH $TARGETARCH
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single

hosting/single/runner.sh

@@ -21,6 +21,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 # Azure App Service customisations
 if [[ "${TARGETBUILD}" = "aas" ]]; then
     DATA_DIR=/home
+    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     /etc/init.d/ssh start
 else
     DATA_DIR=${DATA_DIR:-/data}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.30-alpha.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -3,7 +3,6 @@
   "private": true,
   "devDependencies": {
     "@rollup/plugin-json": "^4.0.2",
-    "@types/mongodb": "3.6.3",
     "@typescript-eslint/parser": "4.28.0",
     "babel-eslint": "^10.0.3",
     "eslint": "^7.28.0",

packages/backend-core/context.js

@@ -6,6 +6,7 @@ const {
   updateAppId,
   doInAppContext,
   doInTenant,
+  doInContext,
 } = require("./src/context")
 
 const identity = require("./src/context/identity")
@@ -19,4 +20,5 @@ module.exports = {
   doInAppContext,
   doInTenant,
   identity,
+  doInContext,
 }

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.30-alpha.7",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -20,7 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "1.4.18-alpha.1",
+    "@budibase/types": "2.0.30-alpha.7",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
@@ -62,6 +62,7 @@
     ]
   },
   "devDependencies": {
+    "@types/chance": "1.1.3",
     "@types/jest": "27.5.1",
     "@types/koa": "2.0.52",
     "@types/lodash": "4.14.180",
@@ -72,6 +73,7 @@
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
+    "chance": "1.1.3",
     "ioredis-mock": "5.8.0",
     "jest": "27.5.1",
     "koa": "2.7.0",

packages/backend-core/src/context/index.ts

@@ -65,7 +65,16 @@ export const getTenantIDFromAppID = (appId: string) => {
   }
 }
 
-// used for automations, API endpoints should always be in context already
+export const doInContext = async (appId: string, task: any) => {
+  // gets the tenant ID from the app ID
+  const tenantId = getTenantIDFromAppID(appId)
+  return doInTenant(tenantId, async () => {
+    return doInAppContext(appId, async () => {
+      return task()
+    })
+  })
+}
+
 export const doInTenant = (tenantId: string | null, task: any) => {
   // make sure default always selected in single tenancy
   if (!env.MULTI_TENANCY) {

packages/backend-core/src/db/constants.ts

@@ -46,6 +46,9 @@ export enum DocumentType {
   AUTOMATION_LOG = "log_au",
   ACCOUNT_METADATA = "acc_metadata",
   PLUGIN = "plg",
+  TABLE = "ta",
+  DATASOURCE = "datasource",
+  DATASOURCE_PLUS = "datasource_plus",
 }
 
 export const StaticDatabases = {

packages/backend-core/src/db/conversions.js

@@ -36,6 +36,7 @@ exports.getDevelopmentAppID = appId => {
   const rest = split.join(APP_PREFIX)
   return `${APP_DEV_PREFIX}${rest}`
 }
+exports.getDevAppID = exports.getDevelopmentAppID
 
 /**
  * Convert a development app ID to a deployed app ID.

packages/backend-core/src/db/utils.ts

@@ -64,6 +64,28 @@ export function getQueryIndex(viewName: ViewName) {
   return `database/${viewName}`
 }
 
+/**
+ * Check if a given ID is that of a table.
+ * @returns {boolean}
+ */
+export const isTableId = (id: string) => {
+  // this includes datasource plus tables
+  return (
+    id &&
+    (id.startsWith(`${DocumentType.TABLE}${SEPARATOR}`) ||
+      id.startsWith(`${DocumentType.DATASOURCE_PLUS}${SEPARATOR}`))
+  )
+}
+
+/**
+ * Check if a given ID is that of a datasource or datasource plus.
+ * @returns {boolean}
+ */
+export const isDatasourceId = (id: string) => {
+  // this covers both datasources and datasource plus
+  return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
+}
+
 /**
  * Generates a new workspace ID.
  * @returns {string} The new workspace ID which the workspace doc can be stored under.

packages/backend-core/src/index.ts

@@ -37,6 +37,7 @@ const core = {
   db,
   ...dbConstants,
   redis,
+  locks: redis.redlock,
   objectStore,
   utils,
   users,

packages/backend-core/src/migrations/definitions.ts

@@ -11,7 +11,7 @@ export const DEFINITIONS: MigrationDefinition[] = [
   },
   {
     type: MigrationType.GLOBAL,
-    name: MigrationName.QUOTAS_1,
+    name: MigrationName.SYNC_QUOTAS,
   },
   {
     type: MigrationType.APP,
@@ -33,8 +33,4 @@ export const DEFINITIONS: MigrationDefinition[] = [
     type: MigrationType.GLOBAL,
     name: MigrationName.GLOBAL_INFO_SYNC_USERS,
   },
-  {
-    type: MigrationType.GLOBAL,
-    name: MigrationName.PLUGIN_COUNT,
-  },
 ]

packages/backend-core/src/objectStore/index.ts

@@ -182,6 +182,11 @@ export const streamUpload = async (
       ...extra,
       ContentType: "application/javascript",
     }
+  } else if (filename?.endsWith(".svg")) {
+    extra = {
+      ...extra,
+      ContentType: "image",
+    }
   }
 
   const params = {

packages/backend-core/src/pkg/context.ts

@@ -8,6 +8,7 @@ import {
   updateAppId,
   doInAppContext,
   doInTenant,
+  doInContext,
 } from "../context"
 
 import * as identity from "../context/identity"
@@ -20,5 +21,6 @@ export = {
   updateAppId,
   doInAppContext,
   doInTenant,
+  doInContext,
   identity,
 }

packages/backend-core/src/pkg/redis.ts

@@ -3,9 +3,11 @@
 import Client from "../redis"
 import utils from "../redis/utils"
 import clients from "../redis/init"
+import * as redlock from "../redis/redlock"
 
 export = {
   Client,
   utils,
   clients,
+  redlock,
 }

packages/backend-core/src/plugin/utils.js

@@ -67,12 +67,8 @@ function validateDatasource(schema) {
       description: joi.string().required(),
       datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
       query: joi
-        .object({
+        .object()
-          create: queryValidator,
+        .pattern(joi.string(), queryValidator)
-          read: queryValidator,
-          update: queryValidator,
-          delete: queryValidator,
-        })
         .unknown(true)
         .required(),
       extra: joi.object().pattern(

packages/backend-core/src/redis/index.ts

@@ -214,6 +214,34 @@ export = class RedisWrapper {
     }
   }
 
+  async bulkGet(keys: string[]) {
+    const db = this._db
+    if (keys.length === 0) {
+      return {}
+    }
+    const prefixedKeys = keys.map(key => addDbPrefix(db, key))
+    let response = await this.getClient().mget(prefixedKeys)
+    if (Array.isArray(response)) {
+      let final: any = {}
+      let count = 0
+      for (let result of response) {
+        if (result) {
+          let parsed
+          try {
+            parsed = JSON.parse(result)
+          } catch (err) {
+            parsed = result
+          }
+          final[keys[count]] = parsed
+        }
+        count++
+      }
+      return final
+    } else {
+      throw new Error(`Invalid response: ${response}`)
+    }
+  }
+
   async store(key: string, value: any, expirySeconds: number | null = null) {
     const db = this._db
     if (typeof value === "object") {

packages/backend-core/src/redis/init.js

@@ -1,27 +1,23 @@
 const Client = require("./index")
 const utils = require("./utils")
-const { getRedlock } = require("./redlock")
 
-let userClient, sessionClient, appClient, cacheClient, writethroughClient
+let userClient,
-let migrationsRedlock
+  sessionClient,
-
+  appClient,
-// turn retry off so that only one instance can ever hold the lock
+  cacheClient,
-const migrationsRedlockConfig = { retryCount: 0 }
+  writethroughClient,
+  lockClient
 
 async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
   cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
+  lockClient = await new Client(utils.Databases.LOCKS).init()
   writethroughClient = await new Client(
     utils.Databases.WRITE_THROUGH,
     utils.SelectableDatabases.WRITE_THROUGH
   ).init()
-  // pass the underlying ioredis client to redlock
-  migrationsRedlock = getRedlock(
-    cacheClient.getClient(),
-    migrationsRedlockConfig
-  )
 }
 
 process.on("exit", async () => {
@@ -30,6 +26,7 @@ process.on("exit", async () => {
   if (appClient) await appClient.finish()
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
+  if (lockClient) await lockClient.finish()
 })
 
 module.exports = {
@@ -63,10 +60,10 @@ module.exports = {
     }
     return writethroughClient
   },
-  getMigrationsRedlock: async () => {
+  getLockClient: async () => {
-    if (!migrationsRedlock) {
+    if (!lockClient) {
       await init()
     }
-    return migrationsRedlock
+    return lockClient
   },
 }

packages/backend-core/src/redis/redlock.ts

@@ -1,14 +1,37 @@
-import Redlock from "redlock"
+import Redlock, { Options } from "redlock"
+import { getLockClient } from "./init"
+import { LockOptions, LockType } from "@budibase/types"
+import * as tenancy from "../tenancy"
 
-export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
+let noRetryRedlock: Redlock | undefined
-  return new Redlock([redisClient], {
+
+const getClient = async (type: LockType): Promise<Redlock> => {
+  switch (type) {
+    case LockType.TRY_ONCE: {
+      if (!noRetryRedlock) {
+        noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
+      }
+      return noRetryRedlock
+    }
+    default: {
+      throw new Error(`Could not get redlock client: ${type}`)
+    }
+  }
+}
+
+export const OPTIONS = {
+  TRY_ONCE: {
+    // immediately throws an error if the lock is already held
+    retryCount: 0,
+  },
+  DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
     driftFactor: 0.01, // multiplied by lock ttl to determine drift time
 
     // the max number of times Redlock will attempt
     // to lock a resource before erroring
-    retryCount: opts.retryCount,
+    retryCount: 10,
 
     // the time in ms between attempts
     retryDelay: 200, // time in ms
@@ -16,6 +39,45 @@ export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
     // the max time in ms randomly added to retries
     // to improve performance under high contention
     // see https://www.awsarchitectureblog.com/2015/03/backoff.html
-    retryJitter: 200, // time in ms
+    retryJitter: 100, // time in ms
-  })
+  },
+}
+
+export const newRedlock = async (opts: Options = {}) => {
+  let options = { ...OPTIONS.DEFAULT, ...opts }
+  const redisWrapper = await getLockClient()
+  const client = redisWrapper.getClient()
+  return new Redlock([client], options)
+}
+
+export const doWithLock = async (opts: LockOptions, task: any) => {
+  const redlock = await getClient(opts.type)
+  let lock
+  try {
+    // aquire lock
+    let name: string = `${tenancy.getTenantId()}_${opts.name}`
+    if (opts.nameSuffix) {
+      name = name + `_${opts.nameSuffix}`
+    }
+    lock = await redlock.lock(name, opts.ttl)
+    // perform locked task
+    return task()
+  } catch (e: any) {
+    // lock limit exceeded
+    if (e.name === "LockError") {
+      if (opts.type === LockType.TRY_ONCE) {
+        // don't throw for try-once locks, they will always error
+        // due to retry count (0) exceeded
+        return
+      } else {
+        throw e
+      }
+    } else {
+      throw e
+    }
+  } finally {
+    if (lock) {
+      await lock.unlock()
+    }
+  }
 }

packages/backend-core/src/redis/utils.js

@@ -28,6 +28,7 @@ exports.Databases = {
   LICENSES: "license",
   GENERIC_CACHE: "data_cache",
   WRITE_THROUGH: "writeThrough",
+  LOCKS: "locks",
 }
 
 /**

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -0,0 +1,23 @@
+import { generator, uuid } from "."
+import { AuthType, CloudAccount, Hosting } from "@budibase/types"
+import * as db from "../../../src/db/utils"
+
+export const cloudAccount = (): CloudAccount => {
+  return {
+    accountId: uuid(),
+    createdAt: Date.now(),
+    verified: true,
+    verificationSent: true,
+    tier: "",
+    email: generator.email(),
+    tenantId: generator.word(),
+    hosting: Hosting.CLOUD,
+    authType: AuthType.PASSWORD,
+    password: generator.word(),
+    tenantName: generator.word(),
+    name: generator.name(),
+    size: "10+",
+    profession: "Software Engineer",
+    budibaseUserId: db.generateGlobalUserID(),
+  }
+}

packages/backend-core/tests/utilities/structures/common.ts

@@ -0,0 +1 @@
+export { v4 as uuid } from "uuid"

packages/backend-core/tests/utilities/structures/index.ts

@@ -1 +1,8 @@
+export * from "./common"
+
+import Chance from "chance"
+export const generator = new Chance()
+
 export * as koa from "./koa"
+export * as accounts from "./accounts"
+export * as licenses from "./licenses"

packages/backend-core/tests/utilities/structures/licenses.ts

@@ -0,0 +1,18 @@
+import { AccountPlan, License, PlanType, Quotas } from "@budibase/types"
+
+const newPlan = (type: PlanType = PlanType.FREE): AccountPlan => {
+  return {
+    type,
+  }
+}
+
+export const newLicense = (opts: {
+  quotas: Quotas
+  planType?: PlanType
+}): License => {
+  return {
+    features: [],
+    quotas: opts.quotas,
+    plan: newPlan(opts.planType),
+  }
+}

packages/backend-core/yarn.lock

@@ -663,6 +663,11 @@
     "@types/connect" "*"
     "@types/node" "*"
 
+"@types/chance@1.1.3":
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/@types/chance/-/chance-1.1.3.tgz#d19fe9391288d60fdccd87632bfc9ab2b4523fea"
+  integrity sha512-X6c6ghhe4/sQh4XzcZWSFaTAUOda38GQHmq9BUanYkOE/EO7ZrkazwKmtsj3xzTjkLWmwULE++23g3d3CCWaWw==
+
 "@types/connect@*":
   version "3.4.35"
   resolved "https://registry.yarnpkg.com/@types/connect/-/connect-3.4.35.tgz#5fcf6ae445e4021d1fc2219a4873cc73a3bb2ad1"
@@ -1555,6 +1560,11 @@ chalk@^4.0.0, chalk@^4.1.0:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
 
+chance@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.3.tgz#414f08634ee479c7a316b569050ea20751b82dd3"
+  integrity sha512-XeJsdoVAzDb1WRPRuMBesRSiWpW1uNTo5Fd7mYxPJsAfgX71+jfuCOHOdbyBz2uAUZ8TwKcXgWk3DMedFfJkbg==
+
 char-regex@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.30-alpha.7",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.30-alpha.7",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/bbui/src/Form/Core/Picker.svelte

@@ -19,6 +19,7 @@
   export let placeholderOption = null
   export let options = []
   export let isOptionSelected = () => false
+  export let isOptionEnabled = () => true
   export let onSelectOption = () => {}
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
@@ -164,6 +165,7 @@
               aria-selected="true"
               tabindex="0"
               on:click={() => onSelectOption(getOptionValue(option, idx))}
+              class:is-disabled={!isOptionEnabled(option)}
             >
               {#if getOptionIcon(option, idx)}
                 <span class="option-extra">
@@ -256,4 +258,7 @@
   .spectrum-Popover :global(.spectrum-Search .spectrum-Textfield-icon) {
     top: 9px;
   }
+  .spectrum-Menu-item.is-disabled {
+    pointer-events: none;
+  }
 </style>

packages/bbui/src/Form/Core/Select.svelte

@@ -12,6 +12,7 @@
   export let getOptionValue = option => option
   export let getOptionIcon = () => null
   export let getOptionColour = () => null
+  export let isOptionEnabled
   export let readonly = false
   export let quiet = false
   export let autoWidth = false
@@ -66,6 +67,7 @@
   {getOptionValue}
   {getOptionIcon}
   {getOptionColour}
+  {isOptionEnabled}
   {autocomplete}
   {sort}
   isPlaceholder={value == null || value === ""}

packages/bbui/src/Form/Select.svelte

@@ -15,6 +15,7 @@
   export let getOptionValue = option => extractProperty(option, "value")
   export let getOptionIcon = option => option?.icon
   export let getOptionColour = option => option?.colour
+  export let isOptionEnabled
   export let quiet = false
   export let autoWidth = false
   export let sort = false
@@ -49,6 +50,7 @@
     {getOptionValue}
     {getOptionIcon}
     {getOptionColour}
+    {isOptionEnabled}
     on:change={onChange}
     on:click
   />

packages/builder/cypress/integration/adminAndManagement/userManagement.spec.js

@@ -20,7 +20,9 @@ filterTests(["smoke", "all"], () => {
       cy.get(".spectrum-Form-itemField").eq(3).should('contain', 'App User')
 
       // User should not have app access
-      cy.get(interact.LIST_ITEMS, { timeout: 500 }).should("contain", "No apps")
+      cy.get(".spectrum-Heading").contains("Apps").parent().within(() => {
+        cy.get(interact.LIST_ITEMS, { timeout: 500 }).should("contain", "This user has access to no apps")
+      })
     })
 
     if (Cypress.env("TEST_ENV")) {

packages/builder/cypress/integration/autoScreensUI.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
-  context("Auto Screens UI", () => {
+  xcontext("Auto Screens UI", () => {
     before(() => {
       cy.login()
       cy.deleteAllApps()
@@ -54,6 +54,7 @@ filterTests(['smoke', 'all'], () => {
       cy.createDatasourceScreen([initialTable, secondTable])
       // Confirm screens have been auto generated
       // Previously generated tables are suffixed with numbers - as expected
+      cy.wait(1000)
       cy.get(interact.BODY).should('contain', 'cypress-tests-2')
         .and('contain', 'cypress-tests-2/:id')
         .and('contain', 'cypress-tests-2/new/row')

packages/builder/cypress/integration/datasources/postgreSql.spec.js

@@ -1,7 +1,7 @@
 import filterTests from "../../support/filterTests"
 
 filterTests(["all"], () => {
-  context("PostgreSQL Datasource Testing", () => {
+  xcontext("PostgreSQL Datasource Testing", () => {
     if (Cypress.env("TEST_ENV")) {
       before(() => {
         cy.login()

packages/builder/cypress/integration/datasources/rest.spec.js

@@ -22,7 +22,7 @@ filterTests(["smoke", "all"], () => {
       cy.wait("@queryError")
       cy.get("@queryError")
         .its("response.body")
-        .should("have.property", "message", "Invalid URL: http://random text?")
+        .should("have.property", "message", "Invalid URL: http://random text")
       cy.get("@queryError")
         .its("response.body")
         .should("have.property", "status", 400)

packages/builder/cypress/integration/queryLevelTransformers.spec.js

@@ -1,5 +1,5 @@
 import filterTests from "../support/filterTests"
-const interact = require('../support/interact')
+const interact = require("../support/interact")
 
 filterTests(["smoke", "all"], () => {
   context("Query Level Transformers", () => {

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.30-alpha.7",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "1.4.18-alpha.1",
+    "@budibase/bbui": "2.0.30-alpha.7",
-    "@budibase/client": "1.4.18-alpha.1",
+    "@budibase/client": "2.0.30-alpha.7",
-    "@budibase/frontend-core": "1.4.18-alpha.1",
+    "@budibase/frontend-core": "2.0.30-alpha.7",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.30-alpha.7",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/componentUtils.js

@@ -143,7 +143,10 @@ export const getComponentSettings = componentType => {
   }
 
   // Ensure whole component name is used
-  if (!componentType.startsWith("@budibase")) {
+  if (
+    !componentType.startsWith("plugin/") &&
+    !componentType.startsWith("@budibase")
+  ) {
     componentType = `@budibase/standard-components/${componentType}`
   }
 
@@ -182,43 +185,42 @@ export const makeComponentUnique = component => {
   // Replace component ID
   const oldId = component._id
   const newId = Helpers.uuid()
-  component._id = newId
+  let definition = JSON.stringify(component)
 
-  if (component._children?.length) {
+  // Replace all instances of this ID in HBS bindings
-    let children = JSON.stringify(component._children)
+  definition = definition.replace(new RegExp(oldId, "g"), newId)
 
-    // Replace all instances of this ID in child HBS bindings
+  // Replace all instances of this ID in JS bindings
-    children = children.replace(new RegExp(oldId, "g"), newId)
+  const bindings = findHBSBlocks(definition)
+  bindings.forEach(binding => {
+    // JSON.stringify will have escaped double quotes, so we need
+    // to account for that
+    let sanitizedBinding = binding.replace(/\\"/g, '"')
 
-    // Replace all instances of this ID in child JS bindings
+    // Check if this is a valid JS binding
-    const bindings = findHBSBlocks(children)
+    let js = decodeJSBinding(sanitizedBinding)
-    bindings.forEach(binding => {
+    if (js != null) {
-      // JSON.stringify will have escaped double quotes, so we need
+      // Replace ID inside JS binding
-      // to account for that
+      js = js.replace(new RegExp(oldId, "g"), newId)
-      let sanitizedBinding = binding.replace(/\\"/g, '"')
 
-      // Check if this is a valid JS binding
+      // Create new valid JS binding
-      let js = decodeJSBinding(sanitizedBinding)
+      let newBinding = encodeJSBinding(js)
-      if (js != null) {
-        // Replace ID inside JS binding
-        js = js.replace(new RegExp(oldId, "g"), newId)
 
-        // Create new valid JS binding
+      // Replace escaped double quotes
-        let newBinding = encodeJSBinding(js)
+      newBinding = newBinding.replace(/"/g, '\\"')
 
-        // Replace escaped double quotes
+      // Insert new JS back into binding.
-        newBinding = newBinding.replace(/"/g, '\\"')
+      // A single string replace here is better than a regex as
+      // the binding contains special characters, and we only need
+      // to replace a single instance.
+      definition = definition.replace(binding, newBinding)
+    }
+  })
 
-        // Insert new JS back into binding.
+  // Recurse on all children
-        // A single string replace here is better than a regex as
+  component = JSON.parse(definition)
-        // the binding contains special characters, and we only need
+  return {
-        // to replace a single instance.
+    ...component,
-        children = children.replace(binding, newBinding)
+    _children: component._children?.map(makeComponentUnique),
-      }
-    })
-
-    // Recurse on all children
-    component._children = JSON.parse(children)
-    component._children.forEach(makeComponentUnique)
   }
 }

packages/builder/src/builderStore/dataBinding.js

@@ -169,7 +169,12 @@ export const getComponentBindableProperties = (asset, componentId) => {
 /**
  * Gets all data provider components above a component.
  */
-export const getContextProviderComponents = (asset, componentId, type) => {
+export const getContextProviderComponents = (
+  asset,
+  componentId,
+  type,
+  options = { includeSelf: false }
+) => {
   if (!asset || !componentId) {
     return []
   }
@@ -177,7 +182,9 @@ export const getContextProviderComponents = (asset, componentId, type) => {
   // Get the component tree leading up to this component, ignoring the component
   // itself
   const path = findComponentPath(asset.props, componentId)
-  path.pop()
+  if (!options?.includeSelf) {
+    path.pop()
+  }
 
   // Filter by only data provider components
   return path.filter(component => {
@@ -243,18 +250,18 @@ export const getDatasourceForProvider = (asset, component) => {
     return null
   }
 
-  // There are different types of setting which can be a datasource, for
+  // For legacy compatibility, we need to be able to handle datasources that are
-  // example an actual datasource object, or a table ID string.
+  // just strings. These are not generated any more, so could be removed in
-  // Convert the datasource setting into a proper datasource object so that
+  // future.
-  // we can use it properly
+  // TODO: remove at some point
-  if (datasourceSetting.type === "table") {
+  const datasource = component[datasourceSetting?.key]
+  if (typeof datasource === "string") {
     return {
-      tableId: component[datasourceSetting?.key],
+      tableId: datasource,
       type: "table",
     }
-  } else {
-    return component[datasourceSetting?.key]
   }
+  return datasource
 }
 
 /**
@@ -396,19 +403,17 @@ export const getUserBindings = () => {
 
   bindings = keys.reduce((acc, key) => {
     const fieldSchema = schema[key]
-    if (fieldSchema.type !== "link") {
+    acc.push({
-      acc.push({
+      type: "context",
-        type: "context",
+      runtimeBinding: `${safeUser}.${makePropSafe(key)}`,
-        runtimeBinding: `${safeUser}.${makePropSafe(key)}`,
+      readableBinding: `Current User.${key}`,
-        readableBinding: `Current User.${key}`,
+      // Field schema and provider are required to construct relationship
-        // Field schema and provider are required to construct relationship
+      // datasource options, based on bindable properties
-        // datasource options, based on bindable properties
+      fieldSchema,
-        fieldSchema,
+      providerId: "user",
-        providerId: "user",
+      category: "Current User",
-        category: "Current User",
+      icon: "User",
-        icon: "User",
+    })
-      })
-    }
     return acc
   }, [])
 
@@ -800,6 +805,17 @@ export const buildFormSchema = component => {
   if (!component) {
     return schema
   }
+
+  // If this is a form block, simply use the fields setting
+  if (component._component.endsWith("formblock")) {
+    let schema = {}
+    component.fields?.forEach(field => {
+      schema[field] = { type: "string" }
+    })
+    return schema
+  }
+
+  // Otherwise find all field component children
   const settings = getComponentSettings(component._component)
   const fieldSetting = settings.find(
     setting => setting.key === "field" && setting.type.startsWith("field/")

packages/builder/src/builderStore/store/frontend.js

@@ -88,27 +88,12 @@ export const getFrontendStore = () => {
     initialise: async pkg => {
       const { layouts, screens, application, clientLibPath } = pkg
 
-      // Fetch component definitions.
+      await store.actions.components.refreshDefinitions(application.appId)
-      // Allow errors to propagate.
-      const components = await API.fetchComponentLibDefinitions(
-        application.appId
-      )
-
-      // Filter out custom component keys so we can flag them
-      const customComponents = Object.keys(components).filter(name =>
-        name.startsWith("plugin/")
-      )
 
       // Reset store state
       store.update(state => ({
         ...state,
         libraries: application.componentLibraries,
-        components,
-        customComponents,
-        clientFeatures: {
-          ...INITIAL_FRONTEND_STATE.clientFeatures,
-          ...components.features,
-        },
         name: application.name,
         description: application.description,
         appId: application.appId,
@@ -345,6 +330,16 @@ export const getFrontendStore = () => {
           return state
         })
       },
+      sendEvent: (name, payload) => {
+        const { previewEventHandler } = get(store)
+        previewEventHandler?.(name, payload)
+      },
+      registerEventHandler: handler => {
+        store.update(state => {
+          state.previewEventHandler = handler
+          return state
+        })
+      },
     },
     layouts: {
       select: layoutId => {
@@ -385,6 +380,29 @@ export const getFrontendStore = () => {
       },
     },
     components: {
+      refreshDefinitions: async appId => {
+        if (!appId) {
+          appId = get(store).appId
+        }
+
+        // Fetch definitions and filter out custom component definitions so we
+        // can flag them
+        const components = await API.fetchComponentLibDefinitions(appId)
+        const customComponents = Object.keys(components).filter(name =>
+          name.startsWith("plugin/")
+        )
+
+        // Update store
+        store.update(state => ({
+          ...state,
+          components,
+          customComponents,
+          clientFeatures: {
+            ...INITIAL_FRONTEND_STATE.clientFeatures,
+            ...components.features,
+          },
+        }))
+      },
       getDefinition: componentName => {
         if (!componentName) {
           return null
@@ -437,12 +455,12 @@ export const getFrontendStore = () => {
             hover: {},
             active: {},
           },
-          _instanceName: `New ${definition.name}`,
+          _instanceName: `New ${definition.friendlyName || definition.name}`,
           ...cloneDeep(props),
           ...extras,
         }
       },
-      create: async (componentName, presetProps) => {
+      create: async (componentName, presetProps, parent, index) => {
         const state = get(store)
         const componentInstance = store.actions.components.createInstance(
           componentName,
@@ -452,48 +470,62 @@ export const getFrontendStore = () => {
           return
         }
 
-        // Patch selected screen
+        // Insert in position if specified
-        await store.actions.screens.patch(screen => {
+        if (parent && index != null) {
-          // Find the selected component
+          await store.actions.screens.patch(screen => {
-          const currentComponent = findComponent(
+            let parentComponent = findComponent(screen.props, parent)
-            screen.props,
+            if (!parentComponent._children?.length) {
-            state.selectedComponentId
+              parentComponent._children = [componentInstance]
-          )
-          if (!currentComponent) {
-            return false
-          }
-
-          // Find parent node to attach this component to
-          let parentComponent
-          if (currentComponent) {
-            // Use selected component as parent if one is selected
-            const definition = store.actions.components.getDefinition(
-              currentComponent._component
-            )
-            if (definition?.hasChildren) {
-              // Use selected component if it allows children
-              parentComponent = currentComponent
             } else {
-              // Otherwise we need to use the parent of this component
+              parentComponent._children.splice(index, 0, componentInstance)
-              parentComponent = findComponentParent(
-                screen.props,
-                currentComponent._id
-              )
             }
-          } else {
+          })
-            // Use screen or layout if no component is selected
+        }
-            parentComponent = screen.props
-          }
 
-          // Attach new component
+        // Otherwise we work out where this component should be inserted
-          if (!parentComponent) {
+        else {
-            return false
+          await store.actions.screens.patch(screen => {
-          }
+            // Find the selected component
-          if (!parentComponent._children) {
+            const currentComponent = findComponent(
-            parentComponent._children = []
+              screen.props,
-          }
+              state.selectedComponentId
-          parentComponent._children.push(componentInstance)
+            )
-        })
+            if (!currentComponent) {
+              return false
+            }
+
+            // Find parent node to attach this component to
+            let parentComponent
+            if (currentComponent) {
+              // Use selected component as parent if one is selected
+              const definition = store.actions.components.getDefinition(
+                currentComponent._component
+              )
+              if (definition?.hasChildren) {
+                // Use selected component if it allows children
+                parentComponent = currentComponent
+              } else {
+                // Otherwise we need to use the parent of this component
+                parentComponent = findComponentParent(
+                  screen.props,
+                  currentComponent._id
+                )
+              }
+            } else {
+              // Use screen or layout if no component is selected
+              parentComponent = screen.props
+            }
+
+            // Attach new component
+            if (!parentComponent) {
+              return false
+            }
+            if (!parentComponent._children) {
+              parentComponent._children = []
+            }
+            parentComponent._children.push(componentInstance)
+          })
+        }
 
         // Select new component
         store.update(state => {
@@ -612,7 +644,7 @@ export const getFrontendStore = () => {
 
           // Make new component unique if copying
           if (!cut) {
-            makeComponentUnique(componentToPaste)
+            componentToPaste = makeComponentUnique(componentToPaste)
           }
           newComponentId = componentToPaste._id
 
@@ -900,6 +932,50 @@ export const getFrontendStore = () => {
           component[name] = value
         })
       },
+      requestEjectBlock: componentId => {
+        store.actions.preview.sendEvent("eject-block", componentId)
+      },
+      handleEjectBlock: async (componentId, ejectedDefinition) => {
+        let nextSelectedComponentId
+
+        await store.actions.screens.patch(screen => {
+          const block = findComponent(screen.props, componentId)
+          const parent = findComponentParent(screen.props, componentId)
+
+          // Sanity check
+          if (!block || !parent?._children?.length) {
+            return false
+          }
+
+          // Attach block children back into ejected definition, using the
+          // _containsSlot flag to know where to insert them
+          const slotContainer = findAllMatchingComponents(
+            ejectedDefinition,
+            x => x._containsSlot
+          )[0]
+          if (slotContainer) {
+            delete slotContainer._containsSlot
+            slotContainer._children = [
+              ...(slotContainer._children || []),
+              ...(block._children || []),
+            ]
+          }
+
+          // Replace block with ejected definition
+          ejectedDefinition = makeComponentUnique(ejectedDefinition)
+          const index = parent._children.findIndex(x => x._id === componentId)
+          parent._children[index] = ejectedDefinition
+          nextSelectedComponentId = ejectedDefinition._id
+        })
+
+        // Select new root component
+        if (nextSelectedComponentId) {
+          store.update(state => {
+            state.selectedComponentId = nextSelectedComponentId
+            return state
+          })
+        }
+      },
     },
     links: {
       save: async (url, title) => {
@@ -945,6 +1021,19 @@ export const getFrontendStore = () => {
         }))
       },
     },
+    dnd: {
+      start: component => {
+        store.actions.preview.sendEvent("dragging-new-component", {
+          dragging: true,
+          component,
+        })
+      },
+      stop: () => {
+        store.actions.preview.sendEvent("dragging-new-component", {
+          dragging: false,
+        })
+      },
+    },
   }
 
   return store

packages/builder/src/builderStore/store/screenTemplates/newRowScreen.js

@@ -1,13 +1,8 @@
 import sanitizeUrl from "./utils/sanitizeUrl"
 import { Screen } from "./utils/Screen"
 import { Component } from "./utils/Component"
-import {
+import { makeBreadcrumbContainer } from "./utils/commonComponents"
-  makeBreadcrumbContainer,
+import { getSchemaForDatasource } from "../../dataBinding"
-  makeMainForm,
-  makeTitleContainer,
-  makeSaveButton,
-  makeDatasourceFormComponents,
-} from "./utils/commonComponents"
 
 export default function (tables) {
   return tables.map(table => {
@@ -23,48 +18,55 @@ export default function (tables) {
 export const newRowUrl = table => sanitizeUrl(`/${table.name}/new/row`)
 export const NEW_ROW_TEMPLATE = "NEW_ROW_TEMPLATE"
 
-function generateTitleContainer(table, formId) {
+const rowListUrl = table => sanitizeUrl(`/${table.name}`)
-  return makeTitleContainer("New Row").addChild(makeSaveButton(table, formId))
+
+const getFields = schema => {
+  let columns = []
+  Object.entries(schema || {}).forEach(([field, fieldSchema]) => {
+    if (!field || !fieldSchema) {
+      return
+    }
+    if (!fieldSchema?.autocolumn) {
+      columns.push(field)
+    }
+  })
+  return columns
 }
 
-const createScreen = table => {
+const generateFormBlock = table => {
-  const screen = new Screen()
+  const datasource = { type: "table", tableId: table._id }
-    .instanceName(`${table.name} - New`)
+  const { schema } = getSchemaForDatasource(null, datasource, {
-    .customProps({
+    formSchema: true,
-      hAlign: "center",
+  })
-    })
+  const formBlock = new Component("@budibase/standard-components/formblock")
-    .route(newRowUrl(table))
+  formBlock
-
-  const form = makeMainForm()
-    .instanceName("Form")
     .customProps({
+      title: "New row",
       actionType: "Create",
+      actionUrl: rowListUrl(table),
+      showDeleteButton: false,
+      showSaveButton: true,
+      fields: getFields(schema),
       dataSource: {
         label: table.name,
         tableId: table._id,
         type: "table",
       },
+      labelPosition: "left",
       size: "spectrum--medium",
     })
-
+    .instanceName(`${table.name} - Form block`)
-  const fieldGroup = new Component("@budibase/standard-components/fieldgroup")
+  return formBlock
-    .instanceName("Field Group")
+}
-    .customProps({
+
-      labelPosition: "left",
+const createScreen = table => {
-    })
+  const formBlock = generateFormBlock(table)
-
+  const screen = new Screen()
-  // Add all form fields from this schema to the field group
+    .instanceName(`${table.name} - New`)
-  const datasource = { type: "table", tableId: table._id }
+    .route(newRowUrl(table))
-  makeDatasourceFormComponents(datasource).forEach(component => {
+
-    fieldGroup.addChild(component)
+  return screen
-  })
+    .addChild(makeBreadcrumbContainer(table.name, "New row"))
-
+    .addChild(formBlock)
-  // Add all children to the form
+    .json()
-  const formId = form._json._id
-  form
-    .addChild(makeBreadcrumbContainer(table.name, "New"))
-    .addChild(generateTitleContainer(table, formId))
-    .addChild(fieldGroup)
-
-  return screen.addChild(form).json()
 }

packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js

@@ -1,15 +1,8 @@
 import sanitizeUrl from "./utils/sanitizeUrl"
-import { rowListUrl } from "./rowListScreen"
 import { Screen } from "./utils/Screen"
 import { Component } from "./utils/Component"
-import { makePropSafe } from "@budibase/string-templates"
+import { makeBreadcrumbContainer } from "./utils/commonComponents"
-import {
+import { getSchemaForDatasource } from "../../dataBinding"
-  makeBreadcrumbContainer,
-  makeTitleContainer,
-  makeSaveButton,
-  makeMainForm,
-  makeDatasourceFormComponents,
-} from "./utils/commonComponents"
 
 export default function (tables) {
   return tables.map(table => {
@@ -25,125 +18,53 @@ export default function (tables) {
 export const ROW_DETAIL_TEMPLATE = "ROW_DETAIL_TEMPLATE"
 export const rowDetailUrl = table => sanitizeUrl(`/${table.name}/:id`)
 
-function generateTitleContainer(table, title, formId, repeaterId) {
+const rowListUrl = table => sanitizeUrl(`/${table.name}`)
-  const saveButton = makeSaveButton(table, formId)
-  const deleteButton = new Component("@budibase/standard-components/button")
-    .text("Delete")
-    .customProps({
-      type: "secondary",
-      quiet: true,
-      size: "M",
-      onClick: [
-        {
-          parameters: {
-            tableId: table._id,
-            rowId: `{{ ${makePropSafe(repeaterId)}.${makePropSafe("_id")} }}`,
-            revId: `{{ ${makePropSafe(repeaterId)}.${makePropSafe("_rev")} }}`,
-            confirm: true,
-          },
-          "##eventHandlerType": "Delete Row",
-        },
-        {
-          parameters: {
-            url: rowListUrl(table),
-          },
-          "##eventHandlerType": "Navigate To",
-        },
-      ],
-    })
-    .instanceName("Delete Button")
 
-  const buttons = new Component("@budibase/standard-components/container")
+const getFields = schema => {
-    .instanceName("Button Container")
+  let columns = []
-    .customProps({
+  Object.entries(schema || {}).forEach(([field, fieldSchema]) => {
-      direction: "row",
+    if (!field || !fieldSchema) {
-      hAlign: "right",
+      return
-      vAlign: "middle",
+    }
-      size: "shrink",
+    if (!fieldSchema?.autocolumn) {
-      gap: "M",
+      columns.push(field)
-    })
+    }
-    .addChild(deleteButton)
+  })
-    .addChild(saveButton)
+  return columns
+}
 
-  return makeTitleContainer(title).addChild(buttons)
+const generateFormBlock = table => {
+  const datasource = { type: "table", tableId: table._id }
+  const { schema } = getSchemaForDatasource(null, datasource, {
+    formSchema: true,
+  })
+
+  const formBlock = new Component("@budibase/standard-components/formblock")
+  formBlock
+    .customProps({
+      title: "Edit row",
+      actionType: "Update",
+      actionUrl: rowListUrl(table),
+      showDeleteButton: true,
+      showSaveButton: true,
+      fields: getFields(schema),
+      dataSource: {
+        label: table.name,
+        tableId: table._id,
+        type: "table",
+      },
+      labelPosition: "left",
+      size: "spectrum--medium",
+    })
+    .instanceName(`${table.name} - Form block`)
+  return formBlock
 }
 
 const createScreen = table => {
-  const provider = new Component("@budibase/standard-components/dataprovider")
-    .instanceName(`Data Provider`)
-    .customProps({
-      dataSource: {
-        label: table.name,
-        name: table._id,
-        tableId: table._id,
-        type: "table",
-      },
-      filter: [
-        {
-          field: "_id",
-          operator: "equal",
-          type: "string",
-          value: `{{ ${makePropSafe("url")}.${makePropSafe("id")} }}`,
-          valueType: "Binding",
-        },
-      ],
-      limit: 1,
-      paginate: false,
-    })
-
-  const repeater = new Component("@budibase/standard-components/repeater")
-    .instanceName("Repeater")
-    .customProps({
-      dataProvider: `{{ literal ${makePropSafe(provider._json._id)} }}`,
-      noRowsMessage: "We couldn't find a row to display",
-    })
-
-  const form = makeMainForm()
-    .instanceName("Form")
-    .customProps({
-      actionType: "Update",
-      size: "spectrum--medium",
-      dataSource: {
-        label: table.name,
-        tableId: table._id,
-        type: "table",
-      },
-    })
-
-  const fieldGroup = new Component("@budibase/standard-components/fieldgroup")
-    .instanceName("Field Group")
-    .customProps({
-      labelPosition: "left",
-    })
-
-  // Add all form fields from this schema to the field group
-  const datasource = { type: "table", tableId: table._id }
-  makeDatasourceFormComponents(datasource).forEach(component => {
-    fieldGroup.addChild(component)
-  })
-
-  // Add all children to the form
-  const formId = form._json._id
-  const repeaterId = repeater._json._id
-  const heading = table.primaryDisplay
-    ? `{{ ${makePropSafe(repeaterId)}.${makePropSafe(table.primaryDisplay)} }}`
-    : null
-  form
-    .addChild(makeBreadcrumbContainer(table.name, heading || "Edit"))
-    .addChild(
-      generateTitleContainer(table, heading || "Edit Row", formId, repeaterId)
-    )
-    .addChild(fieldGroup)
-
-  repeater.addChild(form)
-  provider.addChild(repeater)
-
   return new Screen()
     .instanceName(`${table.name} - Detail`)
     .route(rowDetailUrl(table))
-    .customProps({
+    .addChild(makeBreadcrumbContainer(table.name, "Edit row"))
-      hAlign: "center",
+    .addChild(generateFormBlock(table))
-    })
-    .addChild(provider)
     .json()
 }

packages/builder/src/builderStore/store/screenTemplates/rowListScreen.js

@@ -2,7 +2,6 @@ import sanitizeUrl from "./utils/sanitizeUrl"
 import { newRowUrl } from "./newRowScreen"
 import { Screen } from "./utils/Screen"
 import { Component } from "./utils/Component"
-import { makePropSafe } from "@budibase/string-templates"
 
 export default function (tables) {
   return tables.map(table => {
@@ -18,48 +17,17 @@ export default function (tables) {
 export const ROW_LIST_TEMPLATE = "ROW_LIST_TEMPLATE"
 export const rowListUrl = table => sanitizeUrl(`/${table.name}`)
 
-function generateTitleContainer(table) {
+const generateTableBlock = table => {
-  const newButton = new Component("@budibase/standard-components/button")
+  const tableBlock = new Component("@budibase/standard-components/tableblock")
-    .text("Create New")
+  tableBlock
-    .customProps({
-      size: "M",
-      type: "primary",
-      onClick: [
-        {
-          parameters: {
-            url: newRowUrl(table),
-          },
-          "##eventHandlerType": "Navigate To",
-        },
-      ],
-    })
-    .instanceName("New Button")
-
-  const heading = new Component("@budibase/standard-components/heading")
-    .instanceName("Title")
-    .text(table.name)
-    .customProps({
-      size: "M",
-      align: "left",
-    })
-
-  return new Component("@budibase/standard-components/container")
-    .customProps({
-      direction: "row",
-      hAlign: "stretch",
-      vAlign: "middle",
-      size: "shrink",
-      gap: "M",
-    })
-    .instanceName("Title Container")
-    .addChild(heading)
-    .addChild(newButton)
-}
-
-const createScreen = table => {
-  const provider = new Component("@budibase/standard-components/dataprovider")
-    .instanceName(`Data Provider`)
     .customProps({
+      linkRows: true,
+      linkURL: `${rowListUrl(table)}/:id`,
+      showAutoColumns: false,
+      showTitleButton: true,
+      titleButtonText: "Create new",
+      titleButtonURL: newRowUrl(table),
+      title: table.name,
       dataSource: {
         label: table.name,
         name: table._id,
@@ -68,41 +36,16 @@ const createScreen = table => {
       },
       size: "spectrum--medium",
       paginate: true,
-      limit: 8,
-    })
-
-  const spectrumTable = new Component("@budibase/standard-components/table")
-    .customProps({
-      dataProvider: `{{ literal ${makePropSafe(provider._json._id)} }}`,
-      showAutoColumns: false,
-      quiet: false,
       rowCount: 8,
     })
-    .instanceName(`${table.name} Table`)
+    .instanceName(`${table.name} - Table block`)
-
+  return tableBlock
-  const safeTableId = makePropSafe(spectrumTable._json._id)
+}
-  const safeRowId = makePropSafe("_id")
-  const viewLink = new Component("@budibase/standard-components/link")
-    .customProps({
-      text: "View",
-      url: `${rowListUrl(table)}/{{ ${safeTableId}.${safeRowId} }}`,
-      size: "S",
-      color: "var(--spectrum-global-color-gray-600)",
-      align: "left",
-    })
-    .normalStyle({
-      ["margin-left"]: "16px",
-      ["margin-right"]: "16px",
-    })
-    .instanceName("View Link")
-
-  spectrumTable.addChild(viewLink)
-  provider.addChild(spectrumTable)
 
+const createScreen = table => {
   return new Screen()
     .route(rowListUrl(table))
     .instanceName(`${table.name} - List`)
-    .addChild(generateTitleContainer(table))
+    .addChild(generateTableBlock(table))
-    .addChild(provider)
     .json()
 }

packages/builder/src/builderStore/store/screenTemplates/utils/commonComponents.js

@@ -65,6 +65,11 @@ export function makeBreadcrumbContainer(tableName, text) {
       vAlign: "middle",
       size: "shrink",
     })
+    .normalStyle({
+      width: "600px",
+      "margin-right": "auto",
+      "margin-left": "auto",
+    })
     .instanceName("Breadcrumbs")
     .addChild(link)
     .addChild(arrowText)
@@ -138,6 +143,7 @@ const fieldTypeToComponentMap = {
   attachment: "attachmentfield",
   link: "relationshipfield",
   json: "jsonfield",
+  barcodeqr: "codescanner",
 }
 
 export function makeDatasourceFormComponents(datasource) {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -261,6 +261,7 @@
     } else {
       return [
         FIELDS.STRING,
+        FIELDS.BARCODEQR,
         FIELDS.LONGFORM,
         FIELDS.OPTIONS,
         FIELDS.DATETIME,
@@ -314,7 +315,7 @@
       const relatedTable = $tables.list.find(
         tbl => tbl._id === fieldInfo.tableId
       )
-      if (inUse(relatedTable, fieldInfo.fieldName)) {
+      if (inUse(relatedTable, fieldInfo.fieldName) && !originalName) {
         newError.relatedName = `Column name already in use in table ${relatedTable.name}`
       }
     }

packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte

@@ -17,12 +17,21 @@
   $: selectedRoleId = selectedRole._id
   $: otherRoles = editableRoles.filter(role => role._id !== selectedRoleId)
   $: isCreating = selectedRoleId == null || selectedRoleId === ""
+
+  $: hasUniqueRoleName = !otherRoles
+    ?.map(role => role.name)
+    ?.includes(selectedRole.name)
+
   $: valid =
     selectedRole.name &&
     selectedRole.inherits &&
     selectedRole.permissionId &&
     !builtInRoles.includes(selectedRole.name)
 
+  $: shouldDisableRoleInput =
+    builtInRoles.includes(selectedRole.name) &&
+    selectedRole.name?.toLowerCase() === selectedRoleId?.toLowerCase()
+
   const fetchBasePermissions = async () => {
     try {
       basePermissions = await API.getBasePermissions()
@@ -99,7 +108,7 @@
   title="Edit Roles"
   confirmText={isCreating ? "Create" : "Save"}
   onConfirm={saveRole}
-  disabled={!valid}
+  disabled={!valid || !hasUniqueRoleName}
 >
   {#if errors.length}
     <ErrorsBox {errors} />
@@ -119,15 +128,16 @@
     <Input
       label="Name"
       bind:value={selectedRole.name}
-      disabled={builtInRoles.includes(selectedRole.name)}
+      disabled={shouldDisableRoleInput}
+      error={!hasUniqueRoleName ? "Select a unique role name." : null}
     />
     <Select
       label="Inherits Role"
       bind:value={selectedRole.inherits}
-      options={otherRoles}
+      options={selectedRole._id === "BASIC" ? $roles : otherRoles}
       getOptionValue={role => role._id}
       getOptionLabel={role => role.name}
-      disabled={builtInRoles.includes(selectedRole.name)}
+      disabled={shouldDisableRoleInput}
     />
     <Select
       label="Base Permissions"
@@ -135,11 +145,11 @@
       options={basePermissions}
       getOptionValue={x => x._id}
       getOptionLabel={x => x.name}
-      disabled={builtInRoles.includes(selectedRole.name)}
+      disabled={shouldDisableRoleInput}
     />
   {/if}
   <div slot="footer">
-    {#if !isCreating}
+    {#if !isCreating && !builtInRoles.includes(selectedRole.name)}
       <Button warning on:click={deleteRole}>Delete</Button>
     {/if}
   </div>

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -13,7 +13,7 @@
     customQueryIconColor,
     customQueryText,
   } from "helpers/data/utils"
-  import { getIcon } from "./icons"
+  import IntegrationIcon from "./IntegrationIcon.svelte"
   import { notifications } from "@budibase/bbui"
 
   let openDataSources = []
@@ -123,10 +123,10 @@
         on:iconClick={() => toggleNode(datasource)}
       >
         <div class="datasource-icon" slot="icon">
-          <svelte:component
+          <IntegrationIcon
-            this={getIcon(datasource.source, datasource.schema)}
+            integrationType={datasource.source}
-            height="18"
+            schema={datasource.schema}
-            width="18"
+            size="18"
           />
         </div>
         {#if datasource._id !== BUDIBASE_INTERNAL_DB}

packages/builder/src/components/backend/DatasourceNavigator/IntegrationIcon.svelte

@@ -0,0 +1,32 @@
+<script>
+  import { getIcon } from "./icons"
+  import CustomSVG from "components/common/CustomSVG.svelte"
+  import { admin } from "stores/portal"
+
+  export let integrationType
+  export let schema
+  export let size = "18"
+
+  $: objectStoreUrl = $admin.cloud ? "https://cdn.budi.live" : ""
+  $: pluginsUrl = `${objectStoreUrl}/plugins`
+  $: iconInfo = getIcon(integrationType, schema)
+
+  async function getSvgFromUrl(info) {
+    const url = `${pluginsUrl}/${info.url}`
+    const resp = await fetch(url, {
+      headers: {
+        ["pragma"]: "no-cache",
+        ["cache-control"]: "no-cache",
+      },
+    })
+    return resp.text()
+  }
+</script>
+
+{#if iconInfo.icon}
+  <svelte:component this={iconInfo.icon} height={size} width={size} />
+{:else if iconInfo.url}
+  {#await getSvgFromUrl(iconInfo) then retrievedSvg}
+    <CustomSVG {size} svgHtml={retrievedSvg} />
+  {/await}
+{/if}

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -209,27 +209,29 @@
 {:else}
   <Body size="S"><i>No tables found.</i></Body>
 {/if}
-<Divider />
+{#if integration.relationships !== false}
-<div class="query-header">
+  <Divider />
-  <Heading size="S">Relationships</Heading>
+  <div class="query-header">
-  <Button primary on:click={() => openRelationshipModal()}>
+    <Heading size="S">Relationships</Heading>
-    Define relationship
+    <Button primary on:click={() => openRelationshipModal()}>
-  </Button>
+      Define relationship
-</div>
+    </Button>
-<Body>
+  </div>
-  Tell budibase how your tables are related to get even more smart features.
+  <Body>
-</Body>
+    Tell budibase how your tables are related to get even more smart features.
-{#if relationshipInfo && relationshipInfo.length > 0}
+  </Body>
-  <Table
+  {#if relationshipInfo && relationshipInfo.length > 0}
-    on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
+    <Table
-    schema={relationshipSchema}
+      on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
-    data={relationshipInfo}
+      schema={relationshipSchema}
-    allowEditColumns={false}
+      data={relationshipInfo}
-    allowEditRows={false}
+      allowEditColumns={false}
-    allowSelectRows={false}
+      allowEditRows={false}
-  />
+      allowSelectRows={false}
-{:else}
+    />
-  <Body size="S"><i>No relationships configured.</i></Body>
+  {:else}
+    <Body size="S"><i>No relationships configured.</i></Body>
+  {/if}
 {/if}
 
 <style>

packages/builder/src/components/backend/DatasourceNavigator/_components/DatasourceCard.svelte

@@ -1,7 +1,7 @@
 <script>
   import { createEventDispatcher } from "svelte"
   import { Heading, Detail } from "@budibase/bbui"
-  import { getIcon } from "../icons"
+  import IntegrationIcon from "../IntegrationIcon.svelte"
 
   export let integration
   export let integrationType
@@ -16,11 +16,7 @@
   class="item hoverable"
 >
   <div class="item-body" class:with-type={!!schema.type}>
-    <svelte:component
+    <IntegrationIcon {integrationType} {schema} size="25" />
-      this={getIcon(integrationType, schema)}
-      height="20"
-      width="20"
-    />
     <div class="text">
       <Heading size="XXS">{schema.friendlyName}</Heading>
       {#if schema.type}

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -16,6 +16,8 @@ import Firebase from "./Firebase.svelte"
 import Redis from "./Redis.svelte"
 import Snowflake from "./Snowflake.svelte"
 import Custom from "./Custom.svelte"
+import { integrations } from "stores/backend"
+import { get } from "svelte/store"
 
 const ICONS = {
   BUDIBASE: Budibase,
@@ -41,9 +43,12 @@ const ICONS = {
 export default ICONS
 
 export function getIcon(integrationType, schema) {
-  if (schema?.custom || !ICONS[integrationType]) {
+  const integrationList = get(integrations)
-    return ICONS.CUSTOM
+  if (integrationList[integrationType]?.iconUrl) {
+    return { url: integrationList[integrationType].iconUrl }
+  } else if (schema?.custom || !ICONS[integrationType]) {
+    return { icon: ICONS.CUSTOM }
   } else {
-    return ICONS[integrationType]
+    return { icon: ICONS[integrationType] }
   }
 }

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -124,6 +124,14 @@
       label: "Multi-select",
       value: FIELDS.ARRAY.type,
     },
+    {
+      label: "Barcode/QR",
+      value: FIELDS.BARCODEQR.type,
+    },
+    {
+      label: "Long Form Text",
+      value: FIELDS.LONGFORM.type,
+    },
   ]
 </script>
 

packages/builder/src/components/common/CustomSVG.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { Helpers } from "@budibase/bbui"
+  export let size
+  export let svgHtml
+
+  function substituteSize(svg) {
+    if (svg.includes("height=")) {
+      svg = svg.replace(/height="[^"]+"/, `height="${size}"`)
+    }
+    if (svg.includes("width=")) {
+      svg = svg.replace(/width="[^"]+"/, `width="${size}"`)
+    }
+    if (svg.includes("id=")) {
+      const matches = svg.match(/id="([^"]+)"/g)
+      for (let match of matches) {
+        svg = svg.replace(new RegExp(match, "g"), Helpers.uuid())
+      }
+    }
+    return svg
+  }
+</script>
+
+{@html substituteSize(svgHtml)}

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -43,7 +43,7 @@
   let helpers = handlebarsCompletions()
   let getCaretPosition
   let search = ""
-  let initialValueJS = value?.startsWith("{{ js ")
+  let initialValueJS = typeof value === "string" && value?.startsWith("{{ js ")
   let mode = initialValueJS ? "JavaScript" : "Handlebars"
   let jsValue = initialValueJS ? value : null
   let hbsValue = initialValueJS ? null : value

packages/builder/src/components/design/settings/componentSettings.js

@@ -51,6 +51,7 @@ const componentMap = {
   "field/link": FormFieldSelect,
   "field/array": FormFieldSelect,
   "field/json": FormFieldSelect,
+  "field/barcode/qr": FormFieldSelect,
   // Some validation types are the same as others, so not all types are
   // explicitly listed here. e.g. options uses string validation
   "validation/string": ValidationEditor,

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte

@@ -21,6 +21,7 @@
   export let key
   export let actions
   export let bindings = []
+  export let nested
 
   $: showAvailableActions = !actions?.length
 
@@ -187,6 +188,7 @@
             this={selectedActionComponent}
             parameters={selectedAction.parameters}
             bindings={allBindings}
+            {nested}
           />
         </div>
       {/key}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte

@@ -12,6 +12,7 @@
   export let value = []
   export let name
   export let bindings
+  export let nested
 
   let drawer
   let tmpValue
@@ -90,6 +91,7 @@
     eventType={name}
     {bindings}
     {key}
+    {nested}
   />
 </Drawer>
 

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -10,11 +10,13 @@
 
   export let parameters
   export let bindings = []
+  export let nested
 
   $: formComponents = getContextProviderComponents(
     $currentAsset,
     $store.selectedComponentId,
-    "form"
+    "form",
+    { includeSelf: nested }
   )
   $: schemaComponents = getContextProviderComponents(
     $currentAsset,

packages/builder/src/components/design/settings/controls/EjectBlockButton.svelte

@@ -0,0 +1,13 @@
+<script>
+  import { ActionButton } from "@budibase/bbui"
+
+  const eject = () => {
+    document.dispatchEvent(
+      new KeyboardEvent("keydown", { key: "e", ctrlKey: true })
+    )
+  }
+</script>
+
+<div>
+  <ActionButton secondary on:click={eject}>Eject block</ActionButton>
+</div>

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -20,6 +20,8 @@
   import { createEventDispatcher, onMount } from "svelte"
 
   const dispatch = createEventDispatcher()
+  const { OperatorOptions } = Constants
+  const { getValidOperatorsForType } = LuceneUtils
 
   export let schemaFields
   export let filters = []
@@ -45,7 +47,7 @@
       {
         id: generate(),
         field: null,
-        operator: Constants.OperatorOptions.Equals.value,
+        operator: OperatorOptions.Equals.value,
         value: null,
         valueType: "Value",
       },
@@ -66,49 +68,60 @@
     return schemaFields.find(field => field.name === filter.field)
   }
 
-  const onFieldChange = (expression, field) => {
+  const santizeTypes = filter => {
-    // Update the field types
+    // Update type based on field
-    expression.type = enrichedSchemaFields.find(x => x.name === field)?.type
+    const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field)
-    expression.externalType = getSchema(expression)?.externalType
+    filter.type = fieldSchema?.type
 
-    // Ensure a valid operator is set
+    // Update external type based on field
-    const validOperators = LuceneUtils.getValidOperatorsForType(
+    filter.externalType = getSchema(filter)?.externalType
-      expression.type
+  }
-    ).map(x => x.value)
+
-    if (!validOperators.includes(expression.operator)) {
+  const santizeOperator = filter => {
-      expression.operator =
+    // Ensure a valid operator is selected
-        validOperators[0] ?? Constants.OperatorOptions.Equals.value
+    const operators = getValidOperatorsForType(filter.type).map(x => x.value)
-      onOperatorChange(expression, expression.operator)
+    if (!operators.includes(filter.operator)) {
+      filter.operator = operators[0] ?? OperatorOptions.Equals.value
     }
 
-    // if changed to an array, change default value to empty array
+    // Update the noValue flag if the operator does not take a value
-    const idx = filters.findIndex(x => x.id === expression.id)
+    const noValueOptions = [
-    if (expression.type === "array") {
+      OperatorOptions.Empty.value,
-      filters[idx].value = []
+      OperatorOptions.NotEmpty.value,
-    } else {
+    ]
-      filters[idx].value = null
+    filter.noValue = noValueOptions.includes(filter.operator)
+  }
+
+  const santizeValue = filter => {
+    // Check if the operator allows a value at all
+    if (filter.noValue) {
+      filter.value = null
+      return
+    }
+
+    // Ensure array values are properly set and cleared
+    if (Array.isArray(filter.value)) {
+      if (filter.valueType !== "Value" || filter.type !== "array") {
+        filter.value = null
+      }
+    } else if (filter.type === "array" && filter.valueType === "Value") {
+      filter.value = []
     }
   }
 
-  const onOperatorChange = (expression, operator) => {
+  const onFieldChange = filter => {
-    const noValueOptions = [
+    santizeTypes(filter)
-      Constants.OperatorOptions.Empty.value,
+    santizeOperator(filter)
-      Constants.OperatorOptions.NotEmpty.value,
+    santizeValue(filter)
-    ]
+  }
-    expression.noValue = noValueOptions.includes(operator)
+
-    if (expression.noValue) {
+  const onOperatorChange = filter => {
-      expression.value = null
+    santizeOperator(filter)
-    }
+    santizeValue(filter)
-    if (
+  }
-      operator === Constants.OperatorOptions.In.value &&
+
-      !Array.isArray(expression.value)
+  const onValueTypeChange = filter => {
-    ) {
+    santizeValue(filter)
-      if (expression.value) {
-        expression.value = [expression.value]
-      } else {
-        expression.value = []
-      }
-    }
   }
 
   const getFieldOptions = field => {
@@ -153,23 +166,24 @@
               <Select
                 bind:value={filter.field}
                 options={fieldOptions}
-                on:change={e => onFieldChange(filter, e.detail)}
+                on:change={() => onFieldChange(filter)}
                 placeholder="Column"
               />
               <Select
                 disabled={!filter.field}
-                options={LuceneUtils.getValidOperatorsForType(filter.type)}
+                options={getValidOperatorsForType(filter.type)}
                 bind:value={filter.operator}
-                on:change={e => onOperatorChange(filter, e.detail)}
+                on:change={() => onOperatorChange(filter)}
                 placeholder={null}
               />
               <Select
                 disabled={filter.noValue || !filter.field}
                 options={valueTypeOptions}
                 bind:value={filter.valueType}
+                on:change={() => onValueTypeChange(filter)}
                 placeholder={null}
               />
-              {#if filter.valueType === "Binding"}
+              {#if filter.field && filter.valueType === "Binding"}
                 <DrawerBindableInput
                   disabled={filter.noValue}
                   title={`Value for "${filter.field}"`}
@@ -250,7 +264,7 @@
     column-gap: var(--spacing-l);
     row-gap: var(--spacing-s);
     align-items: center;
-    grid-template-columns: 1fr 120px 120px 1fr auto auto;
+    grid-template-columns: 1fr 150px 120px 1fr 16px 16px;
   }
 
   .filter-label {

packages/builder/src/components/design/settings/controls/FormFieldSelect.svelte

@@ -24,18 +24,17 @@
 
   const getOptions = (schema, type) => {
     let entries = Object.entries(schema ?? {})
-
     let types = []
-    if (type === "field/options") {
+    if (type === "field/options" || type === "field/barcode/qr") {
       // allow options to be used on both options and string fields
       types = [type, "field/string"]
     } else {
       types = [type]
     }
 
-    types = types.map(type => type.split("/")[1])
+    types = types.map(type => type.slice(type.indexOf("/") + 1))
-    entries = entries.filter(entry => types.includes(entry[1].type))
 
+    entries = entries.filter(entry => types.includes(entry[1].type))
     return entries.map(entry => entry[0])
   }
 </script>

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -20,6 +20,7 @@
   export let componentBindings = []
   export let nested = false
   export let highlighted = false
+  export let info = null
 
   $: nullishValue = value == null || value === ""
   $: allBindings = getAllBindings(bindings, componentBindings, nested)
@@ -94,11 +95,15 @@
       bindings={allBindings}
       name={key}
       text={label}
+      {nested}
       {key}
       {type}
       {...props}
     />
   </div>
+  {#if info}
+    <div class="text">{@html info}</div>
+  {/if}
 </div>
 
 <style>
@@ -123,4 +128,9 @@
   .control {
     position: relative;
   }
+  .text {
+    margin-top: var(--spectrum-global-dimension-size-65);
+    font-size: var(--spectrum-global-dimension-font-size-75);
+    color: var(--grey-6);
+  }
 </style>

packages/builder/src/components/design/settings/controls/TableSelect.svelte

@@ -1,26 +1,28 @@
 <script>
   import { Select } from "@budibase/bbui"
-  import { tables } from "stores/backend"
+  import { createEventDispatcher } from "svelte"
+  import { tables as tablesStore } from "stores/backend"
 
   export let value
+
+  const dispatch = createEventDispatcher()
+
+  $: tables = $tablesStore.list.map(m => ({
+    label: m.name,
+    tableId: m._id,
+    type: "table",
+  }))
+
+  const onChange = e => {
+    const dataSource = tables?.find(x => x.tableId === e.detail)
+    dispatch("change", dataSource)
+  }
 </script>
 
-<div>
+<Select
-  <Select extraThin secondary wide on:change {value}>
+  on:change={onChange}
-    <option value="">Choose a table</option>
+  value={value?.tableId}
-    {#each $tables.list as table}
+  options={tables}
-      <option value={table._id}>{table.name}</option>
+  getOptionValue={x => x.tableId}
-    {/each}
+  getOptionLabel={x => x.label}
-  </Select>
+/>
-</div>
-
-<style>
-  div {
-    flex: 1 1 auto;
-    display: flex;
-    flex-direction: row;
-  }
-  div :global(> *) {
-    flex: 1 1 auto;
-  }
-</style>

packages/builder/src/components/design/settings/controls/URLSelect.svelte

@@ -4,6 +4,7 @@
 
   export let value
   export let bindings
+  export let placeholder
 
   $: urlOptions = $store.screens
     .map(screen => screen.routing?.route)
@@ -13,6 +14,7 @@
 <DrawerBindableCombobox
   {value}
   {bindings}
+  {placeholder}
   on:change
   options={urlOptions}
   appendBindingsAsOptions={false}

packages/builder/src/components/integration/index.svelte

@@ -1,7 +1,16 @@
 <script>
   import Editor from "./QueryEditor.svelte"
   import FieldsBuilder from "./QueryFieldsBuilder.svelte"
-  import { Label, Input } from "@budibase/bbui"
+  import {
+    Label,
+    Input,
+    Select,
+    Divider,
+    Layout,
+    Icon,
+    Button,
+    ActionButton,
+  } from "@budibase/bbui"
 
   const QueryTypes = {
     SQL: "sql",
@@ -15,6 +24,8 @@
   export let editable = true
   export let height = 500
 
+  let stepEditors = []
+
   $: urlDisplay =
     schema.urlDisplay &&
     `${datasource.config.url}${
@@ -24,6 +35,39 @@
   function updateQuery({ detail }) {
     query.fields[schema.type] = detail.value
   }
+
+  function updateEditorsOnDelete(deleteIndex) {
+    for (let i = deleteIndex; i < query.fields.steps?.length - 1; i++) {
+      stepEditors[i].update(query.fields.steps[i + 1].value?.value)
+    }
+  }
+  function updateEditorsOnSwap(actionIndex, targetIndex) {
+    const target = query.fields.steps[targetIndex].value?.value
+    stepEditors[targetIndex].update(
+      query.fields.steps[actionIndex].value?.value
+    )
+    stepEditors[actionIndex].update(target)
+  }
+
+  function setEditorTemplate(fromKey, toKey, index) {
+    const currentValue = query.fields.steps[index].value?.value
+    if (
+      !currentValue ||
+      currentValue.toString().replace("\\s", "").length < 3 ||
+      schema.steps.filter(step => step.key === fromKey)[0]?.template ===
+        currentValue
+    ) {
+      query.fields.steps[index].value.value = schema.steps.filter(
+        step => step.key === toKey
+      )[0]?.template
+      stepEditors[index].update(query.fields.steps[index].value.value)
+    }
+    query.fields.steps[index].key = toKey
+  }
+
+  $: shouldDisplayJsonBox =
+    schema.type === QueryTypes.JSON &&
+    query.fields.extra?.actionType !== "pipeline"
 </script>
 
 {#if schema}
@@ -38,7 +82,7 @@
         value={query.fields.sql}
         parameters={query.parameters}
       />
-    {:else if schema.type === QueryTypes.JSON}
+    {:else if shouldDisplayJsonBox}
       <Editor
         editorHeight={height}
         label="Query"
@@ -56,6 +100,118 @@
           <Input thin outline disabled value={urlDisplay} />
         </div>
       {/if}
+    {:else if query.fields.extra?.actionType === "pipeline"}
+      <br />
+      {#if !query.fields.steps?.length}
+        <div class="controls">
+          <Button
+            secondary
+            slot="buttons"
+            on:click={() => {
+              query.fields.steps = [
+                {
+                  key: "$match",
+                  value: "{\n\t\n}",
+                },
+              ]
+            }}>Add stage</Button
+          >
+        </div>
+        <br />
+      {:else}
+        {#each query.fields.steps ?? [] as step, index}
+          <div class="block">
+            <div class="subblock">
+              <Divider noMargin />
+              <div class="blockSection">
+                <div class="block-options">
+                  Stage {index + 1}
+                  <div class="block-actions">
+                    <div style="margin-right: 24px;">
+                      {#if index > 0}
+                        <ActionButton
+                          quiet
+                          on:click={() => {
+                            updateEditorsOnSwap(index, index - 1)
+                            const target = query.fields.steps[index - 1].key
+                            query.fields.steps[index - 1].key =
+                              query.fields.steps[index].key
+                            query.fields.steps[index].key = target
+                          }}
+                          icon="ChevronUp"
+                        />
+                      {/if}
+                      {#if index < query.fields.steps.length - 1}
+                        <ActionButton
+                          quiet
+                          on:click={() => {
+                            updateEditorsOnSwap(index, index + 1)
+                            const target = query.fields.steps[index + 1].key
+                            query.fields.steps[index + 1].key =
+                              query.fields.steps[index].key
+                            query.fields.steps[index].key = target
+                          }}
+                          icon="ChevronDown"
+                        />
+                      {/if}
+                    </div>
+                    <ActionButton
+                      on:click={() => {
+                        updateEditorsOnDelete(index)
+                        query.fields.steps.splice(index, 1)
+                        query.fields.steps = [...query.fields.steps]
+                      }}
+                      icon="DeleteOutline"
+                    />
+                  </div>
+                </div>
+                <Layout noPadding gap="S">
+                  <div class="fields">
+                    <div class="block-field">
+                      <Select
+                        value={step.key}
+                        options={schema.steps.map(s => s.key)}
+                        on:change={({ detail }) => {
+                          setEditorTemplate(step.key, detail, index)
+                        }}
+                      />
+                      <Editor
+                        bind:this={stepEditors[index]}
+                        editorHeight={height / 2}
+                        mode="json"
+                        value={typeof step.value === "string"
+                          ? step.value
+                          : step.value.value}
+                        on:change={({ detail }) => {
+                          query.fields.steps[index].value = detail
+                        }}
+                      />
+                    </div>
+                  </div>
+                </Layout>
+              </div>
+            </div>
+            <div class="separator" />
+            {#if index === query.fields.steps.length - 1}
+              <Icon
+                hoverable
+                name="AddCircle"
+                size="S"
+                on:click={() => {
+                  query.fields.steps = [
+                    ...query.fields.steps,
+                    {
+                      key: "$match",
+                      value: "{\n\t\n}",
+                    },
+                  ]
+                }}
+              />
+              <br />
+            {/if}
+          </div>
+        {/each}
+      {/if}
     {/if}
   {/key}
 {/if}
@@ -67,4 +223,57 @@
     grid-gap: var(--spacing-l);
     align-items: center;
   }
+  .blockSection {
+    padding: var(--spacing-xl);
+  }
+  .block {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: center;
+    margin-top: -6px;
+  }
+  .subblock {
+    width: 480px;
+    font-size: 16px;
+    background-color: var(--background);
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: 4px 4px 4px 4px;
+  }
+  .block-options {
+    justify-content: space-between;
+    display: flex;
+    align-items: center;
+    padding-bottom: 24px;
+  }
+  .block-actions {
+    justify-content: space-between;
+    display: flex;
+    align-items: right;
+  }
+
+  .fields {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    gap: var(--spacing-s);
+  }
+  .block-field {
+    display: grid;
+    grid-gap: 5px;
+  }
+  .separator {
+    width: 1px;
+    height: 25px;
+    border-left: 1px dashed var(--grey-4);
+    color: var(--grey-4);
+    /* center horizontally */
+    align-self: center;
+  }
+  .controls {
+    display: flex;
+    align-items: center;
+    justify-content: right;
+  }
 </style>

packages/builder/src/components/portal/overview/automation/HistoryTab.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Layout, Table, Select, Pagination } from "@budibase/bbui"
+  import { Layout, Table, Select, Pagination, Button } from "@budibase/bbui"
   import DateTimeRenderer from "components/common/renderers/DateTimeRenderer.svelte"
   import StatusRenderer from "./StatusRenderer.svelte"
   import HistoryDetailsPanel from "./HistoryDetailsPanel.svelte"
@@ -7,12 +7,16 @@
   import { createPaginationStore } from "helpers/pagination"
   import { onMount } from "svelte"
   import dayjs from "dayjs"
+  import { auth, licensing, admin } from "stores/portal"
+  import { Constants } from "@budibase/frontend-core"
 
   const ERROR = "error",
     SUCCESS = "success",
     STOPPED = "stopped"
   export let app
 
+  $: licensePlan = $auth.user?.license?.plan
+
   let pageInfo = createPaginationStore()
   let runHistory = null
   let showPanel = false
@@ -26,6 +30,8 @@
   $: fetchLogs(automationId, status, page, timeRange)
 
   const timeOptions = [
+    { value: "90-d", label: "Past 90 days" },
+    { value: "30-d", label: "Past 30 days" },
     { value: "1-w", label: "Past week" },
     { value: "1-d", label: "Past day" },
     { value: "1-h", label: "Past 1 hour" },
@@ -131,10 +137,20 @@
       </div>
       <div class="select">
         <Select
-          placeholder="Past 30 days"
+          placeholder="All"
           label="Date range"
           bind:value={timeRange}
           options={timeOptions}
+          isOptionEnabled={x => {
+            if (licensePlan?.type === Constants.PlanType.FREE) {
+              return ["1-w", "30-d", "90-d"].indexOf(x.value) < 0
+            } else if (licensePlan?.type === Constants.PlanType.TEAM) {
+              return ["90-d"].indexOf(x.value) < 0
+            } else if (licensePlan?.type === Constants.PlanType.PRO) {
+              return ["30-d", "90-d"].indexOf(x.value) < 0
+            }
+            return true
+          }}
         />
       </div>
       <div class="select">
@@ -145,6 +161,14 @@
           options={statusOptions}
         />
       </div>
+      {#if (licensePlan?.type !== Constants.PlanType.ENTERPRISE && $auth.user.accountPortalAccess) || !$admin.cloud}
+        <div class="pro-upgrade">
+          <div class="pro-copy">Expand your automation log history</div>
+          <Button primary newStyles on:click={$licensing.goToUpgradePage()}>
+            Upgrade
+          </Button>
+        </div>
+      {/if}
     </div>
     {#if runHistory}
       <div>
@@ -221,4 +245,15 @@
   .panelOpen {
     grid-template-columns: auto 420px;
   }
+
+  .pro-upgrade {
+    display: flex;
+    align-items: center;
+    justify-content: flex-end;
+    flex: 1;
+  }
+
+  .pro-copy {
+    margin-right: var(--spacing-l);
+  }
 </style>

packages/builder/src/constants/backend/index.js

@@ -8,6 +8,15 @@ export const FIELDS = {
       presence: false,
     },
   },
+  BARCODEQR: {
+    name: "Barcode/QR",
+    type: "barcodeqr",
+    constraints: {
+      type: "string",
+      length: {},
+      presence: false,
+    },
+  },
   LONGFORM: {
     name: "Long Form Text",
     type: "longform",
@@ -148,6 +157,7 @@ export const ALLOWABLE_STRING_OPTIONS = [
   FIELDS.STRING,
   FIELDS.OPTIONS,
   FIELDS.LONGFORM,
+  FIELDS.BARCODEQR,
 ]
 export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map(
   opt => opt.type

packages/builder/src/constants/index.js

@@ -58,13 +58,6 @@ export const DefaultAppTheme = {
   navTextColor: "var(--spectrum-global-color-gray-800)",
 }
 
-export const PlanType = {
-  FREE: "free",
-  PRO: "pro",
-  BUSINESS: "business",
-  ENTERPRISE: "enterprise",
-}
-
 export const PluginSource = {
   URL: "URL",
   NPM: "NPM",

packages/builder/src/helpers/urlStateSync.js

@@ -10,6 +10,7 @@ export const syncURLToState = options => {
     fallbackUrl,
     store,
     routify,
+    beforeNavigate,
   } = options || {}
   if (
     !urlParam ||
@@ -41,6 +42,15 @@ export const syncURLToState = options => {
 
   // Navigate to a certain URL
   const gotoUrl = (url, params) => {
+    if (beforeNavigate) {
+      const res = beforeNavigate(url, params)
+      if (res?.url) {
+        url = res.url
+      }
+      if (res?.params) {
+        params = res.params
+      }
+    }
     log("Navigating to", url, "with params", params)
     cachedGoto(url, params)
   }

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -1,7 +1,16 @@
 <script>
   import { store, automationStore } from "builderStore"
   import { roles, flags } from "stores/backend"
-  import { Icon, Tabs, Tab, Heading, notifications } from "@budibase/bbui"
+  import { apps } from "stores/portal"
+  import {
+    ActionMenu,
+    MenuItem,
+    Icon,
+    Tabs,
+    Tab,
+    Heading,
+    notifications,
+  } from "@budibase/bbui"
   import RevertModal from "components/deploy/RevertModal.svelte"
   import VersionModal from "components/deploy/VersionModal.svelte"
   import DeployNavigation from "components/deploy/DeployNavigation.svelte"
@@ -54,6 +63,9 @@
     })
   }
 
+  $: isPublished =
+    $apps.find(app => app.devId === application)?.status === "published"
+
   onMount(async () => {
     if (!hasSynced && application) {
       try {
@@ -83,12 +95,43 @@
   <div class="root">
     <div class="top-nav">
       <div class="topleftnav">
-        <Icon
+        <ActionMenu>
-          size="M"
+          <div slot="control">
-          name="ArrowLeft"
+            <Icon size="M" hoverable name="ShowMenu" />
-          hoverable
+          </div>
-          on:click={() => $goto("../../portal/apps")}
+          <MenuItem on:click={() => $goto("../../portal/apps")}>
-        />
+            Exit to portal
+          </MenuItem>
+          <MenuItem
+            on:click={() => $goto(`../../portal/overview/${application}`)}
+          >
+            Overview
+          </MenuItem>
+          <MenuItem
+            on:click={() =>
+              $goto(`../../portal/overview/${application}?tab=Access`)}
+          >
+            Access
+          </MenuItem>
+          {#if isPublished}
+            <MenuItem
+              on:click={() =>
+                $goto(
+                  `../../portal/overview/${application}?tab=${encodeURIComponent(
+                    "Automation History"
+                  )}`
+                )}
+            >
+              Automation history
+            </MenuItem>
+          {/if}
+          <MenuItem
+            on:click={() =>
+              $goto(`../../portal/overview/${application}?tab=Settings`)}
+          >
+            Settings
+          </MenuItem>
+        </ActionMenu>
         <Heading size="XS">{$store.name || "App"}</Heading>
       </div>
       <div class="topcenternav">

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -98,11 +98,21 @@
     `./components/${$selectedComponent?._id}/new`
   )
 
+  // Register handler to send custom to the preview
+  $: store.actions.preview.registerEventHandler((name, payload) => {
+    iframe?.contentWindow.postMessage(
+      JSON.stringify({
+        name,
+        payload,
+        isBudibaseEvent: true,
+        runtimeEvent: true,
+      })
+    )
+  })
+
   // Update the iframe with the builder info to render the correct preview
   const refreshContent = message => {
-    if (iframe) {
+    iframe?.contentWindow.postMessage(message)
-      iframe.contentWindow.postMessage(message)
-    }
   }
 
   const receiveMessage = message => {
@@ -200,6 +210,14 @@
             block: "center",
           })
         }
+      } else if (type === "eject-block") {
+        const { id, definition } = data
+        await store.actions.components.handleEjectBlock(id, definition)
+      } else if (type === "reload-plugin") {
+        await store.actions.components.refreshDefinitions()
+      } else if (type === "drop-new-component") {
+        const { component, parent, index } = data
+        await store.actions.components.create(component, null, parent, index)
       } else {
         console.warn(`Client sent unknown event type: ${type}`)
       }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentDropdownMenu.svelte

@@ -4,7 +4,9 @@
 
   export let component
 
+  $: definition = store.actions.components.getDefinition(component?._component)
   $: noPaste = !$store.componentToPaste
+  $: isBlock = definition?.block === true
 
   const keyboardEvent = (key, ctrlKey = false) => {
     document.dispatchEvent(
@@ -30,6 +32,15 @@
   >
     Delete
   </MenuItem>
+  {#if isBlock}
+    <MenuItem
+      icon="Export"
+      keyBind="Ctrl+E"
+      on:click={() => keyboardEvent("e", true)}
+    >
+      Eject block
+    </MenuItem>
+  {/if}
   <MenuItem
     icon="ChevronUp"
     keyBind="Ctrl+!ArrowUp"

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentKeyHandler.svelte

@@ -7,7 +7,9 @@
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
 
   let confirmDeleteDialog
+  let confirmEjectDialog
   let componentToDelete
+  let componentToEject
 
   const keyHandlers = {
     ["^ArrowUp"]: async component => {
@@ -29,6 +31,10 @@
       store.actions.components.copy(component)
       await store.actions.components.paste(component, "below")
     },
+    ["^e"]: component => {
+      componentToEject = component
+      confirmEjectDialog.show()
+    },
     ["^Enter"]: () => {
       $goto("./new")
     },
@@ -124,3 +130,10 @@
   okText="Delete Component"
   onOk={() => store.actions.components.delete(componentToDelete)}
 />
+<ConfirmDialog
+  bind:this={confirmEjectDialog}
+  title="Eject block"
+  body={`Ejecting a block breaks it down into multiple components and cannot be undone. Are you sure you want to eject "${componentToEject?._instanceName}"?`}
+  onOk={() => store.actions.components.requestEjectBlock(componentToEject?._id)}
+  okText="Eject block"
+/>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/navigation/ComponentTree.svelte

@@ -18,6 +18,8 @@
 
   let closedNodes = {}
 
+  $: currentScreen = get(selectedScreen)
+
   $: filteredComponents = components?.filter(component => {
     return (
       !$store.componentToPaste?.isCut ||
@@ -68,9 +70,30 @@
     closedNodes = closedNodes
   }
 
-  const onDrop = async e => {
+  const onDrop = async (e, component) => {
     e.stopPropagation()
     try {
+      const compDef = store.actions.components.getDefinition(
+        $dndStore.source?._component
+      )
+      if (!compDef) {
+        return
+      }
+      const compTypeName = compDef.name.toLowerCase()
+      const path = findComponentPath(currentScreen.props, component._id)
+
+      for (let pathComp of path) {
+        const pathCompDef = store.actions.components.getDefinition(
+          pathComp?._component
+        )
+        if (pathCompDef?.illegalChildren?.indexOf(compTypeName) > -1) {
+          notifications.warning(
+            `${compDef.name} cannot be a child of ${pathCompDef.name} (${pathComp._instanceName})`
+          )
+          return
+        }
+      }
+
       await dndStore.actions.drop()
     } catch (error) {
       console.error(error)
@@ -114,7 +137,9 @@
         on:dragstart={() => dndStore.actions.dragstart(component)}
         on:dragover={dragover(component, index)}
         on:iconClick={() => toggleNodeOpen(component._id)}
-        on:drop={onDrop}
+        on:drop={e => {
+          onDrop(e, component)
+        }}
         text={getComponentText(component)}
         icon={getComponentIcon(component)}
         withArrow={componentHasChildren(component)}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsSection.svelte

@@ -4,6 +4,7 @@
   import { store } from "builderStore"
   import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
   import ResetFieldsButton from "components/design/settings/controls/ResetFieldsButton.svelte"
+  import EjectBlockButton from "components/design/settings/controls/EjectBlockButton.svelte"
   import { getComponentForSetting } from "components/design/settings/componentSettings"
 
   export let componentDefinition
@@ -12,20 +13,29 @@
   export let componentBindings
   export let isScreen = false
 
-  $: sections = getSections(componentDefinition)
+  $: sections = getSections(componentInstance, componentDefinition, isScreen)
 
-  const getSections = definition => {
+  const getSections = (instance, definition, isScreen) => {
     const settings = definition?.settings ?? []
     const generalSettings = settings.filter(setting => !setting.section)
     const customSections = settings.filter(setting => setting.section)
-    return [
+    let sections = [
       {
         name: "General",
-        info: componentDefinition?.info,
         settings: generalSettings,
       },
       ...(customSections || []),
     ]
+
+    // Filter out settings which shouldn't be rendered
+    sections.forEach(section => {
+      section.settings.forEach(setting => {
+        setting.visible = canRenderControl(instance, setting, isScreen)
+      })
+      section.visible = section.settings.some(setting => setting.visible)
+    })
+
+    return sections
   }
 
   const updateSetting = async (key, value) => {
@@ -36,7 +46,7 @@
     }
   }
 
-  const canRenderControl = (setting, isScreen) => {
+  const canRenderControl = (instance, setting, isScreen) => {
     // Prevent rendering on click setting for screens
     if (setting?.type === "event" && isScreen) {
       return false
@@ -51,6 +61,7 @@
     if (setting.dependsOn) {
       let dependantSetting = setting.dependsOn
       let dependantValue = null
+      let invert = !!setting.dependsOn.invert
       if (typeof setting.dependsOn === "object") {
         dependantSetting = setting.dependsOn.setting
         dependantValue = setting.dependsOn.value
@@ -62,7 +73,7 @@
       // If no specific value is depended upon, check if a value exists at all
       // for the dependent setting
       if (dependantValue == null) {
-        const currentValue = componentInstance[dependantSetting]
+        const currentValue = instance[dependantSetting]
         if (currentValue === false) {
           return false
         }
@@ -73,7 +84,11 @@
       }
 
       // Otherwise check the value matches
-      return componentInstance[dependantSetting] === dependantValue
+      if (invert) {
+        return instance[dependantSetting] !== dependantValue
+      } else {
+        return instance[dependantSetting] === dependantValue
+      }
     }
 
     return true
@@ -81,60 +96,54 @@
 </script>
 
 {#each sections as section, idx (section.name)}
-  <DetailSummary name={section.name} collapsible={false}>
+  {#if section.visible}
-    {#if idx === 0 && !componentInstance._component.endsWith("/layout") && !isScreen}
+    <DetailSummary name={section.name} collapsible={false}>
-      <PropertyControl
+      {#if idx === 0 && !componentInstance._component.endsWith("/layout") && !isScreen}
-        control={Input}
-        label="Name"
-        key="_instanceName"
-        value={componentInstance._instanceName}
-        onChange={val => updateSetting("_instanceName", val)}
-      />
-    {/if}
-    {#each section.settings as setting (setting.key)}
-      {#if canRenderControl(setting, isScreen)}
         <PropertyControl
-          type={setting.type}
+          control={Input}
-          control={getComponentForSetting(setting)}
+          label="Name"
-          label={setting.label}
+          key="_instanceName"
-          key={setting.key}
+          value={componentInstance._instanceName}
-          value={componentInstance[setting.key]}
+          onChange={val => updateSetting("_instanceName", val)}
-          defaultValue={setting.defaultValue}
-          nested={setting.nested}
-          onChange={val => updateSetting(setting.key, val)}
-          highlighted={$store.highlightedSettingKey === setting.key}
-          props={{
-            // Generic settings
-            placeholder: setting.placeholder || null,
-
-            // Select settings
-            options: setting.options || [],
-
-            // Number fields
-            min: setting.min || null,
-            max: setting.max || null,
-          }}
-          {bindings}
-          {componentBindings}
-          {componentInstance}
-          {componentDefinition}
         />
       {/if}
-    {/each}
+      {#each section.settings as setting (setting.key)}
-    {#if idx === 0 && componentDefinition?.component?.endsWith("/fieldgroup")}
+        {#if setting.visible}
-      <ResetFieldsButton {componentInstance} />
+          <PropertyControl
-    {/if}
+            type={setting.type}
-    {#if section?.info}
+            control={getComponentForSetting(setting)}
-      <div class="text">
+            label={setting.label}
-        {@html section.info}
+            key={setting.key}
-      </div>
+            value={componentInstance[setting.key]}
-    {/if}
+            defaultValue={setting.defaultValue}
-  </DetailSummary>
+            nested={setting.nested}
-{/each}
+            onChange={val => updateSetting(setting.key, val)}
+            highlighted={$store.highlightedSettingKey === setting.key}
+            info={setting.info}
+            props={{
+              // Generic settings
+              placeholder: setting.placeholder || null,
 
-<style>
+              // Select settings
-  .text {
+              options: setting.options || [],
-    font-size: var(--spectrum-global-dimension-font-size-75);
+
-    color: var(--grey-6);
+              // Number fields
-  }
+              min: setting.min || null,
-</style>
+              max: setting.max || null,
+            }}
+            {bindings}
+            {componentBindings}
+            {componentInstance}
+            {componentDefinition}
+          />
+        {/if}
+      {/each}
+      {#if idx === 0 && componentDefinition?.component?.endsWith("/fieldgroup")}
+        <ResetFieldsButton {componentInstance} />
+      {/if}
+      {#if idx === 0 && componentDefinition?.block}
+        <EjectBlockButton />
+      {/if}
+    </DetailSummary>
+  {/if}
+{/each}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_layout.svelte

@@ -7,6 +7,18 @@
   import ComponentListPanel from "./_components/navigation/ComponentListPanel.svelte"
   import ComponentSettingsPanel from "./_components/settings/ComponentSettingsPanel.svelte"
 
+  const cleanUrl = url => {
+    // Strip trailing slashes
+    if (url?.endsWith("/index")) {
+      url = url.replace("/index", "")
+    }
+    // Hide new component panel whenever component ID changes
+    if (url?.endsWith("/new")) {
+      url = url.replace("/new", "")
+    }
+    return { url }
+  }
+
   // Keep URL and state in sync for selected component ID
   const stopSyncing = syncURLToState({
     urlParam: "componentId",
@@ -15,6 +27,7 @@
     fallbackUrl: "../",
     store,
     routify,
+    beforeNavigate: cleanUrl,
   })
 
   onDestroy(stopSyncing)

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte

@@ -169,6 +169,14 @@
       window.removeEventListener("keydown", handleKeyDown)
     }
   })
+
+  const onDragStart = component => {
+    store.actions.dnd.start(component)
+  }
+
+  const onDragEnd = () => {
+    store.actions.dnd.stop()
+  }
 </script>
 
 <div class="container" transition:fly|local={{ x: 260, duration: 300 }}>
@@ -206,6 +214,9 @@
               <div class="category-label">{category.name}</div>
               {#each category.children as component}
                 <div
+                  draggable="true"
+                  on:dragstart={() => onDragStart(component.component)}
+                  on:dragend={onDragEnd}
                   data-cy={`component-${component.name}`}
                   class="component"
                   class:selected={selectedIndex ===
@@ -229,8 +240,11 @@
         <Layout noPadding gap="XS">
           {#each blocks as block}
             <div
+              draggable="true"
               class="component"
               on:click={() => addComponent(block.component)}
+              on:dragstart={() => onDragStart(block.component)}
+              on:dragend={onDragEnd}
             >
               <Icon name={block.icon} />
               <Body size="XS">{block.name}</Body>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/componentStructure.json

@@ -5,7 +5,8 @@
     "children": [
       "tableblock",
       "cardsblock",
-      "repeaterblock"
+      "repeaterblock",
+      "formblock"
     ]
   },
   {
@@ -66,7 +67,8 @@
       "relationshipfield",
       "datetimefield",
       "multifieldselect",
-      "s3upload"
+      "s3upload",
+      "codescanner"
     ]
   },
   {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/screens/_components/ScreenDropdownMenu.svelte

@@ -38,7 +38,7 @@
     let duplicateScreen = Helpers.cloneDeep(screen)
     delete duplicateScreen._id
     delete duplicateScreen._rev
-    makeComponentUnique(duplicateScreen.props)
+    duplicateScreen.props = makeComponentUnique(duplicateScreen.props)
 
     // Attach the new name and URL
     duplicateScreen.routing.route = sanitizeUrl(screenUrl)

packages/builder/src/pages/builder/apps/index.svelte

@@ -133,7 +133,7 @@
             </Body>
           </Layout>
           <Divider />
-          {#if $licensing.usageMetrics.dayPasses >= 100}
+          {#if $licensing.usageMetrics?.dayPasses >= 100}
             <div>
               <Layout gap="S" justifyItems="center">
                 <img class="spaceman" alt="spaceman" src={Spaceman} />

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -56,7 +56,7 @@
         {
           title: "Plugins",
           href: "/builder/portal/manage/plugins",
-          badge: "Beta",
+          badge: "New",
         },
 
         {

packages/builder/src/pages/builder/portal/manage/groups/[groupId].svelte

@@ -25,6 +25,7 @@
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import CreateEditGroupModal from "./_components/CreateEditGroupModal.svelte"
   import GroupIcon from "./_components/GroupIcon.svelte"
+  import AppAddModal from "./_components/AppAddModal.svelte"
 
   export let groupId
 
@@ -34,15 +35,14 @@
   let prevSearch = undefined
   let pageInfo = createPaginationStore()
   let loaded = false
-  let editModal
+  let editModal, deleteModal, appAddModal
-  let deleteModal
 
   $: page = $pageInfo.page
   $: fetchUsers(page, searchTerm)
   $: group = $groups.find(x => x._id === groupId)
   $: filtered = $users.data
   $: groupApps = $apps.filter(app =>
-    groups.actions.getGroupAppIds(group).includes(apps.getProdAppID(app.appId))
+    groups.actions.getGroupAppIds(group).includes(apps.getProdAppID(app.devId))
   )
   $: {
     if (loaded && !group?._id) {
@@ -182,7 +182,14 @@
     </Layout>
 
     <Layout noPadding gap="S">
-      <Heading size="S">Apps</Heading>
+      <div class="header">
+        <Heading size="S">Apps</Heading>
+        <div>
+          <Button on:click={appAddModal.show()} icon="ExperienceAdd" cta>
+            Add app
+          </Button>
+        </div>
+      </div>
       <List>
         {#if groupApps.length}
           {#each groupApps as app}
@@ -197,12 +204,24 @@
                 <StatusLight
                   square
                   color={RoleUtils.getRoleColour(
-                    group.roles[apps.getProdAppID(app.appId)]
+                    group.roles[apps.getProdAppID(app.devId)]
                   )}
                 >
-                  {getRoleLabel(app.appId)}
+                  {getRoleLabel(app.devId)}
                 </StatusLight>
               </div>
+              <Icon
+                on:click={e => {
+                  groups.actions.removeApp(
+                    groupId,
+                    apps.getProdAppID(app.devId)
+                  )
+                  e.stopPropagation()
+                }}
+                hoverable
+                size="S"
+                name="Close"
+              />
             </ListItem>
           {/each}
         {:else}
@@ -216,6 +235,11 @@
 <Modal bind:this={editModal}>
   <CreateEditGroupModal {group} {saveGroup} />
 </Modal>
+
+<Modal bind:this={appAddModal}>
+  <AppAddModal {group} />
+</Modal>
+
 <ConfirmDialog
   bind:this={deleteModal}
   title="Delete user group"

packages/builder/src/pages/builder/portal/manage/groups/_components/AppAddModal.svelte

@@ -0,0 +1,53 @@
+<script>
+  import { Body, ModalContent, Select } from "@budibase/bbui"
+  import { apps, groups } from "stores/portal"
+  import { roles } from "stores/backend"
+  import RoleSelect from "components/common/RoleSelect.svelte"
+
+  export let group
+
+  $: appOptions = $apps.map(app => ({
+    label: app.name,
+    value: app,
+  }))
+  $: confirmDisabled =
+    (!selectingRole && !selectedApp) || (selectingRole && !selectedRoleId)
+  let selectedApp, selectedRoleId
+  let selectingRole = false
+
+  async function appSelected() {
+    const prodAppId = apps.getProdAppID(selectedApp.devId)
+    if (!selectingRole) {
+      selectingRole = true
+      await roles.fetchByAppId(prodAppId)
+      // return false to stop closing modal
+      return false
+    } else {
+      await groups.actions.addApp(group._id, prodAppId, selectedRoleId)
+    }
+  }
+</script>
+
+<ModalContent
+  onConfirm={appSelected}
+  size="M"
+  title="Add app to group"
+  confirmText={selectingRole ? "Confirm" : "Next"}
+  showSecondaryButton={selectingRole}
+  secondaryButtonText="Back"
+  secondaryAction={() => (selectingRole = false)}
+  disabled={confirmDisabled}
+>
+  {#if !selectingRole}
+    <Body
+      >Select an app to assign roles for members of <i>"{group.name}"</i></Body
+    >
+    <Select bind:value={selectedApp} options={appOptions} />
+  {:else}
+    <Body
+      >Select the role that all members of "<i>{group.name}</i>" will have for
+      <i>"{selectedApp.name}"</i></Body
+    >
+    <RoleSelect allowPublic={false} bind:value={selectedRoleId} />
+  {/if}
+</ModalContent>

packages/builder/src/pages/builder/portal/manage/groups/index.svelte

@@ -27,7 +27,6 @@
     icon: "UserGroup",
     color: "var(--spectrum-global-color-blue-600)",
     users: [],
-    apps: [],
     roles: {},
   }
 
@@ -91,16 +90,14 @@
 
 <Layout noPadding gap="M">
   <Layout gap="XS" noPadding>
-    <Heading size="M">User groups</Heading>
+    <div class="title">
-    {#if !$licensing.groupsEnabled}
+      <Heading size="M">User groups</Heading>
-      <Tags>
+      {#if !$licensing.groupsEnabled}
-        <div class="tags">
+        <Tags>
-          <div class="tag">
+          <Tag icon="LockClosed">Pro plan</Tag>
-            <Tag icon="LockClosed">Pro plan</Tag>
+        </Tags>
-          </div>
+      {/if}
-        </div>
+    </div>
-      </Tags>
-    {/if}
     <Body>
       Easily assign and manage your users' access with user groups.
       {#if !$auth.accountPortalAccess && !$licensing.groupsEnabled && $admin.cloud}
@@ -124,6 +121,7 @@
       {:else}
         <Button
           newStyles
+          primary
           disabled={!$auth.accountPortalAccess && $admin.cloud}
           on:click={$licensing.goToUpgradePage()}
         >
@@ -141,18 +139,22 @@
         </Button>
       {/if}
     </ButtonGroup>
-    <div class="controls-right">
+    {#if $licensing.groupsEnabled}
-      <Search bind:value={searchString} placeholder="Search" />
+      <div class="controls-right">
-    </div>
+        <Search bind:value={searchString} placeholder="Search" />
+      </div>
+    {/if}
   </div>
-  <Table
+  {#if $licensing.groupsEnabled}
-    on:click={({ detail }) => $goto(`./${detail._id}`)}
+    <Table
-    {schema}
+      on:click={({ detail }) => $goto(`./${detail._id}`)}
-    data={filteredGroups}
+      {schema}
-    allowEditColumns={false}
+      data={filteredGroups}
-    allowEditRows={false}
+      allowEditColumns={false}
-    {customRenderers}
+      allowEditRows={false}
-  />
+      {customRenderers}
+    />
+  {/if}
 </Layout>
 
 <Modal bind:this={modal}>
@@ -176,8 +178,11 @@
   .controls-right :global(.spectrum-Search) {
     width: 200px;
   }
-  .tag {
+  .title {
-    margin-top: var(--spacing-xs);
+    display: flex;
-    margin-left: var(--spacing-m);
+    flex-direction: row;
+    align-items: center;
+    justify-content: flex-start;
+    gap: var(--spacing-m);
   }
 </style>

packages/builder/src/pages/builder/portal/manage/plugins/index.svelte

@@ -55,18 +55,20 @@
           Add plugin
         </Button>
       </div>
-      <div class="filters">
+      {#if filteredPlugins?.length}
-        <div class="select">
+        <div class="filters">
-          <Select
+          <div class="select">
-            bind:value={filter}
+            <Select
-            placeholder={null}
+              bind:value={filter}
-            options={filterOptions}
+              placeholder={null}
-            autoWidth
+              options={filterOptions}
-            quiet
+              autoWidth
-          />
+              quiet
+            />
+          </div>
+          <Search bind:value={searchTerm} placeholder="Search plugins" />
         </div>
-        <Search bind:value={searchTerm} placeholder="Search plugins" />
+      {/if}
-      </div>
     </div>
     {#if filteredPlugins?.length}
       <Layout noPadding gap="S">

packages/builder/src/pages/builder/portal/overview/_components/AccessTab.svelte

@@ -156,8 +156,8 @@
               page={$usersFetch.pageNumber + 1}
               hasPrevPage={$usersFetch.hasPrevPage}
               hasNextPage={$usersFetch.hasNextPage}
-              goToPrevPage={$usersFetch.loading ? null : fetch.prevPage}
+              goToPrevPage={$usersFetch.loading ? null : usersFetch.prevPage}
-              goToNextPage={$usersFetch.loading ? null : fetch.nextPage}
+              goToNextPage={$usersFetch.loading ? null : usersFetch.nextPage}
             />
           </div>
         </div>

packages/builder/src/pages/builder/portal/settings/usage.svelte

@@ -11,7 +11,7 @@
   } from "@budibase/bbui"
   import { onMount } from "svelte"
   import { admin, auth, licensing } from "../../../../stores/portal"
-  import { PlanType } from "../../../../constants"
+  import { Constants } from "@budibase/frontend-core"
   import { DashCard, Usage } from "../../../../components/usage"
 
   let staticUsage = []
@@ -125,7 +125,7 @@
   }
 
   const goToAccountPortal = () => {
-    if (license?.plan.type === PlanType.FREE) {
+    if (license?.plan.type === Constants.PlanType.FREE) {
       window.location.href = upgradeUrl
     } else {
       window.location.href = manageUrl
@@ -133,7 +133,7 @@
   }
 
   const setPrimaryActionText = () => {
-    if (license?.plan.type === PlanType.FREE) {
+    if (license?.plan.type === Constants.PlanType.FREE) {
       primaryActionText = "Upgrade"
       return
     }

packages/builder/src/stores/backend/roles.js

@@ -5,16 +5,24 @@ import { RoleUtils } from "@budibase/frontend-core"
 export function createRolesStore() {
   const { subscribe, update, set } = writable([])
 
+  function setRoles(roles) {
+    set(
+      roles.sort((a, b) => {
+        const priorityA = RoleUtils.getRolePriority(a._id)
+        const priorityB = RoleUtils.getRolePriority(b._id)
+        return priorityA > priorityB ? -1 : 1
+      })
+    )
+  }
+
   const actions = {
     fetch: async () => {
       const roles = await API.getRoles()
-      set(
+      setRoles(roles)
-        roles.sort((a, b) => {
+    },
-          const priorityA = RoleUtils.getRolePriority(a._id)
+    fetchByAppId: async appId => {
-          const priorityB = RoleUtils.getRolePriority(b._id)
+      const { roles } = await API.getRolesForApp(appId)
-          return priorityA > priorityB ? -1 : 1
+      setRoles(roles)
-        })
-      )
     },
     delete: async role => {
       await API.deleteRole({

packages/builder/src/stores/portal/apps.js

@@ -21,6 +21,8 @@ const getProdAppID = appId => {
   } else if (!appId.startsWith("app")) {
     rest = appId
     separator = "_"
+  } else {
+    return appId
   }
   return `app${separator}${rest}`
 }

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.4.18-alpha.1",
+  "version": "2.0.30-alpha.7",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "1.4.18-alpha.1",
+    "@budibase/backend-core": "2.0.30-alpha.7",
-    "@budibase/string-templates": "1.4.18-alpha.1",
+    "@budibase/string-templates": "2.0.30-alpha.7",
-    "@budibase/types": "1.4.18-alpha.1",
+    "@budibase/types": "2.0.30-alpha.7",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
@@ -36,6 +36,7 @@
     "docker-compose": "0.23.6",
     "dotenv": "16.0.1",
     "download": "8.0.0",
+    "find-free-port": "^2.0.0",
     "inquirer": "8.0.0",
     "joi": "17.6.0",
     "lookpath": "1.1.0",
@@ -45,7 +46,8 @@
     "pouchdb": "7.3.0",
     "pouchdb-replication-stream": "1.2.9",
     "randomstring": "1.1.5",
-    "tar": "6.1.11"
+    "tar": "6.1.11",
+    "yaml": "^2.1.1"
   },
   "devDependencies": {
     "copyfiles": "^2.4.1",

packages/cli/src/constants.js

@@ -21,3 +21,5 @@ exports.AnalyticsEvents = {
 }
 
 exports.POSTHOG_TOKEN = "phc_yGOn4i7jWKaCTapdGR6lfA4AvmuEQ2ijn5zAVSFYPlS"
+
+exports.GENERATED_USER_EMAIL = "admin@admin.com"

packages/cli/src/exec.js

@@ -22,6 +22,6 @@ exports.runPkgCommand = async (command, dir = "./") => {
     throw new Error("Must have yarn or npm installed to run build.")
   }
   const npmCmd = command === "install" ? `npm ${command}` : `npm run ${command}`
-  const cmd = yarn ? `yarn ${command}` : npmCmd
+  const cmd = yarn ? `yarn ${command} --ignore-engines` : npmCmd
   await exports.exec(cmd, dir)
 }

packages/cli/src/hosting/genUser.js

@@ -0,0 +1,22 @@
+const { success } = require("../utils")
+const { updateDockerComposeService } = require("./utils")
+const randomString = require("randomstring")
+const { GENERATED_USER_EMAIL } = require("../constants")
+
+exports.generateUser = async (password, silent) => {
+  const email = GENERATED_USER_EMAIL
+  if (!password) {
+    password = randomString.generate({ length: 6 })
+  }
+  updateDockerComposeService(service => {
+    service.environment["BB_ADMIN_USER_EMAIL"] = email
+    service.environment["BB_ADMIN_USER_PASSWORD"] = password
+  })
+  if (!silent) {
+    console.log(
+      success(
+        `User admin credentials configured, access with email: ${email} - password: ${password}`
+      )
+    )
+  }
+}

packages/cli/src/hosting/index.js

@@ -1,164 +1,18 @@
 const Command = require("../structures/Command")
-const { CommandWords, InitTypes, AnalyticsEvents } = require("../constants")
+const { CommandWords } = require("../constants")
-const { lookpath } = require("lookpath")
+const { init } = require("./init")
-const {
+const { start } = require("./start")
-  downloadFile,
+const { stop } = require("./stop")
-  logErrorToFile,
+const { status } = require("./status")
-  success,
+const { update } = require("./update")
-  info,
+const { generateUser } = require("./genUser")
-  parseEnv,
+const { watchPlugins } = require("./watch")
-} = require("../utils")
-const { confirmation } = require("../questions")
-const fs = require("fs")
-const compose = require("docker-compose")
-const makeEnv = require("./makeEnv")
-const axios = require("axios")
-const { captureEvent } = require("../events")
-
-const BUDIBASE_SERVICES = ["app-service", "worker-service", "proxy-service"]
-const ERROR_FILE = "docker-error.log"
-const FILE_URLS = [
-  "https://raw.githubusercontent.com/Budibase/budibase/master/hosting/docker-compose.yaml",
-]
-const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
-
-async function downloadFiles() {
-  const promises = []
-  for (let url of FILE_URLS) {
-    const fileName = url.split("/").slice(-1)[0]
-    promises.push(downloadFile(url, `./${fileName}`))
-  }
-  await Promise.all(promises)
-}
-
-async function checkDockerConfigured() {
-  const error =
-    "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
-  const docker = await lookpath("docker")
-  const compose = await lookpath("docker-compose")
-  if (!docker || !compose) {
-    throw error
-  }
-}
-
-function checkInitComplete() {
-  if (!fs.existsSync(makeEnv.filePath)) {
-    throw "Please run the hosting --init command before any other hosting command."
-  }
-}
-
-async function handleError(func) {
-  try {
-    await func()
-  } catch (err) {
-    if (err && err.err) {
-      logErrorToFile(ERROR_FILE, err.err)
-    }
-    throw `Failed to start - logs written to file: ${ERROR_FILE}`
-  }
-}
-
-async function init(type) {
-  const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
-  await checkDockerConfigured()
-  if (!isQuick) {
-    const shouldContinue = await confirmation(
-      "This will create multiple files in current directory, should continue?"
-    )
-    if (!shouldContinue) {
-      console.log("Stopping.")
-      return
-    }
-  }
-  captureEvent(AnalyticsEvents.SelfHostInit, {
-    type,
-  })
-  await downloadFiles()
-  const config = isQuick ? makeEnv.QUICK_CONFIG : {}
-  if (type === InitTypes.DIGITAL_OCEAN) {
-    try {
-      const output = await axios.get(DO_USER_DATA_URL)
-      const response = parseEnv(output.data)
-      for (let [key, value] of Object.entries(makeEnv.ConfigMap)) {
-        if (response[key]) {
-          config[value] = response[key]
-        }
-      }
-    } catch (err) {
-      // don't need to handle error, just don't do anything
-    }
-  }
-  await makeEnv.make(config)
-}
-
-async function start() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(
-    info(
-      "Starting services, this may take a moment - first time this may take a few minutes to download images."
-    )
-  )
-  const port = makeEnv.get("MAIN_PORT")
-  await handleError(async () => {
-    // need to log as it makes it more clear
-    await compose.upAll({ cwd: "./", log: true })
-  })
-  console.log(
-    success(
-      `Services started, please go to http://localhost:${port} for next steps.`
-    )
-  )
-}
-
-async function status() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(info("Budibase status"))
-  await handleError(async () => {
-    const response = await compose.ps()
-    console.log(response.out)
-  })
-}
-
-async function stop() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  console.log(info("Stopping services, this may take a moment."))
-  await handleError(async () => {
-    await compose.stop()
-  })
-  console.log(success("Services have been stopped successfully."))
-}
-
-async function update() {
-  await checkDockerConfigured()
-  checkInitComplete()
-  if (await confirmation("Do you wish to update you docker-compose.yaml?")) {
-    await downloadFiles()
-  }
-  await handleError(async () => {
-    const status = await compose.ps()
-    const parts = status.out.split("\n")
-    const isUp = parts[2] && parts[2].indexOf("Up") !== -1
-    if (isUp) {
-      console.log(info("Stopping services, this may take a moment."))
-      await compose.stop()
-    }
-    console.log(info("Beginning update, this may take a few minutes."))
-    await compose.pullMany(BUDIBASE_SERVICES, { log: true })
-    if (isUp) {
-      console.log(success("Update complete, restarting services..."))
-      await start()
-    }
-  })
-}
 
 const command = new Command(`${CommandWords.HOSTING}`)
   .addHelp("Controls self hosting on the Budibase platform.")
   .addSubOption(
     "--init [type]",
-    "Configure a self hosted platform in current directory, type can be unspecified or 'quick'.",
+    "Configure a self hosted platform in current directory, type can be unspecified, 'quick' or 'single'.",
     init
   )
   .addSubOption(
@@ -181,5 +35,16 @@ const command = new Command(`${CommandWords.HOSTING}`)
     "Update the Budibase images to the latest version.",
     update
   )
+  .addSubOption(
+    "--watch-plugin-dir [directory]",
+    "Add plugin directory watching to a Budibase install.",
+    watchPlugins
+  )
+  .addSubOption(
+    "--gen-user",
+    "Create an admin user automatically as part of first start.",
+    generateUser
+  )
+  .addSubOption("--single", "Specify this with init to use the single image.")
 
 exports.command = command

packages/cli/src/hosting/init.js

@@ -0,0 +1,75 @@
+const { InitTypes, AnalyticsEvents } = require("../constants")
+const { confirmation } = require("../questions")
+const { captureEvent } = require("../events")
+const makeFiles = require("./makeFiles")
+const axios = require("axios")
+const { parseEnv } = require("../utils")
+const { checkDockerConfigured, downloadFiles } = require("./utils")
+const { watchPlugins } = require("./watch")
+const { generateUser } = require("./genUser")
+
+const DO_USER_DATA_URL = "http://169.254.169.254/metadata/v1/user-data"
+
+async function getInitConfig(type, isQuick, port) {
+  const config = isQuick ? makeFiles.QUICK_CONFIG : {}
+  if (type === InitTypes.DIGITAL_OCEAN) {
+    try {
+      const output = await axios.get(DO_USER_DATA_URL)
+      const response = parseEnv(output.data)
+      for (let [key, value] of Object.entries(makeFiles.ConfigMap)) {
+        if (response[key]) {
+          config[value] = response[key]
+        }
+      }
+    } catch (err) {
+      // don't need to handle error, just don't do anything
+    }
+  }
+  // override port
+  if (port) {
+    config[makeFiles.ConfigMap.MAIN_PORT] = port
+  }
+  return config
+}
+
+exports.init = async opts => {
+  let type, isSingle, watchDir, genUser, port, silent
+  if (typeof opts === "string") {
+    type = opts
+  } else {
+    type = opts["init"]
+    isSingle = opts["single"]
+    watchDir = opts["watchPluginDir"]
+    genUser = opts["genUser"]
+    port = opts["port"]
+    silent = opts["silent"]
+  }
+  const isQuick = type === InitTypes.QUICK || type === InitTypes.DIGITAL_OCEAN
+  await checkDockerConfigured()
+  if (!isQuick) {
+    const shouldContinue = await confirmation(
+      "This will create multiple files in current directory, should continue?"
+    )
+    if (!shouldContinue) {
+      console.log("Stopping.")
+      return
+    }
+  }
+  captureEvent(AnalyticsEvents.SelfHostInit, {
+    type,
+  })
+  const config = await getInitConfig(type, isQuick, port)
+  if (!isSingle) {
+    await downloadFiles()
+    await makeFiles.makeEnv(config, silent)
+  } else {
+    await makeFiles.makeSingleCompose(config, silent)
+  }
+  if (watchDir) {
+    await watchPlugins(watchDir, silent)
+  }
+  if (genUser) {
+    const inputPassword = typeof genUser === "string" ? genUser : null
+    await generateUser(inputPassword, silent)
+  }
+}

packages/cli/src/hosting/makeEnv.js

@@ -1,66 +0,0 @@
-const { number } = require("../questions")
-const { success } = require("../utils")
-const fs = require("fs")
-const path = require("path")
-const randomString = require("randomstring")
-
-const FILE_PATH = path.resolve("./.env")
-
-function getContents(port) {
-  return `
-# Use the main port in the builder for your self hosting URL, e.g. localhost:10000
-MAIN_PORT=${port}
-
-# This section contains all secrets pertaining to the system
-JWT_SECRET=${randomString.generate()}
-MINIO_ACCESS_KEY=${randomString.generate()}
-MINIO_SECRET_KEY=${randomString.generate()}
-COUCH_DB_PASSWORD=${randomString.generate()}
-COUCH_DB_USER=${randomString.generate()}
-REDIS_PASSWORD=${randomString.generate()}
-INTERNAL_API_KEY=${randomString.generate()}
-
-# This section contains variables that do not need to be altered under normal circumstances
-APP_PORT=4002
-WORKER_PORT=4003
-MINIO_PORT=4004
-COUCH_DB_PORT=4005
-REDIS_PORT=6379
-WATCHTOWER_PORT=6161
-BUDIBASE_ENVIRONMENT=PRODUCTION`
-}
-
-module.exports.filePath = FILE_PATH
-module.exports.ConfigMap = {
-  MAIN_PORT: "port",
-}
-module.exports.QUICK_CONFIG = {
-  key: "budibase",
-  port: 10000,
-}
-
-module.exports.make = async (inputs = {}) => {
-  const hostingPort =
-    inputs.port ||
-    (await number(
-      "Please enter the port on which you want your installation to run: ",
-      10000
-    ))
-  const fileContents = getContents(hostingPort)
-  fs.writeFileSync(FILE_PATH, fileContents)
-  console.log(
-    success(
-      "Configuration has been written successfully - please check .env file for more details."
-    )
-  )
-}
-
-module.exports.get = property => {
-  const props = fs.readFileSync(FILE_PATH, "utf8").split(property)
-  if (props[0].charAt(0) === "=") {
-    property = props[0]
-  } else {
-    property = props[1]
-  }
-  return property.split("=")[1].split("\n")[0]
-}