Merge branch 'develop' into dnd

.github/workflows/release-develop.yml

@@ -7,6 +7,7 @@ on:
 
 env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
       

.github/workflows/release.yml

@@ -4,9 +4,16 @@ on:
  push:
     branches:
       - master
+ workflow_dispatch:
+    inputs:
+      release_self_host:
+        description: 'Release to self hosters? (Y/N)'     
+        required: true
+        default: 'N'
 
 env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
 
@@ -47,7 +54,19 @@ jobs:
         uses: "WyriHaximus/github-action-get-previous-tag@v1"
 
       - name: Build/release Docker images
-        run: |
+        if: ${{ github.event.inputs.release_self_host != 'Y' }}
+        run: | 
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          yarn build
+          yarn build:docker
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+          BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }}
+
+      - name: Build/release Docker images (Self Host)
+        if: ${{ github.event.inputs.release_self_host == 'Y' }}
+        run: | 
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
           yarn build
           yarn build:docker

hosting/kubernetes/budibase/Chart.yaml

@@ -37,5 +37,5 @@ dependencies:
     condition: services.couchdb.enabled
   - name: ingress-nginx
     version: 3.35.0 
-    repository: https://github.com/kubernetes/ingress-nginx
+    repository: https://kubernetes.github.io/ingress-nginx
     condition: services.ingress.nginx

hosting/kubernetes/budibase/templates/app-service-deployment.yaml

@@ -94,6 +94,8 @@ spec:
           value: {{ .Values.globals.sentryDSN }}
         - name: WORKER_URL
           value: worker-service:{{ .Values.services.worker.port }}
+        - name: COOKIE_DOMAIN
+          value: {{ .Values.globals.cookieDomain | quote }}
         image: budibase/apps
         imagePullPolicy: Always
         name: bbapps

hosting/kubernetes/budibase/templates/worker-service-deployment.yaml

@@ -89,6 +89,8 @@ spec:
           value: {{ .Values.globals.selfHosted | quote }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
+        - name: COOKIE_DOMAIN
+          value: {{ .Values.globals.cookieDomain | quote }}
         image: budibase/worker
         imagePullPolicy: Always
         name: bbworker

hosting/kubernetes/budibase/values.yaml

@@ -90,6 +90,7 @@ globals:
   logLevel: info
   selfHosted: 1
   accountPortalUrL: ""
+  cookieDomain: ""
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 
   # if createSecrets is set to false, you can hard-code your secrets here

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.125-alpha.17",
+  "version": "0.9.146-alpha.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -42,7 +42,8 @@
     "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "test:e2e": "lerna run cy:test",
     "test:e2e:ci": "lerna run cy:ci",
-    "build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION release && cd -",
+    "build:docker": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker:production": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION release && cd -",
     "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "release:helm": "./scripts/release_helm_chart.sh",
     "multi:enable": "lerna run multi:enable",

packages/auth/accounts.js

@@ -0,0 +1 @@
+module.exports = require("./src/cloud/accounts")

packages/auth/deprovision.js

@@ -0,0 +1 @@
+module.exports = require("./src/tenancy/deprovision")

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.125-alpha.17",
+  "version": "0.9.146-alpha.5",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/scripts/jestSetup.js

@@ -1,5 +1,6 @@
 const env = require("../src/environment")
 
+env._set("SELF_HOSTED", "1")
 env._set("NODE_ENV", "jest")
 env._set("JWT_SECRET", "test-jwtsecret")
 env._set("LOG_LEVEL", "silent")

packages/auth/src/cache/user.js

@@ -1,5 +1,7 @@
 const redis = require("../redis/authRedis")
 const { getTenantId, lookupTenantId, getGlobalDB } = require("../tenancy")
+const env = require("../environment")
+const accounts = require("../cloud/accounts")
 
 const EXPIRY_SECONDS = 3600
 
@@ -9,6 +11,15 @@ const EXPIRY_SECONDS = 3600
 const populateFromDB = async (userId, tenantId) => {
   const user = await getGlobalDB(tenantId).get(userId)
   user.budibaseAccess = true
+
+  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
+    const account = await accounts.getAccount(user.email)
+    if (account) {
+      user.account = account
+      user.accountPortalAccess = true
+    }
+  }
+
   return user
 }
 

packages/auth/src/cloud/accounts.js

@@ -0,0 +1,22 @@
+const API = require("./api")
+const env = require("../environment")
+
+const api = new API(env.ACCOUNT_PORTAL_URL)
+
+// TODO: Authorization
+
+exports.getAccount = async email => {
+  const payload = {
+    email,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+  })
+  const json = await response.json()
+
+  if (response.status !== 200) {
+    throw Error(`Error getting account by email ${email}`, json)
+  }
+
+  return json[0]
+}

packages/auth/src/cloud/api.js

@@ -0,0 +1,44 @@
+const fetch = require("node-fetch")
+class API {
+  constructor(host) {
+    this.host = host
+  }
+
+  apiCall =
+    method =>
+    async (url = "", options = {}) => {
+      if (!options.headers) {
+        options.headers = {}
+      }
+
+      if (!options.headers["Content-Type"]) {
+        options.headers = {
+          "Content-Type": "application/json",
+          Accept: "application/json",
+          ...options.headers,
+        }
+      }
+
+      let json = options.headers["Content-Type"] === "application/json"
+
+      const requestOptions = {
+        method: method,
+        body: json ? JSON.stringify(options.body) : options.body,
+        headers: options.headers,
+        // TODO: See if this is necessary
+        credentials: "include",
+      }
+
+      const resp = await fetch(`${this.host}${url}`, requestOptions)
+
+      return resp
+    }
+
+  post = this.apiCall("POST")
+  get = this.apiCall("GET")
+  patch = this.apiCall("PATCH")
+  del = this.apiCall("DELETE")
+  put = this.apiCall("PUT")
+}
+
+module.exports = API

packages/auth/src/db/constants.js

@@ -12,6 +12,7 @@ exports.StaticDatabases = {
     name: "global-info",
     docs: {
       tenants: "tenants",
+      usageQuota: "usage_quota",
     },
   },
 }

packages/auth/src/db/utils.js

@@ -368,8 +368,33 @@ async function getScopedConfig(db, params) {
   return configDoc && configDoc.config ? configDoc.config : configDoc
 }
 
+function generateNewUsageQuotaDoc() {
+  return {
+    _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
+    quotaReset: Date.now() + 2592000000,
+    usageQuota: {
+      automationRuns: 0,
+      rows: 0,
+      storage: 0,
+      apps: 0,
+      users: 0,
+      views: 0,
+      emails: 0,
+    },
+    usageLimits: {
+      automationRuns: 1000,
+      rows: 4000,
+      apps: 4,
+      storage: 1000,
+      users: 10,
+      emails: 50,
+    },
+  }
+}
+
 exports.Replication = Replication
 exports.getScopedConfig = getScopedConfig
 exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams
 exports.getScopedFullConfig = getScopedFullConfig
+exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc

packages/auth/src/environment.js

@@ -16,9 +16,14 @@ module.exports = {
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  AWS_REGION: process.env.AWS_REGION,
   MINIO_URL: process.env.MINIO_URL,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
+  ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
+  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
+  COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   isTest,
   _set(key, value) {
     process.env[key] = value

packages/auth/src/index.js

@@ -12,6 +12,7 @@ const {
   auditLog,
   tenancy,
   appTenancy,
+  authError,
 } = require("./middleware")
 const { setDB } = require("./db")
 const userCache = require("./cache/user")
@@ -60,6 +61,7 @@ module.exports = {
     buildTenancyMiddleware: tenancy,
     buildAppTenancyMiddleware: appTenancy,
     auditLog,
+    authError,
   },
   cache: {
     user: userCache,

packages/auth/src/middleware/index.js

@@ -2,6 +2,7 @@ const jwt = require("./passport/jwt")
 const local = require("./passport/local")
 const google = require("./passport/google")
 const oidc = require("./passport/oidc")
+const { authError } = require("./passport/utils")
 const authenticated = require("./authenticated")
 const auditLog = require("./auditLog")
 const tenancy = require("./tenancy")
@@ -16,4 +17,5 @@ module.exports = {
   auditLog,
   tenancy,
   appTenancy,
+  authError,
 }

packages/auth/src/middleware/passport/google.js

@@ -27,7 +27,11 @@ async function authenticate(accessToken, refreshToken, profile, done) {
  * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport.
  * @returns Dynamically configured Passport Google Strategy
  */
-exports.strategyFactory = async function (config, callbackUrl) {
+exports.strategyFactory = async function (
+  config,
+  callbackUrl,
+  verify = authenticate
+) {
   try {
     const { clientID, clientSecret } = config
 
@@ -43,7 +47,7 @@ exports.strategyFactory = async function (config, callbackUrl) {
         clientSecret: config.clientSecret,
         callbackURL: callbackUrl,
       },
-      authenticate
+      verify
     )
   } catch (err) {
     console.error(err)

packages/auth/src/middleware/passport/tests/third-party-common.spec.js

@@ -104,7 +104,7 @@ describe("third party common", () => {
           _id: id,
           email: email,
         }
-        const response = await db.post(dbUser)
+        const response = await db.put(dbUser)
         dbUser._rev = response.rev
       }
 

packages/auth/src/middleware/passport/third-party-common.js

@@ -71,7 +71,7 @@ exports.authenticateThirdParty = async function (
   dbUser = await syncUser(dbUser, thirdPartyUser)
 
   // create or sync the user
-  const response = await db.post(dbUser)
+  const response = await db.put(dbUser)
   dbUser._rev = response.rev
 
   // authenticate

packages/auth/src/objectStore/index.js

@@ -73,6 +73,7 @@ exports.ObjectStore = bucket => {
   AWS.config.update({
     accessKeyId: env.MINIO_ACCESS_KEY,
     secretAccessKey: env.MINIO_SECRET_KEY,
+    region: env.AWS_REGION,
   })
   const config = {
     s3ForcePathStyle: true,

packages/auth/src/security/permissions.js

@@ -139,8 +139,7 @@ exports.doesHaveResourcePermission = (
   // set foundSub to not subResourceId, incase there is no subResource
   let foundMain = false,
     foundSub = false
-  for (let [resource, level] of Object.entries(permissions)) {
-    const levels = getAllowedLevels(level)
+  for (let [resource, levels] of Object.entries(permissions)) {
     if (resource === resourceId && levels.indexOf(permLevel) !== -1) {
       foundMain = true
     }
@@ -177,10 +176,6 @@ exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => {
   return false
 }
 
-exports.higherPermission = (perm1, perm2) => {
-  return levelToNumber(perm1) > levelToNumber(perm2) ? perm1 : perm2
-}
-
 exports.isPermissionLevelHigherThanRead = level => {
   return levelToNumber(level) > 1
 }

packages/auth/src/security/roles.js

@@ -1,6 +1,6 @@
 const { getDB } = require("../db")
 const { cloneDeep } = require("lodash/fp")
-const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions")
+const { BUILTIN_PERMISSION_IDS } = require("./permissions")
 const {
   generateRoleID,
   getRoleParams,
@@ -193,8 +193,17 @@ exports.getUserPermissions = async (appId, userRoleId) => {
   const permissions = {}
   for (let role of rolesHierarchy) {
     if (role.permissions) {
-      for (let [resource, level] of Object.entries(role.permissions)) {
-        permissions[resource] = higherPermission(permissions[resource], level)
+      for (let [resource, levels] of Object.entries(role.permissions)) {
+        if (!permissions[resource]) {
+          permissions[resource] = []
+        }
+        const permsSet = new Set(permissions[resource])
+        if (Array.isArray(levels)) {
+          levels.forEach(level => permsSet.add(level))
+        } else {
+          permsSet.add(levels)
+        }
+        permissions[resource] = [...permsSet]
       }
     }
   }

packages/auth/src/security/sessions.js

@@ -30,6 +30,10 @@ exports.invalidateSessions = async (userId, sessionId = null) => {
     sessions.push({ key: makeSessionID(userId, sessionId) })
   } else {
     sessions = await getSessionsForUser(userId)
+    sessions.forEach(
+      session =>
+        (session.key = makeSessionID(session.userId, session.sessionId))
+    )
   }
   const client = await redis.getSessionClient()
   const promises = []

packages/auth/src/tenancy/context.js

@@ -53,6 +53,11 @@ exports.setTenantId = (
   // processed later in the chain
   tenantId = user.tenantId || header || tenantId
 
+  // Set the tenantId from the subdomain
+  if (!tenantId) {
+    tenantId = ctx.subdomains && ctx.subdomains[0]
+  }
+
   if (!tenantId && !allowNoTenant) {
     ctx.throw(403, "Tenant id not set")
   }

packages/auth/src/tenancy/deprovision.js

@@ -0,0 +1,97 @@
+const { getGlobalUserParams, getAllApps } = require("../db/utils")
+const { getDB, getCouch } = require("../db")
+const { getGlobalDB } = require("./tenancy")
+const { StaticDatabases } = require("../db/constants")
+
+const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
+const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
+
+const removeTenantFromInfoDB = async tenantId => {
+  try {
+    const infoDb = getDB(PLATFORM_INFO_DB)
+    let tenants = await infoDb.get(TENANT_DOC)
+    tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
+
+    await infoDb.put(tenants)
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} from info db`, err)
+    throw err
+  }
+}
+
+exports.removeUserFromInfoDB = async dbUser => {
+  const infoDb = getDB(PLATFORM_INFO_DB)
+  const keys = [dbUser._id, dbUser.email]
+  const userDocs = await infoDb.allDocs({
+    keys,
+    include_docs: true,
+  })
+  const toDelete = userDocs.rows.map(row => {
+    return {
+      ...row.doc,
+      _deleted: true,
+    }
+  })
+  await infoDb.bulkDocs(toDelete)
+}
+
+const removeUsersFromInfoDB = async tenantId => {
+  try {
+    const globalDb = getGlobalDB(tenantId)
+    const infoDb = getDB(PLATFORM_INFO_DB)
+    const allUsers = await globalDb.allDocs(
+      getGlobalUserParams(null, {
+        include_docs: true,
+      })
+    )
+    const allEmails = allUsers.rows.map(row => row.doc.email)
+    // get the id docs
+    let keys = allUsers.rows.map(row => row.id)
+    // and the email docs
+    keys = keys.concat(allEmails)
+    // retrieve the docs and delete them
+    const userDocs = await infoDb.allDocs({
+      keys,
+      include_docs: true,
+    })
+    const toDelete = userDocs.rows.map(row => {
+      return {
+        ...row.doc,
+        _deleted: true,
+      }
+    })
+    await infoDb.bulkDocs(toDelete)
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} users from info db`, err)
+    throw err
+  }
+}
+
+const removeGlobalDB = async tenantId => {
+  try {
+    const globalDb = getGlobalDB(tenantId)
+    await globalDb.destroy()
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} users from info db`, err)
+    throw err
+  }
+}
+
+const removeTenantApps = async tenantId => {
+  try {
+    const apps = await getAllApps(getCouch(), { all: true })
+    const destroyPromises = apps.map(app => getDB(app.appId).destroy())
+    await Promise.allSettled(destroyPromises)
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} apps`, err)
+    throw err
+  }
+}
+
+// can't live in tenancy package due to circular dependency on db/utils
+exports.deleteTenant = async tenantId => {
+  await removeTenantFromInfoDB(tenantId)
+  await removeUsersFromInfoDB(tenantId)
+  await removeGlobalDB(tenantId)
+  await removeTenantApps(tenantId)
+}

packages/auth/src/tenancy/tenancy.js

@@ -73,7 +73,7 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
   await Promise.all(promises)
 }
 
-exports.getGlobalDB = (tenantId = null) => {
+exports.getGlobalDBName = (tenantId = null) => {
   // tenant ID can be set externally, for example user API where
   // new tenants are being created, this may be the case
   if (!tenantId) {
@@ -81,13 +81,16 @@ exports.getGlobalDB = (tenantId = null) => {
   }
 
   let dbName
-
   if (tenantId === DEFAULT_TENANT_ID) {
     dbName = StaticDatabases.GLOBAL.name
   } else {
     dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
   }
+  return dbName
+}
 
+exports.getGlobalDB = (tenantId = null) => {
+  const dbName = exports.getGlobalDBName(tenantId)
   return getDB(dbName)
 }
 

packages/auth/src/utils.js

@@ -4,6 +4,7 @@ const { options } = require("./middleware/passport/jwt")
 const { createUserEmailView } = require("./db/views")
 const { Headers } = require("./constants")
 const { getGlobalDB } = require("./tenancy")
+const environment = require("./environment")
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -66,17 +67,22 @@ exports.getCookie = (ctx, name) => {
  * @param {string|object} value The value of cookie which will be set.
  */
 exports.setCookie = (ctx, value, name = "builder") => {
-  if (!value) {
-    ctx.cookies.set(name)
-  } else {
+  if (value) {
     value = jwt.sign(value, options.secretOrKey)
-    ctx.cookies.set(name, value, {
-      maxAge: Number.MAX_SAFE_INTEGER,
-      path: "/",
-      httpOnly: false,
-      overwrite: true,
-    })
   }
+
+  const config = {
+    maxAge: Number.MAX_SAFE_INTEGER,
+    path: "/",
+    httpOnly: false,
+    overwrite: true,
+  }
+
+  if (environment.COOKIE_DOMAIN) {
+    config.domain = environment.COOKIE_DOMAIN
+  }
+
+  ctx.cookies.set(name, value, config)
 }
 
 /**

packages/auth/yarn.lock

@@ -4470,9 +4470,9 @@ tmp@^0.0.33:
     os-tmpdir "~1.0.2"
 
 tmpl@1.0.x:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1"
-  integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc"
+  integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==
 
 to-fast-properties@^2.0.0:
   version "2.0.0"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.125-alpha.17",
+  "version": "0.9.146-alpha.5",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Modal/ModalContent.svelte

@@ -14,6 +14,7 @@
   export let showConfirmButton = true
   export let showCloseIcon = true
   export let onConfirm = undefined
+  export let onCancel = undefined
   export let disabled = false
   export let showDivider = true
 
@@ -28,6 +29,14 @@
     }
     loading = false
   }
+
+  async function close() {
+    loading = true
+    if (!onCancel || (await onCancel()) !== false) {
+      cancel()
+    }
+    loading = false
+  }
 </script>
 
 <div
@@ -65,7 +74,7 @@
       >
         <slot name="footer" />
         {#if showCancelButton}
-          <Button group secondary on:click={cancel}>{cancelText}</Button>
+          <Button group secondary on:click={close}>{cancelText}</Button>
         {/if}
         {#if showConfirmButton}
           <Button

packages/bbui/src/Table/Table.svelte

@@ -27,6 +27,7 @@
   export let selectedRows = []
   export let editColumnTitle = "Edit"
   export let customRenderers = []
+  export let disableSorting = false
 
   const dispatch = createEventDispatcher()
 
@@ -63,7 +64,7 @@
   )
 
   // Reset state when data changes
-  $: data.length, reset()
+  $: rows.length, reset()
   const reset = () => {
     nextScrollTop = 0
     scrollTop = 0
@@ -107,7 +108,7 @@
   }
 
   const sortRows = (rows, sortColumn, sortOrder) => {
-    if (!sortColumn || !sortOrder) {
+    if (!sortColumn || !sortOrder || disableSorting) {
       return rows
     }
     return rows.slice().sort((a, b) => {
@@ -131,6 +132,7 @@
       sortColumn = fieldSchema.name
       sortOrder = "Descending"
     }
+    dispatch("sort", { column: sortColumn, order: sortOrder })
   }
 
   const getDisplayName = schema => {

packages/builder/cypress/integration/createTable.spec.js

@@ -6,7 +6,7 @@ context("Create a Table", () => {
 
   it("should create a new Table", () => {
     cy.createTable("dog")
-
+    cy.wait(1000)
     // Check if Table exists
     cy.get(".table-title h1").should("have.text", "dog")
   })
@@ -36,7 +36,8 @@ context("Create a Table", () => {
   it("edits a row", () => {
     cy.contains("button", "Edit").click({ force: true })
     cy.wait(1000)
-    cy.get(".spectrum-Modal input").type("Updated")
+    cy.get(".spectrum-Modal input").clear()
+    cy.get(".spectrum-Modal input").type("RoverUpdated")
     cy.contains("Save").click()
     cy.contains("RoverUpdated").should("have.text", "RoverUpdated")
   })
@@ -62,7 +63,7 @@ context("Create a Table", () => {
 
   it("deletes a table", () => {
     cy.get(".actions > :nth-child(1) > .icon > .spectrum-Icon > use")
-      .first()
+      .eq(1)
       .click({ force: true })
     cy.get(".spectrum-Menu > :nth-child(2)").click()
     cy.contains("Delete Table").click()

packages/builder/cypress/support/commands.js

@@ -35,8 +35,11 @@ Cypress.Commands.add("createApp", name => {
     .within(() => {
       cy.get("input").eq(0).type(name).should("have.value", name).blur()
       cy.get(".spectrum-ButtonGroup").contains("Create app").click()
+      cy.wait(7000)
     })
     .then(() => {
+      // Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future
+      cy.createInitialDatasource("initialTable")
       cy.expandBudibaseConnection()
       cy.get(".nav-item.selected > .content").should("be.visible")
     })
@@ -69,11 +72,28 @@ Cypress.Commands.add("createTestTableWithData", () => {
   cy.addColumn("dog", "age", "Number")
 })
 
-Cypress.Commands.add("createTable", tableName => {
+Cypress.Commands.add("createInitialDatasource", tableName => {
   // Enter table name
+  cy.get(".spectrum-Modal").within(() => {
+    cy.contains("Budibase DB").trigger("mouseover").click().click()
+    cy.wait(1000)
+    cy.contains("Continue").click()
+  })
+
+  cy.get(".spectrum-Modal").within(() => {
+    cy.wait(1000)
+    cy.get("input").first().type(tableName).blur()
+    cy.get(".spectrum-ButtonGroup").contains("Create").click()
+  })
+  cy.contains(tableName).should("be.visible")
+})
+
+Cypress.Commands.add("createTable", tableName => {
   cy.contains("Budibase DB").click()
   cy.contains("Create new table").click()
+
   cy.get(".spectrum-Modal").within(() => {
+    cy.wait(1000)
     cy.get("input").first().type(tableName).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create").click()
   })

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.125-alpha.17",
+  "version": "0.9.146-alpha.5",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.125-alpha.17",
-    "@budibase/client": "^0.9.125-alpha.17",
+    "@budibase/bbui": "^0.9.146-alpha.5",
+    "@budibase/client": "^0.9.146-alpha.5",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.125-alpha.17",
+    "@budibase/string-templates": "^0.9.146-alpha.5",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/App.svelte

@@ -1,16 +1,10 @@
 <script>
-  import { onMount } from "svelte"
   import { Router } from "@roxi/routify"
   import { routes } from "../.routify/routes"
-  import { initialise } from "builderStore"
   import { NotificationDisplay } from "@budibase/bbui"
   import { parse, stringify } from "qs"
   import HelpIcon from "components/common/HelpIcon.svelte"
 
-  onMount(async () => {
-    await initialise()
-  })
-
   const queryHandler = { parse, stringify }
 </script>
 

packages/builder/src/analytics.js

@@ -1,139 +0,0 @@
-import * as Sentry from "@sentry/browser"
-import posthog from "posthog-js"
-import api from "builderStore/api"
-
-let analyticsEnabled
-const posthogConfigured = process.env.POSTHOG_TOKEN && process.env.POSTHOG_URL
-const sentryConfigured = process.env.SENTRY_DSN
-
-const FEEDBACK_SUBMITTED_KEY = "budibase:feedback_submitted"
-const APP_FIRST_STARTED_KEY = "budibase:first_run"
-const feedbackHours = 12
-
-async function activate() {
-  if (analyticsEnabled === undefined) {
-    // only the server knows the true NODE_ENV
-    // this was an issue as NODE_ENV = 'cypress' on the server,
-    // but 'production' on the client
-    const response = await api.get("/api/analytics")
-    analyticsEnabled = (await response.json()).enabled === true
-  }
-  if (!analyticsEnabled) return
-  if (sentryConfigured) Sentry.init({ dsn: process.env.SENTRY_DSN })
-  if (posthogConfigured) {
-    posthog.init(process.env.POSTHOG_TOKEN, {
-      autocapture: false,
-      capture_pageview: false,
-      api_host: process.env.POSTHOG_URL,
-    })
-    posthog.set_config({ persistence: "cookie" })
-  }
-}
-
-function identify(id) {
-  if (!analyticsEnabled || !id) return
-  if (posthogConfigured) posthog.identify(id)
-  if (sentryConfigured)
-    Sentry.configureScope(scope => {
-      scope.setUser({ id: id })
-    })
-}
-
-async function identifyByApiKey(apiKey) {
-  if (!analyticsEnabled) return true
-  try {
-    const response = await fetch(
-      `https://03gaine137.execute-api.eu-west-1.amazonaws.com/prod/account/id?api_key=${apiKey.trim()}`
-    )
-    if (response.status === 200) {
-      const id = await response.json()
-
-      await api.put("/api/keys/userId", { value: id })
-      identify(id)
-      return true
-    }
-
-    return false
-  } catch (error) {
-    console.log(error)
-  }
-}
-
-function captureException(err) {
-  if (!analyticsEnabled) return
-  Sentry.captureException(err)
-  captureEvent("Error", { error: err.message ? err.message : err })
-}
-
-function captureEvent(eventName, props = {}) {
-  if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return
-  props.sourceApp = "builder"
-  posthog.capture(eventName, props)
-}
-
-if (!localStorage.getItem(APP_FIRST_STARTED_KEY)) {
-  localStorage.setItem(APP_FIRST_STARTED_KEY, Date.now())
-}
-
-const isFeedbackTimeElapsed = sinceDateStr => {
-  const sinceDate = parseFloat(sinceDateStr)
-  const feedbackMilliseconds = feedbackHours * 60 * 60 * 1000
-  return Date.now() > sinceDate + feedbackMilliseconds
-}
-
-function submitFeedback(values) {
-  if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return
-  localStorage.setItem(FEEDBACK_SUBMITTED_KEY, Date.now())
-
-  const prefixedValues = Object.entries(values).reduce((obj, [key, value]) => {
-    obj[`feedback_${key}`] = value
-    return obj
-  }, {})
-
-  posthog.capture("Feedback Submitted", prefixedValues)
-}
-
-function requestFeedbackOnDeploy() {
-  if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false
-  const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY)
-  if (!lastSubmittedStr) return true
-  return isFeedbackTimeElapsed(lastSubmittedStr)
-}
-
-function highlightFeedbackIcon() {
-  if (!analyticsEnabled || !process.env.POSTHOG_TOKEN) return false
-  const lastSubmittedStr = localStorage.getItem(FEEDBACK_SUBMITTED_KEY)
-  if (lastSubmittedStr) return isFeedbackTimeElapsed(lastSubmittedStr)
-  const firstRunStr = localStorage.getItem(APP_FIRST_STARTED_KEY)
-  if (!firstRunStr) return false
-  return isFeedbackTimeElapsed(firstRunStr)
-}
-
-// Opt In/Out
-const ifAnalyticsEnabled = func => () => {
-  if (analyticsEnabled && process.env.POSTHOG_TOKEN) {
-    return func()
-  }
-}
-const disabled = () => posthog.has_opted_out_capturing()
-const optIn = () => posthog.opt_in_capturing()
-const optOut = () => posthog.opt_out_capturing()
-
-export default {
-  activate,
-  identify,
-  identifyByApiKey,
-  captureException,
-  captureEvent,
-  requestFeedbackOnDeploy,
-  submitFeedback,
-  highlightFeedbackIcon,
-  disabled: () => {
-    if (analyticsEnabled == null) {
-      return true
-    }
-    return ifAnalyticsEnabled(disabled)
-  },
-  optIn: ifAnalyticsEnabled(optIn),
-  optOut: ifAnalyticsEnabled(optOut),
-}

packages/builder/src/analytics/IntercomClient.js

@@ -0,0 +1,94 @@
+export default class IntercomClient {
+  constructor(token) {
+    this.token = token
+  }
+
+  /**
+   * Instantiate intercom using their provided script.
+   */
+  init() {
+    if (!this.token) return
+
+    const token = this.token
+
+    var w = window
+    var ic = w.Intercom
+    if (typeof ic === "function") {
+      ic("reattach_activator")
+      ic("update", w.intercomSettings)
+    } else {
+      var d = document
+      var i = function () {
+        i.c(arguments)
+      }
+      i.q = []
+      i.c = function (args) {
+        i.q.push(args)
+      }
+      w.Intercom = i
+      var l = function () {
+        var s = d.createElement("script")
+        s.type = "text/javascript"
+        s.async = true
+        s.src = "https://widget.intercom.io/widget/" + token
+        var x = d.getElementsByTagName("script")[0]
+        x.parentNode.insertBefore(s, x)
+      }
+      if (document.readyState === "complete") {
+        l()
+      } else if (w.attachEvent) {
+        w.attachEvent("onload", l)
+      } else {
+        w.addEventListener("load", l, false)
+      }
+
+      this.initialised = true
+    }
+  }
+
+  /**
+   * Show the intercom chat bubble.
+   * @param {Object} user - user to identify
+   * @returns Intercom global object
+   */
+  show(user = {}) {
+    if (!this.initialised) return
+
+    return window.Intercom("boot", {
+      app_id: this.token,
+      ...user,
+    })
+  }
+
+  /**
+   * Update intercom user details and messages.
+   * @returns Intercom global object
+   */
+  update() {
+    if (!this.initialised) return
+
+    return window.Intercom("update")
+  }
+
+  /**
+   * Capture analytics events and send them to intercom.
+   * @param {String} event - event identifier
+   * @param {Object} props - properties for the event
+   * @returns Intercom global object
+   */
+  captureEvent(event, props = {}) {
+    if (!this.initialised) return
+
+    return window.Intercom("trackEvent", event, props)
+  }
+
+  /**
+   * Disassociate the user from the current session.
+   * @returns Intercom global object
+   */
+  logout() {
+    if (!this.initialised) return
+
+    return window.Intercom("shutdown")
+  }
+}

packages/builder/src/analytics/PosthogClient.js

@@ -0,0 +1,80 @@
+import posthog from "posthog-js"
+import { Events } from "./constants"
+
+export default class PosthogClient {
+  constructor(token, url) {
+    this.token = token
+    this.url = url
+  }
+
+  init() {
+    if (!this.token || !this.url) return
+
+    posthog.init(this.token, {
+      autocapture: false,
+      capture_pageview: false,
+      api_host: this.url,
+    })
+    posthog.set_config({ persistence: "cookie" })
+
+    this.initialised = true
+  }
+
+  /**
+   * Set the posthog context to the current user
+   * @param {String} id - unique user id
+   */
+  identify(id) {
+    if (!this.initialised) return
+
+    posthog.identify(id)
+  }
+
+  /**
+   * Update user metadata associated with current user in posthog
+   * @param {Object} meta - user fields
+   */
+  updateUser(meta) {
+    if (!this.initialised) return
+
+    posthog.people.set(meta)
+  }
+
+  /**
+   * Capture analytics events and send them to posthog.
+   * @param {String} event - event identifier
+   * @param {Object} props - properties for the event
+   */
+  captureEvent(eventName, props) {
+    if (!this.initialised) return
+
+    props.sourceApp = "builder"
+    posthog.capture(eventName, props)
+  }
+
+  /**
+   * Submit NPS feedback to posthog.
+   * @param {Object} values - NPS Values
+   */
+  npsFeedback(values) {
+    if (!this.initialised) return
+
+    localStorage.setItem(Events.NPS.SUBMITTED, Date.now())
+
+    const prefixedFeedback = {}
+    for (let key in values) {
+      prefixedFeedback[`feedback_${key}`] = values[key]
+    }
+
+    posthog.capture(Events.NPS.SUBMITTED, prefixedFeedback)
+  }
+
+  /**
+   * Reset posthog user back to initial state on logout.
+   */
+  logout() {
+    if (!this.initialised) return
+
+    posthog.reset()
+  }
+}

packages/builder/src/analytics/SentryClient.js

@@ -0,0 +1,37 @@
+import * as Sentry from "@sentry/browser"
+
+export default class SentryClient {
+  constructor(dsn) {
+    this.dsn = dsn
+  }
+
+  init() {
+    if (this.dsn) {
+      Sentry.init({ dsn: this.dsn })
+
+      this.initalised = true
+    }
+  }
+
+  /**
+   * Capture an exception and send it to sentry.
+   * @param {Error} err - JS error object
+   */
+  captureException(err) {
+    if (!this.initalised) return
+
+    Sentry.captureException(err)
+  }
+
+  /**
+   * Identify user in sentry.
+   * @param {String} id - Unique user id
+   */
+  identify(id) {
+    if (!this.initalised) return
+
+    Sentry.configureScope(scope => {
+      scope.setUser({ id })
+    })
+  }
+}

packages/builder/src/analytics/constants.js

@@ -0,0 +1,49 @@
+export const Events = {
+  BUILDER: {
+    STARTED: "Builder Started",
+  },
+  COMPONENT: {
+    CREATED: "Added Component",
+  },
+  DATASOURCE: {
+    CREATED: "Datasource Created",
+    UPDATED: "Datasource Updated",
+  },
+  TABLE: {
+    CREATED: "Table Created",
+  },
+  VIEW: {
+    CREATED: "View Created",
+    ADDED_FILTER: "Added View Filter",
+    ADDED_CALCULATE: "Added View Calculate",
+  },
+  SCREEN: {
+    CREATED: "Screen Created",
+  },
+  AUTOMATION: {
+    CREATED: "Automation Created",
+    SAVED: "Automation Saved",
+    BLOCK_ADDED: "Added Automation Block",
+  },
+  NPS: {
+    SUBMITTED: "budibase:feedback_submitted",
+  },
+  APP: {
+    CREATED: "budibase:app_created",
+    PUBLISHED: "budibase:app_published",
+    UNPUBLISHED: "budibase:app_unpublished",
+  },
+  ANALYTICS: {
+    OPT_IN: "budibase:analytics_opt_in",
+    OPT_OUT: "budibase:analytics_opt_out",
+  },
+  USER: {
+    INVITE: "budibase:portal_user_invite",
+  },
+  SMTP: {
+    SAVED: "budibase:smtp_saved",
+  },
+  SSO: {
+    SAVED: "budibase:sso_saved",
+  },
+}

packages/builder/src/analytics/index.js

@@ -0,0 +1,63 @@
+import api from "builderStore/api"
+import PosthogClient from "./PosthogClient"
+import IntercomClient from "./IntercomClient"
+import SentryClient from "./SentryClient"
+import { Events } from "./constants"
+
+const posthog = new PosthogClient(
+  process.env.POSTHOG_TOKEN,
+  process.env.POSTHOG_URL
+)
+const sentry = new SentryClient(process.env.SENTRY_DSN)
+const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
+
+class AnalyticsHub {
+  constructor() {
+    this.clients = [posthog, sentry, intercom]
+  }
+
+  async activate() {
+    const analyticsStatus = await api.get("/api/analytics")
+    const json = await analyticsStatus.json()
+
+    // Analytics disabled
+    if (!json.enabled) return
+
+    this.clients.forEach(client => client.init())
+  }
+
+  identify(id, metadata) {
+    posthog.identify(id)
+    if (metadata) {
+      posthog.updateUser(metadata)
+    }
+    sentry.identify(id)
+  }
+
+  captureException(err) {
+    sentry.captureException(err)
+  }
+
+  captureEvent(eventName, props = {}) {
+    posthog.captureEvent(eventName, props)
+    intercom.captureEvent(eventName, props)
+  }
+
+  showChat(user) {
+    intercom.show(user)
+  }
+
+  submitFeedback(values) {
+    posthog.npsFeedback(values)
+  }
+
+  async logout() {
+    posthog.logout()
+    intercom.logout()
+  }
+}
+
+const analytics = new AnalyticsHub()
+
+export { Events }
+export default analytics

packages/builder/src/builderStore/dataBinding.js

@@ -443,7 +443,9 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     for (let from of convertFromProps) {
       if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
-        newBoundValue = newBoundValue.replace(from, binding[convertTo])
+        while (newBoundValue.includes(from)) {
+          newBoundValue = newBoundValue.replace(from, binding[convertTo])
+        }
       }
     }
     result = result.replace(boundValue, newBoundValue)

packages/builder/src/builderStore/index.js

@@ -3,7 +3,6 @@ import { getAutomationStore } from "./store/automation"
 import { getHostingStore } from "./store/hosting"
 import { getThemeStore } from "./store/theme"
 import { derived, writable } from "svelte/store"
-import analytics from "analytics"
 import { FrontendTypes, LAYOUT_NAMES } from "../constants"
 import { findComponent } from "./storeUtils"
 
@@ -55,13 +54,4 @@ export const mainLayout = derived(store, $store => {
 
 export const selectedAccessRole = writable("BASIC")
 
-export const initialise = async () => {
-  try {
-    await analytics.activate()
-    analytics.captureEvent("Builder Started")
-  } catch (err) {
-    console.log(err)
-  }
-}
-
 export const screenSearchString = writable(null)

packages/builder/src/builderStore/store/automation/index.js

@@ -2,7 +2,7 @@ import { writable } from "svelte/store"
 import api from "../../api"
 import Automation from "./Automation"
 import { cloneDeep } from "lodash/fp"
-import analytics from "analytics"
+import analytics, { Events } from "analytics"
 
 const automationActions = store => ({
   fetch: async () => {
@@ -110,7 +110,7 @@ const automationActions = store => ({
       state.selectedBlock = newBlock
       return state
     })
-    analytics.captureEvent("Added Automation Block", {
+    analytics.captureEvent(Events.AUTOMATION.BLOCK_ADDED, {
       name: block.name,
     })
   },

packages/builder/src/builderStore/store/frontend.js

@@ -19,7 +19,7 @@ import {
 import { fetchComponentLibDefinitions } from "../loadComponentLibraries"
 import api from "../api"
 import { FrontendTypes } from "constants"
-import analytics from "analytics"
+import analytics, { Events } from "analytics"
 import {
   findComponentType,
   findComponentParent,
@@ -215,6 +215,13 @@ export const getFrontendStore = () => {
             if (screenToDelete._id === state.selectedScreenId) {
               state.selectedScreenId = null
             }
+            //remove the link for this screen
+            screenDeletePromises.push(
+              store.actions.components.links.delete(
+                screenToDelete.routing.route,
+                screenToDelete.props._instanceName
+              )
+            )
           }
           return state
         })
@@ -443,7 +450,7 @@ export const getFrontendStore = () => {
         })
 
         // Log event
-        analytics.captureEvent("Added Component", {
+        analytics.captureEvent(Events.COMPONENT.CREATED, {
           name: componentInstance._component,
         })
 
@@ -646,6 +653,36 @@ export const getFrontendStore = () => {
           // Save layout
           await store.actions.layouts.save(layout)
         },
+        delete: async (url, title) => {
+          const layout = get(mainLayout)
+          if (!layout) {
+            return
+          }
+
+          // Add link setting to main layout
+          if (layout.props._component.endsWith("layout")) {
+            // If using a new SDK, add to the layout component settings
+            layout.props.links = layout.props.links.filter(
+              link => !(link.text === title && link.url === url)
+            )
+          } else {
+            // If using an old SDK, add to the navigation component
+            // TODO: remove this when we can assume everyone has updated
+            const nav = findComponentType(
+              layout.props,
+              "@budibase/standard-components/navigation"
+            )
+            if (!nav) {
+              return
+            }
+
+            nav._children = nav._children.filter(
+              child => !(child.url === url && child.text === title)
+            )
+          }
+          // Save layout
+          await store.actions.layouts.save(layout)
+        },
       },
     },
   }

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -123,7 +123,7 @@
     padding: var(--spectrum-alias-item-padding-s);
     background: var(--spectrum-alias-background-color-secondary);
     transition: 0.3s all;
-    border: solid #3b3d3c;
+    border: solid var(--spectrum-alias-border-color);
     border-radius: 5px;
     box-sizing: border-box;
     border-width: 2px;

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -1,9 +1,8 @@
 <script>
   import { automationStore } from "builderStore"
-
+  import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import FlowItem from "./FlowItem.svelte"
   import TestDataModal from "./TestDataModal.svelte"
-
   import { flip } from "svelte/animate"
   import { fade, fly } from "svelte/transition"
   import {
@@ -13,13 +12,12 @@
     notifications,
     Modal,
   } from "@budibase/bbui"
-  import { database } from "stores/backend"
 
   export let automation
   export let onSelect
   let testDataModal
   let blocks
-  $: instanceId = $database._id
+  let confirmDeleteDialog
 
   $: {
     blocks = []
@@ -35,6 +33,7 @@
     await automationStore.actions.delete(
       $automationStore.selectedAutomation?.automation
     )
+    notifications.success("Automation deleted.")
   }
 
   async function testAutomation() {
@@ -63,8 +62,14 @@
           style="display:flex;
           color: var(--spectrum-global-color-gray-400);"
         >
-          <span on:click={() => deleteAutomation()} class="iconPadding">
-            <Icon name="DeleteOutline" />
+          <span class="iconPadding">
+            <div class="icon">
+              <Icon
+                on:click={confirmDeleteDialog.show}
+                hoverable
+                name="DeleteOutline"
+              />
+            </div>
           </span>
           <ActionButton
             on:click={() => {
@@ -92,6 +97,17 @@
       </div>
     {/each}
   </div>
+  <ConfirmDialog
+    bind:this={confirmDeleteDialog}
+    okText="Delete Automation"
+    onOk={deleteAutomation}
+    title="Confirm Deletion"
+  >
+    Are you sure you wish to delete the automation
+    <i>{automation.name}?</i>
+    This action cannot be undone.
+  </ConfirmDialog>
+
   <Modal bind:this={testDataModal} width="30%">
     <TestDataModal {testAutomation} />
   </Modal>
@@ -139,7 +155,7 @@
     justify-content: space-between;
   }
 
-  .iconPadding {
+  .icon {
     cursor: pointer;
     display: flex;
     padding-right: var(--spacing-m);

packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte

@@ -4,7 +4,7 @@
   import { automationStore } from "builderStore"
   import { notifications } from "@budibase/bbui"
   import { Input, ModalContent, Layout, Body, Icon } from "@budibase/bbui"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
 
   let name
   let selectedTrigger
@@ -36,7 +36,7 @@
     notifications.success(`Automation ${name} created.`)
 
     $goto(`./${$automationStore.selectedAutomation.automation._id}`)
-    analytics.captureEvent("Automation Created", { name })
+    analytics.captureEvent(Events.AUTOMATION.CREATED, { name })
   }
   $: triggers = Object.entries($automationStore.blockDefinitions.TRIGGER)
 
@@ -102,7 +102,7 @@
     padding: var(--spectrum-alias-item-padding-s);
     background: var(--spectrum-alias-background-color-secondary);
     transition: 0.3s all;
-    border: solid #3b3d3c;
+    border: solid var(--spectrum-alias-border-color);
     border-radius: 5px;
     box-sizing: border-box;
     border-width: 2px;

packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte

@@ -2,7 +2,7 @@
   import { automationStore } from "builderStore"
   import { notifications } from "@budibase/bbui"
   import { Icon, Input, ModalContent, Modal } from "@budibase/bbui"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
 
   let name
   let error = ""
@@ -26,7 +26,7 @@
     }
     await automationStore.actions.save(updatedAutomation)
     notifications.success(`Automation ${name} updated successfully.`)
-    analytics.captureEvent("Automation Saved", { name })
+    analytics.captureEvent(Events.AUTOMATION.SAVED, { name })
     hide()
   }
 

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -20,12 +20,11 @@
   import QueryParamSelector from "./QueryParamSelector.svelte"
   import CronBuilder from "./CronBuilder.svelte"
   import Editor from "components/integration/QueryEditor.svelte"
-  import { database } from "stores/backend"
   import { debounce } from "lodash"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte"
   // need the client lucene builder to convert to the structure API expects
-  import { buildLuceneQuery } from "../../../../../client/src/utils/lucene"
+  import { buildLuceneQuery } from "helpers/lucene"
 
   export let block
   export let webhookModal
@@ -35,13 +34,11 @@
   let drawer
   let tempFilters = lookForFilters(schemaProperties) || []
   let fillWidth = true
-
   $: stepId = block.stepId
   $: bindings = getAvailableBindings(
     block || $automationStore.selectedBlock,
     $automationStore.selectedAutomation?.automation?.definition
   )
-  $: instanceId = $database._id
 
   $: inputData = testData ? testData : block.inputs
   $: tableId = inputData ? inputData.tableId : null
@@ -210,7 +207,7 @@
       {:else if value.customType === "webhookUrl"}
         <WebhookDisplay value={inputData[key]} />
       {:else if value.customType === "triggerSchema"}
-        <SchemaSetup on:change={e => onChange(e, key)} value={value[key]} />
+        <SchemaSetup on:change={e => onChange(e, key)} value={inputData[key]} />
       {:else if value.customType === "code"}
         <CodeEditorModal>
           <pre>{JSON.stringify(bindings, null, 2)}</pre>

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,6 +1,6 @@
 <script>
   import { tables } from "stores/backend"
-  import { Select } from "@budibase/bbui"
+  import { Select, Toggle, DatePicker, Multiselect } from "@budibase/bbui"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
@@ -44,13 +44,31 @@
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
       {#if !schema.autocolumn}
-        {#if schemaHasOptions(schema)}
+        {#if schemaHasOptions(schema) && schema.type !== "array"}
           <Select
             on:change={e => onChange(e, field)}
             label={field}
             value={value[field]}
             options={schema.constraints.inclusion}
           />
+        {:else if schema.type === "datetime"}
+          <DatePicker
+            label={field}
+            value={value[field]}
+            on:change={e => onChange(e, field)}
+          />
+        {:else if schema.type === "boolean"}
+          <Toggle
+            text={field}
+            value={value[field]}
+            on:change={e => onChange(e, field)}
+          />
+        {:else if schema.type === "array"}
+          <Multiselect
+            bind:value={value[field]}
+            label={field}
+            options={schema.constraints.inclusion}
+          />
         {:else if schema.type === "string" || schema.type === "number"}
           {#if $automationStore.selectedAutomation.automation.testData}
             <ModalBindableInput

packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte

@@ -5,10 +5,14 @@
   const dispatch = createEventDispatcher()
 
   export let value = {}
-  $: fieldsArray = Object.entries(value).map(([name, type]) => ({
-    name,
-    type,
-  }))
+
+  $: fieldsArray = value
+    ? Object.entries(value).map(([name, type]) => ({
+        name,
+        type,
+      }))
+    : []
+
   const typeOptions = [
     {
       label: "Text",
@@ -73,7 +77,7 @@
       <Select
         value={field.type}
         on:change={e => {
-          value[field.name] = e.target.value
+          value[field.name] = e.detail
           dispatch("change", value)
         }}
         options={typeOptions}
@@ -88,9 +92,7 @@
 
 <style>
   .root {
-    position: relative;
     max-width: 100%;
-    overflow-x: auto;
     /* so we can show the "+" button beside the "fields" label*/
     top: -26px;
   }
@@ -110,7 +112,6 @@
     /*grid-template-rows: auto auto;
     grid-template-columns: auto;*/
     position: relative;
-    overflow: hidden;
   }
 
   .field :global(select) {

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -1,6 +1,6 @@
 <script>
-  import { tables, views } from "stores/backend"
-
+  import { fade } from "svelte/transition"
+  import { tables } from "stores/backend"
   import CreateRowButton from "./buttons/CreateRowButton.svelte"
   import CreateColumnButton from "./buttons/CreateColumnButton.svelte"
   import CreateViewButton from "./buttons/CreateViewButton.svelte"
@@ -8,72 +8,124 @@
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
-  import * as api from "./api"
+  import TableFilterButton from "./buttons/TableFilterButton.svelte"
   import Table from "./Table.svelte"
   import { TableNames } from "constants"
   import CreateEditRow from "./modals/CreateEditRow.svelte"
+  import { fetchTableData } from "helpers/fetchTableData"
+  import { Pagination } from "@budibase/bbui"
 
   let hideAutocolumns = true
-  let data = []
-  let loading = false
+
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
-  $: title = $tables.selected?.name
   $: schema = $tables.selected?.schema
-  $: tableView = {
-    schema,
-    name: $views.selected?.name,
-  }
   $: type = $tables.selected?.type
   $: isInternal = type !== "external"
+  $: id = $tables.selected?._id
+  $: search = searchTable(id)
+  $: columnOptions = Object.keys($search.schema || {})
 
-  // Fetch rows for specified table
-  $: {
-    loading = true
-    const loadingTableId = $tables.selected?._id
-    api.fetchDataForTable($tables.selected?._id).then(rows => {
-      loading = false
-
-      // If we started a slow request then quickly change table, sometimes
-      // the old data overwrites the new data.
-      // This check ensures that we don't do that.
-      if (loadingTableId !== $tables.selected?._id) {
-        return
-      }
-
-      data = rows || []
+  // Fetches new data whenever the table changes
+  const searchTable = tableId => {
+    return fetchTableData({
+      tableId,
+      schema,
+      limit: 10,
+      paginate: true,
     })
   }
+
+  // Fetch data whenever sorting option changes
+  const onSort = e => {
+    search.update({
+      sortColumn: e.detail.column,
+      sortOrder: e.detail.order,
+    })
+  }
+
+  // Fetch data whenever filters change
+  const onFilter = e => {
+    search.update({
+      filters: e.detail,
+    })
+  }
+
+  // Fetch data whenever schema changes
+  const onUpdateColumns = () => {
+    search.update({
+      schema,
+    })
+  }
+
+  // Fetch data whenever rows are modified. Unfortunately we have to lose
+  // our pagination place, as our bookmarks will have shifted.
+  const onUpdateRows = () => {
+    search.update()
+  }
 </script>
 
-<Table
-  {title}
-  {schema}
-  tableId={$tables.selected?._id}
-  {data}
-  {type}
-  allowEditing={true}
-  bind:hideAutocolumns
-  {loading}
->
-  {#if isInternal}
-    <CreateColumnButton />
-  {/if}
-  {#if schema && Object.keys(schema).length > 0}
-    {#if !isUsersTable}
-      <CreateRowButton
-        title={"Create row"}
-        modalContentComponent={CreateEditRow}
-      />
-    {/if}
+<div>
+  <Table
+    title={$tables.selected?.name}
+    {schema}
+    {type}
+    tableId={id}
+    data={$search.rows}
+    bind:hideAutocolumns
+    loading={$search.loading}
+    on:sort={onSort}
+    allowEditing
+    disableSorting
+    on:updatecolumns={onUpdateColumns}
+    on:updaterows={onUpdateRows}
+  >
     {#if isInternal}
-      <CreateViewButton />
+      <CreateColumnButton on:updatecolumns={onUpdateColumns} />
     {/if}
-    <ManageAccessButton resourceId={$tables.selected?._id} />
-    {#if isUsersTable}
-      <EditRolesButton />
+    {#if schema && Object.keys(schema).length > 0}
+      {#if !isUsersTable}
+        <CreateRowButton
+          on:updaterows={onUpdateRows}
+          title={"Create row"}
+          modalContentComponent={CreateEditRow}
+        />
+      {/if}
+      {#if isInternal}
+        <CreateViewButton />
+      {/if}
+      <ManageAccessButton resourceId={$tables.selected?._id} />
+      {#if isUsersTable}
+        <EditRolesButton />
+      {/if}
+      <HideAutocolumnButton bind:hideAutocolumns />
+      <!-- always have the export last -->
+      <ExportButton view={$tables.selected?._id} />
+      {#key id}
+        <TableFilterButton {schema} on:change={onFilter} />
+      {/key}
     {/if}
-    <HideAutocolumnButton bind:hideAutocolumns />
-    <!-- always have the export last -->
-    <ExportButton view={$tables.selected?._id} />
-  {/if}
-</Table>
+  </Table>
+  {#key id}
+    <div in:fade={{ delay: 200, duration: 100 }}>
+      <div class="pagination">
+        <Pagination
+          page={$search.pageNumber + 1}
+          hasPrevPage={$search.hasPrevPage}
+          hasNextPage={$search.hasNextPage}
+          goToPrevPage={$search.loading ? null : search.prevPage}
+          goToNextPage={$search.loading ? null : search.nextPage}
+        />
+      </div>
+    </div>
+  {/key}
+</div>
+
+<style>
+  .pagination {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    margin-top: var(--spacing-xl);
+  }
+</style>

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -1,8 +1,7 @@
 <script>
   import { fade } from "svelte/transition"
   import { goto, params } from "@roxi/routify"
-  import { Table, Modal, Heading, notifications } from "@budibase/bbui"
-
+  import { Table, Modal, Heading, notifications, Layout } from "@budibase/bbui"
   import api from "builderStore/api"
   import Spinner from "components/common/Spinner.svelte"
   import DeleteRowsButton from "./buttons/DeleteRowsButton.svelte"
@@ -21,6 +20,7 @@
   export let hideAutocolumns
   export let rowCount
   export let type
+  export let disableSorting = false
 
   let selectedRows = []
   let editableColumn
@@ -98,47 +98,57 @@
   }
 </script>
 
-<div>
-  <div class="table-title">
-    {#if title}
-      <Heading size="S">{title}</Heading>
-    {/if}
-    {#if loading}
-      <div transition:fade>
-        <Spinner size="10" />
-      </div>
-    {/if}
+<Layout noPadding gap="S">
+  <div>
+    <div class="table-title">
+      {#if title}
+        <Heading size="S">{title}</Heading>
+      {/if}
+      {#if loading}
+        <div transition:fade|local>
+          <Spinner size="10" />
+        </div>
+      {/if}
+    </div>
+    <div class="popovers">
+      <slot />
+      {#if !isUsersTable && selectedRows.length > 0}
+        <DeleteRowsButton on:updaterows {selectedRows} {deleteRows} />
+      {/if}
+    </div>
   </div>
-  <div class="popovers">
-    <slot />
-    {#if !isUsersTable && selectedRows.length > 0}
-      <DeleteRowsButton {selectedRows} {deleteRows} />
-    {/if}
-  </div>
-</div>
-{#key tableId}
-  <Table
-    {data}
-    {schema}
-    {loading}
-    {customRenderers}
-    {rowCount}
-    bind:selectedRows
-    allowSelectRows={allowEditing && !isUsersTable}
-    allowEditRows={allowEditing}
-    allowEditColumns={allowEditing && isInternal}
-    showAutoColumns={!hideAutocolumns}
-    on:editcolumn={e => editColumn(e.detail)}
-    on:editrow={e => editRow(e.detail)}
-    on:clickrelationship={e => selectRelationship(e.detail)}
-  />
-{/key}
+  {#key tableId}
+    <div class="table-wrapper" in:fade={{ delay: 200, duration: 100 }}>
+      <Table
+        {data}
+        {schema}
+        {loading}
+        {customRenderers}
+        {rowCount}
+        {disableSorting}
+        bind:selectedRows
+        allowSelectRows={allowEditing && !isUsersTable}
+        allowEditRows={allowEditing}
+        allowEditColumns={allowEditing && isInternal}
+        showAutoColumns={!hideAutocolumns}
+        on:editcolumn={e => editColumn(e.detail)}
+        on:editrow={e => editRow(e.detail)}
+        on:clickrelationship={e => selectRelationship(e.detail)}
+        on:sort
+      />
+    </div>
+  {/key}
+</Layout>
 
 <Modal bind:this={editRowModal}>
-  <svelte:component this={editRowComponent} row={editableRow} />
+  <svelte:component this={editRowComponent} on:updaterows row={editableRow} />
 </Modal>
 <Modal bind:this={editColumnModal}>
-  <CreateEditColumn field={editableColumn} onClosed={editColumnModal.hide} />
+  <CreateEditColumn
+    field={editableColumn}
+    on:updatecolumns
+    onClosed={editColumnModal.hide}
+  />
 </Modal>
 
 <style>
@@ -152,6 +162,9 @@
   .table-title > div {
     margin-left: var(--spacing-xs);
   }
+  .table-wrapper {
+    overflow: hidden;
+  }
 
   .popovers {
     display: flex;

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -5,7 +5,7 @@
   import Table from "./Table.svelte"
   import CalculateButton from "./buttons/CalculateButton.svelte"
   import GroupByButton from "./buttons/GroupByButton.svelte"
-  import FilterButton from "./buttons/FilterButton.svelte"
+  import ViewFilterButton from "./buttons/ViewFilterButton.svelte"
   import ExportButton from "./buttons/ExportButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
@@ -61,7 +61,7 @@
   allowEditing={!view?.calculation}
   bind:hideAutocolumns
 >
-  <FilterButton {view} />
+  <ViewFilterButton {view} />
   <CalculateButton {view} />
   {#if view.calculation}
     <GroupByButton {view} />

packages/builder/src/components/backend/DataTable/buttons/CreateColumnButton.svelte

@@ -9,5 +9,5 @@
   Create column
 </ActionButton>
 <Modal bind:this={modal}>
-  <CreateEditColumn />
+  <CreateEditColumn on:updatecolumns />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/CreateRowButton.svelte

@@ -12,5 +12,5 @@
   {title}
 </ActionButton>
 <Modal bind:this={modal}>
-  <svelte:component this={modalContentComponent} />
+  <svelte:component this={modalContentComponent} on:updaterows />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/DeleteRowsButton.svelte

@@ -1,15 +1,18 @@
 <script>
+  import { createEventDispatcher } from "svelte"
   import { Button } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
 
   export let selectedRows
   export let deleteRows
 
+  const dispatch = createEventDispatcher()
   let modal
 
   async function confirmDeletion() {
     await deleteRows()
     modal?.hide()
+    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -0,0 +1,46 @@
+<script>
+  import { createEventDispatcher } from "svelte"
+  import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
+  import FilterDrawer from "components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte"
+
+  export let schema
+  export let filters
+
+  const dispatch = createEventDispatcher()
+  let modal
+  let tempValue = filters || []
+
+  $: schemaFields = Object.values(schema || {})
+</script>
+
+<ActionButton
+  icon="Filter"
+  size="S"
+  quiet
+  on:click={modal.show}
+  active={tempValue?.length > 0}
+>
+  Filter
+</ActionButton>
+<Modal bind:this={modal}>
+  <ModalContent
+    title="Filter"
+    confirmText="Save"
+    size="XL"
+    onConfirm={() => dispatch("change", tempValue)}
+  >
+    <div class="wrapper">
+      <FilterDrawer
+        allowBindings={false}
+        bind:filters={tempValue}
+        {schemaFields}
+      />
+    </div>
+  </ModalContent>
+</Modal>
+
+<style>
+  .wrapper :global(.main) {
+    padding: 0;
+  }
+</style>

packages/builder/src/components/backend/DataTable/modals/CalculateModal.svelte

@@ -1,7 +1,7 @@
 <script>
   import { Select, Label, notifications, ModalContent } from "@budibase/bbui"
   import { tables, views } from "stores/backend"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
   import { FIELDS } from "constants/backend"
 
   const CALCULATIONS = [
@@ -40,7 +40,7 @@
   function saveView() {
     views.save(view)
     notifications.success(`View ${view.name} saved.`)
-    analytics.captureEvent("Added View Calculate", { field: view.field })
+    analytics.captureEvent(Events.VIEW.ADDED_CALCULATE, { field: view.field })
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -10,6 +10,7 @@
     ModalContent,
     Context,
   } from "@budibase/bbui"
+  import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables } from "stores/backend"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
@@ -30,8 +31,9 @@
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
   const LINK_TYPE = FIELDS.LINK.type
-  let fieldDefinitions = cloneDeep(FIELDS)
+  const dispatch = createEventDispatcher()
   const { hide } = getContext(Context.Modal)
+  let fieldDefinitions = cloneDeep(FIELDS)
 
   export let field = {
     type: "string",
@@ -81,12 +83,13 @@
     if (field.type === AUTO_TYPE) {
       field = buildAutoColumn($tables.draft.name, field.name, field.subtype)
     }
-    tables.saveField({
+    await tables.saveField({
       originalName,
       field,
       primaryDisplay,
       indexes,
     })
+    dispatch("updatecolumns")
   }
 
   function deleteColumn() {
@@ -99,6 +102,7 @@
       hide()
       deletion = false
     }
+    dispatch("updatecolumns")
   }
 
   function handleTypeChange(event) {

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { createEventDispatcher } from "svelte"
   import { tables, rows } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import RowFieldControl from "../RowFieldControl.svelte"
@@ -12,6 +13,7 @@
   export let row = {}
 
   let errors = []
+  const dispatch = createEventDispatcher()
 
   $: creating = row?._id == null
   $: table = row.tableId
@@ -43,6 +45,7 @@
 
     notifications.success("Row saved successfully.")
     rows.save(rowResponse)
+    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -1,4 +1,5 @@
 <script>
+  import { createEventDispatcher } from "svelte"
   import { tables, rows } from "stores/backend"
   import { roles } from "stores/backend"
   import { notifications } from "@budibase/bbui"
@@ -9,6 +10,7 @@
 
   export let row = {}
 
+  const dispatch = createEventDispatcher()
   let errors = []
 
   $: creating = row?._id == null
@@ -71,6 +73,7 @@
 
     notifications.success("User saved successfully")
     rows.save(rowResponse)
+    dispatch("updaterows")
   }
 </script>
 

packages/builder/src/components/backend/DataTable/modals/CreateViewModal.svelte

@@ -3,7 +3,7 @@
   import { goto } from "@roxi/routify"
   import { views as viewsStore } from "stores/backend"
   import { tables } from "stores/backend"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
 
   let name
   let field
@@ -21,7 +21,7 @@
       field,
     })
     notifications.success(`View ${name} created`)
-    analytics.captureEvent("View Created", { name })
+    analytics.captureEvent(Events.VIEW.CREATED, { name })
     $goto(`../../view/${name}`)
   }
 </script>

packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte

@@ -11,7 +11,7 @@
     Icon,
   } from "@budibase/bbui"
   import { tables, views } from "stores/backend"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
 
   const CONDITIONS = [
     {
@@ -65,7 +65,7 @@
   function saveView() {
     views.save(view)
     notifications.success(`View ${view.name} saved.`)
-    analytics.captureEvent("Added View Filter", {
+    analytics.captureEvent(Events.VIEW.ADDED_FILTER, {
       filters: JSON.stringify(view.filters),
     })
   }

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,8 +1,9 @@
 <script>
   import { onMount } from "svelte"
-  import { goto } from "@roxi/routify"
+  import { get } from "svelte/store"
+  import { goto, params } from "@roxi/routify"
   import { BUDIBASE_INTERNAL_DB } from "constants"
-  import { database, datasources, queries } from "stores/backend"
+  import { database, datasources, queries, tables, views } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
   import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
@@ -10,9 +11,27 @@
   import ICONS from "./icons"
 
   let openDataSources = []
+  $: enrichedDataSources = $datasources.list.map(datasource => {
+    const selected = $datasources.selected === datasource._id
+    const open = openDataSources.includes(datasource._id)
+    const containsSelected = containsActiveEntity(datasource)
+    return {
+      ...datasource,
+      selected,
+      open: selected || open || containsSelected,
+    }
+  })
+  $: openDataSource = enrichedDataSources.find(x => x.open)
+  $: {
+    // Ensure the open data source is always included in the list of open
+    // data sources
+    if (openDataSource) {
+      openNode(openDataSource)
+    }
+  }
 
   function selectDatasource(datasource) {
-    toggleNode(datasource)
+    openNode(datasource)
     datasources.select(datasource._id)
     $goto(`./datasource/${datasource._id}`)
   }
@@ -22,12 +41,22 @@
     $goto(`./datasource/${query.datasourceId}/${query._id}`)
   }
 
+  function closeNode(datasource) {
+    openDataSources = openDataSources.filter(id => datasource._id !== id)
+  }
+
+  function openNode(datasource) {
+    if (!openDataSources.includes(datasource._id)) {
+      openDataSources = [...openDataSources, datasource._id]
+    }
+  }
+
   function toggleNode(datasource) {
     const isOpen = openDataSources.includes(datasource._id)
     if (isOpen) {
-      openDataSources = openDataSources.filter(id => datasource._id !== id)
+      closeNode(datasource)
     } else {
-      openDataSources = [...openDataSources, datasource._id]
+      openNode(datasource)
     }
   }
 
@@ -35,16 +64,47 @@
     datasources.fetch()
     queries.fetch()
   })
+
+  const containsActiveEntity = datasource => {
+    // If we're view a query then the data source ID is in the URL
+    if ($params.selectedDatasource === datasource._id) {
+      return true
+    }
+
+    // If there are no entities it can't contain anything
+    if (!datasource.entities) {
+      return false
+    }
+
+    // Get a list of table options
+    let options = datasource.entities
+    if (!Array.isArray(options)) {
+      options = Object.values(options)
+    }
+
+    // Check for a matching table
+    if ($params.selectedTable) {
+      const selectedTable = get(tables).selected?._id
+      return options.find(x => x._id === selectedTable) != null
+    }
+
+    // Check for a matching view
+    const selectedView = get(views).selected?.name
+    const table = options.find(table => {
+      return table.views?.[selectedView] != null
+    })
+    return table != null
+  }
 </script>
 
 {#if $database?._id}
   <div class="hierarchy-items-container">
-    {#each $datasources.list as datasource, idx}
+    {#each enrichedDataSources as datasource, idx}
       <NavItem
         border={idx > 0}
         text={datasource.name}
-        opened={openDataSources.includes(datasource._id)}
-        selected={$datasources.selected === datasource._id}
+        opened={datasource.open}
+        selected={datasource.selected}
         withArrow={true}
         on:click={() => selectDatasource(datasource)}
         on:iconClick={() => toggleNode(datasource)}
@@ -61,22 +121,21 @@
         {/if}
       </NavItem>
 
-      {#if openDataSources.includes(datasource._id)}
+      {#if datasource.open}
         <TableNavigator sourceId={datasource._id} />
+        {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
+          <NavItem
+            indentLevel={1}
+            icon="SQLQuery"
+            text={query.name}
+            opened={$queries.selected === query._id}
+            selected={$queries.selected === query._id}
+            on:click={() => onClickQuery(query)}
+          >
+            <EditQueryPopover {query} />
+          </NavItem>
+        {/each}
       {/if}
-
-      {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
-        <NavItem
-          indentLevel={1}
-          icon="SQLQuery"
-          text={query.name}
-          opened={$queries.selected === query._id}
-          selected={$queries.selected === query._id}
-          on:click={() => onClickQuery(query)}
-        >
-          <EditQueryPopover {query} />
-        </NavItem>
-      {/each}
     {/each}
   </div>
 {/if}

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte

@@ -1,20 +1,28 @@
 <script>
-  import { Label, Input, Layout, Toggle } from "@budibase/bbui"
+  import { Label, Input, Layout, Toggle, Button } from "@budibase/bbui"
   import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { capitalise } from "helpers"
 
   export let integration
   export let schema
+  let addButton
 </script>
 
 <form>
   <Layout gap="S">
     {#each Object.keys(schema) as configKey}
       {#if schema[configKey].type === "object"}
-        <Label>{capitalise(configKey)}</Label>
+        <div class="form-row ssl">
+          <Label>{capitalise(configKey)}</Label>
+          <Button secondary thin outline on:click={addButton.addEntry()}
+            >Add</Button
+          >
+        </div>
         <KeyValueBuilder
+          bind:this={addButton}
           defaults={schema[configKey].default}
           bind:object={integration[configKey]}
+          noAddButton={true}
         />
       {:else if schema[configKey].type === "boolean"}
         <div class="form-row">
@@ -42,4 +50,11 @@
     grid-gap: var(--spacing-l);
     align-items: center;
   }
+
+  .form-row.ssl {
+    display: grid;
+    grid-template-columns: 20% 20%;
+    grid-gap: var(--spacing-l);
+    align-items: center;
+  }
 </style>

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -1,74 +1,160 @@
 <script>
-  import { goto } from "@roxi/routify"
-  import { datasources } from "stores/backend"
-  import { notifications } from "@budibase/bbui"
-  import { Input, Label, ModalContent, Modal, Context } from "@budibase/bbui"
-  import TableIntegrationMenu from "../TableIntegrationMenu/index.svelte"
+  import { ModalContent, Modal, Body, Layout, Detail } from "@budibase/bbui"
+  import { onMount } from "svelte"
+  import ICONS from "../icons"
+  import api from "builderStore/api"
+  import { IntegrationNames } from "constants"
   import CreateTableModal from "components/backend/TableNavigator/modals/CreateTableModal.svelte"
-  import analytics from "analytics"
-  import { getContext } from "svelte"
+  import DatasourceConfigModal from "components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte"
 
-  const modalContext = getContext(Context.Modal)
+  export let modal
+  let integrations = []
+  let integration = {}
+  let internalTableModal
+  let externalDatasourceModal
 
-  let tableModal
-  let name
-  let error = ""
-  let integration
+  const INTERNAL = "BUDIBASE"
 
-  $: checkOpenModal(integration && integration.type === "BUDIBASE")
+  onMount(() => {
+    fetchIntegrations()
+  })
 
-  function checkValid(evt) {
-    const datasourceName = evt.target.value
-    if (
-      $datasources?.list.some(datasource => datasource.name === datasourceName)
-    ) {
-      error = `Datasource with name ${datasourceName} already exists. Please choose another name.`
-      return
+  function selectIntegration(integrationType) {
+    const selected = integrations[integrationType]
+
+    // build the schema
+    const config = {}
+    for (let key of Object.keys(selected.datasource)) {
+      config[key] = selected.datasource[key].default
     }
-    error = ""
-  }
-
-  function checkOpenModal(isInternal) {
-    if (isInternal) {
-      tableModal.show()
-    }
-  }
-
-  async function saveDatasource() {
-    const { type, plus, ...config } = integration
-
-    // Create datasource
-    const response = await datasources.save({
-      name,
-      source: type,
+    integration = {
+      type: integrationType,
+      plus: selected.plus,
       config,
-      plus,
-    })
-    notifications.success(`Datasource ${name} created successfully.`)
-    analytics.captureEvent("Datasource Created", { name, type })
+      schema: selected.datasource,
+    }
+  }
 
-    // Navigate to new datasource
-    $goto(`./datasource/${response._id}`)
+  function chooseNextModal() {
+    if (integration.type === INTERNAL) {
+      externalDatasourceModal.hide()
+      internalTableModal.show()
+    } else {
+      externalDatasourceModal.show()
+    }
+  }
+
+  async function fetchIntegrations() {
+    const response = await api.get("/api/integrations")
+    const json = await response.json()
+    integrations = {
+      [INTERNAL]: { datasource: {}, name: "INTERNAL/CSV" },
+      ...json,
+    }
+    return json
   }
 </script>
 
-<Modal bind:this={tableModal} on:hide={modalContext.hide}>
-  <CreateTableModal bind:name />
+<Modal bind:this={internalTableModal}>
+  <CreateTableModal />
 </Modal>
-<ModalContent
-  title="Create Datasource"
-  size="L"
-  confirmText="Create"
-  onConfirm={saveDatasource}
-  disabled={error || !name || !integration?.type}
->
-  <Input
-    data-cy="datasource-name-input"
-    label="Datasource Name"
-    on:input={checkValid}
-    bind:value={name}
-    {error}
-  />
-  <Label>Datasource Type</Label>
-  <TableIntegrationMenu bind:integration />
-</ModalContent>
+
+<Modal bind:this={externalDatasourceModal}>
+  <DatasourceConfigModal {integration} />
+</Modal>
+
+<Modal bind:this={modal}>
+  <ModalContent
+    disabled={!Object.keys(integration).length}
+    title="Data"
+    confirmText="Continue"
+    showCancelButton={false}
+    size="M"
+    onConfirm={() => {
+      chooseNextModal()
+    }}
+  >
+    <Layout noPadding>
+      <Body size="XS"
+        >All apps need data. You can connect to a data source below, or add data
+        to your app using Budibase's built-in database.
+      </Body>
+      <div
+        class:selected={integration.type === INTERNAL}
+        on:click={() => selectIntegration(INTERNAL)}
+        class="item hoverable"
+      >
+        <div class="item-body">
+          <svelte:component this={ICONS.BUDIBASE} height="18" width="18" />
+          <span class="icon-spacing"> <Body size="S">Budibase DB</Body></span>
+        </div>
+      </div>
+    </Layout>
+
+    <Layout gap="XS" noPadding>
+      <div class="title-spacing">
+        <Detail size="S">Connect to data source</Detail>
+      </div>
+      <div class="item-list">
+        {#each Object.entries(integrations).filter(([key]) => key !== INTERNAL) as [integrationType, schema]}
+          <div
+            class:selected={integration.type === integrationType}
+            on:click={() => selectIntegration(integrationType)}
+            class="item hoverable"
+          >
+            <div class="item-body">
+              <svelte:component
+                this={ICONS[integrationType]}
+                height="18"
+                width="18"
+              />
+
+              <span class="icon-spacing">
+                <Body size="S"
+                  >{schema.name || IntegrationNames[integrationType]}</Body
+                ></span
+              >
+            </div>
+          </div>
+        {/each}
+      </div>
+    </Layout>
+  </ModalContent>
+</Modal>
+
+<style>
+  .icon-spacing {
+    margin-left: var(--spacing-m);
+  }
+  .item-body {
+    display: flex;
+    margin-left: var(--spacing-m);
+  }
+  .item-list {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
+    grid-gap: var(--spectrum-alias-grid-baseline);
+  }
+
+  .item {
+    cursor: pointer;
+    display: grid;
+    grid-gap: var(--spectrum-alias-grid-margin-xsmall);
+    padding: var(--spectrum-alias-item-padding-s);
+    background: var(--spectrum-alias-background-color-secondary);
+    transition: 0.3s all;
+    border: solid var(--spectrum-alias-border-color);
+    border-radius: 5px;
+    box-sizing: border-box;
+    border-width: 2px;
+  }
+
+  .selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+
+  .item:hover,
+  .selected {
+    background: var(--spectrum-alias-background-color-tertiary);
+  }
+</style>

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -0,0 +1,72 @@
+<script>
+  import { goto } from "@roxi/routify"
+  import { ModalContent, notifications, Body, Layout } from "@budibase/bbui"
+  import analytics, { Events } from "analytics"
+  import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
+  import { datasources } from "stores/backend"
+  import { IntegrationNames } from "constants"
+
+  export let integration
+
+  function prepareData() {
+    let datasource = {}
+    let existingTypeCount = $datasources.list.filter(
+      ds => ds.source == integration.type
+    ).length
+
+    let baseName = IntegrationNames[integration.type]
+    let name =
+      existingTypeCount == 0 ? baseName : `${baseName}-${existingTypeCount + 1}`
+
+    datasource.type = "datasource"
+    datasource.source = integration.type
+    datasource.config = integration.config
+    datasource.name = name
+    datasource.plus = integration.plus
+
+    return datasource
+  }
+  async function saveDatasource() {
+    const datasource = prepareData()
+    try {
+      // Create datasource
+      const resp = await datasources.save(datasource, datasource.plus)
+
+      await datasources.select(resp._id)
+      $goto(`./datasource/${resp._id}`)
+      notifications.success(`Datasource updated successfully.`)
+      analytics.captureEvent(Events.DATASOURCE.CREATED, {
+        name: resp.name,
+        source: resp.source,
+      })
+      return true
+    } catch (err) {
+      notifications.error(`Error saving datasource: ${err}`)
+      return false
+    }
+  }
+</script>
+
+<ModalContent
+  title={`Connect to ${IntegrationNames[integration.type]}`}
+  onConfirm={() => saveDatasource()}
+  confirmText={integration.plus
+    ? "Fetch tables from database"
+    : "Save and continue to query"}
+  cancelText="Back"
+  size="M"
+>
+  <Layout noPadding>
+    <Body size="XS"
+      >Connect your database to Budibase using the config below.
+    </Body>
+  </Layout>
+
+  <IntegrationConfigForm
+    schema={integration.schema}
+    bind:integration={integration.config}
+  />
+</ModalContent>
+
+<style>
+</style>

packages/builder/src/components/backend/DatasourceNavigator/modals/UpdateDatasourceModal.svelte

@@ -2,7 +2,7 @@
   import { datasources } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import { Input, ModalContent, Modal } from "@budibase/bbui"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
 
   let error = ""
   let modal
@@ -35,7 +35,7 @@
     }
     await datasources.save(updatedDatasource)
     notifications.success(`Datasource ${name} updated successfully.`)
-    analytics.captureEvent("Datasource Updated", updatedDatasource)
+    analytics.captureEvent(Events.DATASOURCE.UPDATED, updatedDatasource)
     hide()
   }
 </script>

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -12,7 +12,7 @@
     Layout,
   } from "@budibase/bbui"
   import TableDataImport from "../TableDataImport.svelte"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
   import screenTemplates from "builderStore/store/screenTemplates"
   import { buildAutoColumn, getAutoColumnInformation } from "builderStore/utils"
   import { NEW_ROW_TEMPLATE } from "builderStore/store/screenTemplates/newRowScreen"
@@ -67,7 +67,7 @@
     // Create table
     const table = await tables.save(newTable)
     notifications.success(`Table ${name} created successfully.`)
-    analytics.captureEvent("Table Created", { name })
+    analytics.captureEvent(Events.TABLE.CREATED, { name })
 
     // Create auto screens
     if (createAutoscreens) {

packages/builder/src/components/deploy/DeployModal.svelte

@@ -2,7 +2,8 @@
   import { onMount, onDestroy } from "svelte"
   import { Button, Modal, notifications, ModalContent } from "@budibase/bbui"
   import api from "builderStore/api"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
+  import { store } from "builderStore"
 
   const DeploymentStatus = {
     SUCCESS: "SUCCESS",
@@ -23,6 +24,9 @@
       if (response.status !== 200) {
         throw new Error(`status ${response.status}`)
       } else {
+        analytics.captureEvent(Events.APP.PUBLISHED, {
+          appId: $store.appId,
+        })
         notifications.success(`Application published successfully`)
       }
     } catch (err) {

packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte

@@ -4,7 +4,7 @@
   import { roles } from "stores/backend"
   import { Input, Select, ModalContent, Toggle } from "@budibase/bbui"
   import getTemplates from "builderStore/store/screenTemplates"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
 
   const CONTAINER = "@budibase/standard-components/container"
 
@@ -66,7 +66,7 @@
 
     if (templateIndex !== undefined) {
       const template = templates[templateIndex]
-      analytics.captureEvent("Screen Created", {
+      analytics.captureEvent(Events.SCREEN.CREATED, {
         template: template.id || template.name,
       })
     }

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ConditionalUIDrawer.svelte

@@ -12,7 +12,7 @@
   import { dndzone } from "svelte-dnd-action"
   import { generate } from "shortid"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
-  import { OperatorOptions, getValidOperatorsForType } from "helpers/lucene"
+  import { OperatorOptions, getValidOperatorsForType } from "constants/lucene"
   import { selectedComponent, store } from "builderStore"
   import { getComponentForSettingType } from "./componentSettings"
   import PropertyControl from "./PropertyControl.svelte"

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset, store } from "builderStore"
+  import { getActionProviderComponents } from "builderStore/dataBinding"
+
+  export let parameters
+
+  $: actionProviders = getActionProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId,
+    "RefreshDatasource"
+  )
+</script>
+
+<div class="root">
+  <Label small>Data Provider</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={actionProviders}
+    getOptionLabel={x => x._instanceName}
+    getOptionValue={x => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 70px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -12,6 +12,7 @@ import ClearForm from "./ClearForm.svelte"
 import CloseScreenModal from "./CloseScreenModal.svelte"
 import ChangeFormStep from "./ChangeFormStep.svelte"
 import UpdateStateStep from "./UpdateState.svelte"
+import RefreshDataProvider from "./RefreshDataProvider.svelte"
 
 // Defines which actions are available to configure in the front end.
 // Unfortunately the "name" property is used as the identifier so please don't
@@ -62,6 +63,10 @@ export const getAvailableActions = () => {
       name: "Change Form Step",
       component: ChangeFormStep,
     },
+    {
+      name: "Refresh Data Provider",
+      component: RefreshDataProvider,
+    },
   ]
 
   if (get(store).clientFeatures?.state) {

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -13,18 +13,20 @@
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { generate } from "shortid"
-  import { getValidOperatorsForType, OperatorOptions } from "helpers/lucene"
+  import { getValidOperatorsForType, OperatorOptions } from "constants/lucene"
 
   export let schemaFields
   export let filters = []
   export let bindings = []
   export let panel = BindingPanel
+  export let allowBindings = true
 
   const BannedTypes = ["link", "attachment", "formula"]
 
   $: fieldOptions = (schemaFields ?? [])
     .filter(field => !BannedTypes.includes(field.type))
     .map(field => field.name)
+  $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
 
   const addFilter = () => {
     filters = [
@@ -93,7 +95,7 @@
     <Layout noPadding>
       <Body size="S">
         {#if !filters?.length}
-          Add your first filter column.
+          Add your first filter expression.
         {:else}
           Results are filtered to only those which match all of the following
           constraints.
@@ -117,7 +119,7 @@
             />
             <Select
               disabled={filter.noValue || !filter.field}
-              options={["Value", "Binding"]}
+              options={valueTypeOptions}
               bind:value={filter.valueType}
               placeholder={null}
             />
@@ -133,7 +135,7 @@
               />
             {:else if ["string", "longform", "number"].includes(filter.type)}
               <Input disabled={filter.noValue} bind:value={filter.value} />
-            {:else if filter.type === "options" || "array"}
+            {:else if ["options", "array"].includes(filter.type)}
               <Combobox
                 disabled={filter.noValue}
                 options={getFieldOptions(filter.field)}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterEditor.svelte

@@ -20,7 +20,6 @@
   $: dataSource = getDatasourceForProvider($currentAsset, componentInstance)
   $: schema = getSchemaForDatasource($currentAsset, dataSource)?.schema
   $: schemaFields = Object.values(schema || {})
-  $: internalTable = dataSource?.type === "table"
 
   const saveFilter = async () => {
     dispatch("change", tempValue)

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -4,6 +4,7 @@
   export let defaults
   export let object = defaults || {}
   export let readOnly
+  export let noAddButton
 
   let fields = Object.entries(object).map(([name, value]) => ({ name, value }))
 
@@ -12,7 +13,7 @@
     {}
   )
 
-  function addEntry() {
+  export function addEntry() {
     fields = [...fields, {}]
   }
 
@@ -32,7 +33,7 @@
     {/if}
   {/each}
 </div>
-{#if !readOnly}
+{#if !readOnly && !noAddButton}
   <div>
     <Button secondary thin outline on:click={addEntry}>Add</Button>
   </div>

packages/builder/src/components/start/CreateAppModal.svelte

@@ -12,7 +12,7 @@
   import { admin } from "stores/portal"
   import { string, mixed, object } from "yup"
   import api, { get, post } from "builderStore/api"
-  import analytics from "analytics"
+  import analytics, { Events } from "analytics"
   import { onMount } from "svelte"
   import { capitalise } from "helpers"
   import { goto } from "@roxi/routify"
@@ -98,9 +98,9 @@
         throw new Error(appJson.message)
       }
 
-      analytics.captureEvent("App Created", {
+      analytics.captureEvent(Events.APP.CREATED, {
         name: $values.name,
-        appId: appJson._id,
+        appId: appJson.instance._id,
         template,
       })
 

packages/builder/src/components/upgrade/UpgradeModal.svelte

@@ -4,7 +4,7 @@
   let upgradeModal
 
   const onConfirm = () => {
-    window.open("https://accounts.budibase.com/install", "_blank")
+    window.open("https://account.budibase.app/portal/install", "_blank")
   }
 </script>
 

packages/builder/src/constants/index.js

@@ -15,6 +15,20 @@ export const AppStatus = {
   DEPLOYED: "published",
 }
 
+export const IntegrationNames = {
+  POSTGRES: "PostgreSQL",
+  MONGODB: "MongoDB",
+  COUCHDB: "CouchDB",
+  S3: "S3",
+  MYSQL: "MySQL",
+  REST: "REST",
+  DYNAMODB: "DynamoDB",
+  ELASTICSEARCH: "ElasticSearch",
+  SQL_SERVER: "SQL Server",
+  AIRTABLE: "Airtable",
+  ARANGODB: "ArangoDB",
+}
+
 // fields on the user table that cannot be edited
 export const UNEDITABLE_USER_FIELDS = [
   "email",

packages/builder/src/constants/lucene.js

@@ -0,0 +1,97 @@
+/**
+ * Operator options for lucene queries
+ */
+export const OperatorOptions = {
+  Equals: {
+    value: "equal",
+    label: "Equals",
+  },
+  NotEquals: {
+    value: "notEqual",
+    label: "Not equals",
+  },
+  Empty: {
+    value: "empty",
+    label: "Is empty",
+  },
+  NotEmpty: {
+    value: "notEmpty",
+    label: "Is not empty",
+  },
+  StartsWith: {
+    value: "string",
+    label: "Starts with",
+  },
+  Like: {
+    value: "fuzzy",
+    label: "Like",
+  },
+  MoreThan: {
+    value: "rangeLow",
+    label: "More than",
+  },
+  LessThan: {
+    value: "rangeHigh",
+    label: "Less than",
+  },
+  Contains: {
+    value: "equal",
+    label: "Contains",
+  },
+  NotContains: {
+    value: "notEqual",
+    label: "Does Not Contain",
+  },
+}
+
+/**
+ * Returns the valid operator options for a certain data type
+ * @param type the data type
+ */
+export const getValidOperatorsForType = type => {
+  const Op = OperatorOptions
+  if (type === "string") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.StartsWith,
+      Op.Like,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  } else if (type === "number") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.MoreThan,
+      Op.LessThan,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  } else if (type === "options") {
+    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "array") {
+    return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty]
+  } else if (type === "boolean") {
+    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "longform") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.StartsWith,
+      Op.Like,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  } else if (type === "datetime") {
+    return [
+      Op.Equals,
+      Op.NotEquals,
+      Op.MoreThan,
+      Op.LessThan,
+      Op.Empty,
+      Op.NotEmpty,
+    ]
+  }
+  return []
+}

packages/builder/src/helpers/fetchTableData.js

@@ -0,0 +1,206 @@
+// Do not use any aliased imports in common files, as these will be bundled
+// by multiple bundlers which may not be able to resolve them
+import { writable, derived, get } from "svelte/store"
+import * as API from "../builderStore/api"
+import { buildLuceneQuery } from "./lucene"
+
+const defaultOptions = {
+  tableId: null,
+  filters: null,
+  limit: 10,
+  sortColumn: null,
+  sortOrder: "ascending",
+  paginate: true,
+  schema: null,
+}
+
+export const fetchTableData = opts => {
+  // Save option set so we can override it later rather than relying on params
+  let options = {
+    ...defaultOptions,
+    ...opts,
+  }
+
+  // Local non-observable state
+  let query
+  let sortType
+  let lastBookmark
+
+  // Local observable state
+  const store = writable({
+    rows: [],
+    schema: null,
+    loading: false,
+    loaded: false,
+    bookmarks: [],
+    pageNumber: 0,
+  })
+
+  // Derive certain properties to return
+  const derivedStore = derived(store, $store => {
+    return {
+      ...$store,
+      hasNextPage: $store.bookmarks[$store.pageNumber + 1] != null,
+      hasPrevPage: $store.pageNumber > 0,
+    }
+  })
+
+  const fetchPage = async bookmark => {
+    lastBookmark = bookmark
+    const { tableId, limit, sortColumn, sortOrder, paginate } = options
+    store.update($store => ({ ...$store, loading: true }))
+    const res = await API.post(`/api/${options.tableId}/search`, {
+      tableId,
+      query,
+      limit,
+      sort: sortColumn,
+      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortType,
+      paginate,
+      bookmark,
+    })
+    store.update($store => ({ ...$store, loading: false, loaded: true }))
+    return await res.json()
+  }
+
+  // Fetches a fresh set of results from the server
+  const fetchData = async () => {
+    const { tableId, schema, sortColumn, filters } = options
+
+    // Ensure table ID exists
+    if (!tableId) {
+      return
+    }
+
+    // Get and enrich schema.
+    // Ensure there are "name" properties for all fields and that field schema
+    // are objects
+    let enrichedSchema = schema
+    if (!enrichedSchema) {
+      const definition = await API.get(`/api/tables/${tableId}`)
+      enrichedSchema = definition?.schema ?? null
+    }
+    if (enrichedSchema) {
+      Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
+        if (typeof fieldSchema === "string") {
+          enrichedSchema[fieldName] = {
+            type: fieldSchema,
+            name: fieldName,
+          }
+        } else {
+          enrichedSchema[fieldName] = {
+            ...fieldSchema,
+            name: fieldName,
+          }
+        }
+      })
+
+      // Save fixed schema so we can provide it later
+      options.schema = enrichedSchema
+    }
+
+    // Ensure schema exists
+    if (!schema) {
+      return
+    }
+    store.update($store => ({ ...$store, schema }))
+
+    // Work out what sort type to use
+    if (!sortColumn || !schema[sortColumn]) {
+      sortType = "string"
+    }
+    const type = schema?.[sortColumn]?.type
+    sortType = type === "number" ? "number" : "string"
+
+    // Build the lucene query
+    query = buildLuceneQuery(filters)
+
+    // Actually fetch data
+    const page = await fetchPage()
+    store.update($store => ({
+      ...$store,
+      loading: false,
+      loaded: true,
+      pageNumber: 0,
+      rows: page.rows,
+      bookmarks: page.hasNextPage ? [null, page.bookmark] : [null],
+    }))
+  }
+
+  // Fetches the next page of data
+  const nextPage = async () => {
+    const state = get(derivedStore)
+    if (state.loading || !options.paginate || !state.hasNextPage) {
+      return
+    }
+
+    // Fetch next page
+    const page = await fetchPage(state.bookmarks[state.pageNumber + 1])
+
+    // Update state
+    store.update($store => {
+      let { bookmarks, pageNumber } = $store
+      if (page.hasNextPage) {
+        bookmarks[pageNumber + 2] = page.bookmark
+      }
+      return {
+        ...$store,
+        pageNumber: pageNumber + 1,
+        rows: page.rows,
+        bookmarks,
+      }
+    })
+  }
+
+  // Fetches the previous page of data
+  const prevPage = async () => {
+    const state = get(derivedStore)
+    if (state.loading || !options.paginate || !state.hasPrevPage) {
+      return
+    }
+
+    // Fetch previous page
+    const page = await fetchPage(state.bookmarks[state.pageNumber - 1])
+
+    // Update state
+    store.update($store => {
+      return {
+        ...$store,
+        pageNumber: $store.pageNumber - 1,
+        rows: page.rows,
+      }
+    })
+  }
+
+  // Resets the data set and updates options
+  const update = async newOptions => {
+    if (newOptions) {
+      options = {
+        ...options,
+        ...newOptions,
+      }
+    }
+    await fetchData()
+  }
+
+  // Loads the same page again
+  const refresh = async () => {
+    if (get(store).loading) {
+      return
+    }
+    const page = await fetchPage(lastBookmark)
+    store.update($store => ({ ...$store, rows: page.rows }))
+  }
+
+  // Initially fetch data but don't bother waiting for the result
+  fetchData()
+
+  // Return our derived store which will be updated over time
+  return {
+    subscribe: derivedStore.subscribe,
+    nextPage,
+    prevPage,
+    update,
+    refresh,
+  }
+}

packages/builder/src/helpers/lucene.js

@@ -1,90 +1,179 @@
-export const OperatorOptions = {
-  Equals: {
-    value: "equal",
-    label: "Equals",
-  },
-  NotEquals: {
-    value: "notEqual",
-    label: "Not equals",
-  },
-  Empty: {
-    value: "empty",
-    label: "Is empty",
-  },
-  NotEmpty: {
-    value: "notEmpty",
-    label: "Is not empty",
-  },
-  StartsWith: {
-    value: "string",
-    label: "Starts with",
-  },
-  Like: {
-    value: "fuzzy",
-    label: "Like",
-  },
-  MoreThan: {
-    value: "rangeLow",
-    label: "More than",
-  },
-  LessThan: {
-    value: "rangeHigh",
-    label: "Less than",
-  },
-  Contains: {
-    value: "equal",
-    label: "Contains",
-  },
-  NotContains: {
-    value: "notEqual",
-    label: "Does Not Contain",
-  },
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = filter => {
+  let query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(expression => {
+      let { operator, field, type, value } = expression
+      // Parse all values into correct types
+      if (type === "datetime" && value) {
+        value = new Date(value).toISOString()
+      }
+      if (type === "number") {
+        value = parseFloat(value)
+      }
+      if (type === "boolean") {
+        value = `${value}`?.toLowerCase() === "true"
+      }
+      if (operator.startsWith("range")) {
+        if (!query.range[field]) {
+          query.range[field] = {
+            low:
+              type === "number"
+                ? Number.MIN_SAFE_INTEGER
+                : "0000-00-00T00:00:00.000Z",
+            high:
+              type === "number"
+                ? Number.MAX_SAFE_INTEGER
+                : "9999-00-00T00:00:00.000Z",
+          }
+        }
+        if (operator === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (operator === "rangeHigh" && value != null && value !== "") {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        if (type === "boolean") {
+          // Transform boolean filters to cope with null.
+          // "equals false" needs to be "not equals true"
+          // "not equals false" needs to be "equals true"
+          if (operator === "equal" && value === false) {
+            query.notEqual[field] = true
+          } else if (operator === "notEqual" && value === false) {
+            query.equal[field] = true
+          } else {
+            query[operator][field] = value
+          }
+        } else {
+          query[operator][field] = value
+        }
+      }
+    })
+  }
+
+  return query
 }
 
-export const getValidOperatorsForType = type => {
-  const Op = OperatorOptions
-  if (type === "string") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
-  } else if (type === "number") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
-  } else if (type === "options") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
-  } else if (type === "array") {
-    return [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty]
-  } else if (type === "boolean") {
-    return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
-  } else if (type === "longform") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
-  } else if (type === "datetime") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const luceneQuery = (docs, query) => {
+  if (!query) {
+    return docs
   }
-  return []
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match = (type, failFn) => doc => {
+    const filters = Object.entries(query[type] || {})
+    for (let i = 0; i < filters.length; i++) {
+      if (failFn(filters[i][0], filters[i][1], doc)) {
+        return false
+      }
+    }
+    return true
+  }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (key, value, doc) => {
+    return !doc[key] || !doc[key].startsWith(value)
+  })
+
+  // Process a fuzzy match (treat the same as starts with when running locally)
+  const fuzzyMatch = match("fuzzy", (key, value, doc) => {
+    return !doc[key] || !doc[key].startsWith(value)
+  })
+
+  // Process a range match
+  const rangeMatch = match("range", (key, value, doc) => {
+    return !doc[key] || doc[key] < value.low || doc[key] > value.high
+  })
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match("equal", (key, value, doc) => {
+    return value != null && value !== "" && doc[key] !== value
+  })
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match("notEqual", (key, value, doc) => {
+    return value != null && value !== "" && doc[key] === value
+  })
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (key, value, doc) => {
+    return doc[key] != null && doc[key] !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (key, value, doc) => {
+    return doc[key] == null || doc[key] === ""
+  })
+
+  // Match a document against all criteria
+  const docMatch = doc => {
+    return (
+      stringMatch(doc) &&
+      fuzzyMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
+  return docs.slice().sort((a, b) => {
+    const colA = parse(a[sort])
+    const colB = parse(b[sort])
+    if (sortOrder === "Descending") {
+      return colA > colB ? -1 : 1
+    } else {
+      return colA > colB ? 1 : -1
+    }
+  })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs, limit) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
 }

packages/builder/src/helpers/warnings.js

@@ -0,0 +1,16 @@
+export const suppressWarnings = warnings => {
+  if (!warnings?.length) {
+    return
+  }
+  const regex = new RegExp(warnings.map(x => `(${x})`).join("|"), "gi")
+  const warn = console.warn
+  console.warn = (...params) => {
+    const msg = params[0]
+    if (msg && typeof msg === "string") {
+      if (msg.match(regex)) {
+        return
+      }
+    }
+    warn(...params)
+  }
+}

packages/builder/src/main.js

@@ -7,11 +7,19 @@ import "@spectrum-css/vars/dist/spectrum-light.css"
 import "@spectrum-css/vars/dist/spectrum-lightest.css"
 import "@spectrum-css/page/dist/index-vars.css"
 import "./global.css"
-
+import { suppressWarnings } from "./helpers/warnings"
 import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js"
+import App from "./App.svelte"
+
+// Init spectrum icons
 loadSpectrumIcons()
 
-import App from "./App.svelte"
+// Suppress svelte runtime warnings
+suppressWarnings([
+  "was created with unknown prop",
+  "was created without expected prop",
+  "received an unexpected slot",
+])
 
 export default new App({
   target: document.getElementById("app"),

packages/builder/src/pages/builder/_layout.svelte

@@ -4,44 +4,73 @@
   import { onMount } from "svelte"
 
   let loaded = false
+  // don't react to these
+  let cloud = $admin.cloud
+  let shouldRedirect = !cloud || $admin.disableAccountPortal
 
   $: multiTenancyEnabled = $admin.multiTenancy
   $: hasAdminUser = $admin?.checklist?.adminUser?.checked
   $: tenantSet = $auth.tenantSet
   $: cloud = $admin.cloud
+  $: user = $auth.user
+
+  const validateTenantId = async () => {
+    // set the tenant from the url in the cloud
+    const tenantId = window.location.host.split(".")[0]
+
+    if (!tenantId.includes("localhost:")) {
+      // user doesn't have permission to access this tenant - kick them out
+      if (user?.tenantId !== tenantId) {
+        await auth.logout()
+        await auth.setOrganisation(null)
+      } else {
+        await auth.setOrganisation(tenantId)
+      }
+    }
+  }
 
   onMount(async () => {
     await auth.checkAuth()
     await admin.init()
+
+    if (cloud && multiTenancyEnabled) {
+      await validateTenantId()
+    }
+
     loaded = true
   })
 
   $: {
     // We should never see the org or admin user creation screens in the cloud
-    if (!cloud) {
-      const apiReady = $admin.loaded && $auth.loaded
-      // if tenant is not set go to it
-      if (loaded && apiReady && multiTenancyEnabled && !tenantSet) {
-        $redirect("./auth/org")
-      }
-      // Force creation of an admin user if one doesn't exist
-      else if (loaded && apiReady && !hasAdminUser) {
-        $redirect("./admin")
-      }
-    }
-  }
-  // Redirect to log in at any time if the user isn't authenticated
-  $: {
+    const apiReady = $admin.loaded && $auth.loaded
+    // if tenant is not set go to it
     if (
+      loaded &&
+      shouldRedirect &&
+      apiReady &&
+      multiTenancyEnabled &&
+      !tenantSet
+    ) {
+      $redirect("./auth/org")
+    }
+    // Force creation of an admin user if one doesn't exist
+    else if (loaded && shouldRedirect && apiReady && !hasAdminUser) {
+      $redirect("./admin")
+    }
+    // Redirect to log in at any time if the user isn't authenticated
+    else if (
       loaded &&
       (hasAdminUser || cloud) &&
       !$auth.user &&
       !$isActive("./auth") &&
-      !$isActive("./invite")
+      !$isActive("./invite") &&
+      !$isActive("./admin")
     ) {
       const returnUrl = encodeURIComponent(window.location.pathname)
       $redirect("./auth?", { returnUrl })
-    } else if ($auth?.user?.forceResetPassword) {
+    }
+    // check if password reset required for user
+    else if ($auth.user?.forceResetPassword) {
       $redirect("./auth/reset")
     }
   }

packages/builder/src/pages/builder/admin/_components/ImportAppsModal.svelte

@@ -0,0 +1,50 @@
+<script>
+  import { notifications, ModalContent, Dropzone, Body } from "@budibase/bbui"
+  import { post } from "builderStore/api"
+
+  let submitting = false
+
+  $: value = { file: null }
+
+  async function importApps() {
+    submitting = true
+
+    try {
+      // Create form data to create app
+      let data = new FormData()
+      data.append("importFile", value.file)
+
+      // Create App
+      const importResp = await post("/api/cloud/import", data, {})
+      const importJson = await importResp.json()
+      if (!importResp.ok) {
+        throw new Error(importJson.message)
+      }
+      // now reload to get to login
+      window.location.reload()
+    } catch (error) {
+      notifications.error(error)
+      submitting = false
+    }
+  }
+</script>
+
+<ModalContent
+  title="Import apps"
+  confirmText="Import apps"
+  onConfirm={importApps}
+  disabled={!value.file}
+>
+  <Body
+    >Please upload the file that was exported from your Cloud environment to get
+    started</Body
+  >
+  <Dropzone
+    gallery={false}
+    label="File to import"
+    value={[value.file]}
+    on:change={e => {
+      value.file = e.detail?.[0]
+    }}
+  />
+</ModalContent>

packages/builder/src/pages/builder/admin/index.svelte

@@ -7,18 +7,22 @@
     Input,
     Body,
     ActionButton,
+    Modal,
   } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
   import api from "builderStore/api"
   import { admin, auth } from "stores/portal"
   import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
+  import ImportAppsModal from "./_components/ImportAppsModal.svelte"
   import Logo from "assets/bb-emblem.svg"
 
   let adminUser = {}
   let error
+  let modal
 
   $: tenantId = $auth.tenantId
   $: multiTenancyEnabled = $admin.multiTenancy
+  $: cloud = $admin.cloud
 
   async function save() {
     try {
@@ -38,6 +42,9 @@
   }
 </script>
 
+<Modal bind:this={modal} padding={false} width="600px">
+  <ImportAppsModal />
+</Modal>
 <section>
   <div class="container">
     <Layout>
@@ -66,6 +73,15 @@
           >
             Change organisation
           </ActionButton>
+        {:else if !cloud}
+          <ActionButton
+            quiet
+            on:click={() => {
+              modal.show()
+            }}
+          >
+            Import from cloud
+          </ActionButton>
         {/if}
       </Layout>
     </Layout>

packages/builder/src/pages/builder/app/[application]/data/_layout.svelte

@@ -1,12 +1,14 @@
 <script>
   import { goto, params } from "@roxi/routify"
-  import { Icon, Modal, Tabs, Tab } from "@budibase/bbui"
+  import { Icon, Tabs, Tab } from "@budibase/bbui"
   import { BUDIBASE_INTERNAL_DB } from "constants"
   import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
   import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
 
   let selected = "Sources"
+
   let modal
+
   $: isExternal =
     $params.selectedDatasource &&
     $params.selectedDatasource !== BUDIBASE_INTERNAL_DB
@@ -23,9 +25,7 @@
       <Tab title="Sources">
         <div class="tab-content-padding">
           <DatasourceNavigator />
-          <Modal bind:this={modal}>
-            <CreateDatasourceModal />
-          </Modal>
+          <CreateDatasourceModal bind:modal />
         </div>
       </Tab>
     </Tabs>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/[query]/_layout.svelte

@@ -1 +1,13 @@
+<script>
+  import { params } from "@roxi/routify"
+  import { queries } from "stores/backend"
+
+  if ($params.query) {
+    const query = $queries.list.find(q => q._id === $params.query)
+    if (query) {
+      queries.select(query)
+    }
+  }
+</script>
+
 <slot />

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_layout.svelte

@@ -2,7 +2,7 @@
   import { params } from "@roxi/routify"
   import { datasources } from "stores/backend"
 
-  if ($params.selectedDatasource) {
+  if ($params.selectedDatasource && !$params.query) {
     const datasource = $datasources.list.find(
       m => m._id === $params.selectedDatasource
     )

packages/builder/src/pages/builder/app/[application]/data/datasource/_layout.svelte

@@ -1,14 +1 @@
-<script>
-  import { datasources } from "stores/backend"
-  import { goto, leftover } from "@roxi/routify"
-  import { onMount } from "svelte"
-
-  onMount(async () => {
-    // navigate to first datasource in list, if not already selected
-    if (!$leftover && $datasources.list.length > 0 && !$datasources.selected) {
-      $goto(`./${$datasources.list[0]._id}`)
-    }
-  })
-</script>
-
 <slot />