Merge branch 'develop' of github.com:Budibase/budibase into frontend-core

charts/budibase/templates/app-service-deployment.yaml

@@ -108,6 +108,8 @@ spec:
           value: {{ .Values.globals.accountPortalApiKey | quote }}
         - name: COOKIE_DOMAIN
           value: {{ .Values.globals.cookieDomain | quote }}
+        - name: HTTP_MIGRATIONS
+          value: {{ .Values.globals.httpMigrations | quote }}
         image: budibase/apps:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
         name: bbapps

charts/budibase/values.yaml

@@ -99,6 +99,7 @@ globals:
   accountPortalApiKey: ""
   cookieDomain: ""
   platformUrl: ""
+  httpMigrations: "0"
 
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -36,10 +36,10 @@
     "dev:server": "lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
     "test": "lerna run test",
     "lint:eslint": "eslint packages",
-    "lint:prettier": "prettier --check \"packages/**/*.{js,svelte}\"",
+    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix packages",
-    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,svelte}\"",
+    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\"",
     "lint:fix:ts": "lerna run lint:fix",
     "lint:fix": "yarn run lint:fix:ts && yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "test:e2e": "lerna run cy:test",

packages/backend-core/context.js

@@ -0,0 +1,17 @@
+const {
+  getAppDB,
+  getDevAppDB,
+  getProdAppDB,
+  getAppId,
+  updateAppId,
+  doInAppContext,
+} = require("./src/context")
+
+module.exports = {
+  getAppDB,
+  getDevAppDB,
+  getProdAppDB,
+  getAppId,
+  updateAppId,
+  doInAppContext,
+}

packages/backend-core/db.js

@@ -1,4 +1,6 @@
 module.exports = {
   ...require("./src/db/utils"),
   ...require("./src/db/constants"),
+  ...require("./src/db"),
+  ...require("./src/db/views"),
 }

packages/backend-core/deprovision.js

@@ -1 +1 @@
-module.exports = require("./src/tenancy/deprovision")
+module.exports = require("./src/context/deprovision")

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/auth.js

@@ -13,6 +13,7 @@ const {
   appTenancy,
   authError,
   csrf,
+  internalApi,
 } = require("./middleware")
 
 // Strategies
@@ -44,4 +45,5 @@ module.exports = {
   auditLog,
   authError,
   buildCsrfMiddleware: csrf,
+  internalApi,
 }

packages/backend-core/src/context/FunctionContext.js

@@ -4,8 +4,8 @@ const { newid } = require("../hashing")
 const REQUEST_ID_KEY = "requestId"
 
 class FunctionContext {
-  static getMiddleware(updateCtxFn = null) {
-    const namespace = this.createNamespace()
+  static getMiddleware(updateCtxFn = null, contextName = "session") {
+    const namespace = this.createNamespace(contextName)
 
     return async function (ctx, next) {
       await new Promise(
@@ -24,14 +24,14 @@ class FunctionContext {
     }
   }
 
-  static run(callback) {
-    const namespace = this.createNamespace()
+  static run(callback, contextName = "session") {
+    const namespace = this.createNamespace(contextName)
 
     return namespace.runAndReturn(callback)
   }
 
-  static setOnContext(key, value) {
-    const namespace = this.createNamespace()
+  static setOnContext(key, value, contextName = "session") {
+    const namespace = this.createNamespace(contextName)
     namespace.set(key, value)
   }
 
@@ -55,16 +55,16 @@ class FunctionContext {
     }
   }
 
-  static destroyNamespace() {
+  static destroyNamespace(name = "session") {
     if (this._namespace) {
-      cls.destroyNamespace("session")
+      cls.destroyNamespace(name)
       this._namespace = null
     }
   }
 
-  static createNamespace() {
+  static createNamespace(name = "session") {
     if (!this._namespace) {
-      this._namespace = cls.createNamespace("session")
+      this._namespace = cls.createNamespace(name)
     }
     return this._namespace
   }

packages/backend-core/src/context/deprovision.js

@@ -1,6 +1,6 @@
 const { getGlobalUserParams, getAllApps } = require("../db/utils")
 const { getDB, getCouch } = require("../db")
-const { getGlobalDB } = require("./tenancy")
+const { getGlobalDB } = require("../tenancy")
 const { StaticDatabases } = require("../db/constants")
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants

packages/backend-core/src/context/index.js

@@ -0,0 +1,195 @@
+const env = require("../environment")
+const { Headers } = require("../../constants")
+const cls = require("./FunctionContext")
+const { getCouch } = require("../db")
+const { getProdAppID, getDevelopmentAppID } = require("../db/conversions")
+const { isEqual } = require("lodash")
+
+// some test cases call functions directly, need to
+// store an app ID to pretend there is a context
+let TEST_APP_ID = null
+
+const ContextKeys = {
+  TENANT_ID: "tenantId",
+  APP_ID: "appId",
+  // whatever the request app DB was
+  CURRENT_DB: "currentDb",
+  // get the prod app DB from the request
+  PROD_DB: "prodDb",
+  // get the dev app DB from the request
+  DEV_DB: "devDb",
+  DB_OPTS: "dbOpts",
+}
+
+exports.DEFAULT_TENANT_ID = "default"
+
+exports.isDefaultTenant = () => {
+  return exports.getTenantId() === exports.DEFAULT_TENANT_ID
+}
+
+exports.isMultiTenant = () => {
+  return env.MULTI_TENANCY
+}
+
+// used for automations, API endpoints should always be in context already
+exports.doInTenant = (tenantId, task) => {
+  return cls.run(() => {
+    // set the tenant id
+    cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+
+    // invoke the task
+    return task()
+  })
+}
+
+exports.doInAppContext = (appId, task) => {
+  return cls.run(() => {
+    // set the app ID
+    cls.setOnContext(ContextKeys.APP_ID, appId)
+
+    // invoke the task
+    return task()
+  })
+}
+
+exports.updateTenantId = tenantId => {
+  cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+}
+
+exports.updateAppId = appId => {
+  try {
+    cls.setOnContext(ContextKeys.APP_ID, appId)
+    cls.setOnContext(ContextKeys.PROD_DB, null)
+    cls.setOnContext(ContextKeys.DEV_DB, null)
+    cls.setOnContext(ContextKeys.CURRENT_DB, null)
+    cls.setOnContext(ContextKeys.DB_OPTS, null)
+  } catch (err) {
+    if (env.isTest()) {
+      TEST_APP_ID = appId
+    } else {
+      throw err
+    }
+  }
+}
+
+exports.setTenantId = (
+  ctx,
+  opts = { allowQs: false, allowNoTenant: false }
+) => {
+  let tenantId
+  // exit early if not multi-tenant
+  if (!exports.isMultiTenant()) {
+    cls.setOnContext(ContextKeys.TENANT_ID, this.DEFAULT_TENANT_ID)
+    return
+  }
+
+  const allowQs = opts && opts.allowQs
+  const allowNoTenant = opts && opts.allowNoTenant
+  const header = ctx.request.headers[Headers.TENANT_ID]
+  const user = ctx.user || {}
+  if (allowQs) {
+    const query = ctx.request.query || {}
+    tenantId = query.tenantId
+  }
+  // override query string (if allowed) by user, or header
+  // URL params cannot be used in a middleware, as they are
+  // processed later in the chain
+  tenantId = user.tenantId || header || tenantId
+
+  // Set the tenantId from the subdomain
+  if (!tenantId) {
+    tenantId = ctx.subdomains && ctx.subdomains[0]
+  }
+
+  if (!tenantId && !allowNoTenant) {
+    ctx.throw(403, "Tenant id not set")
+  }
+  // check tenant ID just incase no tenant was allowed
+  if (tenantId) {
+    cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+  }
+}
+
+exports.isTenantIdSet = () => {
+  const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
+  return !!tenantId
+}
+
+exports.getTenantId = () => {
+  if (!exports.isMultiTenant()) {
+    return exports.DEFAULT_TENANT_ID
+  }
+  const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
+  if (!tenantId) {
+    throw Error("Tenant id not found")
+  }
+  return tenantId
+}
+
+exports.getAppId = () => {
+  const foundId = cls.getFromContext(ContextKeys.APP_ID)
+  if (!foundId && env.isTest() && TEST_APP_ID) {
+    return TEST_APP_ID
+  } else {
+    return foundId
+  }
+}
+
+function getDB(key, opts) {
+  const dbOptsKey = `${key}${ContextKeys.DB_OPTS}`
+  let storedOpts = cls.getFromContext(dbOptsKey)
+  let db = cls.getFromContext(key)
+  if (db && isEqual(opts, storedOpts)) {
+    return db
+  }
+  const appId = exports.getAppId()
+  const CouchDB = getCouch()
+  let toUseAppId
+  switch (key) {
+    case ContextKeys.CURRENT_DB:
+      toUseAppId = appId
+      break
+    case ContextKeys.PROD_DB:
+      toUseAppId = getProdAppID(appId)
+      break
+    case ContextKeys.DEV_DB:
+      toUseAppId = getDevelopmentAppID(appId)
+      break
+  }
+  db = new CouchDB(toUseAppId, opts)
+  try {
+    cls.setOnContext(key, db)
+    if (opts) {
+      cls.setOnContext(dbOptsKey, opts)
+    }
+  } catch (err) {
+    if (!env.isTest()) {
+      throw err
+    }
+  }
+  return db
+}
+
+/**
+ * Opens the app database based on whatever the request
+ * contained, dev or prod.
+ */
+exports.getAppDB = opts => {
+  return getDB(ContextKeys.CURRENT_DB, opts)
+}
+
+/**
+ * This specifically gets the prod app ID, if the request
+ * contained a development app ID, this will open the prod one.
+ */
+exports.getProdAppDB = opts => {
+  return getDB(ContextKeys.PROD_DB, opts)
+}
+
+/**
+ * This specifically gets the dev app ID, if the request
+ * contained a prod app ID, this will open the dev one.
+ */
+exports.getDevAppDB = opts => {
+  return getDB(ContextKeys.DEV_DB, opts)
+}

packages/backend-core/src/db/constants.js

@@ -32,3 +32,7 @@ exports.StaticDatabases = {
     },
   },
 }
+
+exports.APP_PREFIX = exports.DocumentTypes.APP + exports.SEPARATOR
+exports.APP_DEV = exports.APP_DEV_PREFIX =
+  exports.DocumentTypes.APP_DEV + exports.SEPARATOR

packages/backend-core/src/db/conversions.js

@@ -0,0 +1,46 @@
+const NO_APP_ERROR = "No app provided"
+const { APP_DEV_PREFIX, APP_PREFIX } = require("./constants")
+
+exports.isDevAppID = appId => {
+  if (!appId) {
+    throw NO_APP_ERROR
+  }
+  return appId.startsWith(APP_DEV_PREFIX)
+}
+
+exports.isProdAppID = appId => {
+  if (!appId) {
+    throw NO_APP_ERROR
+  }
+  return appId.startsWith(APP_PREFIX) && !exports.isDevAppID(appId)
+}
+
+exports.isDevApp = app => {
+  if (!app) {
+    throw NO_APP_ERROR
+  }
+  return exports.isDevAppID(app.appId)
+}
+
+/**
+ * Convert a development app ID to a deployed app ID.
+ */
+exports.getProdAppID = appId => {
+  // if dev, convert it
+  if (appId.startsWith(APP_DEV_PREFIX)) {
+    const id = appId.split(APP_DEV_PREFIX)[1]
+    return `${APP_PREFIX}${id}`
+  }
+  return appId
+}
+
+/**
+ * Convert a deployed app ID to a development app ID.
+ */
+exports.getDevelopmentAppID = appId => {
+  if (!appId.startsWith(APP_DEV_PREFIX)) {
+    const id = appId.split(APP_PREFIX)[1]
+    return `${APP_DEV_PREFIX}${id}`
+  }
+  return appId
+}

packages/backend-core/src/db/utils.js

@@ -2,7 +2,13 @@ const { newid } = require("../hashing")
 const Replication = require("./Replication")
 const { DEFAULT_TENANT_ID, Configs } = require("../constants")
 const env = require("../environment")
-const { StaticDatabases, SEPARATOR, DocumentTypes } = require("./constants")
+const {
+  StaticDatabases,
+  SEPARATOR,
+  DocumentTypes,
+  APP_PREFIX,
+  APP_DEV,
+} = require("./constants")
 const {
   getTenantId,
   getTenantIDFromAppID,
@@ -12,8 +18,13 @@ const fetch = require("node-fetch")
 const { getCouch } = require("./index")
 const { getAppMetadata } = require("../cache/appMetadata")
 const { checkSlashesInUrl } = require("../helpers")
-
-const NO_APP_ERROR = "No app provided"
+const {
+  isDevApp,
+  isProdAppID,
+  isDevAppID,
+  getDevelopmentAppID,
+  getProdAppID,
+} = require("./conversions")
 
 const UNICODE_MAX = "\ufff0"
 
@@ -24,10 +35,15 @@ exports.ViewNames = {
 exports.StaticDatabases = StaticDatabases
 
 exports.DocumentTypes = DocumentTypes
-exports.APP_PREFIX = DocumentTypes.APP + SEPARATOR
-exports.APP_DEV = exports.APP_DEV_PREFIX = DocumentTypes.APP_DEV + SEPARATOR
+exports.APP_PREFIX = APP_PREFIX
+exports.APP_DEV = exports.APP_DEV_PREFIX = APP_DEV
 exports.SEPARATOR = SEPARATOR
 exports.getTenantIDFromAppID = getTenantIDFromAppID
+exports.isDevApp = isDevApp
+exports.isProdAppID = isProdAppID
+exports.isDevAppID = isDevAppID
+exports.getDevelopmentAppID = getDevelopmentAppID
+exports.getProdAppID = getProdAppID
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
@@ -52,27 +68,6 @@ function getDocParams(docType, docId = null, otherProps = {}) {
   }
 }
 
-exports.isDevAppID = appId => {
-  if (!appId) {
-    throw NO_APP_ERROR
-  }
-  return appId.startsWith(exports.APP_DEV_PREFIX)
-}
-
-exports.isProdAppID = appId => {
-  if (!appId) {
-    throw NO_APP_ERROR
-  }
-  return appId.startsWith(exports.APP_PREFIX) && !exports.isDevAppID(appId)
-}
-
-function isDevApp(app) {
-  if (!app) {
-    throw NO_APP_ERROR
-  }
-  return exports.isDevAppID(app.appId)
-}
-
 /**
  * Generates a new workspace ID.
  * @returns {string} The new workspace ID which the workspace doc can be stored under.
@@ -157,29 +152,6 @@ exports.getRoleParams = (roleId = null, otherProps = {}) => {
   return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
 }
 
-/**
- * Convert a development app ID to a deployed app ID.
- */
-exports.getDeployedAppID = appId => {
-  // if dev, convert it
-  if (appId.startsWith(exports.APP_DEV_PREFIX)) {
-    const id = appId.split(exports.APP_DEV_PREFIX)[1]
-    return `${exports.APP_PREFIX}${id}`
-  }
-  return appId
-}
-
-/**
- * Convert a deployed app ID to a development app ID.
- */
-exports.getDevelopmentAppID = appId => {
-  if (!appId.startsWith(exports.APP_DEV_PREFIX)) {
-    const id = appId.split(exports.APP_PREFIX)[1]
-    return `${exports.APP_DEV_PREFIX}${id}`
-  }
-  return appId
-}
-
 exports.getCouchUrl = () => {
   if (!env.COUCH_DB_URL) return
 
@@ -225,7 +197,7 @@ exports.getAllDbs = async () => {
   }
   let couchUrl = `${exports.getCouchUrl()}/_all_dbs`
   let tenantId = getTenantId()
-  if (!env.MULTI_TENANCY || tenantId == DEFAULT_TENANT_ID) {
+  if (!env.MULTI_TENANCY || tenantId === DEFAULT_TENANT_ID) {
     // just get all DBs when:
     // - single tenancy
     // - default tenant
@@ -250,11 +222,10 @@ exports.getAllDbs = async () => {
 /**
  * Lots of different points in the system need to find the full list of apps, this will
  * enumerate the entire CouchDB cluster and get the list of databases (every app).
- * NOTE: this operation is fine in self hosting, but cannot be used when hosting many
- * different users/companies apps as there is no security around it - all apps are returned.
  * @return {Promise<object[]>} returns the app information document stored in each app database.
  */
-exports.getAllApps = async (CouchDB, { dev, all, idsOnly } = {}) => {
+exports.getAllApps = async ({ dev, all, idsOnly } = {}) => {
+  const CouchDB = getCouch()
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY && !tenantId) {
     tenantId = DEFAULT_TENANT_ID
@@ -310,8 +281,8 @@ exports.getAllApps = async (CouchDB, { dev, all, idsOnly } = {}) => {
 /**
  * Utility function for getAllApps but filters to production apps only.
  */
-exports.getDeployedAppIDs = async CouchDB => {
-  return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter(
+exports.getProdAppIDs = async () => {
+  return (await exports.getAllApps({ idsOnly: true })).filter(
     id => !exports.isDevAppID(id)
   )
 }
@@ -319,13 +290,14 @@ exports.getDeployedAppIDs = async CouchDB => {
 /**
  * Utility function for the inverse of above.
  */
-exports.getDevAppIDs = async CouchDB => {
-  return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter(id =>
+exports.getDevAppIDs = async () => {
+  return (await exports.getAllApps({ idsOnly: true })).filter(id =>
     exports.isDevAppID(id)
   )
 }
 
-exports.dbExists = async (CouchDB, dbName) => {
+exports.dbExists = async dbName => {
+  const CouchDB = getCouch()
   let exists = false
   try {
     const db = CouchDB(dbName, { skip_setup: true })

packages/backend-core/src/middleware/appTenancy.js

@@ -3,8 +3,9 @@ const {
   updateTenantId,
   isTenantIdSet,
   DEFAULT_TENANT_ID,
+  updateAppId,
 } = require("../tenancy")
-const ContextFactory = require("../tenancy/FunctionContext")
+const ContextFactory = require("../context/FunctionContext")
 const { getTenantIDFromAppID } = require("../db/utils")
 
 module.exports = () => {
@@ -21,5 +22,6 @@ module.exports = () => {
     const appId = ctx.appId ? ctx.appId : ctx.user ? ctx.user.appId : null
     const tenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
     updateTenantId(tenantId)
+    updateAppId(appId)
   })
 }

packages/backend-core/src/middleware/index.js

@@ -7,6 +7,7 @@ const authenticated = require("./authenticated")
 const auditLog = require("./auditLog")
 const tenancy = require("./tenancy")
 const appTenancy = require("./appTenancy")
+const internalApi = require("./internalApi")
 const datasourceGoogle = require("./passport/datasource/google")
 const csrf = require("./csrf")
 
@@ -20,6 +21,7 @@ module.exports = {
   tenancy,
   appTenancy,
   authError,
+  internalApi,
   datasource: {
     google: datasourceGoogle,
   },

packages/backend-core/src/middleware/internalApi.js

@@ -0,0 +1,14 @@
+const env = require("../environment")
+const { Headers } = require("../constants")
+
+/**
+ * API Key only endpoint.
+ */
+module.exports = async (ctx, next) => {
+  const apiKey = ctx.request.headers[Headers.API_KEY]
+  if (apiKey !== env.INTERNAL_API_KEY) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  return next()
+}

packages/backend-core/src/middleware/tenancy.js

@@ -1,5 +1,5 @@
 const { setTenantId } = require("../tenancy")
-const ContextFactory = require("../tenancy/FunctionContext")
+const ContextFactory = require("../context/FunctionContext")
 const { buildMatcherRegex, matches } = require("./matchers")
 
 module.exports = (

packages/backend-core/src/migrations/index.js

@@ -1,20 +1,17 @@
+const { DEFAULT_TENANT_ID } = require("../constants")
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB, getTenantId } = require("../tenancy")
+const { getAllApps } = require("../db/utils")
+const environment = require("../environment")
+const {
+  doInTenant,
+  getTenantIds,
+  getGlobalDBName,
+  getTenantId,
+} = require("../tenancy")
 
-exports.MIGRATION_DBS = {
-  GLOBAL_DB: "GLOBAL_DB",
-}
-
-exports.MIGRATIONS = {
-  USER_EMAIL_VIEW_CASING: "user_email_view_casing",
-  QUOTAS_1: "quotas_1",
-}
-
-const DB_LOOKUP = {
-  [exports.MIGRATION_DBS.GLOBAL_DB]: [
-    exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
-    exports.MIGRATIONS.QUOTAS_1,
-  ],
+exports.MIGRATION_TYPES = {
+  GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
+  APP: "app", // run per app, recorded in each app db, app db is provided as an argument
 }
 
 exports.getMigrationsDoc = async db => {
@@ -28,40 +25,90 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+const runMigration = async (CouchDB, migration, options = {}) => {
   const tenantId = getTenantId()
-  try {
-    let db
-    if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
-      db = getGlobalDB()
-    } else {
-      throw new Error(`Unrecognised migration db [${migrationDb}]`)
-    }
+  const migrationType = migration.type
+  const migrationName = migration.name
 
-    if (!DB_LOOKUP[migrationDb].includes(migrationName)) {
-      throw new Error(
-        `Unrecognised migration name [${migrationName}] for db [${migrationDb}]`
-      )
-    }
-
-    const doc = await exports.getMigrationsDoc(db)
-    // exit if the migration has been performed
-    if (doc[migrationName]) {
-      return
-    }
-
-    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
-    await migrateFn()
-    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
-
-    // mark as complete
-    doc[migrationName] = Date.now()
-    await db.put(doc)
-  } catch (err) {
-    console.error(
-      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
-      err
+  // get the db to store the migration in
+  let dbNames
+  if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
+    dbNames = [getGlobalDBName()]
+  } else if (migrationType === exports.MIGRATION_TYPES.APP) {
+    const apps = await getAllApps(CouchDB, migration.opts)
+    dbNames = apps.map(app => app.appId)
+  } else {
+    throw new Error(
+      `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
     )
-    throw err
+  }
+
+  // run the migration against each db
+  for (const dbName of dbNames) {
+    const db = new CouchDB(dbName)
+    try {
+      const doc = await exports.getMigrationsDoc(db)
+
+      // exit if the migration has been performed already
+      if (doc[migrationName]) {
+        if (
+          options.force &&
+          options.force[migrationType] &&
+          options.force[migrationType].includes(migrationName)
+        ) {
+          console.log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
+          )
+        } else {
+          // the migration has already been performed
+          continue
+        }
+      }
+
+      console.log(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
+      )
+      // run the migration with tenant context
+      await migration.fn(db)
+      console.log(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+      )
+
+      // mark as complete
+      doc[migrationName] = Date.now()
+      await db.put(doc)
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        err
+      )
+      throw err
+    }
   }
 }
+
+exports.runMigrations = async (CouchDB, migrations, options = {}) => {
+  console.log("Running migrations")
+  let tenantIds
+  if (environment.MULTI_TENANCY) {
+    if (!options.tenantIds || !options.tenantIds.length) {
+      // run for all tenants
+      tenantIds = await getTenantIds()
+    }
+  } else {
+    // single tenancy
+    tenantIds = [DEFAULT_TENANT_ID]
+  }
+
+  // for all tenants
+  for (const tenantId of tenantIds) {
+    // for all migrations
+    for (const migration of migrations) {
+      // run the migration
+      await doInTenant(tenantId, () =>
+        runMigration(CouchDB, migration, options)
+      )
+    }
+  }
+  console.log("Migrations complete")
+}

packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap

@@ -3,7 +3,7 @@
 exports[`migrations should match snapshot 1`] = `
 Object {
   "_id": "migrations",
-  "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40",
-  "user_email_view_casing": 1487076708000,
+  "_rev": "1-6277abc4e3db950221768e5a2618a059",
+  "test": 1487076708000,
 }
 `;

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,7 +1,7 @@
 require("../../tests/utilities/dbConfig")
 
-const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index")
-const database = require("../../db")
+const { runMigrations, getMigrationsDoc } = require("../index")
+const CouchDB = require("../../db").getCouch()
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -13,8 +13,14 @@ describe("migrations", () => {
 
   const migrationFunction = jest.fn()
 
+  const MIGRATIONS = [{
+    type: "global",
+    name: "test",
+    fn: migrationFunction
+  }]
+
   beforeEach(() => {
-    db = database.getDB(StaticDatabases.GLOBAL.name)
+    db = new CouchDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {
@@ -22,39 +28,29 @@ describe("migrations", () => {
     await db.destroy()
   })
 
-  const validMigration = () => {
-    return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
+  const migrate = () => {
+    return runMigrations(CouchDB, MIGRATIONS)
   }
 
   it("should run a new migration", async () => {
-    await validMigration()
+    await migrate()
     expect(migrationFunction).toHaveBeenCalled()
+    const doc = await getMigrationsDoc(db)
+    expect(doc.test).toBeDefined()
   })
 
   it("should match snapshot", async () => {
-    await validMigration()
+    await migrate()
     const doc = await getMigrationsDoc(db)
     expect(doc).toMatchSnapshot()
   })
 
   it("should skip a previously run migration", async () => {
-    await validMigration()
-    await validMigration()
+    await migrate()
+    const previousMigrationTime = await getMigrationsDoc(db).test
+    await migrate()
+    const currentMigrationTime = await getMigrationsDoc(db).test
     expect(migrationFunction).toHaveBeenCalledTimes(1)
+    expect(currentMigrationTime).toBe(previousMigrationTime)
   })
-
-  it("should reject an unknown migration name", async () => {
-    expect(async () => { 
-      await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
-  it("should reject an unknown database name", async () => {
-    expect(async () => { 
-      await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
 })

packages/backend-core/src/security/roles.js

@@ -1,4 +1,3 @@
-const { getDB } = require("../db")
 const { cloneDeep } = require("lodash/fp")
 const { BUILTIN_PERMISSION_IDS } = require("./permissions")
 const {
@@ -7,6 +6,8 @@ const {
   DocumentTypes,
   SEPARATOR,
 } = require("../db/utils")
+const { getAppDB } = require("../context")
+const { getDB } = require("../db")
 
 const BUILTIN_IDS = {
   ADMIN: "ADMIN",
@@ -111,11 +112,10 @@ exports.lowerBuiltinRoleID = (roleId1, roleId2) => {
 /**
  * Gets the role object, this is mainly useful for two purposes, to check if the level exists and
  * to check if the role inherits any others.
- * @param {string} appId The app in which to look for the role.
  * @param {string|null} roleId The level ID to lookup.
  * @returns {Promise<Role|object|null>} The role object, which may contain an "inherits" property.
  */
-exports.getRole = async (appId, roleId) => {
+exports.getRole = async roleId => {
   if (!roleId) {
     return null
   }
@@ -128,7 +128,7 @@ exports.getRole = async (appId, roleId) => {
     )
   }
   try {
-    const db = getDB(appId)
+    const db = getAppDB()
     const dbRole = await db.get(exports.getDBRoleID(roleId))
     role = Object.assign(role, dbRole)
     // finalise the ID
@@ -145,11 +145,11 @@ exports.getRole = async (appId, roleId) => {
 /**
  * Simple function to get all the roles based on the top level user role ID.
  */
-async function getAllUserRoles(appId, userRoleId) {
+async function getAllUserRoles(userRoleId) {
   if (!userRoleId) {
     return [BUILTIN_IDS.BASIC]
   }
-  let currentRole = await exports.getRole(appId, userRoleId)
+  let currentRole = await exports.getRole(userRoleId)
   let roles = currentRole ? [currentRole] : []
   let roleIds = [userRoleId]
   // get all the inherited roles
@@ -159,7 +159,7 @@ async function getAllUserRoles(appId, userRoleId) {
     roleIds.indexOf(currentRole.inherits) === -1
   ) {
     roleIds.push(currentRole.inherits)
-    currentRole = await exports.getRole(appId, currentRole.inherits)
+    currentRole = await exports.getRole(currentRole.inherits)
     roles.push(currentRole)
   }
   return roles
@@ -168,29 +168,23 @@ async function getAllUserRoles(appId, userRoleId) {
 /**
  * Returns an ordered array of the user's inherited role IDs, this can be used
  * to determine if a user can access something that requires a specific role.
- * @param {string} appId The ID of the application from which roles should be obtained.
  * @param {string} userRoleId The user's role ID, this can be found in their access token.
  * @param {object} opts Various options, such as whether to only retrieve the IDs (default true).
  * @returns {Promise<string[]>} returns an ordered array of the roles, with the first being their
  * highest level of access and the last being the lowest level.
  */
-exports.getUserRoleHierarchy = async (
-  appId,
-  userRoleId,
-  opts = { idOnly: true }
-) => {
+exports.getUserRoleHierarchy = async (userRoleId, opts = { idOnly: true }) => {
   // special case, if they don't have a role then they are a public user
-  const roles = await getAllUserRoles(appId, userRoleId)
+  const roles = await getAllUserRoles(userRoleId)
   return opts.idOnly ? roles.map(role => role._id) : roles
 }
 
 /**
  * Given an app ID this will retrieve all of the roles that are currently within that app.
- * @param {string} appId The ID of the app to retrieve the roles from.
  * @return {Promise<object[]>} An array of the role objects that were found.
  */
 exports.getAllRoles = async appId => {
-  const db = getDB(appId)
+  const db = appId ? getDB(appId) : getAppDB()
   const body = await db.allDocs(
     getRoleParams(null, {
       include_docs: true,
@@ -218,19 +212,17 @@ exports.getAllRoles = async appId => {
 }
 
 /**
- * This retrieves the required role/
- * @param appId
+ * This retrieves the required role
  * @param permLevel
  * @param resourceId
  * @param subResourceId
  * @return {Promise<{permissions}|Object>}
  */
 exports.getRequiredResourceRole = async (
-  appId,
   permLevel,
   { resourceId, subResourceId }
 ) => {
-  const roles = await exports.getAllRoles(appId)
+  const roles = await exports.getAllRoles()
   let main = [],
     sub = []
   for (let role of roles) {
@@ -251,8 +243,7 @@ exports.getRequiredResourceRole = async (
 }
 
 class AccessController {
-  constructor(appId) {
-    this.appId = appId
+  constructor() {
     this.userHierarchies = {}
   }
 
@@ -270,7 +261,7 @@ class AccessController {
     }
     let roleIds = this.userHierarchies[userRoleId]
     if (!roleIds) {
-      roleIds = await exports.getUserRoleHierarchy(this.appId, userRoleId)
+      roleIds = await exports.getUserRoleHierarchy(userRoleId)
       this.userHierarchies[userRoleId] = roleIds
     }
 

packages/backend-core/src/tenancy/context.js

@@ -1,84 +0,0 @@
-const env = require("../environment")
-const { Headers } = require("../../constants")
-const cls = require("./FunctionContext")
-
-exports.DEFAULT_TENANT_ID = "default"
-
-exports.isDefaultTenant = () => {
-  return exports.getTenantId() === exports.DEFAULT_TENANT_ID
-}
-
-exports.isMultiTenant = () => {
-  return env.MULTI_TENANCY
-}
-
-const TENANT_ID = "tenantId"
-
-// used for automations, API endpoints should always be in context already
-exports.doInTenant = (tenantId, task) => {
-  return cls.run(() => {
-    // set the tenant id
-    cls.setOnContext(TENANT_ID, tenantId)
-
-    // invoke the task
-    return task()
-  })
-}
-
-exports.updateTenantId = tenantId => {
-  cls.setOnContext(TENANT_ID, tenantId)
-}
-
-exports.setTenantId = (
-  ctx,
-  opts = { allowQs: false, allowNoTenant: false }
-) => {
-  let tenantId
-  // exit early if not multi-tenant
-  if (!exports.isMultiTenant()) {
-    cls.setOnContext(TENANT_ID, this.DEFAULT_TENANT_ID)
-    return
-  }
-
-  const allowQs = opts && opts.allowQs
-  const allowNoTenant = opts && opts.allowNoTenant
-  const header = ctx.request.headers[Headers.TENANT_ID]
-  const user = ctx.user || {}
-  if (allowQs) {
-    const query = ctx.request.query || {}
-    tenantId = query.tenantId
-  }
-  // override query string (if allowed) by user, or header
-  // URL params cannot be used in a middleware, as they are
-  // processed later in the chain
-  tenantId = user.tenantId || header || tenantId
-
-  // Set the tenantId from the subdomain
-  if (!tenantId) {
-    tenantId = ctx.subdomains && ctx.subdomains[0]
-  }
-
-  if (!tenantId && !allowNoTenant) {
-    ctx.throw(403, "Tenant id not set")
-  }
-  // check tenant ID just incase no tenant was allowed
-  if (tenantId) {
-    cls.setOnContext(TENANT_ID, tenantId)
-  }
-}
-
-exports.isTenantIdSet = () => {
-  const tenantId = cls.getFromContext(TENANT_ID)
-  return !!tenantId
-}
-
-exports.getTenantId = () => {
-  if (!exports.isMultiTenant()) {
-    return exports.DEFAULT_TENANT_ID
-  }
-  const tenantId = cls.getFromContext(TENANT_ID)
-  if (!tenantId) {
-    throw Error("Tenant id not found")
-  }
-  return tenantId
-}

packages/backend-core/src/tenancy/index.js

@@ -1,4 +1,4 @@
 module.exports = {
-  ...require("./context"),
+  ...require("../context"),
   ...require("./tenancy"),
 }

packages/backend-core/src/tenancy/tenancy.js

@@ -1,6 +1,6 @@
 const { getDB } = require("../db")
 const { SEPARATOR, StaticDatabases, DocumentTypes } = require("../db/constants")
-const { getTenantId, DEFAULT_TENANT_ID, isMultiTenant } = require("./context")
+const { getTenantId, DEFAULT_TENANT_ID, isMultiTenant } = require("../context")
 const env = require("../environment")
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
@@ -148,3 +148,15 @@ exports.isUserInAppTenant = (appId, user = null) => {
   const tenantId = exports.getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
   return tenantId === userTenantId
 }
+
+exports.getTenantIds = async () => {
+  const db = getDB(PLATFORM_INFO_DB)
+  let tenants
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (err) {
+    // if theres an error the doc doesn't exist, no tenants exist
+    return []
+  }
+  return (tenants && tenants.tenantIds) || []
+}

packages/backend-core/src/utils.js

@@ -20,9 +20,6 @@ const { hash } = require("./hashing")
 const userCache = require("./cache/user")
 const env = require("./environment")
 const { getUserSessions, invalidateSessions } = require("./security/sessions")
-const { migrateIfRequired } = require("./migrations")
-const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS
-const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -144,11 +141,6 @@ exports.getGlobalUserByEmail = async email => {
   }
   const db = getGlobalDB()
 
-  await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => {
-    // re-create the view with latest changes
-    await createUserEmailView(db)
-  })
-
   try {
     let users = (
       await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Field.svelte

@@ -6,11 +6,12 @@
   export let label = null
   export let labelPosition = "above"
   export let error = null
+  export let tooltip = ""
 </script>
 
 <div class="spectrum-Form-item" class:above={labelPosition === "above"}>
   {#if label}
-    <FieldLabel forId={id} {label} position={labelPosition} />
+    <FieldLabel forId={id} {label} position={labelPosition} {tooltip} />
   {/if}
   <div class="spectrum-Form-itemField">
     <slot />

packages/bbui/src/Form/FieldLabel.svelte

@@ -1,19 +1,24 @@
 <script>
+  import TooltipWrapper from "../Tooltip/TooltipWrapper.svelte"
+
   import "@spectrum-css/fieldlabel/dist/index-vars.css"
 
   export let forId
   export let label
   export let position = "above"
+  export let tooltip = ""
 
   $: className = position === "above" ? "" : `spectrum-FieldLabel--${position}`
 </script>
 
-<label
-  for={forId}
-  class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${className}`}
->
-  {label || ""}
-</label>
+<TooltipWrapper {tooltip} size="S">
+  <label
+    for={forId}
+    class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${className}`}
+  >
+    {label || ""}
+  </label>
+</TooltipWrapper>
 
 <style>
   label {

packages/bbui/src/Form/Select.svelte

@@ -17,6 +17,7 @@
   export let quiet = false
   export let autoWidth = false
   export let sort = false
+  export let tooltip = ""
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -32,7 +33,7 @@
   }
 </script>
 
-<Field {label} {labelPosition} {error}>
+<Field {label} {labelPosition} {error} {tooltip}>
   <Select
     {quiet}
     {error}

packages/bbui/src/Label/Label.svelte

@@ -1,74 +1,20 @@
 <script>
   import "@spectrum-css/fieldlabel/dist/index-vars.css"
-  import Tooltip from "../Tooltip/Tooltip.svelte"
-  import Icon from "../Icon/Icon.svelte"
+  import TooltipWrapper from "../Tooltip/TooltipWrapper.svelte"
 
   export let size = "M"
   export let tooltip = ""
-  export let showTooltip = false
 </script>
 
-{#if tooltip}
-  <div class="container">
-    <label
-      for=""
-      class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}
-    >
-      <slot />
-    </label>
-    <div class="icon-container">
-      <div
-        class="icon"
-        class:icon-small={size === "M" || size === "S"}
-        on:mouseover={() => (showTooltip = true)}
-        on:focus={() => (showTooltip = true)}
-        on:mouseleave={() => (showTooltip = false)}
-      >
-        <Icon name="InfoOutline" size="S" disabled={true} />
-      </div>
-      {#if showTooltip}
-        <div class="tooltip">
-          <Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
-        </div>
-      {/if}
-    </div>
-  </div>
-{:else}
+<TooltipWrapper {tooltip} {size}>
   <label for="" class={`spectrum-FieldLabel spectrum-FieldLabel--size${size}`}>
     <slot />
   </label>
-{/if}
+</TooltipWrapper>
 
 <style>
   label {
     padding: 0;
     white-space: nowrap;
   }
-  .container {
-    display: flex;
-    align-items: center;
-  }
-  .icon-container {
-    position: relative;
-    display: flex;
-    justify-content: center;
-    margin-top: 1px;
-    margin-left: 5px;
-    margin-right: 5px;
-  }
-  .tooltip {
-    position: absolute;
-    display: flex;
-    justify-content: center;
-    top: 15px;
-    z-index: 1;
-    width: 160px;
-  }
-  .icon {
-    transform: scale(0.75);
-  }
-  .icon-small {
-    margin-top: -2px;
-    margin-bottom: -5px;
-  }
 </style>

packages/bbui/src/Tooltip/TooltipWrapper.svelte

@@ -0,0 +1,60 @@
+<script>
+  import Tooltip from "./Tooltip.svelte"
+  import Icon from "../Icon/Icon.svelte"
+
+  export let tooltip = ""
+  export let size = "M"
+
+  let showTooltip = false
+</script>
+
+<div class:container={!!tooltip}>
+  <slot />
+  {#if tooltip}
+    <div class="icon-container">
+      <div
+        class="icon"
+        class:icon-small={size === "M" || size === "S"}
+        on:mouseover={() => (showTooltip = true)}
+        on:mouseleave={() => (showTooltip = false)}
+      >
+        <Icon name="InfoOutline" size="S" disabled={true} />
+      </div>
+      {#if showTooltip}
+        <div class="tooltip">
+          <Tooltip textWrapping={true} direction={"bottom"} text={tooltip} />
+        </div>
+      {/if}
+    </div>
+  {/if}
+</div>
+
+<style>
+  .container {
+    display: flex;
+    align-items: center;
+  }
+  .icon-container {
+    position: relative;
+    display: flex;
+    justify-content: center;
+    margin-top: 1px;
+    margin-left: 5px;
+    margin-right: 5px;
+  }
+  .tooltip {
+    position: absolute;
+    display: flex;
+    justify-content: center;
+    top: 15px;
+    z-index: 1;
+    width: 160px;
+  }
+  .icon {
+    transform: scale(0.75);
+  }
+  .icon-small {
+    margin-top: -2px;
+    margin-bottom: -5px;
+  }
+</style>

packages/builder/cypress/setup.js

@@ -3,9 +3,6 @@ const path = require("path")
 
 const tmpdir = path.join(require("os").tmpdir(), ".budibase")
 
-// these run on ports we don't normally use so that they can run alongside the
-const fs = require("fs")
-
 // normal development system
 const WORKER_PORT = "10002"
 const MAIN_PORT = cypressConfig.env.PORT
@@ -29,22 +26,20 @@ process.env.ALLOW_DEV_AUTOMATIONS = 1
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"
 
-async function run() {
+exports.run = (
+  serverLoc = "../../server/dist",
+  workerLoc = "../../worker/dist"
+) => {
   // require("dotenv").config({ path: resolve(dir, ".env") })
-  if (!fs.existsSync("../server/dist")) {
-    console.error("Unable to run cypress, need to build server first")
-    process.exit(-1)
-  }
-
   // don't make this a variable or top level require
   // it will cause environment module to be loaded prematurely
-  const server = require("../../server/dist/app")
+  require(serverLoc)
   process.env.PORT = WORKER_PORT
-  const worker = require("../../worker/src/index")
+  require(workerLoc)
   // reload main port for rest of system
   process.env.PORT = MAIN_PORT
-  server.on("close", () => console.log("Server Closed"))
-  worker.on("close", () => console.log("Worker Closed"))
 }
 
-run()
+if (require.main === module) {
+  exports.run()
+}

packages/builder/cypress/ts/setup.ts

@@ -0,0 +1,4 @@
+// @ts-ignore
+import { run } from "../setup"
+
+run("../../server/src/index", "../../worker/src/index")

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -9,12 +9,13 @@
     "dev:builder": "routify -c dev:vite",
     "dev:vite": "vite --host 0.0.0.0",
     "rollup": "rollup -c -w",
-    "cy:setup": "node ./cypress/setup.js",
+    "cy:setup": "ts-node ./cypress/ts/setup.ts",
+    "cy:setup:ci": "node ./cypress/setup.js",
     "cy:run": "cypress run",
     "cy:open": "cypress open",
     "cy:run:ci": "cypress run --record",
     "cy:test": "start-server-and-test cy:setup http://localhost:10001/builder cy:run",
-    "cy:ci": "start-server-and-test cy:setup http://localhost:10001/builder cy:run",
+    "cy:ci": "start-server-and-test cy:setup:ci http://localhost:10001/builder cy:run",
     "cy:debug": "start-server-and-test cy:setup http://localhost:10001/builder cy:open"
   },
   "jest": {
@@ -63,11 +64,11 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.8",
-    "@budibase/client": "^1.0.46-alpha.8",
-    "@budibase/frontend-core": "^1.0.46-alpha.8",
+    "@budibase/bbui": "^1.0.49-alpha.5",
+    "@budibase/client": "^1.0.49-alpha.5",
+    "@budibase/frontend-core": "^1.0.49-alpha.5",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.46-alpha.8",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
@@ -105,6 +106,8 @@
     "start-server-and-test": "^1.12.1",
     "svelte": "^3.38.2",
     "svelte-jester": "^1.3.2",
+    "ts-node": "^10.4.0",
+    "typescript": "^4.5.5",
     "vite": "^2.1.5"
   },
   "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -22,8 +22,10 @@
     RelationshipTypes,
     ALLOWABLE_STRING_OPTIONS,
     ALLOWABLE_NUMBER_OPTIONS,
+    ALLOWABLE_JSON_OPTIONS,
     ALLOWABLE_STRING_TYPES,
     ALLOWABLE_NUMBER_TYPES,
+    ALLOWABLE_JSON_TYPES,
     SWITCHABLE_TYPES,
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "builderStore/utils"
@@ -154,6 +156,7 @@
     delete field.subtype
     delete field.tableId
     delete field.relationshipType
+    delete field.formulaType
 
     // Add in defaults and initial definition
     const definition = fieldDefinitions[event.detail?.toUpperCase()]
@@ -165,6 +168,9 @@
     if (field.type === LINK_TYPE) {
       field.relationshipType = RelationshipTypes.MANY_TO_MANY
     }
+    if (field.type === FORMULA_TYPE) {
+      field.formulaType = "dynamic"
+    }
   }
 
   function onChangeRequired(e) {
@@ -245,6 +251,11 @@
       ALLOWABLE_NUMBER_TYPES.indexOf(field.type) !== -1
     ) {
       return ALLOWABLE_NUMBER_OPTIONS
+    } else if (
+      originalName &&
+      ALLOWABLE_JSON_TYPES.indexOf(field.type) !== -1
+    ) {
+      return ALLOWABLE_JSON_OPTIONS
     } else if (!external) {
       return [
         ...Object.values(fieldDefinitions),
@@ -435,8 +446,22 @@
       error={errors.relatedName}
     />
   {:else if field.type === FORMULA_TYPE}
+    {#if !table.sql}
+      <Select
+        label="Formula type"
+        bind:value={field.formulaType}
+        options={[
+          { label: "Dynamic", value: "dynamic" },
+          { label: "Static", value: "static" },
+        ]}
+        getOptionLabel={option => option.label}
+        getOptionValue={option => option.value}
+        tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered,
+         while static formula are calculated when the row is saved."
+      />
+    {/if}
     <ModalBindableInput
-      title="Handlebars Formula"
+      title="Formula"
       label="Formula"
       value={field.formula}
       on:change={e => (field.formula = e.detail)}
@@ -445,7 +470,7 @@
     />
   {:else if field.type === AUTO_TYPE}
     <Select
-      label="Auto Column Type"
+      label="Auto column type"
       value={field.subtype}
       on:change={e => (field.subtype = e.detail)}
       options={Object.entries(getAutoColumnInformation())}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte

@@ -132,7 +132,7 @@
                 {bindings}
                 on:change={event => (filter.value = event.detail)}
               />
-            {:else if ["string", "longform", "number"].includes(filter.type)}
+            {:else if ["string", "longform", "number", "formula"].includes(filter.type)}
               <Input disabled={filter.noValue} bind:value={filter.value} />
             {:else if ["options", "array"].includes(filter.type)}
               <Combobox

packages/builder/src/components/integration/AccessLevelSelect.svelte

@@ -1,16 +1,14 @@
 <script>
   import { Label, notifications, Select } from "@budibase/bbui"
   import { permissions, roles } from "stores/backend"
-  import { onMount } from "svelte"
   import { Roles } from "constants/backend"
 
   export let query
-  export let saveId
   export let label
 
-  $: updateRole(roleId, saveId)
+  $: getPermissions(query)
 
-  let roleId, loaded
+  let roleId, loaded, fetched
 
   async function updateRole(role, id) {
     try {
@@ -30,19 +28,23 @@
     }
   }
 
-  onMount(async () => {
-    if (!query || !query._id) {
+  async function getPermissions(queryToFetch) {
+    if (fetched?._id === queryToFetch?._id) {
+      return
+    }
+    fetched = queryToFetch
+    if (!queryToFetch || !queryToFetch._id) {
       roleId = Roles.BASIC
       loaded = true
       return
     }
     try {
-      roleId = (await permissions.forResource(query._id))["read"]
+      roleId = (await permissions.forResource(queryToFetch._id))["read"]
     } catch (err) {
       roleId = Roles.BASIC
     }
     loaded = true
-  })
+  }
 </script>
 
 {#if loaded}

packages/builder/src/constants/backend/index.js

@@ -148,20 +148,23 @@ export const RelationshipTypes = {
 }
 
 export const ALLOWABLE_STRING_OPTIONS = [FIELDS.STRING, FIELDS.OPTIONS]
-
 export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map(
   opt => opt.type
 )
 
 export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN]
-
 export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map(
   opt => opt.type
 )
 
-export const SWITCHABLE_TYPES = ALLOWABLE_NUMBER_TYPES.concat(
-  ALLOWABLE_STRING_TYPES
-)
+export const ALLOWABLE_JSON_OPTIONS = [FIELDS.JSON, FIELDS.ARRAY]
+export const ALLOWABLE_JSON_TYPES = ALLOWABLE_JSON_OPTIONS.map(opt => opt.type)
+
+export const SWITCHABLE_TYPES = [
+  ...ALLOWABLE_STRING_TYPES,
+  ...ALLOWABLE_NUMBER_TYPES,
+  ...ALLOWABLE_JSON_TYPES,
+]
 
 export const IntegrationTypes = {
   POSTGRES: "POSTGRES",

packages/builder/src/helpers/searchFields.js

@@ -27,5 +27,8 @@ export function getFields(fields, { allowLinks } = { allowLinks: true }) {
       filteredFields = filteredFields.concat(getTableFields(linkField))
     }
   }
-  return filteredFields
+  const staticFormulaFields = fields.filter(
+    field => field.type === "formula" && field.formulaType === "static"
+  )
+  return filteredFields.concat(staticFormulaFields)
 }

packages/builder/src/pages/builder/_layout.svelte

@@ -56,7 +56,7 @@
 
   onMount(async () => {
     try {
-      await auth.checkAuth()
+      await auth.getSelf()
       await admin.init()
 
       // Set init info if present

packages/builder/src/pages/builder/auth/reset.svelte

@@ -32,7 +32,7 @@
 
   onMount(async () => {
     try {
-      await auth.checkAuth()
+      await auth.getSelf()
       await organisation.init()
     } catch (error) {
       notifications.error("Error getting org config")

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -315,6 +315,14 @@
             }}
             class="template-card"
           >
+            <a
+              href={item.url}
+              target="_blank"
+              class="external-link"
+              on:click|stopPropagation
+            >
+              <Icon name="LinkOut" size="S" />
+            </a>
             <div class="card-body">
               <div style="color: {item.background}" class="iconAlign">
                 <svg
@@ -496,6 +504,7 @@
     border: 1px solid var(--spectrum-global-color-gray-300);
     cursor: pointer;
     display: flex;
+    position: relative;
   }
 
   .template-card:hover {
@@ -506,6 +515,18 @@
     align-items: center;
     padding: 12px;
   }
+
+  .external-link {
+    position: absolute;
+    top: 5px;
+    right: 5px;
+    color: var(--spectrum-global-color-gray-300);
+    z-index: 99;
+  }
+  .external-link:hover {
+    color: var(--spectrum-global-color-gray-500);
+  }
+
   .iconAlign {
     padding: 0 0 0 var(--spacing-m);
     display: inline-block;

packages/builder/src/stores/portal/auth.js

@@ -114,11 +114,7 @@ export function createAuthStore() {
     return info
   }
 
-  return {
-    subscribe: store.subscribe,
-    setOrganisation,
-    getInitInfo,
-    setInitInfo,
+  const actions = {
     checkQueryString: async () => {
       const urlParams = new URLSearchParams(window.location.search)
       if (urlParams.has("tenantId")) {
@@ -129,7 +125,7 @@ export function createAuthStore() {
     setOrg: async tenantId => {
       await setOrganisation(tenantId)
     },
-    checkAuth: async () => {
+    getSelf: async () => {
       // We need to catch this locally as we never want this to fail, even
       // though normally we never want to swallow API errors at the store level.
       // We're either logged in or we aren't.
@@ -143,12 +139,12 @@ export function createAuthStore() {
     },
     login: async creds => {
       const tenantId = get(store).tenantId
-      const response = await API.logIn({
+      await API.logIn({
         username: creds.username,
         password: creds.password,
         tenantId,
       })
-      setUser(response.user)
+      await actions.getSelf()
     },
     logout: async () => {
       setUser(null)
@@ -177,6 +173,14 @@ export function createAuthStore() {
       })
     },
   }
+
+  return {
+    subscribe: store.subscribe,
+    setOrganisation,
+    getInitInfo,
+    setInitInfo,
+    ...actions,
+  }
 }
 
 export const auth = createAuthStore()

packages/builder/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "module": "commonjs",
+    "lib": ["es2019"],
+    "allowJs": true,
+    "outDir": "dist",
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "incremental": true
+  },
+  "include": [
+    "./src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "**/*.json",
+    "**/*.spec.ts",
+    "**/*.spec.js"
+  ]
+}

packages/builder/yarn.lock

@@ -970,10 +970,10 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/bbui@^1.0.46", "@budibase/bbui@^1.0.46-alpha.3":
-  version "1.0.46"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.46.tgz#7306d4eda7f2c827577a4affa1fd314b38ba1198"
-  integrity sha512-padm0qq2SBNIslXEQW+HIv32pkIHFzloR93FDzSXh0sO43Q+/d2gbAhjI9ZUSAVncx9JNc46dolL1CwrvHFElg==
+"@budibase/bbui@^1.0.46-alpha.6", "@budibase/bbui@^1.0.47":
+  version "1.0.47"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.47.tgz#df2848b89f881fe603e7156855d6a6c31d4f58bf"
+  integrity sha512-RRm/BgK5aSx2/vGjMGljw240/48Ksc3/h4yB1nhQj8Xx3fKhlGnWDvWNy+sakvA6+fJvEXuti8RoxHtQ6lXmqA==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"
@@ -1020,14 +1020,14 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/client@^1.0.46-alpha.3":
-  version "1.0.46"
-  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.46.tgz#e6ef8945b9d7046b6e6d6761628aa1d85387acca"
-  integrity sha512-jI3z1G/EsfJNCQCvrqzsR4vR1zLoVefzCXCEASIPg9BPzdiAFSwuUJVLijLFIIKfuDVeveUll94fgu7XNY8U2w==
+"@budibase/client@^1.0.46-alpha.6":
+  version "1.0.47"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.47.tgz#ce9e2fbd300e5dc389ea29a3a3347897f096c824"
+  integrity sha512-jB/al8v+nY/VLc6sH5Jt9JzWONVo+24/cI95iXlZSV5xwiKIVGj4+2F5QjKZ0c9Gm7SrrfP2T571N+4XaXNCGg==
   dependencies:
-    "@budibase/bbui" "^1.0.46"
+    "@budibase/bbui" "^1.0.47"
     "@budibase/standard-components" "^0.9.139"
-    "@budibase/string-templates" "^1.0.46"
+    "@budibase/string-templates" "^1.0.47"
     regexparam "^1.3.0"
     shortid "^2.2.15"
     svelte-spa-router "^3.0.5"
@@ -1082,10 +1082,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@^1.0.46", "@budibase/string-templates@^1.0.46-alpha.3":
-  version "1.0.46"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.46.tgz#5beef1687b451e4512a465b4e143c8ab46234006"
-  integrity sha512-t4ZAUkSz2XatjAN0faex5ovmD3mFz672lV/aBk7tfLFzZiKlWjngqdwpLLQNnsqeGvYo75JP2J06j86SX6O83w==
+"@budibase/string-templates@^1.0.46-alpha.6", "@budibase/string-templates@^1.0.47":
+  version "1.0.47"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.47.tgz#626b9fc4542c7b36a0ae24e820d25a704c527bec"
+  integrity sha512-87BUfOPr8FGKH8Pt88jhKNGT9PcOmkLRCeen4xi1dI113pAQznBO9vgV+cXOChUBBEQka9Rrt85LMJXidiwVgg==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.7"
     dayjs "^1.10.4"
@@ -1102,6 +1102,18 @@
     exec-sh "^0.3.2"
     minimist "^1.2.0"
 
+"@cspotcode/source-map-consumer@0.8.0":
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"
+  integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg==
+
+"@cspotcode/source-map-support@0.7.0":
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.7.0.tgz#4789840aa859e46d2f3173727ab707c66bf344f5"
+  integrity sha512-X4xqRHqN8ACt2aHVe51OxeA2HjbcL4MqFqXkrmQszJ1NOUuUu5u6Vqx/0lZSVNku7velL5FC/s5uEAj1lsBMhA==
+  dependencies:
+    "@cspotcode/source-map-consumer" "0.8.0"
+
 "@cypress/listr-verbose-renderer@^0.4.1":
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/@cypress/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz#a77492f4b11dcc7c446a34b3e28721afd33c642a"
@@ -1795,6 +1807,26 @@
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
   integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
 
+"@tsconfig/node10@^1.0.7":
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.8.tgz#c1e4e80d6f964fbecb3359c43bd48b40f7cadad9"
+  integrity sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg==
+
+"@tsconfig/node12@^1.0.7":
+  version "1.0.9"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.9.tgz#62c1f6dee2ebd9aead80dc3afa56810e58e1a04c"
+  integrity sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw==
+
+"@tsconfig/node14@^1.0.0":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2"
+  integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg==
+
+"@tsconfig/node16@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e"
+  integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==
+
 "@types/aria-query@^4.2.0":
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-4.2.2.tgz#ed4e0ad92306a704f9fb132a0cfcf77486dbe2bc"
@@ -1971,6 +2003,11 @@ acorn-walk@^7.1.1:
   resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
   integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
 
+acorn-walk@^8.1.1:
+  version "8.2.0"
+  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1"
+  integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==
+
 acorn@^7.1.1:
   version "7.4.1"
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
@@ -1981,6 +2018,11 @@ acorn@^8.2.4:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.5.0.tgz#4512ccb99b3698c752591e9bb4472e38ad43cee2"
   integrity sha512-yXbYeFy+jUuYd3/CDcg2NkIYE991XYX/bje7LmjJigUciaeO1JR4XxXgCIV1/Zc/dRuFEyw1L0pbA+qynJkW5Q==
 
+acorn@^8.4.1:
+  version "8.7.0"
+  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf"
+  integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ==
+
 agent-base@6:
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
@@ -2087,6 +2129,11 @@ arch@^2.1.2:
   resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11"
   integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==
 
+arg@^4.1.0:
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
+  integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
+
 argparse@^1.0.10, argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@@ -2720,6 +2767,11 @@ core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85"
   integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==
 
+create-require@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
+  integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==
+
 cross-spawn@^6.0.0:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -2965,6 +3017,11 @@ diff-sequences@^27.0.6:
   resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.0.6.tgz#3305cb2e55a033924054695cc66019fd7f8e5723"
   integrity sha512-ag6wfpBFyNXZ0p8pcuIDS//D8H062ZQJ3fzYxjpmeKjnz8W4pekL3AI8VohmyZmsWW2PWaHgjsmqR6L13101VQ==
 
+diff@^4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
+  integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
+
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
@@ -5004,6 +5061,11 @@ make-dir@^3.0.0:
   dependencies:
     semver "^6.0.0"
 
+make-error@^1.1.1:
+  version "1.3.6"
+  resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
+  integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==
+
 makeerror@1.0.12:
   version "1.0.12"
   resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.12.tgz#3e5dd2079a82e812e983cc6610c4a2cb0eaa801a"
@@ -6587,6 +6649,24 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
 
+ts-node@^10.4.0:
+  version "10.4.0"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.4.0.tgz#680f88945885f4e6cf450e7f0d6223dd404895f7"
+  integrity sha512-g0FlPvvCXSIO1JDF6S232P5jPYqBkRL9qly81ZgAOSU7rwI0stphCgd2kLiCrU9DjQCrJMWEqcNSjQL02s6d8A==
+  dependencies:
+    "@cspotcode/source-map-support" "0.7.0"
+    "@tsconfig/node10" "^1.0.7"
+    "@tsconfig/node12" "^1.0.7"
+    "@tsconfig/node14" "^1.0.0"
+    "@tsconfig/node16" "^1.0.2"
+    acorn "^8.4.1"
+    acorn-walk "^8.1.1"
+    arg "^4.1.0"
+    create-require "^1.1.0"
+    diff "^4.0.1"
+    make-error "^1.1.1"
+    yn "3.1.1"
+
 tslib@^1.9.0, tslib@^1.9.3:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
@@ -6655,6 +6735,11 @@ typeof-article@^0.1.1:
   dependencies:
     kind-of "^3.1.0"
 
+typescript@^4.5.5:
+  version "4.5.5"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"
+  integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==
+
 uglify-js@^3.1.4:
   version "3.14.5"
   resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.14.5.tgz#cdabb7d4954231d80cb4a927654c4655e51f4859"
@@ -7011,6 +7096,11 @@ year@^0.2.1:
   resolved "https://registry.yarnpkg.com/year/-/year-0.2.1.tgz#4083ae520a318b23ec86037f3000cb892bdf9bb0"
   integrity sha1-QIOuUgoxiyPshgN/MADLiSvfm7A=
 
+yn@3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"
+  integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==
+
 yup@0.29.2:
   version "0.29.2"
   resolved "https://registry.yarnpkg.com/yup/-/yup-0.29.2.tgz#5302abd9024cca335b987793f8df868e410b7b67"

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.8",
-    "@budibase/frontend-core": "^1.0.46-alpha.8",
-    "@budibase/string-templates": "^1.0.46-alpha.8",
+    "@budibase/bbui": "^1.0.49-alpha.5",
+    "@budibase/frontend-core": "^1.0.49-alpha.5",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
     "regexparam": "^1.3.0",
     "rollup-plugin-polyfill-node": "^0.8.0",
     "shortid": "^2.2.15",

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -39,6 +39,7 @@
     number: "numberfield",
     datetime: "datetimefield",
     boolean: "booleanfield",
+    formula: "stringfield",
   }
 
   let formId

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -35,6 +35,7 @@
     number: "numberfield",
     datetime: "datetimefield",
     boolean: "booleanfield",
+    formula: "stringfield",
   }
 
   let formId
@@ -60,10 +61,11 @@
     let enrichedFilter = [...(filter || [])]
     columns?.forEach(column => {
       const safePath = column.name.split(".").map(safe).join(".")
+      const stringType = column.type === "string" || column.type === "formula"
       enrichedFilter.push({
         field: column.name,
-        operator: column.type === "string" ? "string" : "equal",
         type: column.type,
+        operator: stringType ? "string" : "equal",
         valueType: "Binding",
         value: `{{ ${safe(formId)}.${safePath} }}`,
       })

packages/client/src/components/app/dynamic-filter/FilterModal.svelte

@@ -16,10 +16,14 @@
   export let schemaFields
   export let filters = []
 
-  const BannedTypes = ["link", "attachment", "formula", "json"]
+  const BannedTypes = ["link", "attachment", "json"]
 
   $: fieldOptions = (schemaFields ?? [])
-    .filter(field => !BannedTypes.includes(field.type))
+    .filter(
+      field =>
+        !BannedTypes.includes(field.type) ||
+        (field.type === "formula" && field.formulaType === "static")
+    )
     .map(field => field.name)
 
   const addFilter = () => {
@@ -112,7 +116,7 @@
               on:change={e => onOperatorChange(filter, e.detail)}
               placeholder={null}
             />
-            {#if ["string", "longform", "number"].includes(filter.type)}
+            {#if ["string", "longform", "number", "formula"].includes(filter.type)}
               <Input disabled={filter.noValue} bind:value={filter.value} />
             {:else if ["options", "array"].includes(filter.type)}
               <Combobox

packages/client/src/components/app/forms/Field.svelte

@@ -32,6 +32,7 @@
     validation,
     formStep
   )
+  $: schemaType = fieldSchema?.type !== "formula" ? fieldSchema?.type : "string"
 
   // Focus label when editing
   let labelNode
@@ -72,7 +73,7 @@
         <Placeholder
           text="Add the Field setting to start using your component"
         />
-      {:else if fieldSchema?.type && fieldSchema?.type !== type && type !== "options"}
+      {:else if schemaType && schemaType !== type && type !== "options"}
         <Placeholder
           text="This Field setting is the wrong data type for this component"
         />

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.8",
+    "@budibase/bbui": "^1.0.49-alpha.5",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/fetch/QueryFetch.js

@@ -16,7 +16,14 @@ export default class QueryFetch extends DataFetch {
       return null
     }
     try {
-      return await this.API.fetchQueryDefinition(datasource._id)
+      const definition = await this.API.fetchQueryDefinition(datasource._id)
+      // After getting the definition of query, it loses "fields" attribute
+      // because of security reason from the server. However, this attribute
+      // needs to be inside the definition for pagination.
+      if (!definition.fields) {
+        definition.fields = datasource.fields
+      }
+      return definition
     } catch (error) {
       return null
     }

packages/frontend-core/src/utils/lucene.js

@@ -7,24 +7,26 @@ import { OperatorOptions } from "../constants"
  */
 export const getValidOperatorsForType = type => {
   const Op = OperatorOptions
+  const stringOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.StartsWith,
+    Op.Like,
+    Op.Empty,
+    Op.NotEmpty,
+  ]
+  const numOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.MoreThan,
+    Op.LessThan,
+    Op.Empty,
+    Op.NotEmpty,
+  ]
   if (type === "string") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return stringOps
   } else if (type === "number") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return numOps
   } else if (type === "options") {
     return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
   } else if (type === "array") {
@@ -32,23 +34,11 @@ export const getValidOperatorsForType = type => {
   } else if (type === "boolean") {
     return [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
   } else if (type === "longform") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.StartsWith,
-      Op.Like,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return stringOps
   } else if (type === "datetime") {
-    return [
-      Op.Equals,
-      Op.NotEquals,
-      Op.MoreThan,
-      Op.LessThan,
-      Op.Empty,
-      Op.NotEmpty,
-    ]
+    return numOps
+  } else if (type === "formula") {
+    return stringOps.concat([Op.MoreThan, Op.LessThan])
   }
   return []
 }

packages/server/__mocks__/arangojs.ts

@@ -9,7 +9,7 @@ module ArangoMock {
     this.close = jest.fn()
   }
 
-// @ts-ignore
+  // @ts-ignore
   arangodb.aql = (strings, ...args) => {
     let str = strings.join("{}")
 

packages/server/__mocks__/aws-sdk.ts

@@ -1,7 +1,7 @@
 module AwsMock {
   const aws: any = {}
 
-  const response = (body: any) => () => ({promise: () => body})
+  const response = (body: any) => () => ({ promise: () => body })
 
   function DocumentClient() {
     // @ts-ignore
@@ -39,9 +39,9 @@ module AwsMock {
     )
   }
 
-  aws.DynamoDB = {DocumentClient}
+  aws.DynamoDB = { DocumentClient }
   aws.S3 = S3
-  aws.config = {update: jest.fn()}
+  aws.config = { update: jest.fn() }
 
   module.exports = aws
-}
+}

packages/server/__mocks__/mongodb.ts

@@ -5,15 +5,14 @@ module MongoMock {
     this.connect = jest.fn()
     this.close = jest.fn()
     this.insertOne = jest.fn()
-    this.insertMany = jest.fn(() => ({toArray: () => []}))
-    this.find = jest.fn(() => ({toArray: () => []}))
+    this.insertMany = jest.fn(() => ({ toArray: () => [] }))
+    this.find = jest.fn(() => ({ toArray: () => [] }))
     this.findOne = jest.fn()
     this.count = jest.fn()
     this.deleteOne = jest.fn()
-    this.deleteMany = jest.fn(() => ({toArray: () => []}))
+    this.deleteMany = jest.fn(() => ({ toArray: () => [] }))
     this.updateOne = jest.fn()
-    this.updateMany = jest.fn(() => ({toArray: () => []}))
-
+    this.updateMany = jest.fn(() => ({ toArray: () => [] }))
 
     this.collection = jest.fn(() => ({
       insertOne: this.insertOne,

packages/server/__mocks__/mssql.ts

@@ -10,15 +10,15 @@ module MsSqlMock {
     ],
   }))
 
-// mssql.connect = jest.fn(() => ({ recordset: [] }))
+  // mssql.connect = jest.fn(() => ({ recordset: [] }))
 
   mssql.ConnectionPool = jest.fn(() => ({
     connect: jest.fn(() => ({
       request: jest.fn(() => ({
-        query: jest.fn(sql => ({ recordset: [ sql ] })),
+        query: jest.fn(sql => ({ recordset: [sql] })),
       })),
     })),
   }))
 
   module.exports = mssql
-}
+}

packages/server/__mocks__/node-fetch.ts

@@ -62,7 +62,8 @@ module FetchMock {
       return json({
         url,
         opts,
-        value: "<!doctype html><html itemscope=\"\" itemtype=\"http://schema.org/WebPage\" lang=\"en-GB\"></html>",
+        value:
+          '<!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="en-GB"></html>',
       })
     } else if (url.includes("failonce.com")) {
       failCount++

packages/server/__mocks__/pg.ts

@@ -10,16 +10,14 @@ module PgMock {
     ],
   }))
 
-// constructor
-  function Client() {
-  }
+  // constructor
+  function Client() {}
 
   Client.prototype.query = query
   Client.prototype.connect = jest.fn()
   Client.prototype.release = jest.fn()
 
-  function Pool() {
-  }
+  function Pool() {}
 
   const on = jest.fn()
   Pool.prototype.query = query

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.8",
+  "version": "1.0.49-alpha.5",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.8",
-    "@budibase/client": "^1.0.46-alpha.8",
-    "@budibase/string-templates": "^1.0.46-alpha.8",
+    "@budibase/backend-core": "^1.0.49-alpha.5",
+    "@budibase/client": "^1.0.49-alpha.5",
+    "@budibase/string-templates": "^1.0.49-alpha.5",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/likeCypress.ts

@@ -30,4 +30,4 @@ process.env.ALLOW_DEV_AUTOMATIONS = "1"
 const server = require("../src/app")
 process.env.PORT = WORKER_PORT
 const worker = require("../../worker/src/index")
-process.env.PORT = MAIN_PORT
+process.env.PORT = MAIN_PORT

packages/server/src/api/controllers/application.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const env = require("../../environment")
 const packageJson = require("../../../package.json")
 const {
@@ -29,7 +28,7 @@ const { processObject } = require("@budibase/string-templates")
 const {
   getAllApps,
   isDevAppID,
-  getDeployedAppID,
+  getProdAppID,
   Replication,
 } = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
@@ -45,11 +44,17 @@ const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 const { syncGlobalUsers } = require("./user")
 const { app: appCache } = require("@budibase/backend-core/cache")
 const { cleanupAutomations } = require("../../automations/utils")
+const {
+  getAppDB,
+  getProdAppDB,
+  updateAppId,
+} = require("@budibase/backend-core/context")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
 // utility function, need to do away with this
-async function getLayouts(db) {
+async function getLayouts() {
+  const db = getAppDB()
   return (
     await db.allDocs(
       getLayoutParams(null, {
@@ -59,7 +64,8 @@ async function getLayouts(db) {
   ).rows.map(row => row.doc)
 }
 
-async function getScreens(db) {
+async function getScreens() {
+  const db = getAppDB()
   return (
     await db.allDocs(
       getScreenParams(null, {
@@ -75,7 +81,7 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrl(ctx) {
+exports.getAppUrl = ctx => {
   // construct the url
   let url
   if (ctx.request.body.url) {
@@ -117,8 +123,9 @@ async function createInstance(template) {
   const tenantId = isMultiTenant() ? getTenantId() : null
   const baseAppId = generateAppID(tenantId)
   const appId = generateDevAppID(baseAppId)
+  updateAppId(appId)
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   await db.put({
     _id: "_design/database",
     // view collation information, read before writing any complex views:
@@ -128,9 +135,9 @@ async function createInstance(template) {
 
   // NOTE: indexes need to be created before any tables/templates
   // add view for linked rows
-  await createLinkView(appId)
-  await createRoutingView(appId)
-  await createAllSearchIndex(appId)
+  await createLinkView()
+  await createRoutingView()
+  await createAllSearchIndex()
 
   // replicate the template data to the instance DB
   // this is currently very hard to test, downloading and importing template files
@@ -156,7 +163,7 @@ async function createInstance(template) {
 exports.fetch = async ctx => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
-  const apps = await getAllApps(CouchDB, { dev, all })
+  const apps = await getAllApps({ dev, all })
 
   // get the locks for all the dev apps
   if (dev || all) {
@@ -179,12 +186,11 @@ exports.fetch = async ctx => {
 }
 
 exports.fetchAppDefinition = async ctx => {
-  const db = new CouchDB(ctx.params.appId)
-  const layouts = await getLayouts(db)
+  const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
-  const accessController = new AccessController(ctx.params.appId)
+  const accessController = new AccessController()
   const screens = await accessController.checkScreensAccess(
-    await getScreens(db),
+    await getScreens(),
     userRoleId
   )
   ctx.body = {
@@ -195,15 +201,15 @@ exports.fetchAppDefinition = async ctx => {
 }
 
 exports.fetchAppPackage = async ctx => {
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
-  const layouts = await getLayouts(db)
-  let screens = await getScreens(db)
+  const layouts = await getLayouts()
+  let screens = await getScreens()
 
   // Only filter screens if the user is not a builder
   if (!(ctx.user.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
-    const accessController = new AccessController(ctx.params.appId)
+    const accessController = new AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
   }
 
@@ -216,10 +222,10 @@ exports.fetchAppPackage = async ctx => {
 }
 
 exports.create = async ctx => {
-  const apps = await getAllApps(CouchDB, { dev: true })
+  const apps = await getAllApps({ dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
@@ -234,7 +240,7 @@ exports.create = async ctx => {
   const instance = await createInstance(instanceConfig)
   const appId = instance._id
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let _rev
   try {
     // if template there will be an existing doc
@@ -280,13 +286,13 @@ exports.create = async ctx => {
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
 exports.update = async ctx => {
-  const apps = await getAllApps(CouchDB, { dev: true })
+  const apps = await getAllApps({ dev: true })
   // validation
   const name = ctx.request.body.name
   if (name) {
     checkAppName(ctx, apps, name, ctx.params.appId)
   }
-  const url = await getAppUrl(ctx)
+  const url = await exports.getAppUrl(ctx)
   if (url) {
     checkAppUrl(ctx, apps, url, ctx.params.appId)
     ctx.request.body.url = url
@@ -299,7 +305,7 @@ exports.update = async ctx => {
 
 exports.updateClient = async ctx => {
   // Get current app version
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
   const currentVersion = application.version
 
@@ -321,7 +327,7 @@ exports.updateClient = async ctx => {
 
 exports.revertClient = async ctx => {
   // Check app can be reverted
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
   if (!application.revertableVersion) {
     ctx.throw(400, "There is no version to revert to")
@@ -343,7 +349,7 @@ exports.revertClient = async ctx => {
 }
 
 exports.delete = async ctx => {
-  const db = new CouchDB(ctx.params.appId)
+  const db = getAppDB()
 
   const result = await db.destroy()
   /* istanbul ignore next */
@@ -368,10 +374,11 @@ exports.sync = async (ctx, next) => {
   }
 
   // replicate prod to dev
-  const prodAppId = getDeployedAppID(appId)
+  const prodAppId = getProdAppID(appId)
 
   try {
-    const prodDb = new CouchDB(prodAppId, { skip_setup: true })
+    // specific case, want to make sure setup is skipped
+    const prodDb = getProdAppDB({ skip_setup: true })
     const info = await prodDb.info()
     if (info.error) throw info.error
   } catch (err) {
@@ -399,7 +406,7 @@ exports.sync = async (ctx, next) => {
   }
 
   // sync the users
-  await syncGlobalUsers(appId)
+  await syncGlobalUsers()
 
   if (error) {
     ctx.throw(400, error)
@@ -411,7 +418,7 @@ exports.sync = async (ctx, next) => {
 }
 
 const updateAppPackage = async (appPackage, appId) => {
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const application = await db.get(DocumentTypes.APP_METADATA)
 
   const newAppPackage = { ...application, ...appPackage }
@@ -430,7 +437,7 @@ const updateAppPackage = async (appPackage, appId) => {
 }
 
 const createEmptyAppPackage = async (ctx, app) => {
-  const db = new CouchDB(app.appId)
+  const db = getAppDB()
 
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {

packages/server/src/api/controllers/auth.js

@@ -1,11 +1,10 @@
-const CouchDB = require("../../db")
 const { outputProcessing } = require("../../utilities/rowProcessor")
 const { InternalTables } = require("../../db/utils")
 const { getFullUser } = require("../../utilities/users")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 
 exports.fetchSelf = async ctx => {
-  const appId = ctx.appId
   let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
   if (!userId) {
@@ -19,8 +18,8 @@ exports.fetchSelf = async ctx => {
   // forward the csrf token from the session
   user.csrfToken = ctx.user.csrfToken
 
-  if (appId) {
-    const db = new CouchDB(appId)
+  if (getAppId()) {
+    const db = getAppDB()
     // remove the full roles structure
     delete user.roles
     try {
@@ -29,7 +28,7 @@ exports.fetchSelf = async ctx => {
       // make sure there is never a stale csrf token
       delete metadata.csrfToken
       // specifically needs to make sure is enriched
-      ctx.body = await outputProcessing(ctx, userTable, {
+      ctx.body = await outputProcessing(userTable, {
         ...user,
         ...metadata,
       })

packages/server/src/api/controllers/automation.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const actions = require("../../automations/actions")
 const triggers = require("../../automations/triggers")
 const { getAutomationParams, generateAutomationID } = require("../../db/utils")
@@ -10,6 +9,7 @@ const {
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
 const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@@ -20,14 +20,9 @@ const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
  *                       *
  *************************/
 
-async function cleanupAutomationMetadata(appId, automationId) {
+async function cleanupAutomationMetadata(automationId) {
+  await deleteEntityMetadata(MetadataTypes.AUTOMATION_TEST_INPUT, automationId)
   await deleteEntityMetadata(
-    appId,
-    MetadataTypes.AUTOMATION_TEST_INPUT,
-    automationId
-  )
-  await deleteEntityMetadata(
-    appId,
     MetadataTypes.AUTOMATION_TEST_HISTORY,
     automationId
   )
@@ -58,7 +53,7 @@ function cleanAutomationInputs(automation) {
 }
 
 exports.create = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
 
@@ -72,7 +67,6 @@ exports.create = async function (ctx) {
   automation.type = "automation"
   automation = cleanAutomationInputs(automation)
   automation = await checkForWebhooks({
-    appId: ctx.appId,
     newAuto: automation,
   })
   const response = await db.put(automation)
@@ -89,13 +83,12 @@ exports.create = async function (ctx) {
 }
 
 exports.update = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
   const oldAutomation = await db.get(automation._id)
   automation = cleanAutomationInputs(automation)
   automation = await checkForWebhooks({
-    appId: ctx.appId,
     oldAuto: oldAutomation,
     newAuto: automation,
   })
@@ -131,7 +124,7 @@ exports.update = async function (ctx) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const response = await db.allDocs(
     getAutomationParams(null, {
       include_docs: true,
@@ -141,20 +134,19 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   ctx.body = await db.get(ctx.params.id)
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const automationId = ctx.params.id
   const oldAutomation = await db.get(automationId)
   await checkForWebhooks({
-    appId: ctx.appId,
     oldAuto: oldAutomation,
   })
   // delete metadata first
-  await cleanupAutomationMetadata(ctx.appId, automationId)
+  await cleanupAutomationMetadata(automationId)
   ctx.body = await db.remove(automationId, ctx.params.rev)
 }
 
@@ -180,12 +172,11 @@ module.exports.getDefinitionList = async function (ctx) {
  *********************/
 
 exports.trigger = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let automation = await db.get(ctx.params.id)
   await triggers.externalTrigger(automation, {
     ...ctx.request.body,
-    appId,
+    appId: ctx.appId,
   })
   ctx.body = {
     message: `Automation ${automation._id} has been triggered.`,
@@ -205,8 +196,7 @@ function prepareTestInput(input) {
 }
 
 exports.test = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let automation = await db.get(ctx.params.id)
   await setTestFlag(automation._id)
   const testInput = prepareTestInput(ctx.request.body)
@@ -214,7 +204,7 @@ exports.test = async function (ctx) {
     automation,
     {
       ...testInput,
-      appId,
+      appId: ctx.appId,
     },
     { getResponses: true }
   )

packages/server/src/api/controllers/cloud.js

@@ -1,6 +1,5 @@
 const env = require("../../environment")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
 const {
   exportDB,
   sendTempFile,
@@ -30,7 +29,7 @@ exports.exportApps = async ctx => {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
-  const apps = await getAllApps(CouchDB, { all: true })
+  const apps = await getAllApps({ all: true })
   const globalDBString = await exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentTypes.USER),
   })
@@ -63,7 +62,7 @@ async function hasBeenImported() {
   if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
     return true
   }
-  const apps = await getAllApps(CouchDB, { all: true })
+  const apps = await getAllApps({ all: true })
   return apps.length !== 0
 }
 

packages/server/src/api/controllers/component.js

@@ -1,15 +1,14 @@
-const CouchDB = require("../../db")
 const { DocumentTypes } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
-  const appId = ctx.params.appId || ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const app = await db.get(DocumentTypes.APP_METADATA)
 
   let componentManifests = await Promise.all(
     app.componentLibraries.map(async library => {
-      let manifest = await getComponentLibraryManifest(appId, library)
+      let manifest = await getComponentLibraryManifest(library)
 
       return {
         manifest,

packages/server/src/api/controllers/datasource.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const {
   generateDatasourceID,
   getDatasourceParams,
@@ -11,12 +10,11 @@ const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
-
   // Get internal tables
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const internalTables = await db.allDocs(
     getTableParams(null, {
       include_docs: true,
@@ -31,7 +29,7 @@ exports.fetch = async function (ctx) {
 
   // Get external datasources
   const datasources = (
-    await database.allDocs(
+    await db.allDocs(
       getDatasourceParams(null, {
         include_docs: true,
       })
@@ -49,7 +47,7 @@ exports.fetch = async function (ctx) {
 }
 
 exports.buildSchemaFromDb = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const datasource = await db.get(ctx.params.datasourceId)
 
   const { tables, error } = await buildSchemaHelper(datasource)
@@ -98,7 +96,7 @@ const invalidateVariables = async (existingDatasource, updatedDatasource) => {
 }
 
 exports.update = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
   const auth = datasource.config.auth
@@ -126,7 +124,7 @@ exports.update = async function (ctx) {
 }
 
 exports.save = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
 
@@ -162,7 +160,7 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
   // Delete all queries for the datasource
   const queries = await db.allDocs(
@@ -184,7 +182,7 @@ exports.destroy = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
+  const database = getAppDB()
   ctx.body = await database.get(ctx.params.datasourceId)
 }
 
@@ -192,7 +190,7 @@ exports.find = async function (ctx) {
 exports.query = async function (ctx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await getDatasourceAndQuery(ctx.appId, queryJson)
+    ctx.body = await getDatasourceAndQuery(queryJson)
   } catch (err) {
     ctx.throw(400, err)
   }

packages/server/src/api/controllers/deploy/Deployment.js

@@ -1,18 +1,14 @@
 const newid = require("../../../db/newid")
+const { getAppId } = require("@budibase/backend-core/context")
 
 /**
  * This is used to pass around information about the deployment that is occurring
  */
 class Deployment {
-  constructor(appId, id = null) {
-    this.appId = appId
+  constructor(id = null) {
     this._id = id || newid()
   }
 
-  getAppId() {
-    return this.appId
-  }
-
   setVerification(verification) {
     if (!verification) {
       return
@@ -43,7 +39,7 @@ class Deployment {
   getJSON() {
     const obj = {
       _id: this._id,
-      appId: this.appId,
+      appId: getAppId(),
       status: this.status,
     }
     if (this.err) {

packages/server/src/api/controllers/deploy/index.js

@@ -1,12 +1,20 @@
-const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication, getDeployedAppID } = require("@budibase/backend-core/db")
+const {
+  Replication,
+  getProdAppID,
+  getDevelopmentAppID,
+} = require("@budibase/backend-core/db")
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
 const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
 const { app: appCache } = require("@budibase/backend-core/cache")
+const {
+  getAppId,
+  getAppDB,
+  getProdAppDB,
+} = require("@budibase/backend-core/context")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -34,9 +42,8 @@ async function checkAllDeployments(deployments) {
 }
 
 async function storeDeploymentHistory(deployment) {
-  const appId = deployment.getAppId()
   const deploymentJSON = deployment.getJSON()
-  const db = new CouchDB(appId)
+  const db = getAppDB()
 
   let deploymentDoc
   try {
@@ -64,7 +71,7 @@ async function storeDeploymentHistory(deployment) {
 }
 
 async function initDeployedApp(prodAppId) {
-  const db = new CouchDB(prodAppId)
+  const db = getProdAppDB()
   console.log("Reading automation docs")
   const automations = (
     await db.allDocs(
@@ -88,10 +95,12 @@ async function initDeployedApp(prodAppId) {
 
 async function deployApp(deployment) {
   try {
-    const productionAppId = getDeployedAppID(deployment.appId)
+    const appId = getAppId()
+    const devAppId = getDevelopmentAppID(appId)
+    const productionAppId = getProdAppID(appId)
 
     const replication = new Replication({
-      source: deployment.appId,
+      source: devAppId,
       target: productionAppId,
     })
 
@@ -99,7 +108,7 @@ async function deployApp(deployment) {
 
     await replication.replicate()
     console.log("replication complete.. replacing app meta doc")
-    const db = new CouchDB(productionAppId)
+    const db = getProdAppDB()
     const appDoc = await db.get(DocumentTypes.APP_METADATA)
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
@@ -122,8 +131,7 @@ async function deployApp(deployment) {
 
 exports.fetchDeployments = async function (ctx) {
   try {
-    const appId = ctx.appId
-    const db = new CouchDB(appId)
+    const db = getAppDB()
     const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
     const { updated, deployments } = await checkAllDeployments(
       deploymentDoc,
@@ -140,8 +148,7 @@ exports.fetchDeployments = async function (ctx) {
 
 exports.deploymentProgress = async function (ctx) {
   try {
-    const appId = ctx.appId
-    const db = new CouchDB(appId)
+    const db = getAppDB()
     const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
     ctx.body = deploymentDoc[ctx.params.deploymentId]
   } catch (err) {
@@ -153,7 +160,7 @@ exports.deploymentProgress = async function (ctx) {
 }
 
 exports.deployApp = async function (ctx) {
-  let deployment = new Deployment(ctx.appId)
+  let deployment = new Deployment()
   console.log("Deployment object created")
   deployment.setStatus(DeploymentStatus.PENDING)
   console.log("Deployment object set to pending")

packages/server/src/api/controllers/dev.js

@@ -1,12 +1,12 @@
 const fetch = require("node-fetch")
-const CouchDB = require("../../db")
 const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication } = require("@budibase/backend-core/db")
+const { Replication, getProdAppID } = require("@budibase/backend-core/db")
 const { DocumentTypes } = require("../../db/utils")
 const { app: appCache } = require("@budibase/backend-core/cache")
+const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
@@ -77,11 +77,11 @@ exports.clearLock = async ctx => {
 
 exports.revert = async ctx => {
   const { appId } = ctx.params
-  const productionAppId = appId.replace("_dev", "")
+  const productionAppId = getProdAppID(appId)
 
   // App must have been deployed first
   try {
-    const db = new CouchDB(productionAppId, { skip_setup: true })
+    const db = getProdAppDB({ skip_setup: true })
     const info = await db.info()
     if (info.error) throw info.error
     const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
@@ -103,7 +103,7 @@ exports.revert = async ctx => {
 
     await replication.rollback()
     // update appID in reverted app to be dev version again
-    const db = new CouchDB(appId)
+    const db = getAppDB()
     const appDoc = await db.get(DocumentTypes.APP_METADATA)
     appDoc.appId = appId
     appDoc.instance._id = appId

packages/server/src/api/controllers/layout.js

@@ -2,11 +2,11 @@ const {
   EMPTY_LAYOUT,
   BASE_LAYOUT_PROP_IDS,
 } = require("../../constants/layouts")
-const CouchDB = require("../../db")
 const { generateLayoutID, getScreenParams } = require("../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.save = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let layout = ctx.request.body
 
   if (!layout.props) {
@@ -26,7 +26,7 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const layoutId = ctx.params.layoutId,
     layoutRev = ctx.params.layoutRev
 

packages/server/src/api/controllers/metadata.js

@@ -1,7 +1,7 @@
 const { MetadataTypes } = require("../../constants")
-const CouchDB = require("../../db")
 const { generateMetadataID } = require("../../db/utils")
 const { saveEntityMetadata, deleteEntityMetadata } = require("../../utilities")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.getTypes = async ctx => {
   ctx.body = {
@@ -14,17 +14,12 @@ exports.saveMetadata = async ctx => {
   if (type === MetadataTypes.AUTOMATION_TEST_HISTORY) {
     ctx.throw(400, "Cannot save automation history type")
   }
-  ctx.body = await saveEntityMetadata(
-    ctx.appId,
-    type,
-    entityId,
-    ctx.request.body
-  )
+  ctx.body = await saveEntityMetadata(type, entityId, ctx.request.body)
 }
 
 exports.deleteMetadata = async ctx => {
   const { type, entityId } = ctx.params
-  await deleteEntityMetadata(ctx.appId, type, entityId)
+  await deleteEntityMetadata(type, entityId)
   ctx.body = {
     message: "Metadata deleted successfully",
   }
@@ -32,7 +27,7 @@ exports.deleteMetadata = async ctx => {
 
 exports.getMetadata = async ctx => {
   const { type, entityId } = ctx.params
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const id = generateMetadataID(type, entityId)
   try {
     ctx.body = await db.get(id)

packages/server/src/api/controllers/migrations.js

@@ -0,0 +1,13 @@
+const { migrate, MIGRATIONS } = require("../../migrations")
+
+exports.migrate = async ctx => {
+  const options = ctx.request.body
+  // don't await as can take a while, just return
+  migrate(options)
+  ctx.status = 200
+}
+
+exports.fetchDefinitions = async ctx => {
+  ctx.body = MIGRATIONS
+  ctx.status = 200
+}

packages/server/src/api/controllers/permission.js

@@ -6,12 +6,12 @@ const {
   getBuiltinRoles,
 } = require("@budibase/backend-core/roles")
 const { getRoleParams } = require("../../db/utils")
-const CouchDB = require("../../db")
 const {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
 } = require("../../utilities/security")
 const { removeFromArray } = require("../../utilities")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const PermissionUpdateType = {
   REMOVE: "remove",
@@ -35,7 +35,7 @@ async function updatePermissionOnRole(
   { roleId, resourceId, level },
   updateType
 ) {
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const remove = updateType === PermissionUpdateType.REMOVE
   const isABuiltin = isBuiltin(roleId)
   const dbRoleId = getDBRoleID(roleId)
@@ -106,7 +106,7 @@ exports.fetchLevels = function (ctx) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const roles = await getAllDBRoles(db)
   let permissions = {}
   // create an object with structure role ID -> resource ID -> level
@@ -133,7 +133,7 @@ exports.fetch = async function (ctx) {
 
 exports.getResourcePerms = async function (ctx) {
   const resourceId = ctx.params.resourceId
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const body = await db.allDocs(
     getRoleParams(null, {
       include_docs: true,

packages/server/src/api/controllers/query/import/index.ts

@@ -1,10 +1,11 @@
-import CouchDB from "../../../../db"
 import { queryValidation } from "../validation"
 import { generateQueryID } from "../../../../db/utils"
 import { ImportInfo, ImportSource } from "./sources/base"
 import { OpenAPI2 } from "./sources/openapi2"
 import { Query } from "./../../../../definitions/common"
 import { Curl } from "./sources/curl"
+// @ts-ignore
+import { getAppDB } from "@budibase/backend-core/context"
 interface ImportResult {
   errorQueries: Query[]
   queries: Query[]
@@ -33,10 +34,7 @@ export class RestImporter {
     return this.source.getInfo()
   }
 
-  importQueries = async (
-    appId: string,
-    datasourceId: string
-  ): Promise<ImportResult> => {
+  importQueries = async (datasourceId: string): Promise<ImportResult> => {
     // constuct the queries
     let queries = await this.source.getQueries(datasourceId)
 
@@ -58,7 +56,7 @@ export class RestImporter {
       })
 
     // persist queries
-    const db = new CouchDB(appId)
+    const db = getAppDB()
     const response = await db.bulkDocs(queries)
 
     // create index to seperate queries and errors

packages/server/src/api/controllers/query/import/tests/index.spec.js

@@ -6,6 +6,7 @@ const db = jest.fn(() => {
   }
 })
 jest.mock("../../../../../db", () => db)
+require("@budibase/backend-core").init(require("../../../../../db"))
 
 const { RestImporter } = require("../index")
 
@@ -77,7 +78,7 @@ describe("Rest Importer", () => {
   const testImportQueries = async (key, data, assertions) => {
     await init(data)
     bulkDocs.mockReturnValue([])
-    const importResult = await restImporter.importQueries("appId", "datasourceId")
+    const importResult = await restImporter.importQueries("datasourceId")
     expect(importResult.errorQueries.length).toBe(0)
     expect(importResult.queries.length).toBe(assertions[key].count)
     expect(bulkDocs).toHaveBeenCalledTimes(1)

packages/server/src/api/controllers/query/index.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const {
   generateQueryID,
   getQueryParams,
@@ -10,6 +9,7 @@ const { save: saveDatasource } = require("../datasource")
 const { RestImporter } = require("./import")
 const { invalidateDynamicVariables } = require("../../../threads/utils")
 const environment = require("../../../environment")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: environment.QUERY_THREAD_TIMEOUT || 10000,
@@ -28,7 +28,7 @@ function enrichQueries(input) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
   const body = await db.allDocs(
     getQueryParams(null, {
@@ -69,7 +69,7 @@ exports.import = async ctx => {
     datasourceId = body.datasourceId
   }
 
-  const importResult = await importer.importQueries(ctx.appId, datasourceId)
+  const importResult = await importer.importQueries(datasourceId)
 
   ctx.body = {
     ...importResult,
@@ -79,7 +79,7 @@ exports.import = async ctx => {
 }
 
 exports.save = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const query = ctx.request.body
 
   if (!query._id) {
@@ -94,7 +94,7 @@ exports.save = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const query = enrichQueries(await db.get(ctx.params.queryId))
   // remove properties that could be dangerous in real app
   if (isProdAppID(ctx.appId)) {
@@ -105,7 +105,7 @@ exports.find = async function (ctx) {
 }
 
 exports.preview = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
   const datasource = await db.get(ctx.request.body.datasourceId)
   // preview may not have a queryId as it hasn't been saved, but if it does
@@ -136,7 +136,7 @@ exports.preview = async function (ctx) {
 }
 
 async function execute(ctx, opts = { rowsOnly: false }) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
@@ -181,7 +181,8 @@ exports.executeV2 = async function (ctx) {
   return execute(ctx, { rowsOnly: false })
 }
 
-const removeDynamicVariables = async (db, queryId) => {
+const removeDynamicVariables = async queryId => {
+  const db = getAppDB()
   const query = await db.get(queryId)
   const datasource = await db.get(query.datasourceId)
   const dynamicVariables = datasource.config.dynamicVariables
@@ -202,8 +203,8 @@ const removeDynamicVariables = async (db, queryId) => {
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
-  await removeDynamicVariables(db, ctx.params.queryId)
+  const db = getAppDB()
+  await removeDynamicVariables(ctx.params.queryId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

packages/server/src/api/controllers/role.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const {
   Role,
   getRole,
@@ -10,6 +9,7 @@ const {
   getUserMetadataParams,
   InternalTables,
 } = require("../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -40,15 +40,15 @@ async function updateRolesOnUserTable(db, roleId, updateOption) {
 }
 
 exports.fetch = async function (ctx) {
-  ctx.body = await getAllRoles(ctx.appId)
+  ctx.body = await getAllRoles()
 }
 
 exports.find = async function (ctx) {
-  ctx.body = await getRole(ctx.appId, ctx.params.roleId)
+  ctx.body = await getRole(ctx.params.roleId)
 }
 
 exports.save = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let { _id, name, inherits, permissionId } = ctx.request.body
   if (!_id) {
     _id = generateRoleID()
@@ -69,7 +69,7 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const roleId = ctx.params.roleId
   if (isBuiltin(roleId)) {
     ctx.throw(400, "Cannot delete builtin role.")

packages/server/src/api/controllers/routing.js

@@ -39,12 +39,11 @@ Routing.prototype.addScreenId = function (fullpath, roleId, screenId) {
 
 /**
  * Gets the full routing structure by querying the routing view and processing the result into the tree.
- * @param {string} appId The application to produce the routing structure for.
  * @returns {Promise<object>} The routing structure, this is the full structure designed for use in the builder,
  * if the client routing is required then the updateRoutingStructureForUserRole should be used.
  */
-async function getRoutingStructure(appId) {
-  const screenRoutes = await getRoutingInfo(appId)
+async function getRoutingStructure() {
+  const screenRoutes = await getRoutingInfo()
   const routing = new Routing()
 
   for (let screenRoute of screenRoutes) {
@@ -57,13 +56,13 @@ async function getRoutingStructure(appId) {
 }
 
 exports.fetch = async ctx => {
-  ctx.body = await getRoutingStructure(ctx.appId)
+  ctx.body = await getRoutingStructure()
 }
 
 exports.clientFetch = async ctx => {
-  const routing = await getRoutingStructure(ctx.appId)
+  const routing = await getRoutingStructure()
   let roleId = ctx.user.role._id
-  const roleIds = await getUserRoleHierarchy(ctx.appId, roleId)
+  const roleIds = await getUserRoleHierarchy(roleId)
   for (let topLevel of Object.values(routing.routes)) {
     for (let subpathKey of Object.keys(topLevel.subpaths)) {
       let found = false

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -19,6 +19,19 @@ import {
   isRowId,
   convertRowId,
 } from "../../../integrations/utils"
+import { getDatasourceAndQuery } from "./utils"
+import {
+  DataSourceOperation,
+  FieldTypes,
+  RelationshipTypes,
+} from "../../../constants"
+import { breakExternalTableId, isSQL } from "../../../integrations/utils"
+import { processObjectSync } from "@budibase/string-templates"
+// @ts-ignore
+import { cloneDeep } from "lodash/fp"
+import { processFormulas } from "../../../utilities/rowProcessor/utils"
+// @ts-ignore
+import { getAppDB } from "@budibase/backend-core/context"
 
 interface ManyRelationship {
   tableId?: string
@@ -38,18 +51,6 @@ interface RunConfig {
 }
 
 module External {
-  const { getDatasourceAndQuery } = require("./utils")
-  const {
-    DataSourceOperation,
-    FieldTypes,
-    RelationshipTypes,
-  } = require("../../../constants")
-  const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
-  const { processObjectSync } = require("@budibase/string-templates")
-  const { cloneDeep } = require("lodash/fp")
-  const CouchDB = require("../../../db")
-  const { processFormulas } = require("../../../utilities/rowProcessor/utils")
-
   function buildFilters(
     id: string | undefined,
     filters: SearchFilters,
@@ -210,19 +211,12 @@ module External {
   }
 
   class ExternalRequest {
-    private readonly appId: string
     private operation: Operation
     private tableId: string
     private datasource: Datasource
     private tables: { [key: string]: Table } = {}
 
-    constructor(
-      appId: string,
-      operation: Operation,
-      tableId: string,
-      datasource: Datasource
-    ) {
-      this.appId = appId
+    constructor(operation: Operation, tableId: string, datasource: Datasource) {
       this.operation = operation
       this.tableId = tableId
       this.datasource = datasource
@@ -231,12 +225,14 @@ module External {
       }
     }
 
-    getTable(tableId: string | undefined): Table {
+    getTable(tableId: string | undefined): Table | undefined {
       if (!tableId) {
         throw "Table ID is unknown, cannot find table"
       }
       const { tableName } = breakExternalTableId(tableId)
-      return this.tables[tableName]
+      if (tableName) {
+        return this.tables[tableName]
+      }
     }
 
     inputProcessing(row: Row | undefined, table: Table) {
@@ -272,9 +268,11 @@ module External {
           newRow[key] = row[key]
           continue
         }
-        const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+        const { tableName: linkTableName } = breakExternalTableId(
+          field?.tableId
+        )
         // table has to exist for many to many
-        if (!this.tables[linkTableName]) {
+        if (!linkTableName || !this.tables[linkTableName]) {
           continue
         }
         const linkTable = this.tables[linkTableName]
@@ -422,7 +420,7 @@ module External {
         }
         const { tableName: linkTableName } = breakExternalTableId(field.tableId)
         // no table to link to, this is not a valid relationships
-        if (!this.tables[linkTableName]) {
+        if (!linkTableName || !this.tables[linkTableName]) {
           continue
         }
         const linkTable = this.tables[linkTableName]
@@ -460,6 +458,9 @@ module External {
     async lookupRelations(tableId: string, row: Row) {
       const related: { [key: string]: any } = {}
       const { tableName } = breakExternalTableId(tableId)
+      if (!tableName) {
+        return related
+      }
       const table = this.tables[tableName]
       // @ts-ignore
       const primaryKey = table.primary[0]
@@ -484,7 +485,7 @@ module External {
         if (!lookupField || !row[lookupField]) {
           continue
         }
-        const response = await getDatasourceAndQuery(this.appId, {
+        const response = await getDatasourceAndQuery({
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {
             equal: {
@@ -515,28 +516,30 @@ module External {
       row: Row,
       relationships: ManyRelationship[]
     ) {
-      const { appId } = this
       // if we're creating (in a through table) need to wipe the existing ones first
       const promises = []
       const related = await this.lookupRelations(mainTableId, row)
       for (let relationship of relationships) {
         const { key, tableId, isUpdate, id, ...rest } = relationship
-        const body = processObjectSync(rest, row)
+        const body: { [key: string]: any } = processObjectSync(rest, row, {})
         const linkTable = this.getTable(tableId)
         // @ts-ignore
-        const linkPrimary = linkTable.primary[0]
+        const linkPrimary = linkTable?.primary[0]
+        if (!linkTable || !linkPrimary) {
+          return
+        }
         const rows = related[key].rows || []
         const found = rows.find(
           (row: { [key: string]: any }) =>
             row[linkPrimary] === relationship.id ||
-            row[linkPrimary] === body[linkPrimary]
+            row[linkPrimary] === body?.[linkPrimary]
         )
         const operation = isUpdate
           ? DataSourceOperation.UPDATE
           : DataSourceOperation.CREATE
         if (!found) {
           promises.push(
-            getDatasourceAndQuery(appId, {
+            getDatasourceAndQuery({
               endpoint: getEndpoint(tableId, operation),
               // if we're doing many relationships then we're writing, only one response
               body,
@@ -552,9 +555,12 @@ module External {
       for (let [colName, { isMany, rows, tableId }] of Object.entries(
         related
       )) {
-        const table: Table = this.getTable(tableId)
+        const table: Table | undefined = this.getTable(tableId)
         // if its not the foreign key skip it, nothing to do
-        if (table.primary && table.primary.indexOf(colName) !== -1) {
+        if (
+          !table ||
+          (table.primary && table.primary.indexOf(colName) !== -1)
+        ) {
           continue
         }
         for (let row of rows) {
@@ -566,7 +572,7 @@ module External {
               : DataSourceOperation.UPDATE
             const body = isMany ? null : { [colName]: null }
             promises.push(
-              getDatasourceAndQuery(this.appId, {
+              getDatasourceAndQuery({
                 endpoint: getEndpoint(tableId, op),
                 body,
                 filters,
@@ -605,20 +611,25 @@ module External {
           continue
         }
         const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-        const linkTable = this.tables[linkTableName]
-        if (linkTable) {
-          const linkedFields = extractRealFields(linkTable, fields)
-          fields = fields.concat(linkedFields)
+        if (linkTableName) {
+          const linkTable = this.tables[linkTableName]
+          if (linkTable) {
+            const linkedFields = extractRealFields(linkTable, fields)
+            fields = fields.concat(linkedFields)
+          }
         }
       }
       return fields
     }
 
     async run(config: RunConfig) {
-      const { appId, operation, tableId } = this
+      const { operation, tableId } = this
       let { datasourceId, tableName } = breakExternalTableId(tableId)
+      if (!tableName) {
+        throw "Unable to run without a table name"
+      }
       if (!this.datasource) {
-        const db = new CouchDB(appId)
+        const db = getAppDB()
         this.datasource = await db.get(datasourceId)
         if (!this.datasource || !this.datasource.entities) {
           throw "No tables found, fetch tables before query."
@@ -670,7 +681,7 @@ module External {
         },
       }
       // can't really use response right now
-      const response = await getDatasourceAndQuery(appId, json)
+      const response = await getDatasourceAndQuery(json)
       // handle many to many relationships now if we know the ID (could be auto increment)
       if (
         operation !== DataSourceOperation.READ &&

packages/server/src/api/controllers/row/external.js

@@ -9,9 +9,9 @@ const {
   breakRowIdField,
 } = require("../../../integrations/utils")
 const ExternalRequest = require("./ExternalRequest")
-const CouchDB = require("../../../db")
+const { getAppDB } = require("@budibase/backend-core/context")
 
-async function handleRequest(appId, operation, tableId, opts = {}) {
+async function handleRequest(operation, tableId, opts = {}) {
   // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
   if (opts && opts.filters) {
     for (let filterField of NoEmptyFilterStrings) {
@@ -25,9 +25,7 @@ async function handleRequest(appId, operation, tableId, opts = {}) {
       }
     }
   }
-  return new ExternalRequest(appId, operation, tableId, opts.datasource).run(
-    opts
-  )
+  return new ExternalRequest(operation, tableId, opts.datasource).run(opts)
 }
 
 exports.handleRequest = handleRequest
@@ -181,7 +179,7 @@ exports.fetchEnrichedRow = async ctx => {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")

packages/server/src/api/controllers/row/internal.js

@@ -1,8 +1,7 @@
-const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const {
-  getRowParams,
   generateRowID,
+  getRowParams,
   DocumentTypes,
   InternalTables,
 } = require("../../../db/utils")
@@ -10,11 +9,9 @@ const userController = require("../user")
 const {
   inputProcessing,
   outputProcessing,
-  processAutoColumn,
   cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
-const { isEqual } = require("lodash")
 const { validate, findRow } = require("./utils")
 const { fullSearch, paginatedSearch } = require("./internalSearch")
 const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
@@ -27,6 +24,8 @@ const {
   getFromMemoryDoc,
 } = require("../view/utils")
 const { cloneDeep } = require("lodash/fp")
+const { getAppDB } = require("@budibase/backend-core/context")
+const { finaliseRow, updateRelatedFormula } = require("./staticFormula")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -34,51 +33,6 @@ const CALCULATION_TYPES = {
   STATS: "stats",
 }
 
-async function storeResponse(ctx, db, row, oldTable, table) {
-  row.type = "row"
-  // don't worry about rev, tables handle rev/lastID updates
-  // if another row has been written since processing this will
-  // handle the auto ID clash
-  if (!isEqual(oldTable, table)) {
-    try {
-      await db.put(table)
-    } catch (err) {
-      if (err.status === 409) {
-        const updatedTable = await db.get(table._id)
-        let response = processAutoColumn(null, updatedTable, row, {
-          reprocessing: true,
-        })
-        await db.put(response.table)
-        row = response.row
-      } else {
-        throw err
-      }
-    }
-  }
-  const response = await db.put(row)
-  row._rev = response.rev
-  // process the row before return, to include relationships
-  row = await outputProcessing(ctx, table, row, { squash: false })
-  return { row, table }
-}
-
-// doesn't do the outputProcessing
-async function getRawTableData(ctx, db, tableId) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows
-}
-
 async function getView(db, viewName) {
   let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
   let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
@@ -105,9 +59,24 @@ async function getView(db, viewName) {
   return viewInfo
 }
 
+async function getRawTableData(ctx, db, tableId) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    await userController.fetchMetadata(ctx)
+    rows = ctx.body
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows
+}
+
 exports.patch = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const inputs = ctx.request.body
   const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
@@ -146,14 +115,13 @@ exports.patch = async ctx => {
 
   // returned row is cleaned and prepared for writing to DB
   row = await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.ROW_UPDATE,
     row,
     tableId: row.tableId,
     table,
   })
   // check if any attachments removed
-  await cleanupAttachments(appId, table, { oldRow, row })
+  await cleanupAttachments(table, { oldRow, row })
 
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
@@ -162,12 +130,14 @@ exports.patch = async ctx => {
     return { row: ctx.body, table }
   }
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.save = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let inputs = ctx.request.body
   inputs.tableId = ctx.params.tableId
 
@@ -189,18 +159,19 @@ exports.save = async function (ctx) {
 
   // make sure link rows are up to date
   row = await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.ROW_SAVE,
     row,
     tableId: row.tableId,
     table,
   })
 
-  return storeResponse(ctx, db, row, dbTable, table)
+  return finaliseRow(table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
 }
 
 exports.fetchView = async ctx => {
-  const appId = ctx.appId
   const viewName = ctx.params.viewName
 
   // if this is a table view being looked for just transfer to that
@@ -209,7 +180,7 @@ exports.fetchView = async ctx => {
     return exports.fetch(ctx)
   }
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { calculation, group, field } = ctx.query
   const viewInfo = await getView(db, viewName)
   let response
@@ -236,7 +207,7 @@ exports.fetchView = async ctx => {
         schema: {},
       }
     }
-    rows = await outputProcessing(ctx, table, response.rows)
+    rows = await outputProcessing(table, response.rows)
   }
 
   if (calculation === CALCULATION_TYPES.STATS) {
@@ -263,27 +234,24 @@ exports.fetchView = async ctx => {
 }
 
 exports.fetch = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
 
   const tableId = ctx.params.tableId
   let table = await db.get(tableId)
   let rows = await getRawTableData(ctx, db, tableId)
-  return outputProcessing(ctx, table, rows)
+  return outputProcessing(table, rows)
 }
 
 exports.find = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const table = await db.get(ctx.params.tableId)
-  let row = await findRow(ctx, db, ctx.params.tableId, ctx.params.rowId)
-  row = await outputProcessing(ctx, table, row)
+  let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId)
+  row = await outputProcessing(table, row)
   return row
 }
 
 exports.destroy = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { _id, _rev } = ctx.request.body
   let row = await db.get(_id)
 
@@ -292,16 +260,17 @@ exports.destroy = async function (ctx) {
   }
   const table = await db.get(row.tableId)
   // update the row to include full relationships before deleting them
-  row = await outputProcessing(ctx, table, row, { squash: false })
+  row = await outputProcessing(table, row, { squash: false })
   // now remove the relationships
   await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.ROW_DELETE,
     row,
     tableId: row.tableId,
   })
   // remove any attachments that were on the row from object storage
-  await cleanupAttachments(appId, table, { row })
+  await cleanupAttachments(table, { row })
+  // remove any static formula
+  await updateRelatedFormula(table, row)
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -317,20 +286,18 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkDestroy = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const tableId = ctx.params.tableId
   const table = await db.get(tableId)
   let { rows } = ctx.request.body
 
   // before carrying out any updates, make sure the rows are ready to be returned
   // they need to be the full rows (including previous relationships) for automations
-  rows = await outputProcessing(ctx, table, rows, { squash: false })
+  rows = await outputProcessing(table, rows, { squash: false })
 
   // remove the relationships first
   let updates = rows.map(row =>
     linkRows.updateLinks({
-      appId,
       eventType: linkRows.EventType.ROW_DELETE,
       row,
       tableId: row.tableId,
@@ -349,7 +316,8 @@ exports.bulkDestroy = async ctx => {
     await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
   }
   // remove any attachments that were on the rows from object storage
-  await cleanupAttachments(appId, table, { rows })
+  await cleanupAttachments(table, { rows })
+  await updateRelatedFormula(table, rows)
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }
@@ -360,28 +328,27 @@ exports.search = async ctx => {
     return { rows: await exports.fetch(ctx) }
   }
 
-  const appId = ctx.appId
   const { tableId } = ctx.params
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { paginate, query, ...params } = ctx.request.body
   params.version = ctx.version
   params.tableId = tableId
 
   let response
   if (paginate) {
-    response = await paginatedSearch(appId, query, params)
+    response = await paginatedSearch(query, params)
   } else {
-    response = await fullSearch(appId, query, params)
+    response = await fullSearch(query, params)
   }
 
   // Enrich search results with relationships
   if (response.rows && response.rows.length) {
     // enrich with global users if from users table
     if (tableId === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(appId, response.rows)
+      response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
     const table = await db.get(tableId)
-    response.rows = await outputProcessing(ctx, table, response.rows)
+    response.rows = await outputProcessing(table, response.rows)
   }
 
   return response
@@ -389,25 +356,22 @@ exports.search = async ctx => {
 
 exports.validate = async ctx => {
   return validate({
-    appId: ctx.appId,
     tableId: ctx.params.tableId,
     row: ctx.request.body,
   })
 }
 
 exports.fetchEnrichedRow = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const tableId = ctx.params.tableId
   const rowId = ctx.params.rowId
   // need table to work out where links go in row
   let [table, row] = await Promise.all([
     db.get(tableId),
-    findRow(ctx, db, tableId, rowId),
+    findRow(ctx, tableId, rowId),
   ])
   // get the link docs
   const linkVals = await linkRows.getLinkDocuments({
-    appId,
     tableId,
     rowId,
   })
@@ -434,7 +398,7 @@ exports.fetchEnrichedRow = async ctx => {
   for (let [tableId, rows] of Object.entries(groups)) {
     // need to include the IDs in these rows for any links they may have
     linkedRows = linkedRows.concat(
-      await outputProcessing(ctx, tables[tableId], rows)
+      await outputProcessing(tables[tableId], rows)
     )
   }
 

packages/server/src/api/controllers/row/internalSearch.js

@@ -1,14 +1,14 @@
 const { SearchIndexes } = require("../../../db/utils")
 const fetch = require("node-fetch")
 const { getCouchUrl } = require("@budibase/backend-core/db")
+const { getAppId } = require("@budibase/backend-core/context")
 
 /**
  * Class to build lucene query URLs.
  * Optionally takes a base lucene query object.
  */
 class QueryBuilder {
-  constructor(appId, base) {
-    this.appId = appId
+  constructor(base) {
     this.query = {
       string: {},
       fuzzy: {},
@@ -37,22 +37,30 @@ class QueryBuilder {
   }
 
   setLimit(limit) {
-    this.limit = limit
+    if (limit != null) {
+      this.limit = limit
+    }
     return this
   }
 
   setSort(sort) {
-    this.sort = sort
+    if (sort != null) {
+      this.sort = sort
+    }
     return this
   }
 
   setSortOrder(sortOrder) {
-    this.sortOrder = sortOrder
+    if (sortOrder != null) {
+      this.sortOrder = sortOrder
+    }
     return this
   }
 
   setSortType(sortType) {
-    this.sortType = sortType
+    if (sortType != null) {
+      this.sortType = sortType
+    }
     return this
   }
 
@@ -233,7 +241,8 @@ class QueryBuilder {
   }
 
   async run() {
-    const url = `${getCouchUrl()}/${this.appId}/_design/database/_search/${
+    const appId = getAppId()
+    const url = `${getCouchUrl()}/${appId}/_design/database/_search/${
       SearchIndexes.ROWS
     }`
     const body = this.buildSearchBody()
@@ -270,7 +279,6 @@ const runQuery = async (url, body) => {
  * Gets round the fixed limit of 200 results from a query by fetching as many
  * pages as required and concatenating the results. This recursively operates
  * until enough results have been found.
- * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -283,7 +291,7 @@ const runQuery = async (url, body) => {
  *   rows {array|null} Current results in the recursive search
  * @returns {Promise<*[]|*>}
  */
-const recursiveSearch = async (appId, query, params) => {
+const recursiveSearch = async (query, params) => {
   const bookmark = params.bookmark
   const rows = params.rows || []
   if (rows.length >= params.limit) {
@@ -293,7 +301,7 @@ const recursiveSearch = async (appId, query, params) => {
   if (rows.length > params.limit - 200) {
     pageSize = params.limit - rows.length
   }
-  const page = await new QueryBuilder(appId, query)
+  const page = await new QueryBuilder(query)
     .setVersion(params.version)
     .setTable(params.tableId)
     .setBookmark(bookmark)
@@ -313,14 +321,13 @@ const recursiveSearch = async (appId, query, params) => {
     bookmark: page.bookmark,
     rows: [...rows, ...page.rows],
   }
-  return await recursiveSearch(appId, query, newParams)
+  return await recursiveSearch(query, newParams)
 }
 
 /**
  * Performs a paginated search. A bookmark will be returned to allow the next
  * page to be fetched. There is a max limit off 200 results per page in a
  * paginated search.
- * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -332,13 +339,13 @@ const recursiveSearch = async (appId, query, params) => {
  *   bookmark {string} The bookmark to resume from
  * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
  */
-exports.paginatedSearch = async (appId, query, params) => {
+exports.paginatedSearch = async (query, params) => {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
   }
   limit = Math.min(limit, 200)
-  const search = new QueryBuilder(appId, query)
+  const search = new QueryBuilder(query)
     .setVersion(params.version)
     .setTable(params.tableId)
     .setSort(params.sort)
@@ -367,7 +374,6 @@ exports.paginatedSearch = async (appId, query, params) => {
  * desired amount of results. There is a limit of 1000 results to avoid
  * heavy performance hits, and to avoid client components breaking from
  * handling too much data.
- * @param appId {string} The app ID to search
  * @param query {object} The JSON query structure
  * @param params {object} The search params including:
  *   tableId {string} The table ID to search
@@ -378,12 +384,12 @@ exports.paginatedSearch = async (appId, query, params) => {
  *   limit {number} The desired number of results
  * @returns {Promise<{rows: *}>}
  */
-exports.fullSearch = async (appId, query, params) => {
+exports.fullSearch = async (query, params) => {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 1000
   }
   params.limit = Math.min(limit, 1000)
-  const rows = await recursiveSearch(appId, query, params)
+  const rows = await recursiveSearch(query, params)
   return { rows }
 }

packages/server/src/api/controllers/row/staticFormula.js

@@ -0,0 +1,156 @@
+const { getRowParams } = require("../../../db/utils")
+const {
+  outputProcessing,
+  processAutoColumn,
+  processFormulas,
+} = require("../../../utilities/rowProcessor")
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { isEqual } = require("lodash")
+const { cloneDeep } = require("lodash/fp")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+/**
+ * This function runs through a list of enriched rows, looks at the rows which
+ * are related and then checks if they need the state of their formulas
+ * updated.
+ * NOTE: this will only for affect static formulas.
+ */
+exports.updateRelatedFormula = async (table, enrichedRows) => {
+  const db = getAppDB()
+  // no formula to update, we're done
+  if (!table.relatedFormula) {
+    return
+  }
+  let promises = []
+  for (let enrichedRow of Array.isArray(enrichedRows)
+    ? enrichedRows
+    : [enrichedRows]) {
+    // the related rows by tableId
+    let relatedRows = {}
+    for (let [key, field] of Object.entries(enrichedRow)) {
+      const columnDefinition = table.schema[key]
+      if (columnDefinition && columnDefinition.type === FieldTypes.LINK) {
+        const relatedTableId = columnDefinition.tableId
+        if (!relatedRows[relatedTableId]) {
+          relatedRows[relatedTableId] = []
+        }
+        relatedRows[relatedTableId] = relatedRows[relatedTableId].concat(field)
+      }
+    }
+    for (let tableId of table.relatedFormula) {
+      let relatedTable
+      try {
+        // no rows to update, skip
+        if (!relatedRows[tableId] || relatedRows[tableId].length === 0) {
+          continue
+        }
+        relatedTable = await db.get(tableId)
+      } catch (err) {
+        // no error scenario, table doesn't seem to exist anymore, ignore
+      }
+      for (let column of Object.values(relatedTable.schema)) {
+        // needs updated in related rows
+        if (
+          column.type === FieldTypes.FORMULA &&
+          column.formulaType === FormulaTypes.STATIC
+        ) {
+          // re-enrich rows for all the related, don't update the related formula for them
+          promises = promises.concat(
+            relatedRows[tableId].map(related =>
+              exports.finaliseRow(relatedTable, related, {
+                updateFormula: false,
+              })
+            )
+          )
+        }
+      }
+    }
+  }
+  await Promise.all(promises)
+}
+
+exports.updateAllFormulasInTable = async table => {
+  const db = getAppDB()
+  // start by getting the raw rows (which will be written back to DB after update)
+  let rows = (
+    await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+  // now enrich the rows, note the clone so that we have the base state of the
+  // rows so that we don't write any of the enriched information back
+  let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
+    squash: false,
+  })
+  const updatedRows = []
+  for (let row of rows) {
+    // find the enriched row, if found process the formulas
+    const enrichedRow = enrichedRows.find(enriched => enriched._id === row._id)
+    if (enrichedRow) {
+      const processed = processFormulas(table, cloneDeep(row), {
+        dynamic: false,
+        contextRows: enrichedRow,
+      })
+      // values have changed, need to add to bulk docs to update
+      if (!isEqual(processed, row)) {
+        updatedRows.push(processed)
+      }
+    }
+  }
+  await db.bulkDocs(updatedRows)
+}
+
+/**
+ * This function runs at the end of the save/patch functions of the row controller, all this
+ * really does is enrich the row, handle any static formula processing, then return the enriched
+ * row. The reason we need to return the enriched row is that the automation row created trigger
+ * expects the row to be totally enriched/contain all relationships.
+ */
+exports.finaliseRow = async (
+  table,
+  row,
+  { oldTable, updateFormula } = { updateFormula: true }
+) => {
+  const db = getAppDB()
+  row.type = "row"
+  // process the row before return, to include relationships
+  let enrichedRow = await outputProcessing(table, cloneDeep(row), {
+    squash: false,
+  })
+  // use enriched row to generate formulas for saving, specifically only use as context
+  row = processFormulas(table, row, {
+    dynamic: false,
+    contextRows: enrichedRow,
+  })
+
+  // don't worry about rev, tables handle rev/lastID updates
+  // if another row has been written since processing this will
+  // handle the auto ID clash
+  if (oldTable && !isEqual(oldTable, table)) {
+    try {
+      await db.put(table)
+    } catch (err) {
+      if (err.status === 409) {
+        const updatedTable = await db.get(table._id)
+        let response = processAutoColumn(null, updatedTable, row, {
+          reprocessing: true,
+        })
+        await db.put(response.table)
+        row = response.row
+      } else {
+        throw err
+      }
+    }
+  }
+  const response = await db.put(row)
+  // for response, calculate the formulas for the enriched row
+  enrichedRow._rev = response.rev
+  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  // this updates the related formulas in other rows based on the relations to this row
+  if (updateFormula) {
+    await exports.updateRelatedFormula(table, enrichedRow)
+  }
+  return { row: enrichedRow, table }
+}

packages/server/src/api/controllers/row/utils.js

@@ -1,11 +1,11 @@
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
-const CouchDB = require("../../../db")
 const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
 const { processStringSync } = require("@budibase/string-templates")
 const { makeExternalQuery } = require("../../../integrations/base/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -17,14 +17,15 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-exports.getDatasourceAndQuery = async (appId, json) => {
+exports.getDatasourceAndQuery = async json => {
   const datasourceId = json.endpoint.datasourceId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 
-exports.findRow = async (ctx, db, tableId, rowId) => {
+exports.findRow = async (ctx, tableId, rowId) => {
+  const db = getAppDB()
   let row
   // TODO remove special user case in future
   if (tableId === InternalTables.USER_METADATA) {
@@ -42,9 +43,9 @@ exports.findRow = async (ctx, db, tableId, rowId) => {
   return row
 }
 
-exports.validate = async ({ appId, tableId, row, table }) => {
+exports.validate = async ({ tableId, row, table }) => {
   if (!table) {
-    const db = new CouchDB(appId)
+    const db = getAppDB()
     table = await db.get(tableId)
   }
   const errors = {}

packages/server/src/api/controllers/screen.js

@@ -1,10 +1,9 @@
-const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
 const { AccessController } = require("@budibase/backend-core/roles")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
 
   const screens = (
     await db.allDocs(
@@ -14,15 +13,14 @@ exports.fetch = async ctx => {
     )
   ).rows.map(element => element.doc)
 
-  ctx.body = await new AccessController(appId).checkScreensAccess(
+  ctx.body = await new AccessController().checkScreensAccess(
     screens,
     ctx.user.role._id
   )
 }
 
 exports.save = async ctx => {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   let screen = ctx.request.body
 
   if (!screen._id) {
@@ -39,7 +37,7 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   await db.remove(ctx.params.screenId, ctx.params.screenRev)
   ctx.body = {
     message: "Screen deleted successfully",

packages/server/src/api/controllers/static/index.js

@@ -6,7 +6,6 @@ const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
@@ -17,6 +16,7 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
 const AWS = require("aws-sdk")
 const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
@@ -44,7 +44,7 @@ async function getAppIdFromUrl(ctx) {
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
 
   // search prod apps for a url that matches, exclude dev where id is always used
-  const apps = await getAllApps(CouchDB, { dev: false })
+  const apps = await getAllApps({ dev: false })
   const app = apps.filter(
     a => a.url && a.url.toLowerCase() === possibleAppUrl
   )[0]
@@ -85,7 +85,7 @@ exports.uploadFile = async function (ctx) {
 exports.serveApp = async function (ctx) {
   let appId = await getAppIdFromUrl(ctx)
   const App = require("./templates/BudibaseApp.svelte").default
-  const db = new CouchDB(appId, { skip_setup: true })
+  const db = getAppDB({ skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
 
   const { head, html, css } = App.render({
@@ -111,7 +111,7 @@ exports.serveClientLibrary = async function (ctx) {
 }
 
 exports.getSignedUploadURL = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
+  const database = getAppDB()
 
   // Ensure datasource is valid
   let datasource

packages/server/src/api/controllers/table/bulkFormula.js

@@ -0,0 +1,173 @@
+const { FieldTypes, FormulaTypes } = require("../../../constants")
+const { getAllInternalTables, clearColumns } = require("./utils")
+const { doesContainStrings } = require("@budibase/string-templates")
+const { cloneDeep } = require("lodash/fp")
+const { isEqual, uniq } = require("lodash")
+const { updateAllFormulasInTable } = require("../row/staticFormula")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+function isStaticFormula(column) {
+  return (
+    column.type === FieldTypes.FORMULA &&
+    column.formulaType === FormulaTypes.STATIC
+  )
+}
+
+/**
+ * This retrieves the formula columns from a table schema that use a specified column name
+ * in the formula.
+ */
+function getFormulaThatUseColumn(table, columnNames) {
+  let formula = []
+  columnNames = Array.isArray(columnNames) ? columnNames : [columnNames]
+  for (let column of Object.values(table.schema)) {
+    // not a static formula, or doesn't contain a relationship
+    if (!isStaticFormula(column)) {
+      continue
+    }
+    if (!doesContainStrings(column.formula, columnNames)) {
+      continue
+    }
+    formula.push(column.name)
+  }
+  return formula
+}
+
+/**
+ * This functions checks for when a related table, column or related column is deleted, if any
+ * tables need to have the formula column removed.
+ */
+async function checkIfFormulaNeedsCleared(table, { oldTable, deletion }) {
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables()).filter(
+    tbl => tbl._id !== table._id
+  )
+  const schemaToUse = oldTable ? oldTable.schema : table.schema
+  let removedColumns = Object.values(schemaToUse).filter(
+    column => deletion || !table.schema[column.name]
+  )
+  // remove any formula columns that used related columns
+  for (let removed of removedColumns) {
+    let tableToUse = table
+    // if relationship, get the related table
+    if (removed.type === FieldTypes.LINK) {
+      tableToUse = tables.find(table => table._id === removed.tableId)
+    }
+    const columnsToDelete = getFormulaThatUseColumn(tableToUse, removed.name)
+    if (columnsToDelete.length > 0) {
+      await clearColumns(table, columnsToDelete)
+    }
+    // need a special case, where a column has been removed from this table, but was used
+    // in a different, related tables formula
+    if (!table.relatedFormula) {
+      continue
+    }
+    for (let relatedTableId of table.relatedFormula) {
+      const relatedColumns = Object.values(table.schema).filter(
+        column => column.tableId === relatedTableId
+      )
+      const relatedTable = tables.find(table => table._id === relatedTableId)
+      // look to see if the column was used in a relationship formula,
+      // relationships won't be used for this
+      if (relatedTable && relatedColumns && removed.type !== FieldTypes.LINK) {
+        let relatedFormulaToRemove = []
+        for (let column of relatedColumns) {
+          relatedFormulaToRemove = relatedFormulaToRemove.concat(
+            getFormulaThatUseColumn(relatedTable, [
+              column.fieldName,
+              removed.name,
+            ])
+          )
+        }
+        if (relatedFormulaToRemove.length > 0) {
+          await clearColumns(relatedTable, uniq(relatedFormulaToRemove))
+        }
+      }
+    }
+  }
+}
+
+/**
+ * This function adds a note to related tables that they are
+ * used in a static formula - so that the link controller
+ * can manage hydrating related rows formula fields. This is
+ * specifically only for static formula.
+ */
+async function updateRelatedFormulaLinksOnTables(
+  table,
+  { deletion } = { deletion: false }
+) {
+  const db = getAppDB()
+  // start by retrieving all tables, remove the current table from the list
+  const tables = (await getAllInternalTables()).filter(
+    tbl => tbl._id !== table._id
+  )
+  // clone the tables, so we can compare at end
+  const initialTables = cloneDeep(tables)
+  // first find the related column names
+  const relatedColumns = Object.values(table.schema).filter(
+    col => col.type === FieldTypes.LINK
+  )
+  // we start by removing the formula field from all tables
+  for (let otherTable of tables) {
+    if (!otherTable.relatedFormula) {
+      continue
+    }
+    const index = otherTable.relatedFormula.indexOf(table._id)
+    if (index !== -1) {
+      otherTable.relatedFormula.splice(index, 1)
+    }
+  }
+  // if deleting, just remove the table IDs, don't try add
+  if (!deletion) {
+    for (let relatedCol of relatedColumns) {
+      let columns = getFormulaThatUseColumn(table, relatedCol.name)
+      if (!columns || columns.length === 0) {
+        continue
+      }
+      const relatedTable = tables.find(
+        related => related._id === relatedCol.tableId
+      )
+      // check if the table is already in the list of related formula, if it isn't, then add it
+      if (
+        relatedTable &&
+        (!relatedTable.relatedFormula ||
+          !relatedTable.relatedFormula.includes(table._id))
+      ) {
+        relatedTable.relatedFormula = relatedTable.relatedFormula
+          ? [...relatedTable.relatedFormula, table._id]
+          : [table._id]
+      }
+    }
+  }
+  // now we just need to compare all the tables and see if any need saved
+  for (let initial of initialTables) {
+    const found = tables.find(tbl => initial._id === tbl._id)
+    if (found && !isEqual(initial, found)) {
+      await db.put(found)
+    }
+  }
+}
+
+async function checkIfFormulaUpdated(table, { oldTable }) {
+  // look to see if any formula values have changed
+  const shouldUpdate = Object.values(table.schema).find(
+    column =>
+      isStaticFormula(column) &&
+      (!oldTable ||
+        !oldTable.schema[column.name] ||
+        !isEqual(oldTable.schema[column.name], column))
+  )
+  // if a static formula column has updated, then need to run the update
+  if (shouldUpdate != null) {
+    await updateAllFormulasInTable(table)
+  }
+}
+
+exports.runStaticFormulaChecks = async (table, { oldTable, deletion }) => {
+  await updateRelatedFormulaLinksOnTables(table, { deletion })
+  await checkIfFormulaNeedsCleared(table, { oldTable, deletion })
+  if (!deletion) {
+    await checkIfFormulaUpdated(table, { oldTable })
+  }
+}

packages/server/src/api/controllers/table/external.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const {
   buildExternalTableId,
   breakExternalTableId,
@@ -19,6 +18,7 @@ const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
 const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 async function makeTableRequest(
   datasource,
@@ -159,7 +159,6 @@ function isRelationshipSetup(column) {
 }
 
 exports.save = async function (ctx) {
-  const appId = ctx.appId
   const table = ctx.request.body
   // can't do this right now
   delete table.dataImport
@@ -176,14 +175,14 @@ exports.save = async function (ctx) {
 
   let oldTable
   if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await getTable(appId, ctx.request.body._id)
+    oldTable = await getTable(ctx.request.body._id)
   }
 
   if (hasTypeChanged(tableToSave, oldTable)) {
     ctx.throw(400, "A column type has changed.")
   }
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   const oldTables = cloneDeep(datasource.entities)
   const tables = datasource.entities
@@ -267,14 +266,13 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const appId = ctx.appId
-  const tableToDelete = await getTable(appId, ctx.params.tableId)
+  const tableToDelete = await getTable(ctx.params.tableId)
   if (!tableToDelete || !tableToDelete.created) {
     ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
   }
   const datasourceId = getDatasourceId(tableToDelete)
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   const tables = datasource.entities
 
@@ -290,8 +288,7 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkImport = async function (ctx) {
-  const appId = ctx.appId
-  const table = await getTable(appId, ctx.params.tableId)
+  const table = await getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
   if (!dataImport || !dataImport.schema || !dataImport.csvString) {
     ctx.throw(400, "Provided data import information is invalid.")
@@ -300,7 +297,7 @@ exports.bulkImport = async function (ctx) {
     ...dataImport,
     existingTable: table,
   })
-  await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
+  await handleRequest(DataSourceOperation.BULK_CREATE, table._id, {
     rows,
   })
   return table

packages/server/src/api/controllers/table/index.js

@@ -1,14 +1,10 @@
-const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
 const { isExternalTable, isSQL } = require("../../../integrations/utils")
-const {
-  getTableParams,
-  getDatasourceParams,
-  BudibaseInternalDB,
-} = require("../../../db/utils")
-const { getTable } = require("./utils")
+const { getDatasourceParams } = require("../../../db/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
+const { getTable, getAllInternalTables } = require("./utils")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -24,19 +20,9 @@ function pickApi({ tableId, table }) {
 
 // covers both internal and external
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
 
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-
-  const internal = internalTables.rows.map(tableDoc => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
+  const internal = await getAllInternalTables()
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {
@@ -63,7 +49,7 @@ exports.fetch = async function (ctx) {
 
 exports.find = async function (ctx) {
   const tableId = ctx.params.id
-  ctx.body = await getTable(ctx.appId, tableId)
+  ctx.body = await getTable(tableId)
 }
 
 exports.save = async function (ctx) {
@@ -102,7 +88,7 @@ exports.validateCSVSchema = async function (ctx) {
   const { csvString, schema = {}, tableId } = ctx.request.body
   let existingTable
   if (tableId) {
-    existingTable = await getTable(ctx.appId, tableId)
+    existingTable = await getTable(tableId)
   }
   let result = await csvParser.parse(csvString, schema)
   if (existingTable) {

packages/server/src/api/controllers/table/internal.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const { getRowParams, generateTableID } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
@@ -9,10 +8,13 @@ const {
   handleDataImport,
 } = require("./utils")
 const usageQuota = require("../../../utilities/usageQuota")
+const { getAppDB } = require("@budibase/backend-core/context")
+const env = require("../../../environment")
+const { cleanupAttachments } = require("../../../utilities/rowProcessor")
+const { runStaticFormulaChecks } = require("./bulkFormula")
 
 exports.save = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const { dataImport, ...rest } = ctx.request.body
   let tableToSave = {
     type: "table",
@@ -34,8 +36,7 @@ exports.save = async function (ctx) {
   // saving a table is a complex operation, involving many different steps, this
   // has been broken out into a utility to make it more obvious/easier to manipulate
   const tableSaveFunctions = new TableSaveFunctions({
-    db,
-    ctx,
+    user: ctx.user,
     oldTable,
     dataImport,
   })
@@ -80,7 +81,6 @@ exports.save = async function (ctx) {
   // update linked rows
   try {
     const linkResp = await linkRows.updateLinks({
-      appId,
       eventType: oldTable
         ? linkRows.EventType.TABLE_UPDATED
         : linkRows.EventType.TABLE_SAVE,
@@ -104,13 +104,13 @@ exports.save = async function (ctx) {
   tableToSave._rev = result.rev
 
   tableToSave = await tableSaveFunctions.after(tableToSave)
-
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(tableToSave, { oldTable })
   return tableToSave
 }
 
 exports.destroy = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const tableToDelete = await db.get(ctx.params.tableId)
 
   // Delete all rows for that table
@@ -124,7 +124,6 @@ exports.destroy = async function (ctx) {
 
   // update linked rows
   await linkRows.updateLinks({
-    appId,
     eventType: linkRows.EventType.TABLE_DELETE,
     table: tableToDelete,
   })
@@ -133,21 +132,25 @@ exports.destroy = async function (ctx) {
   await db.remove(tableToDelete)
 
   // remove table search index
-  const currentIndexes = await db.getIndexes()
-  const existingIndex = currentIndexes.indexes.find(
-    existing => existing.name === `search:${ctx.params.tableId}`
-  )
-  if (existingIndex) {
-    await db.deleteIndex(existingIndex)
+  if (!env.isTest()) {
+    const currentIndexes = await db.getIndexes()
+    const existingIndex = currentIndexes.indexes.find(
+      existing => existing.name === `search:${ctx.params.tableId}`
+    )
+    if (existingIndex) {
+      await db.deleteIndex(existingIndex)
+    }
   }
 
+  // has to run after, make sure it has _id
+  await runStaticFormulaChecks(tableToDelete, { deletion: true })
+  await cleanupAttachments(tableToDelete, { rows })
   return tableToDelete
 }
 
 exports.bulkImport = async function (ctx) {
-  const appId = ctx.appId
-  const table = await getTable(appId, ctx.params.tableId)
+  const table = await getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
-  await handleDataImport(appId, ctx.user, table, dataImport)
+  await handleDataImport(ctx.user, table, dataImport)
   return table
 }

packages/server/src/api/controllers/table/utils.js

@@ -1,14 +1,22 @@
-const CouchDB = require("../../../db")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
   generateRowID,
   InternalTables,
+  getTableParams,
+  BudibaseInternalDB,
 } = require("../../../db/utils")
-const { isEqual } = require("lodash/fp")
+const { isEqual } = require("lodash")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
-const { inputProcessing } = require("../../../utilities/rowProcessor")
-const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
+const {
+  inputProcessing,
+  cleanupAttachments,
+} = require("../../../utilities/rowProcessor")
+const {
+  USERS_TABLE_SCHEMA,
+  SwitchableTypes,
+  CanSwitchTypes,
+} = require("../../../constants")
 const {
   isExternalTable,
   breakExternalTableId,
@@ -17,8 +25,26 @@ const {
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
 const usageQuota = require("../../../utilities/usageQuota")
+const { getAppDB } = require("@budibase/backend-core/context")
+const { cloneDeep } = require("lodash/fp")
 
-exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
+exports.clearColumns = async (table, columnNames) => {
+  const db = getAppDB()
+  const rows = await db.allDocs(
+    getRowParams(table._id, null, {
+      include_docs: true,
+    })
+  )
+  return db.bulkDocs(
+    rows.rows.map(({ doc }) => {
+      columnNames.forEach(colName => delete doc[colName])
+      return doc
+    })
+  )
+}
+
+exports.checkForColumnUpdates = async (oldTable, updatedTable) => {
+  const db = getAppDB()
   let updatedRows = []
   const rename = updatedTable._rename
   let deletedColumns = []
@@ -35,18 +61,22 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
         include_docs: true,
       })
     )
-    updatedRows = rows.rows.map(({ doc }) => {
+    const rawRows = rows.rows.map(({ doc }) => doc)
+    updatedRows = rawRows.map(row => {
+      row = cloneDeep(row)
       if (rename) {
-        doc[rename.updated] = doc[rename.old]
-        delete doc[rename.old]
+        row[rename.updated] = row[rename.old]
+        delete row[rename.old]
       } else if (deletedColumns.length !== 0) {
-        deletedColumns.forEach(colName => delete doc[colName])
+        deletedColumns.forEach(colName => delete row[colName])
       }
-      return doc
+      return row
     })
 
+    // cleanup any attachments from object storage for deleted attachment columns
+    await cleanupAttachments(updatedTable, { oldTable, rows: rawRows })
     // Update views
-    await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
+    await exports.checkForViewUpdates(updatedTable, rename, deletedColumns)
     delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
@@ -73,12 +103,12 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
   return tableToSave
 }
 
-exports.handleDataImport = async (appId, user, table, dataImport) => {
+exports.handleDataImport = async (user, table, dataImport) => {
   if (!dataImport || !dataImport.csvString) {
     return table
   }
 
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   // Populate the table with rows imported from CSV in a bulk update
   const data = await csvParser.transform({
     ...dataImport,
@@ -123,8 +153,8 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
   return table
 }
 
-exports.handleSearchIndexes = async (appId, table) => {
-  const db = new CouchDB(appId)
+exports.handleSearchIndexes = async table => {
+  const db = getAppDB()
   // create relevant search indexes
   if (table.indexes && table.indexes.length > 0) {
     const currentIndexes = await db.getIndexes()
@@ -181,12 +211,9 @@ exports.checkStaticTables = table => {
 }
 
 class TableSaveFunctions {
-  constructor({ db, ctx, oldTable, dataImport }) {
-    this.db = db
-    this.ctx = ctx
-    if (this.ctx && this.ctx.user) {
-      this.appId = this.ctx.appId
-    }
+  constructor({ user, oldTable, dataImport }) {
+    this.db = getAppDB()
+    this.user = user
     this.oldTable = oldTable
     this.dataImport = dataImport
     // any rows that need updated
@@ -204,24 +231,15 @@ class TableSaveFunctions {
 
   // when confirmed valid
   async mid(table) {
-    let response = await exports.checkForColumnUpdates(
-      this.db,
-      this.oldTable,
-      table
-    )
+    let response = await exports.checkForColumnUpdates(this.oldTable, table)
     this.rows = this.rows.concat(response.rows)
     return table
   }
 
   // after saving
   async after(table) {
-    table = await exports.handleSearchIndexes(this.appId, table)
-    table = await exports.handleDataImport(
-      this.appId,
-      this.ctx.user,
-      table,
-      this.dataImport
-    )
+    table = await exports.handleSearchIndexes(table)
+    table = await exports.handleDataImport(this.user, table, this.dataImport)
     return table
   }
 
@@ -230,8 +248,22 @@ class TableSaveFunctions {
   }
 }
 
-exports.getAllExternalTables = async (appId, datasourceId) => {
-  const db = new CouchDB(appId)
+exports.getAllInternalTables = async () => {
+  const db = getAppDB()
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map(tableDoc => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+exports.getAllExternalTables = async datasourceId => {
+  const db = getAppDB()
   const datasource = await db.get(datasourceId)
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
@@ -239,25 +271,25 @@ exports.getAllExternalTables = async (appId, datasourceId) => {
   return datasource.entities
 }
 
-exports.getExternalTable = async (appId, datasourceId, tableName) => {
-  const entities = await exports.getAllExternalTables(appId, datasourceId)
+exports.getExternalTable = async (datasourceId, tableName) => {
+  const entities = await exports.getAllExternalTables(datasourceId)
   return entities[tableName]
 }
 
-exports.getTable = async (appId, tableId) => {
-  const db = new CouchDB(appId)
+exports.getTable = async tableId => {
+  const db = getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     const datasource = await db.get(datasourceId)
-    const table = await exports.getExternalTable(appId, datasourceId, tableName)
+    const table = await exports.getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {
     return db.get(tableId)
   }
 }
 
-exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
-  const views = await getViews(db)
+exports.checkForViewUpdates = async (table, rename, deletedColumns) => {
+  const views = await getViews()
   const tableViews = views.filter(view => view.meta.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
@@ -319,7 +351,7 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
     // Update view if required
     if (needsUpdated) {
       const newViewTemplate = viewTemplate(view.meta)
-      await saveView(db, null, view.name, newViewTemplate)
+      await saveView(null, view.name, newViewTemplate)
       if (!newViewTemplate.meta.schema) {
         newViewTemplate.meta.schema = table.schema
       }
@@ -348,6 +380,23 @@ exports.foreignKeyStructure = (keyName, meta = null) => {
   return structure
 }
 
+exports.areSwitchableTypes = (type1, type2) => {
+  if (
+    SwitchableTypes.indexOf(type1) === -1 &&
+    SwitchableTypes.indexOf(type2) === -1
+  ) {
+    return false
+  }
+  for (let option of CanSwitchTypes) {
+    const index1 = option.indexOf(type1),
+      index2 = option.indexOf(type2)
+    if (index1 !== -1 && index2 !== -1 && index1 !== index2) {
+      return true
+    }
+  }
+  return false
+}
+
 exports.hasTypeChanged = (table, oldTable) => {
   if (!oldTable) {
     return false
@@ -358,7 +407,7 @@ exports.hasTypeChanged = (table, oldTable) => {
       continue
     }
     const newType = table.schema[key].type
-    if (oldType !== newType && SwitchableTypes.indexOf(oldType) === -1) {
+    if (oldType !== newType && !exports.areSwitchableTypes(oldType, newType)) {
       return true
     }
   }

packages/server/src/api/controllers/user.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../db")
 const {
   generateUserMetadataID,
   getUserMetadataParams,
@@ -11,12 +10,14 @@ const { isEqual } = require("lodash")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const {
   getDevelopmentAppID,
-  getDeployedAppIDs,
+  getProdAppIDs,
+  dbExists,
 } = require("@budibase/backend-core/db")
-const { doesDatabaseExist } = require("../../utilities")
 const { UserStatus } = require("@budibase/backend-core/constants")
+const { getAppDB } = require("@budibase/backend-core/context")
 
-async function rawMetadata(db) {
+async function rawMetadata() {
+  const db = getAppDB()
   return (
     await db.allDocs(
       getUserMetadataParams(null, {
@@ -54,13 +55,10 @@ function combineMetadataAndUser(user, metadata) {
   return null
 }
 
-exports.syncGlobalUsers = async appId => {
+exports.syncGlobalUsers = async () => {
   // sync user metadata
-  const db = new CouchDB(appId)
-  const [users, metadata] = await Promise.all([
-    getGlobalUsers(appId),
-    rawMetadata(db),
-  ])
+  const db = getAppDB()
+  const [users, metadata] = await Promise.all([getGlobalUsers(), rawMetadata()])
   const toWrite = []
   for (let user of users) {
     const combined = await combineMetadataAndUser(user, metadata)
@@ -94,7 +92,7 @@ exports.syncUser = async function (ctx) {
   let prodAppIds
   // if they are a builder then get all production app IDs
   if ((user.builder && user.builder.global) || deleting) {
-    prodAppIds = await getDeployedAppIDs(CouchDB)
+    prodAppIds = await getProdAppIDs()
   } else {
     prodAppIds = Object.entries(roles)
       .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
@@ -104,10 +102,10 @@ exports.syncUser = async function (ctx) {
     const roleId = roles[prodAppId]
     const devAppId = getDevelopmentAppID(prodAppId)
     for (let appId of [prodAppId, devAppId]) {
-      if (!(await doesDatabaseExist(appId))) {
+      if (!(await dbExists(appId))) {
         continue
       }
-      const db = new CouchDB(appId)
+      const db = getAppDB()
       const metadataId = generateUserMetadataID(userId)
       let metadata
       try {
@@ -143,8 +141,8 @@ exports.syncUser = async function (ctx) {
 }
 
 exports.fetchMetadata = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
-  const global = await getGlobalUsers(ctx.appId)
+  const database = getAppDB()
+  const global = await getGlobalUsers()
   const metadata = await rawMetadata(database)
   const users = []
   for (let user of global) {
@@ -173,8 +171,7 @@ exports.updateSelfMetadata = async function (ctx) {
 }
 
 exports.updateMetadata = async function (ctx) {
-  const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const db = getAppDB()
   const user = ctx.request.body
   // this isn't applicable to the user
   delete user.roles
@@ -186,7 +183,7 @@ exports.updateMetadata = async function (ctx) {
 }
 
 exports.destroyMetadata = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   try {
     const dbUser = await db.get(ctx.params.id)
     await db.remove(dbUser._id, dbUser._rev)
@@ -209,7 +206,7 @@ exports.setFlag = async function (ctx) {
     ctx.throw(400, "Must supply a 'flag' field in request body.")
   }
   const flagDocId = generateUserFlagID(userId)
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let doc
   try {
     doc = await db.get(flagDocId)
@@ -224,7 +221,7 @@ exports.setFlag = async function (ctx) {
 exports.getFlags = async function (ctx) {
   const userId = ctx.user._id
   const docId = generateUserFlagID(userId)
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   let doc
   try {
     doc = await db.get(docId)

packages/server/src/api/controllers/view/index.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../../../db")
 const viewTemplate = require("./viewBuilder")
 const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
@@ -6,14 +5,14 @@ const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
 const { getTable } = require("../table/utils")
 const { FieldTypes } = require("../../../constants")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 exports.fetch = async ctx => {
-  const db = new CouchDB(ctx.appId)
-  ctx.body = await getViews(db)
+  ctx.body = await getViews()
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const { originalName, ...viewToSave } = ctx.request.body
   const view = viewTemplate(viewToSave)
 
@@ -21,7 +20,7 @@ exports.save = async ctx => {
     ctx.throw(400, "Cannot create view without a name")
   }
 
-  await saveView(db, originalName, viewToSave.name, view)
+  await saveView(originalName, viewToSave.name, view)
 
   // add views to table document
   const table = await db.get(ctx.request.body.tableId)
@@ -42,9 +41,9 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const viewName = decodeURI(ctx.params.viewName)
-  const view = await deleteView(db, viewName)
+  const view = await deleteView(viewName)
   const table = await db.get(view.meta.tableId)
   delete table.views[viewName]
   await db.put(table)
@@ -53,9 +52,8 @@ exports.destroy = async ctx => {
 }
 
 exports.exportView = async ctx => {
-  const db = new CouchDB(ctx.appId)
   const viewName = decodeURI(ctx.query.view)
-  const view = await getView(db, viewName)
+  const view = await getView(viewName)
 
   const format = ctx.query.format
   if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
@@ -83,7 +81,7 @@ exports.exportView = async ctx => {
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
     const tableId = ctx.params.tableId || view.meta.tableId
-    const table = await getTable(ctx.appId, tableId)
+    const table = await getTable(tableId)
     schema = table.schema
   }
 

packages/server/src/api/controllers/view/utils.js

@@ -6,8 +6,10 @@ const {
   SEPARATOR,
 } = require("../../../db/utils")
 const env = require("../../../environment")
+const { getAppDB } = require("@budibase/backend-core/context")
 
-exports.getView = async (db, viewName) => {
+exports.getView = async viewName => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     return designDoc.views[viewName]
@@ -22,7 +24,8 @@ exports.getView = async (db, viewName) => {
   }
 }
 
-exports.getViews = async db => {
+exports.getViews = async () => {
+  const db = getAppDB()
   const response = []
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
@@ -54,7 +57,8 @@ exports.getViews = async db => {
   return response
 }
 
-exports.saveView = async (db, originalName, viewName, viewTemplate) => {
+exports.saveView = async (originalName, viewName, viewTemplate) => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     designDoc.views = {
@@ -91,7 +95,8 @@ exports.saveView = async (db, originalName, viewName, viewTemplate) => {
   }
 }
 
-exports.deleteView = async (db, viewName) => {
+exports.deleteView = async viewName => {
+  const db = getAppDB()
   if (env.SELF_HOSTED) {
     const designDoc = await db.get("_design/database")
     const view = designDoc.views[viewName]

packages/server/src/api/controllers/webhook.js

@@ -1,9 +1,9 @@
-const CouchDB = require("../../db")
 const { generateWebhookID, getWebhookParams } = require("../../db/utils")
 const toJsonSchema = require("to-json-schema")
 const validate = require("jsonschema").validate
 const triggers = require("../../automations/triggers")
-const { getDeployedAppID } = require("@budibase/backend-core/db")
+const { getProdAppID } = require("@budibase/backend-core/db")
+const { getAppDB, updateAppId } = require("@budibase/backend-core/context")
 
 const AUTOMATION_DESCRIPTION = "Generated from Webhook Schema"
 
@@ -23,7 +23,7 @@ exports.WebhookType = {
 }
 
 exports.fetch = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const response = await db.allDocs(
     getWebhookParams(null, {
       include_docs: true,
@@ -33,7 +33,7 @@ exports.fetch = async ctx => {
 }
 
 exports.save = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   const webhook = ctx.request.body
   webhook.appId = ctx.appId
 
@@ -52,12 +52,13 @@ exports.save = async ctx => {
 }
 
 exports.destroy = async ctx => {
-  const db = new CouchDB(ctx.appId)
+  const db = getAppDB()
   ctx.body = await db.remove(ctx.params.id, ctx.params.rev)
 }
 
 exports.buildSchema = async ctx => {
-  const db = new CouchDB(ctx.params.instance)
+  updateAppId(ctx.params.instance)
+  const db = getAppDB()
   const webhook = await db.get(ctx.params.id)
   webhook.bodySchema = toJsonSchema(ctx.request.body)
   // update the automation outputs
@@ -81,9 +82,10 @@ exports.buildSchema = async ctx => {
 }
 
 exports.trigger = async ctx => {
-  const prodAppId = getDeployedAppID(ctx.params.instance)
+  const prodAppId = getProdAppID(ctx.params.instance)
+  updateAppId(prodAppId)
   try {
-    const db = new CouchDB(prodAppId)
+    const db = getAppDB()
     const webhook = await db.get(ctx.params.id)
     // validate against the schema
     if (webhook.bodySchema) {

packages/server/src/api/routes/index.js

@@ -24,6 +24,7 @@ const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -53,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes