Merge branch 'feature/draft-apps' into admin/user-management-ui

packages/auth/README.md

@@ -1 +1,12 @@
-# Budibase Authentication Library
+# Budibase Core backend library
+
+This library contains core functionality, like auth and security features
+which are shared between backend services.
+
+#### Note about top level JS files
+For the purposes of being able to do say `require("@budibase/auth/permissions")` we need to
+specify the exports at the top-level of the module.
+
+For these files they should be limited to a single `require` of the file that should
+be exported and then a single `module.exports = ...` to export the file in
+commonJS.

packages/auth/db.js

@@ -0,0 +1 @@
+module.exports = require("./src/db/utils")

packages/auth/package.json

@@ -11,6 +11,7 @@
     "ioredis": "^4.27.1",
     "jsonwebtoken": "^8.5.1",
     "koa-passport": "^4.1.4",
+    "lodash": "^4.17.21",
     "node-fetch": "^2.6.1",
     "passport-google-auth": "^1.0.2",
     "passport-google-oauth": "^2.0.0",

packages/auth/permissions.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/permissions")

packages/auth/redis.js

@@ -0,0 +1,4 @@
+module.exports = {
+  Client: require("./src/redis"),
+  utils: require("./src/redis/utils"),
+}

packages/auth/roles.js

@@ -0,0 +1 @@
+module.exports = require("./src/security/roles")

packages/auth/src/constants.js

@@ -14,3 +14,10 @@ exports.GlobalRoles = {
   BUILDER: "builder",
   GROUP_MANAGER: "group_manager",
 }
+
+exports.Configs = {
+  SETTINGS: "settings",
+  ACCOUNT: "account",
+  SMTP: "smtp",
+  GOOGLE: "google",
+}

packages/auth/src/db/Replication.js

@@ -0,0 +1,74 @@
+const { getDB } = require(".")
+
+class Replication {
+  /**
+   *
+   * @param {String} source - the DB you want to replicate or rollback to
+   * @param {String} target - the DB you want to replicate to, or rollback from
+   */
+  constructor({ source, target }) {
+    this.source = getDB(source)
+    this.target = getDB(target)
+  }
+
+  promisify(operation, opts = {}) {
+    return new Promise(resolve => {
+      operation(this.target, opts)
+        .on("denied", function (err) {
+          // a document failed to replicate (e.g. due to permissions)
+          throw new Error(`Denied: Document failed to replicate ${err}`)
+        })
+        .on("complete", function (info) {
+          return resolve(info)
+        })
+        .on("error", function (err) {
+          throw new Error(`Replication Error: ${err}`)
+        })
+    })
+  }
+
+  /**
+   * Two way replication operation, intended to be promise based.
+   * @param {Object} opts - PouchDB replication options
+   */
+  sync(opts) {
+    this.replication = this.promisify(this.source.sync, opts)
+    return this.replication
+  }
+
+  /**
+   * One way replication operation, intended to be promise based.
+   * @param {Object} opts - PouchDB replication options
+   */
+  replicate(opts) {
+    this.replication = this.promisify(this.source.replicate.to, opts)
+    return this.replication
+  }
+
+  /**
+   * Set up an ongoing live sync between 2 CouchDB databases.
+   * @param {Object} opts - PouchDB replication options
+   */
+  subscribe(opts = {}) {
+    this.replication = this.source.replicate
+      .to(this.target, {
+        live: true,
+        retry: true,
+        ...opts,
+      })
+      .on("error", function (err) {
+        throw new Error(`Replication Error: ${err}`)
+      })
+  }
+
+  async rollback() {
+    await this.target.destroy()
+    await this.replicate()
+  }
+
+  cancel() {
+    this.replication.cancel()
+  }
+}
+
+module.exports = Replication

packages/auth/src/db/index.js

@@ -7,3 +7,7 @@ module.exports.setDB = pouch => {
 module.exports.getDB = dbName => {
   return new Pouch(dbName)
 }
+
+module.exports.getCouch = () => {
+  return Pouch
+}

packages/auth/src/db/utils.js

@@ -1,4 +1,9 @@
 const { newid } = require("../hashing")
+const Replication = require("./Replication")
+const { getCouch } = require("./index")
+
+const UNICODE_MAX = "\ufff0"
+const SEPARATOR = "_"
 
 exports.ViewNames = {
   USER_BY_EMAIL: "by_email",
@@ -12,19 +17,42 @@ exports.StaticDatabases = {
 
 const DocumentTypes = {
   USER: "us",
-  APP: "app",
   GROUP: "group",
   CONFIG: "config",
   TEMPLATE: "template",
+  APP: "app",
+  APP_DEV: "app_dev",
+  ROLE: "role",
 }
 
 exports.DocumentTypes = DocumentTypes
-
-const UNICODE_MAX = "\ufff0"
-const SEPARATOR = "_"
-
+exports.APP_PREFIX = DocumentTypes.APP + SEPARATOR
+exports.APP_DEV_PREFIX = DocumentTypes.APP_DEV + SEPARATOR
 exports.SEPARATOR = SEPARATOR
 
+/**
+ * If creating DB allDocs/query params with only a single top level ID this can be used, this
+ * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
+ * More complex cases such as link docs and rows which have multiple levels of IDs that their
+ * ID consists of need their own functions to build the allDocs parameters.
+ * @param {string} docType The type of document which input params are being built for, e.g. user,
+ * link, app, table and so on.
+ * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
+ * for a singular document.
+ * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+function getDocParams(docType, docId = null, otherProps = {}) {
+  if (docId == null) {
+    docId = ""
+  }
+  return {
+    ...otherProps,
+    startkey: `${docType}${SEPARATOR}${docId}`,
+    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
+  }
+}
+
 /**
  * Generates a new group ID.
  * @returns {string} The new group ID which the group doc can be stored under.
@@ -94,6 +122,49 @@ exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
   }
 }
 
+/**
+ * Generates a new role ID.
+ * @returns {string} The new role ID which the role doc can be stored under.
+ */
+exports.generateRoleID = id => {
+  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
+}
+
+/**
+ * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
+ */
+exports.getRoleParams = (roleId = null, otherProps = {}) => {
+  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
+}
+
+/**
+ * Lots of different points in the system need to find the full list of apps, this will
+ * enumerate the entire CouchDB cluster and get the list of databases (every app).
+ * NOTE: this operation is fine in self hosting, but cannot be used when hosting many
+ * different users/companies apps as there is no security around it - all apps are returned.
+ * @return {Promise<object[]>} returns the app information document stored in each app database.
+ */
+exports.getAllApps = async (devApps = false) => {
+  const CouchDB = getCouch()
+  let allDbs = await CouchDB.allDbs()
+  const appDbNames = allDbs.filter(dbName => dbName.startsWith(exports.APP_PREFIX))
+  const appPromises = appDbNames.map(db => new CouchDB(db).get(db))
+  if (appPromises.length === 0) {
+    return []
+  } else {
+    const response = await Promise.allSettled(appPromises)
+    const apps = response
+      .filter(result => result.status === "fulfilled")
+      .map(({ value }) => value)
+    return apps.filter(app => {
+      if (devApps) {
+        return app._id.startsWith(exports.APP_DEV_PREFIX)
+      }
+      return !app._id.startsWith(exports.APP_DEV_PREFIX)
+    })
+  }
+}
+
 /**
  * Generates a new configuration ID.
  * @returns {string} The new configuration ID which the config doc can be stored under.
@@ -165,6 +236,7 @@ async function getScopedConfig(db, params) {
   return configDoc && configDoc.config ? configDoc.config : configDoc
 }
 
+exports.Replication = Replication
 exports.getScopedConfig = getScopedConfig
 exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams

packages/auth/src/objectStore/index.js

@@ -10,6 +10,7 @@ const fs = require("fs")
 const env = require("../environment")
 const { budibaseTempDir, ObjectStoreBuckets } = require("./utils")
 const { v4 } = require("uuid")
+const { APP_PREFIX, APP_DEV_PREFIX } = require("../db/utils")
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@@ -28,6 +29,16 @@ const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.js,
 ]
 
+// does normal sanitization and then swaps dev apps to apps
+function sanitizeKey(input) {
+  return sanitize(sanitizeBucket(input)).replace(/\\/g, "/")
+}
+
+// simply handles the dev app to app conversion
+function sanitizeBucket(input) {
+  return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
+}
+
 function publicPolicy(bucketName) {
   return {
     Version: "2012-10-17",
@@ -61,7 +72,7 @@ exports.ObjectStore = bucket => {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
     params: {
-      Bucket: bucket,
+      Bucket: sanitizeBucket(bucket),
     },
   }
   if (env.MINIO_URL) {
@@ -75,6 +86,7 @@ exports.ObjectStore = bucket => {
  * if it does not exist then it will create it.
  */
 exports.makeSureBucketExists = async (client, bucketName) => {
+  bucketName = sanitizeBucket(bucketName)
   try {
     await client
       .headBucket({
@@ -114,16 +126,22 @@ exports.makeSureBucketExists = async (client, bucketName) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-exports.upload = async ({ bucket, filename, path, type, metadata }) => {
+exports.upload = async ({
+  bucket: bucketName,
+  filename,
+  path,
+  type,
+  metadata,
+}) => {
   const extension = [...filename.split(".")].pop()
   const fileBytes = fs.readFileSync(path)
 
-  const objectStore = exports.ObjectStore(bucket)
-  await exports.makeSureBucketExists(objectStore, bucket)
+  const objectStore = exports.ObjectStore(bucketName)
+  await exports.makeSureBucketExists(objectStore, bucketName)
 
   const config = {
     // windows file paths need to be converted to forward slashes for s3
-    Key: sanitize(filename).replace(/\\/g, "/"),
+    Key: sanitizeKey(filename),
     Body: fileBytes,
     ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()],
   }
@@ -137,13 +155,13 @@ exports.upload = async ({ bucket, filename, path, type, metadata }) => {
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-exports.streamUpload = async (bucket, filename, stream) => {
-  const objectStore = exports.ObjectStore(bucket)
-  await exports.makeSureBucketExists(objectStore, bucket)
+exports.streamUpload = async (bucketName, filename, stream) => {
+  const objectStore = exports.ObjectStore(bucketName)
+  await exports.makeSureBucketExists(objectStore, bucketName)
 
   const params = {
-    Bucket: bucket,
-    Key: sanitize(filename).replace(/\\/g, "/"),
+    Bucket: sanitizeBucket(bucketName),
+    Key: sanitizeKey(filename),
     Body: stream,
   }
   return objectStore.upload(params).promise()
@@ -153,11 +171,11 @@ exports.streamUpload = async (bucket, filename, stream) => {
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-exports.retrieve = async (bucket, filepath) => {
-  const objectStore = exports.ObjectStore(bucket)
+exports.retrieve = async (bucketName, filepath) => {
+  const objectStore = exports.ObjectStore(bucketName)
   const params = {
-    Bucket: bucket,
-    Key: sanitize(filepath).replace(/\\/g, "/"),
+    Bucket: sanitizeBucket(bucketName),
+    Key: sanitizeKey(filepath),
   }
   const response = await objectStore.getObject(params).promise()
   // currently these are all strings
@@ -171,17 +189,21 @@ exports.retrieve = async (bucket, filepath) => {
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-exports.retrieveToTmp = async (bucket, filepath) => {
-  const data = await exports.retrieve(bucket, filepath)
+exports.retrieveToTmp = async (bucketName, filepath) => {
+  bucketName = sanitizeBucket(bucketName)
+  filepath = sanitizeKey(filepath)
+  const data = await exports.retrieve(bucketName, filepath)
   const outputPath = join(budibaseTempDir(), v4())
   fs.writeFileSync(outputPath, data)
   return outputPath
 }
 
-exports.deleteFolder = async (bucket, folder) => {
-  const client = exports.ObjectStore(bucket)
+exports.deleteFolder = async (bucketName, folder) => {
+  bucketName = sanitizeBucket(bucketName)
+  folder = sanitizeKey(folder)
+  const client = exports.ObjectStore(bucketName)
   const listParams = {
-    Bucket: bucket,
+    Bucket: bucketName,
     Prefix: folder,
   }
 
@@ -190,7 +212,7 @@ exports.deleteFolder = async (bucket, folder) => {
     return
   }
   const deleteParams = {
-    Bucket: bucket,
+    Bucket: bucketName,
     Delete: {
       Objects: [],
     },
@@ -203,28 +225,31 @@ exports.deleteFolder = async (bucket, folder) => {
   response = await client.deleteObjects(deleteParams).promise()
   // can only empty 1000 items at once
   if (response.Deleted.length === 1000) {
-    return exports.deleteFolder(bucket, folder)
+    return exports.deleteFolder(bucketName, folder)
   }
 }
 
-exports.uploadDirectory = async (bucket, localPath, bucketPath) => {
+exports.uploadDirectory = async (bucketName, localPath, bucketPath) => {
+  bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
   for (let file of files) {
-    const path = join(bucketPath, file.name)
+    const path = sanitizeKey(join(bucketPath, file.name))
     const local = join(localPath, file.name)
     if (file.isDirectory()) {
-      uploads.push(exports.uploadDirectory(bucket, local, path))
+      uploads.push(exports.uploadDirectory(bucketName, local, path))
     } else {
       uploads.push(
-        exports.streamUpload(bucket, path, fs.createReadStream(local))
+        exports.streamUpload(bucketName, path, fs.createReadStream(local))
       )
     }
   }
   await Promise.all(uploads)
 }
 
-exports.downloadTarball = async (url, bucket, path) => {
+exports.downloadTarball = async (url, bucketName, path) => {
+  bucketName = sanitizeBucket(bucketName)
+  path = sanitizeKey(path)
   const response = await fetch(url)
   if (!response.ok) {
     throw new Error(`unexpected response ${response.statusText}`)
@@ -233,7 +258,7 @@ exports.downloadTarball = async (url, bucket, path) => {
   const tmpPath = join(budibaseTempDir(), path)
   await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
   if (!env.isTest()) {
-    await exports.uploadDirectory(bucket, tmpPath, path)
+    await exports.uploadDirectory(bucketName, tmpPath, path)
   }
   // return the temporary path incase there is a use for it
   return tmpPath

packages/auth/src/redis/index.js

@@ -144,8 +144,8 @@ class RedisWrapper {
 
   async clear() {
     const db = this._db
-    let items = await this.scan(db)
-    await Promise.all(items.map(obj => this.delete(db, obj.key)))
+    let items = await this.scan()
+    await Promise.all(items.map(obj => this.delete(obj.key)))
   }
 }
 

packages/auth/src/redis/utils.js

@@ -9,6 +9,7 @@ const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD
 exports.Databases = {
   PW_RESETS: "pwReset",
   INVITATIONS: "invitation",
+  DEV_LOCKS: "devLocks",
 }
 
 exports.getRedisOptions = (clustered = false) => {
@@ -31,6 +32,9 @@ exports.getRedisOptions = (clustered = false) => {
 }
 
 exports.addDbPrefix = (db, key) => {
+  if (key.includes(db)) {
+    return key
+  }
   return `${db}${SEPARATOR}${key}`
 }
 

packages/auth/src/security/roles.js

@@ -1,7 +1,7 @@
-const CouchDB = require("../../db")
+const { getDB } = require("../db")
 const { cloneDeep } = require("lodash/fp")
 const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions")
-const { generateRoleID, DocumentTypes, SEPARATOR } = require("../../db/utils")
+const { generateRoleID, getRoleParams, DocumentTypes, SEPARATOR } = require("../db/utils")
 
 const BUILTIN_IDS = {
   ADMIN: "ADMIN",
@@ -11,6 +11,14 @@ const BUILTIN_IDS = {
   BUILDER: "BUILDER",
 }
 
+// exclude internal roles like builder
+const EXTERNAL_BUILTIN_ROLE_IDS = [
+  BUILTIN_IDS.ADMIN,
+  BUILTIN_IDS.POWER,
+  BUILTIN_IDS.BASIC,
+  BUILTIN_IDS.PUBLIC,
+]
+
 function Role(id, name) {
   this._id = id
   this.name = name
@@ -116,7 +124,7 @@ exports.getRole = async (appId, roleId) => {
     )
   }
   try {
-    const db = new CouchDB(appId)
+    const db = getDB(appId)
     const dbRole = await db.get(exports.getDBRoleID(roleId))
     role = Object.assign(role, dbRole)
     // finalise the ID
@@ -145,7 +153,7 @@ async function getAllUserRoles(appId, userRoleId) {
     currentRole &&
     currentRole.inherits &&
     roleIds.indexOf(currentRole.inherits) === -1
-  ) {
+    ) {
     roleIds.push(currentRole.inherits)
     currentRole = await exports.getRole(appId, currentRole.inherits)
     roles.push(currentRole)
@@ -192,6 +200,39 @@ exports.getUserPermissions = async (appId, userRoleId) => {
   }
 }
 
+/**
+ * Given an app ID this will retrieve all of the roles that are currently within that app.
+ * @param {string} appId The ID of the app to retrieve the roles from.
+ * @return {Promise<object[]>} An array of the role objects that were found.
+ */
+exports.getAllRoles = async appId => {
+  const db = getDB(appId)
+  const body = await db.allDocs(
+    getRoleParams(null, {
+      include_docs: true,
+    })
+  )
+  let roles = body.rows.map(row => row.doc)
+  const builtinRoles = exports.getBuiltinRoles()
+
+  // need to combine builtin with any DB record of them (for sake of permissions)
+  for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) {
+    const builtinRole = builtinRoles[builtinRoleId]
+    const dbBuiltin = roles.filter(
+      dbRole => exports.getExternalRoleID(dbRole._id) === builtinRoleId
+    )[0]
+    if (dbBuiltin == null) {
+      roles.push(builtinRole)
+    } else {
+      // remove role and all back after combining with the builtin
+      roles = roles.filter(role => role._id !== dbBuiltin._id)
+      dbBuiltin._id = exports.getExternalRoleID(dbBuiltin._id)
+      roles.push(Object.assign(builtinRole, dbBuiltin))
+    }
+  }
+  return roles
+}
+
 class AccessController {
   constructor(appId) {
     this.appId = appId

packages/bbui/src/Link/Link.svelte

@@ -11,6 +11,7 @@
 </script>
 
 <a
+  on:click
   {href}
   {target}
   class:spectrum-Link--primary={primary}

packages/builder/src/builderStore/store/hosting.js

@@ -2,7 +2,6 @@ import { writable } from "svelte/store"
 import api, { get } from "../api"
 
 const INITIAL_HOSTING_UI_STATE = {
-  hostingInfo: {},
   appUrl: "",
   deployedApps: {},
   deployedAppNames: [],
@@ -13,28 +12,12 @@ export const getHostingStore = () => {
   const store = writable({ ...INITIAL_HOSTING_UI_STATE })
   store.actions = {
     fetch: async () => {
-      const responses = await Promise.all([
-        api.get("/api/hosting/"),
-        api.get("/api/hosting/urls"),
-      ])
-      const [info, urls] = await Promise.all(responses.map(resp => resp.json()))
+      const response = await api.get("/api/hosting/urls")
+      const urls = await response.json()
       store.update(state => {
-        state.hostingInfo = info
         state.appUrl = urls.app
         return state
       })
-      return info
-    },
-    save: async hostingInfo => {
-      const response = await api.post("/api/hosting", hostingInfo)
-      const revision = (await response.json()).rev
-      store.update(state => {
-        state.hostingInfo = {
-          ...hostingInfo,
-          _rev: revision,
-        }
-        return state
-      })
     },
     fetchDeployedApps: async () => {
       let deployments = await (await get("/api/hosting/apps")).json()

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -50,7 +50,7 @@
       <div class="section">
         {#each categories as [categoryName, bindings]}
           <Heading size="XS">{categoryName}</Heading>
-          {#each bindableProperties.filter(binding =>
+          {#each bindings.filter(binding =>
             binding.label.match(searchRgx)
           ) as binding}
             <div

packages/builder/src/components/deploy/DeploymentHistory.svelte

@@ -36,8 +36,7 @@
   let errorReason
   let poll
   let deployments = []
-  let urlComponent =
-    $hostingStore.hostingInfo.type === "self" ? $store.url : `/${appId}`
+  let urlComponent = $store.url || `/${appId}`
   let deploymentUrl = `${$hostingStore.appUrl}${urlComponent}`
 
   const formatDate = (date, format) =>

packages/builder/src/components/settings/SettingsModal.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { General, DangerZone, APIKeys } from "./tabs"
+  import { General, DangerZone } from "./tabs"
   import { ModalContent, Tab, Tabs } from "@budibase/bbui"
 </script>
 
@@ -13,9 +13,9 @@
       <Tab title="General">
         <General />
       </Tab>
-      <Tab title="API Keys">
+      <!-- <Tab title="API Keys">
         <APIKeys />
-      </Tab>
+      </Tab> -->
       <Tab title="Danger Zone">
         <DangerZone />
       </Tab>

packages/builder/src/components/settings/tabs/General.svelte

@@ -49,27 +49,22 @@
   onMount(async () => {
     const nameError = "Your application must have a name.",
       urlError = "Your application must have a URL."
-    let hostingInfo = await hostingStore.actions.fetch()
-    if (hostingInfo.type === "self") {
-      await hostingStore.actions.fetchDeployedApps()
-      const existingAppNames = get(hostingStore).deployedAppNames
-      const existingAppUrls = get(hostingStore).deployedAppUrls
-      const nameIdx = existingAppNames.indexOf(get(store).name)
-      const urlIdx = existingAppUrls.indexOf(get(store).url)
-      if (nameIdx !== -1) {
-        existingAppNames.splice(nameIdx, 1)
-      }
-      if (urlIdx !== -1) {
-        existingAppUrls.splice(urlIdx, 1)
-      }
-      nameValidation = {
-        name: string().required(nameError).notOneOf(existingAppNames),
-      }
-      urlValidation = {
-        url: string().required(urlError).notOneOf(existingAppUrls),
-      }
-    } else {
-      nameValidation = { name: string().required(nameError) }
+    await hostingStore.actions.fetchDeployedApps()
+    const existingAppNames = get(hostingStore).deployedAppNames
+    const existingAppUrls = get(hostingStore).deployedAppUrls
+    const nameIdx = existingAppNames.indexOf(get(store).name)
+    const urlIdx = existingAppUrls.indexOf(get(store).url)
+    if (nameIdx !== -1) {
+      existingAppNames.splice(nameIdx, 1)
+    }
+    if (urlIdx !== -1) {
+      existingAppUrls.splice(urlIdx, 1)
+    }
+    nameValidation = {
+      name: string().required(nameError).notOneOf(existingAppNames),
+    }
+    urlValidation = {
+      url: string().required(urlError).notOneOf(existingAppUrls),
     }
   })
 </script>
@@ -81,14 +76,12 @@
     error={nameError}
     label="App Name"
   />
-  {#if $hostingStore.hostingInfo.type === "self"}
-    <Input
-      on:change={e => updateApplication({ url: e.detail })}
-      value={$store.url}
-      error={urlError}
-      label="App URL"
-    />
-  {/if}
+  <Input
+    on:change={e => updateApplication({ url: e.detail })}
+    value={$store.url}
+    error={urlError}
+    label="App URL"
+  />
   <TextArea
     on:change={e => updateApplication({ description: e.detail })}
     value={$store.description}

packages/builder/src/components/start/AppCard.svelte

@@ -9,18 +9,23 @@
     Link,
   } from "@budibase/bbui"
   import { gradient } from "actions"
+  import { AppStatus } from "constants"
   import { url } from "@roxi/routify"
+  import { auth } from "stores/backend"
 
   export let app
   export let exportApp
+  export let openApp
   export let deleteApp
+  export let releaseLock
+  export let deletable
 </script>
 
 <div class="wrapper">
   <Layout noPadding gap="XS" alignContent="start">
     <div class="preview" use:gradient={{ seed: app.name }} />
     <div class="title">
-      <Link href={$url(`../../app/${app._id}`)}>
+      <Link on:click={() => openApp(app)}>
         <Heading size="XS">
           {app.name}
         </Heading>
@@ -30,16 +35,23 @@
         <MenuItem on:click={() => exportApp(app)} icon="Download">
           Export
         </MenuItem>
-        <MenuItem on:click={() => deleteApp(app)} icon="Delete">
-          Delete
-        </MenuItem>
+        {#if deletable}
+          <MenuItem on:click={() => deleteApp(app)} icon="Delete">
+            Delete
+          </MenuItem>
+        {/if}
+        {#if app.lockedBy && app.lockedBy?.email === $auth.user?.email}
+          <MenuItem on:click={() => releaseLock(app._id)} icon="LockOpen">
+            Release Lock
+          </MenuItem>
+        {/if}
       </ActionMenu>
     </div>
     <div class="status">
       <Body noPadding size="S">
         Edited {Math.floor(1 + Math.random() * 10)} months ago
       </Body>
-      {#if Math.random() > 0.5}
+      {#if app.lockedBy}
         <Icon name="LockClosed" />
       {/if}
     </div>

packages/builder/src/components/start/AppRow.svelte

@@ -8,18 +8,22 @@
     MenuItem,
     Link,
   } from "@budibase/bbui"
+  import { AppStatus } from "constants"
   import { url } from "@roxi/routify"
+  import { auth } from "stores/backend"
 
   export let app
   export let openApp
   export let exportApp
   export let deleteApp
+  export let releaseLock
   export let last
+  export let deletable
 </script>
 
 <div class="title" class:last>
   <div class="preview" use:gradient={{ seed: app.name }} />
-  <Link href={$url(`../../app/${app._id}`)}>
+  <Link on:click={() => openApp(app)}>
     <Heading size="XS">
       {app.name}
     </Heading>
@@ -29,15 +33,12 @@
   Edited {Math.round(Math.random() * 10 + 1)} months ago
 </div>
 <div class:last>
-  {#if Math.random() < 0.33}
+  {#if app.lockedBy}
+    <div class="status status--locked-you" />
+    Locked by {app.lockedBy.email}
+  {:else}
     <div class="status status--open" />
     Open
-  {:else if Math.random() < 0.33}
-    <div class="status status--locked-other" />
-    Locked by Will Wheaton
-  {:else}
-    <div class="status status--locked-you" />
-    Locked by you
   {/if}
 </div>
 <div class:last>
@@ -45,7 +46,14 @@
   <ActionMenu align="right">
     <Icon hoverable slot="control" name="More" />
     <MenuItem on:click={() => exportApp(app)} icon="Download">Export</MenuItem>
-    <MenuItem on:click={() => deleteApp(app)} icon="Delete">Delete</MenuItem>
+    {#if deletable}
+      <MenuItem on:click={() => deleteApp(app)} icon="Delete">Delete</MenuItem>
+    {/if}
+    {#if app.lockedBy && app.lockedBy?.email === $auth.user?.email}
+      <MenuItem on:click={() => releaseLock(app._id)} icon="LockOpen">
+        Release Lock
+      </MenuItem>
+    {/if}
   </ActionMenu>
 </div>
 

packages/builder/src/components/start/BuilderSettingsModal.svelte

@@ -1,47 +1,19 @@
 <script>
-  import { hostingStore } from "builderStore"
-  import { HostingTypes } from "constants/backend"
   import {
     Heading,
     Divider,
     notifications,
-    Input,
     ModalContent,
     Toggle,
     Body,
   } from "@budibase/bbui"
   import ThemeEditor from "components/settings/ThemeEditor.svelte"
   import analytics from "analytics"
-  import { onMount } from "svelte"
-
-  let hostingInfo
-  let selfhosted = false
 
   $: analyticsDisabled = analytics.disabled()
 
   async function save() {
-    hostingInfo.type = selfhosted ? HostingTypes.SELF : HostingTypes.CLOUD
-    if (!selfhosted && hostingInfo._rev) {
-      hostingInfo = {
-        type: hostingInfo.type,
-        _id: hostingInfo._id,
-        _rev: hostingInfo._rev,
-      }
-    }
-    try {
-      await hostingStore.actions.save(hostingInfo)
-      notifications.success(`Settings saved.`)
-    } catch (err) {
-      notifications.error(`Failed to update builder settings.`)
-    }
-  }
-
-  function updateSelfHosting(event) {
-    if (hostingInfo.type === HostingTypes.CLOUD && event.detail) {
-      hostingInfo.hostingUrl = "localhost:10000"
-      hostingInfo.useHttps = false
-      hostingInfo.selfHostKey = "budibase"
-    }
+    notifications.success(`Settings saved.`)
   }
 
   function toggleAnalytics() {
@@ -51,33 +23,12 @@
       analytics.optOut()
     }
   }
-
-  onMount(async () => {
-    hostingInfo = await hostingStore.actions.fetch()
-    selfhosted = hostingInfo.type === "self"
-  })
 </script>
 
 <ModalContent title="Builder settings" confirmText="Save" onConfirm={save}>
   <Heading size="XS">Theme</Heading>
   <ThemeEditor />
   <Divider noMargin noGrid />
-  <Heading size="XS">Hosting</Heading>
-  <Body size="S">
-    This section contains settings that relate to the deployment and hosting of
-    apps made in this builder.
-  </Body>
-  <Toggle
-    text="Self hosted"
-    on:change={updateSelfHosting}
-    bind:value={selfhosted}
-  />
-  {#if selfhosted}
-    <Input bind:value={hostingInfo.hostingUrl} label="Hosting URL" />
-    <Input bind:value={hostingInfo.selfHostKey} label="Hosting Key" />
-    <Toggle text="HTTPS" bind:value={hostingInfo.useHttps} />
-  {/if}
-  <Divider noMargin noGrid />
   <Heading size="XS">Analytics</Heading>
   <Body size="S">
     If you would like to send analytics that help us make budibase better,

packages/builder/src/constants/index.js

@@ -9,6 +9,11 @@ export const FrontendTypes = {
   NONE: "none",
 }
 
+export const AppStatus = {
+  DEV: "dev",
+  PUBLISHED: "published",
+}
+
 // fields on the user table that cannot be edited
 export const UNEDITABLE_USER_FIELDS = ["email", "password", "roleId", "status"]
 

packages/builder/src/pages/builder/app/[application]/deploy/index.svelte

@@ -1,6 +1,6 @@
 <script>
   import { Button, Modal, notifications, Heading } from "@budibase/bbui"
-  import { store, hostingStore } from "builderStore"
+  import { store } from "builderStore"
   import api from "builderStore/api"
   import DeploymentHistory from "components/deploy/DeploymentHistory.svelte"
   import analytics from "analytics"
@@ -15,41 +15,18 @@
   $: appId = $store.appId
 
   async function deployApp() {
-    // Must have cloud or self host API key to deploy
-    if (!$hostingStore.hostingInfo?.selfHostKey) {
-      const response = await api.get(`/api/keys/`)
-      const userKeys = await response.json()
-      if (!userKeys.budibase) {
-        notifications.error(
-          "No budibase API Keys configured. You must set either a self hosted or cloud API key to deploy your budibase app."
-        )
-        return
-      }
-    }
-
-    const DEPLOY_URL = `/api/deploy`
-
     try {
       notifications.info(`Deployment started. Please wait.`)
-      const response = await api.post(DEPLOY_URL)
+      const response = await api.post("/api/deploy")
       const json = await response.json()
       if (response.status !== 200) {
         throw new Error()
       }
 
-      analytics.captureEvent("Deployed App", {
-        appId,
-        hostingType: $hostingStore.hostingInfo?.type,
-      })
-
       if (analytics.requestFeedbackOnDeploy()) {
         feedbackModal.show()
       }
     } catch (err) {
-      analytics.captureEvent("Deploy App Failed", {
-        appId,
-        hostingType: $hostingStore.hostingInfo?.type,
-      })
       analytics.captureException(err)
       notifications.error("Deployment unsuccessful. Please try again later.")
     }
@@ -60,7 +37,7 @@
   <img src={Rocket} alt="Rocket flying through sky" />
   <div>
     <Heading size="M">It's time to shine!</Heading>
-    <Button size="XL" cta medium on:click={deployApp}>Deploy App</Button>
+    <Button size="XL" cta medium on:click={deployApp}>Publish App</Button>
   </div>
 </section>
 <Modal bind:this={feedbackModal}>

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -23,8 +23,10 @@
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import AppCard from "components/start/AppCard.svelte"
   import AppRow from "components/start/AppRow.svelte"
+  import { AppStatus } from "constants"
 
   let layout = "grid"
+  let appStatus = AppStatus.PUBLISHED
   let template
   let appToDelete
   let creationModal
@@ -32,6 +34,8 @@
   let creatingApp = false
   let loaded = false
 
+  $: appStatus && apps.load(appStatus)
+
   const checkKeys = async () => {
     const response = await api.get(`/api/keys/`)
     const keys = await response.json()
@@ -57,7 +61,11 @@
   }
 
   const openApp = app => {
-    $goto(`../../app/${app._id}`)
+    if (appStatus === AppStatus.DEV) {
+      $goto(`../../app/${app._id}`)
+    } else {
+      window.open(`/${app._id}`, "_blank")
+    }
   }
 
   const exportApp = app => {
@@ -83,47 +91,68 @@
     if (!appToDelete) {
       return
     }
+
     await del(`/api/applications/${appToDelete?._id}`)
     await apps.load()
     appToDelete = null
+    notifications.success("App deleted successfully.")
+  }
+
+  const releaseLock = async appId => {
+    try {
+      const response = await del(`/api/dev/${appId}/lock`)
+      const json = await response.json()
+      if (response.status !== 200) throw json.message
+
+      notifications.success("Lock released")
+      await apps.load(appStatus)
+    } catch (err) {
+      notifications.error(`Error releasing lock: ${err}`)
+    }
   }
 
   onMount(async () => {
     checkKeys()
-    await apps.load()
+    await apps.load(appStatus)
     loaded = true
   })
 </script>
 
 <Page wide>
-  {#if $apps.length}
-    <Layout noPadding>
-      <div class="title">
-        <Heading>Apps</Heading>
-        <ButtonGroup>
-          <Button secondary on:click={initiateAppImport}>Import app</Button>
-          <Button cta on:click={initiateAppCreation}>Create new app</Button>
-        </ButtonGroup>
-      </div>
-      <div class="filter">
-        <div class="select">
-          <Select quiet placeholder="Filter by groups" />
-        </div>
-        <ActionGroup>
-          <ActionButton
-            on:click={() => (layout = "grid")}
-            selected={layout === "grid"}
-            quiet
-            icon="ClassicGridView"
-          />
-          <ActionButton
-            on:click={() => (layout = "table")}
-            selected={layout === "table"}
-            quiet
-            icon="ViewRow"
-          />
-        </ActionGroup>
+  <Layout noPadding>
+    <div class="title">
+      <Heading>Apps</Heading>
+      <ButtonGroup>
+        <Button secondary on:click={initiateAppImport}>Import app</Button>
+        <Button cta on:click={initiateAppCreation}>Create new app</Button>
+      </ButtonGroup>
+    </div>
+    <div class="filter">
+      <div class="select">
+        <Select
+          bind:value={appStatus}
+          options={[
+            { label: "Published", value: AppStatus.PUBLISHED },
+            { label: "In Development", value: AppStatus.DEV },
+          ]}
+        />
       </div>
+      <ActionGroup>
+        <ActionButton
+          on:click={() => (layout = "grid")}
+          selected={layout === "grid"}
+          quiet
+          icon="ClassicGridView"
+        />
+        <ActionButton
+          on:click={() => (layout = "table")}
+          selected={layout === "table"}
+          quiet
+          icon="ViewRow"
+        />
+      </ActionGroup>
+    </div>
+    {#if $apps.length}
       <div
         class:appGrid={layout === "grid"}
         class:appTable={layout === "table"}
@@ -131,6 +160,8 @@
         {#each $apps as app, idx (app._id)}
           <svelte:component
             this={layout === "grid" ? AppCard : AppRow}
+            deletable={appStatus === AppStatus.PUBLISHED}
+            {releaseLock}
             {app}
             {openApp}
             {exportApp}
@@ -139,8 +170,8 @@
           />
         {/each}
       </div>
-    </Layout>
-  {/if}
+    {/if}
+  </Layout>
   {#if !$apps.length && !creatingApp && loaded}
     <div class="empty-wrapper">
       <Modal inline>

packages/builder/src/stores/portal/apps.js

@@ -4,9 +4,9 @@ import { get } from "builderStore/api"
 export function createAppStore() {
   const store = writable([])
 
-  async function load() {
+  async function load(status = "") {
     try {
-      const res = await get("/api/applications")
+      const res = await get(`/api/applications?status=${status}`)
       const json = await res.json()
       if (res.ok && Array.isArray(json)) {
         store.set(json)

packages/server/src/api/controllers/application.js

@@ -16,11 +16,14 @@ const {
   getLayoutParams,
   getScreenParams,
   generateScreenID,
+  generateDevAppID,
+  DocumentTypes,
+  AppStatus,
 } = require("../../db/utils")
 const {
   BUILTIN_ROLE_IDS,
   AccessController,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const {
   createHomeScreen,
@@ -30,8 +33,9 @@ const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
 const { getAllApps } = require("../../utilities")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const { getDeployedApps } = require("../../utilities/builder/hosting")
+const { getDeployedApps } = require("../../utilities/workerRequests")
 const { clientLibraryPath } = require("../../utilities")
+const { getAllLocks } = require("../../utilities/redis")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -84,7 +88,10 @@ async function getAppUrlIfNotInUse(ctx) {
 }
 
 async function createInstance(template) {
-  const appId = generateAppID()
+  // TODO: Do we need the normal app ID?
+  const baseAppId = generateAppID()
+  const appId = generateDevAppID(baseAppId)
+
   const db = new CouchDB(appId)
   await db.put({
     _id: "_design/database",
@@ -114,7 +121,24 @@ async function createInstance(template) {
 }
 
 exports.fetch = async function (ctx) {
-  ctx.body = await getAllApps()
+  const isDev = ctx.query && ctx.query.status === AppStatus.DEV
+  const apps = await getAllApps(isDev)
+
+  // get the locks for all the dev apps
+  if (isDev) {
+    const locks = await getAllLocks()
+    for (let app of apps) {
+      const lock = locks.find(lock => lock.appId === app._id)
+      if (lock) {
+        app.lockedBy = lock.user
+      } else {
+        // make sure its definitely not present
+        delete app.lockedBy
+      }
+    }
+  }
+
+  ctx.body = apps
 }
 
 exports.fetchAppDefinition = async function (ctx) {
@@ -194,6 +218,12 @@ exports.update = async function (ctx) {
   const data = ctx.request.body
   const newData = { ...application, ...data, url }
 
+  // the locked by property is attached by server but generated from
+  // Redis, shouldn't ever store it
+  if (newData.lockedBy) {
+    delete newData.lockedBy
+  }
+
   const response = await db.put(newData)
   data._rev = response.rev
 

packages/server/src/api/controllers/deploy/Deployment.js

@@ -1,5 +1,3 @@
-const { getAppQuota } = require("./quota")
-const env = require("../../../environment")
 const newid = require("../../../db/newid")
 
 /**
@@ -11,24 +9,6 @@ class Deployment {
     this._id = id || newid()
   }
 
-  // purely so that we can do quota stuff outside the main deployment context
-  async init() {
-    if (!env.SELF_HOSTED) {
-      this.setQuota(await getAppQuota(this.appId))
-    }
-  }
-
-  setQuota(quota) {
-    if (!quota) {
-      return
-    }
-    this.quota = quota
-  }
-
-  getQuota() {
-    return this.quota
-  }
-
   getAppId() {
     return this.appId
   }
@@ -38,9 +18,6 @@ class Deployment {
       return
     }
     this.verification = verification
-    if (this.verification.quota) {
-      this.quota = this.verification.quota
-    }
   }
 
   getVerification() {
@@ -58,9 +35,6 @@ class Deployment {
     if (json.verification) {
       this.setVerification(json.verification)
     }
-    if (json.quota) {
-      this.setQuota(json.quota)
-    }
     if (json.status) {
       this.setStatus(json.status, json.err)
     }
@@ -78,9 +52,6 @@ class Deployment {
     if (this.verification && this.verification.cfDistribution) {
       obj.cfDistribution = this.verification.cfDistribution
     }
-    if (this.quota) {
-      obj.quota = this.quota
-    }
     return obj
   }
 }

packages/server/src/api/controllers/deploy/awsDeploy.js

@@ -1,80 +0,0 @@
-const AWS = require("aws-sdk")
-const fetch = require("node-fetch")
-const env = require("../../../environment")
-const {
-  deployToObjectStore,
-  performReplication,
-  fetchCredentials,
-} = require("./utils")
-
-/**
- * Verifies the users API key and
- * Verifies that the deployment fits within the quota of the user
- * Links to the "check-api-key" lambda.
- * @param {object} deployment - information about the active deployment, including the appId and quota.
- */
-exports.preDeployment = async function (deployment) {
-  const json = await fetchCredentials(env.DEPLOYMENT_CREDENTIALS_URL, {
-    apiKey: env.BUDIBASE_API_KEY,
-    appId: deployment.getAppId(),
-    quota: deployment.getQuota(),
-  })
-
-  // set credentials here, means any time we're verified we're ready to go
-  if (json.credentials) {
-    AWS.config.update({
-      accessKeyId: json.credentials.AccessKeyId,
-      secretAccessKey: json.credentials.SecretAccessKey,
-      sessionToken: json.credentials.SessionToken,
-    })
-  }
-
-  return json
-}
-
-/**
- * Finalises the deployment, updating the quota for the user API key
- * The verification process returns the levels to update to.
- * Calls the "deployment-success" lambda.
- * @param {object} deployment information about the active deployment, including the quota info.
- * @returns {Promise<object>} The usage has been updated against the user API key.
- */
-exports.postDeployment = async function (deployment) {
-  const DEPLOYMENT_SUCCESS_URL =
-    env.DEPLOYMENT_CREDENTIALS_URL + "deploy/success"
-
-  const response = await fetch(DEPLOYMENT_SUCCESS_URL, {
-    method: "POST",
-    body: JSON.stringify({
-      apiKey: env.BUDIBASE_API_KEY,
-      quota: deployment.getQuota(),
-    }),
-    headers: {
-      "Content-Type": "application/json",
-      Accept: "application/json",
-    },
-  })
-
-  if (response.status !== 200) {
-    throw new Error(`Error updating deployment quota for API Key`)
-  }
-
-  return await response.json()
-}
-
-exports.deploy = async function (deployment) {
-  const appId = deployment.getAppId()
-  const { bucket, accountId } = deployment.getVerification()
-  const metadata = { accountId }
-  await deployToObjectStore(appId, bucket, metadata)
-}
-
-exports.replicateDb = async function (deployment) {
-  const appId = deployment.getAppId()
-  const verification = deployment.getVerification()
-  return performReplication(
-    appId,
-    verification.couchDbSession,
-    env.DEPLOYMENT_DB_URL
-  )
-}

packages/server/src/api/controllers/deploy/index.js

@@ -1,9 +1,6 @@
 const PouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const {
-  getHostingInfo,
-  HostingTypes,
-} = require("../../../utilities/builder/hosting")
+const { Replication } = require("@budibase/auth/db")
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
 const DeploymentStatus = {
@@ -12,9 +9,6 @@ const DeploymentStatus = {
   FAILURE: "FAILURE",
 }
 
-// default to AWS deployment, this will be updated before use (if required)
-let deploymentService = require("./awsDeploy")
-
 // checks that deployments are in a good state, any pending will be updated
 async function checkAllDeployments(deployments) {
   let updated = false
@@ -62,22 +56,30 @@ async function storeLocalDeploymentHistory(deployment) {
 }
 
 async function deployApp(deployment) {
-  const appId = deployment.getAppId()
   try {
-    await deployment.init()
-    deployment.setVerification(
-      await deploymentService.preDeployment(deployment)
-    )
+    const productionAppId = deployment.appId.replace("_dev", "")
 
-    console.log(`Uploading assets for appID ${appId}..`)
+    const replication = new Replication({
+      source: deployment.appId,
+      target: productionAppId,
+    })
 
-    await deploymentService.deploy(deployment)
+    await replication.replicate()
 
-    // replicate the DB to the main couchDB cluster
-    console.log("Replicating local PouchDB to CouchDB..")
-    await deploymentService.replicateDb(deployment)
+    // Strip the _dev prefix and update the appID document in the new DB
+    const db = new PouchDB(productionAppId)
+    const appDoc = await db.get(deployment.appId)
+    appDoc._id = productionAppId
+    delete appDoc._rev
+    appDoc.instance._id = productionAppId
+    await db.put(appDoc)
 
-    await deploymentService.postDeployment(deployment)
+    // Set up live sync between the live and dev instances
+    const liveReplication = new Replication({
+      source: productionAppId,
+      target: deployment.appId,
+    })
+    liveReplication.subscribe()
 
     deployment.setStatus(DeploymentStatus.SUCCESS)
     await storeLocalDeploymentHistory(deployment)
@@ -122,12 +124,6 @@ exports.deploymentProgress = async function (ctx) {
 }
 
 exports.deployApp = async function (ctx) {
-  // start by checking whether to deploy local or to cloud
-  const hostingInfo = await getHostingInfo()
-  deploymentService =
-    hostingInfo.type === HostingTypes.CLOUD
-      ? require("./awsDeploy")
-      : require("./selfDeploy")
   let deployment = new Deployment(ctx.appId)
   deployment.setStatus(DeploymentStatus.PENDING)
   deployment = await storeLocalDeploymentHistory(deployment)

packages/server/src/api/controllers/deploy/quota.js

@@ -1,39 +0,0 @@
-const PouchDB = require("../../../db")
-const {
-  DocumentTypes,
-  SEPARATOR,
-  UNICODE_MAX,
-  ViewNames,
-} = require("../../../db/utils")
-
-exports.getAppQuota = async function (appId) {
-  const db = new PouchDB(appId)
-
-  const rows = await db.allDocs({
-    startkey: DocumentTypes.ROW + SEPARATOR,
-    endkey: DocumentTypes.ROW + SEPARATOR + UNICODE_MAX,
-  })
-
-  const users = await db.allDocs({
-    startkey: DocumentTypes.USER + SEPARATOR,
-    endkey: DocumentTypes.USER + SEPARATOR + UNICODE_MAX,
-  })
-
-  const existingRows = rows.rows.length
-  const existingUsers = users.rows.length
-
-  const designDoc = await db.get("_design/database")
-
-  let views = 0
-  for (let viewName of Object.keys(designDoc.views)) {
-    if (Object.values(ViewNames).indexOf(viewName) === -1) {
-      views++
-    }
-  }
-
-  return {
-    rows: existingRows,
-    users: existingUsers,
-    views: views,
-  }
-}

packages/server/src/api/controllers/deploy/selfDeploy.js

@@ -1,60 +0,0 @@
-const AWS = require("aws-sdk")
-const {
-  deployToObjectStore,
-  performReplication,
-  fetchCredentials,
-} = require("./utils")
-const {
-  getWorkerUrl,
-  getCouchUrl,
-  getSelfHostKey,
-} = require("../../../utilities/builder/hosting")
-
-exports.preDeployment = async function () {
-  const url = `${await getWorkerUrl()}/api/deploy`
-  try {
-    const json = await fetchCredentials(url, {
-      selfHostKey: await getSelfHostKey(),
-    })
-
-    // response contains:
-    // couchDbSession, bucket, objectStoreSession
-
-    // set credentials here, means any time we're verified we're ready to go
-    if (json.objectStoreSession) {
-      AWS.config.update({
-        accessKeyId: json.objectStoreSession.accessKeyId,
-        secretAccessKey: json.objectStoreSession.secretAccessKey,
-      })
-    }
-
-    return json
-  } catch (err) {
-    throw {
-      message: "Unauthorised to deploy, check self hosting key",
-      status: 401,
-    }
-  }
-}
-
-exports.postDeployment = async function () {
-  // we don't actively need to do anything after deployment in self hosting
-}
-
-exports.deploy = async function (deployment) {
-  const appId = deployment.getAppId()
-  const verification = deployment.getVerification()
-  // no metadata, aws has account ID in metadata
-  const metadata = {}
-  await deployToObjectStore(appId, verification.bucket, metadata)
-}
-
-exports.replicateDb = async function (deployment) {
-  const appId = deployment.getAppId()
-  const verification = deployment.getVerification()
-  return performReplication(
-    appId,
-    verification.couchDbSession,
-    await getCouchUrl()
-  )
-}

packages/server/src/api/controllers/deploy/utils.js

@@ -1,136 +0,0 @@
-const { join } = require("../../../utilities/centralPath")
-const fs = require("fs")
-const { budibaseAppsDir } = require("../../../utilities/budibaseDir")
-const fetch = require("node-fetch")
-const PouchDB = require("../../../db")
-const CouchDB = require("pouchdb")
-const { upload } = require("../../../utilities/fileSystem")
-const { attachmentsRelativeURL } = require("../../../utilities")
-
-// TODO: everything in this file is to be removed
-
-function walkDir(dirPath, callback) {
-  for (let filename of fs.readdirSync(dirPath)) {
-    const filePath = `${dirPath}/${filename}`
-    const stat = fs.lstatSync(filePath)
-
-    if (stat.isFile()) {
-      callback(filePath)
-    } else {
-      walkDir(filePath, callback)
-    }
-  }
-}
-
-exports.fetchCredentials = async function (url, body) {
-  const response = await fetch(url, {
-    method: "POST",
-    body: JSON.stringify(body),
-    headers: { "Content-Type": "application/json" },
-  })
-
-  const json = await response.json()
-  if (json.errors) {
-    throw new Error(json.errors)
-  }
-
-  if (response.status !== 200) {
-    throw new Error(
-      `Error fetching temporary credentials: ${JSON.stringify(json)}`
-    )
-  }
-
-  return json
-}
-
-exports.prepareUpload = async function ({ s3Key, bucket, metadata, file }) {
-  const response = await upload({
-    bucket,
-    metadata,
-    filename: s3Key,
-    path: file.path,
-    type: file.type,
-  })
-
-  // don't store a URL, work this out on the way out as the URL could change
-  return {
-    size: file.size,
-    name: file.name,
-    url: attachmentsRelativeURL(response.Key),
-    extension: [...file.name.split(".")].pop(),
-    key: response.Key,
-  }
-}
-
-exports.deployToObjectStore = async function (appId, bucket, metadata) {
-  const appAssetsPath = join(budibaseAppsDir(), appId, "public")
-
-  let uploads = []
-
-  // Upload HTML, CSS and JS for each page of the web app
-  walkDir(appAssetsPath, function (filePath) {
-    const filePathParts = filePath.split("/")
-    const appAssetUpload = exports.prepareUpload({
-      bucket,
-      file: {
-        path: filePath,
-        name: filePathParts.pop(),
-      },
-      s3Key: filePath.replace(appAssetsPath, `assets/${appId}`),
-      metadata,
-    })
-    uploads.push(appAssetUpload)
-  })
-
-  // Upload file attachments
-  const db = new PouchDB(appId)
-  let fileUploads
-  try {
-    fileUploads = await db.get("_local/fileuploads")
-  } catch (err) {
-    fileUploads = { _id: "_local/fileuploads", uploads: [] }
-  }
-
-  for (let file of fileUploads.uploads) {
-    if (file.uploaded) continue
-
-    const attachmentUpload = exports.prepareUpload({
-      file,
-      s3Key: `assets/${appId}/attachments/${file.processedFileName}`,
-      bucket,
-      metadata,
-    })
-
-    uploads.push(attachmentUpload)
-
-    // mark file as uploaded
-    file.uploaded = true
-  }
-
-  db.put(fileUploads)
-
-  try {
-    return await Promise.all(uploads)
-  } catch (err) {
-    console.error("Error uploading budibase app assets to s3", err)
-    throw err
-  }
-}
-
-exports.performReplication = (appId, session, dbUrl) => {
-  return new Promise((resolve, reject) => {
-    const local = new PouchDB(appId)
-
-    const remote = new CouchDB(`${dbUrl}/${appId}`, {
-      fetch: function (url, opts) {
-        opts.headers.set("Cookie", `${session};`)
-        return CouchDB.fetch(url, opts)
-      },
-    })
-
-    const replication = local.sync(remote)
-
-    replication.on("complete", () => resolve())
-    replication.on("error", err => reject(err))
-  })
-}

packages/server/src/api/controllers/dev.js

@@ -2,6 +2,7 @@ const fetch = require("node-fetch")
 const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
+const { clearLock } = require("../../utilities/redis")
 
 async function redirect(ctx, method) {
   const { devPath } = ctx.params
@@ -32,3 +33,15 @@ exports.redirectPost = async ctx => {
 exports.redirectDelete = async ctx => {
   await redirect(ctx, "DELETE")
 }
+
+exports.clearLock = async ctx => {
+  const { appId } = ctx.params
+  try {
+    await clearLock(appId, ctx.user)
+  } catch (err) {
+    ctx.throw(400, `Unable to remove lock. ${err}`)
+  }
+  ctx.body = {
+    message: "Lock released successfully.",
+  }
+}

packages/server/src/api/controllers/hosting.js

@@ -1,41 +1,19 @@
 const CouchDB = require("../../db")
-const {
-  getHostingInfo,
-  getDeployedApps,
-  HostingTypes,
-  getAppUrl,
-} = require("../../utilities/builder/hosting")
-const { StaticDatabases } = require("../../db/utils")
-
-exports.fetchInfo = async ctx => {
-  ctx.body = {
-    types: Object.values(HostingTypes),
-  }
-}
-
-exports.save = async ctx => {
-  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
-  const { type } = ctx.request.body
-  if (type === HostingTypes.CLOUD && ctx.request.body._rev) {
-    ctx.body = await db.remove({
-      ...ctx.request.body,
-      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
-    })
-  } else {
-    ctx.body = await db.put({
-      ...ctx.request.body,
-      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
-    })
-  }
-}
-
-exports.fetch = async ctx => {
-  ctx.body = await getHostingInfo()
-}
+const { getDeployedApps } = require("../../utilities/workerRequests")
+const { getScopedConfig } = require("@budibase/auth/db")
+const { Configs } = require("@budibase/auth").constants
+const { checkSlashesInUrl } = require("../../utilities")
 
 exports.fetchUrls = async ctx => {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const settings = await getScopedConfig(db, { type: Configs.SETTINGS })
+  let appUrl = "http://localhost:10000/app"
+  if (settings && settings["platformUrl"]) {
+    appUrl = checkSlashesInUrl(`${settings["platformUrl"]}/app`)
+  }
   ctx.body = {
-    app: await getAppUrl(ctx.appId),
+    app: appUrl,
   }
 }
 

packages/server/src/api/controllers/permission.js

@@ -3,19 +3,19 @@ const {
   PermissionLevels,
   isPermissionLevelHigherThanRead,
   higherPermission,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const {
   isBuiltin,
   getDBRoleID,
   getExternalRoleID,
   getBuiltinRoles,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 const { getRoleParams } = require("../../db/utils")
 const CouchDB = require("../../db")
 const {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
-} = require("../../utilities/security/utilities")
+} = require("../../utilities/security")
 
 const PermissionUpdateType = {
   REMOVE: "remove",

packages/server/src/api/controllers/query.js

@@ -58,6 +58,9 @@ async function enrichQueryFields(fields, parameters) {
 
   // enrich the fields with dynamic parameters
   for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      continue
+    }
     if (typeof fields[key] === "object") {
       // enrich nested fields object
       enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)

packages/server/src/api/controllers/role.js

@@ -1,12 +1,12 @@
 const CouchDB = require("../../db")
 const {
   getBuiltinRoles,
-  BUILTIN_ROLE_IDS,
   Role,
   getRole,
   isBuiltin,
   getExternalRoleID,
-} = require("../../utilities/security/roles")
+  getAllRoles,
+} = require("@budibase/auth/roles")
 const {
   generateRoleID,
   getRoleParams,
@@ -19,14 +19,6 @@ const UpdateRolesOptions = {
   REMOVED: "removed",
 }
 
-// exclude internal roles like builder
-const EXTERNAL_BUILTIN_ROLE_IDS = [
-  BUILTIN_ROLE_IDS.ADMIN,
-  BUILTIN_ROLE_IDS.POWER,
-  BUILTIN_ROLE_IDS.BASIC,
-  BUILTIN_ROLE_IDS.PUBLIC,
-]
-
 async function updateRolesOnUserTable(db, roleId, updateOption) {
   const table = await db.get(InternalTables.USER_METADATA)
   const schema = table.schema
@@ -51,31 +43,7 @@ async function updateRolesOnUserTable(db, roleId, updateOption) {
 }
 
 exports.fetch = async function (ctx) {
-  const db = new CouchDB(ctx.appId)
-  const body = await db.allDocs(
-    getRoleParams(null, {
-      include_docs: true,
-    })
-  )
-  let roles = body.rows.map(row => row.doc)
-  const builtinRoles = getBuiltinRoles()
-
-  // need to combine builtin with any DB record of them (for sake of permissions)
-  for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) {
-    const builtinRole = builtinRoles[builtinRoleId]
-    const dbBuiltin = roles.filter(
-      dbRole => getExternalRoleID(dbRole._id) === builtinRoleId
-    )[0]
-    if (dbBuiltin == null) {
-      roles.push(builtinRole)
-    } else {
-      // remove role and all back after combining with the builtin
-      roles = roles.filter(role => role._id !== dbBuiltin._id)
-      dbBuiltin._id = getExternalRoleID(dbBuiltin._id)
-      roles.push(Object.assign(builtinRole, dbBuiltin))
-    }
-  }
-  ctx.body = roles
+  ctx.body = await getAllRoles(ctx.appId)
 }
 
 exports.find = async function (ctx) {

packages/server/src/api/controllers/routing.js

@@ -2,7 +2,7 @@ const { getRoutingInfo } = require("../../utilities/routing")
 const {
   getUserRoleHierarchy,
   BUILTIN_ROLE_IDS,
-} = require("../../utilities/security/roles")
+} = require("@budibase/auth/roles")
 
 const URL_SEPARATOR = "/"
 

packages/server/src/api/controllers/screen.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../db")
 const { getScreenParams, generateScreenID } = require("../../db/utils")
-const { AccessController } = require("../../utilities/security/roles")
+const { AccessController } = require("@budibase/auth/roles")
 
 exports.fetch = async ctx => {
   const appId = ctx.appId

packages/server/src/api/controllers/script.js

@@ -9,14 +9,16 @@ class ScriptExecutor {
   }
 
   execute() {
-    const returnValue = this.script.runInContext(this.context)
-    return returnValue
+    return this.script.runInContext(this.context)
   }
 }
 
 exports.execute = async function (ctx) {
   const executor = new ScriptExecutor(ctx.request.body)
 
-  const result = executor.execute()
-  ctx.body = result
+  ctx.body = executor.execute()
+}
+
+exports.save = async function (ctx) {
+  ctx.throw(501, "Not currently implemented")
 }

packages/server/src/api/controllers/static/index.js

@@ -5,10 +5,9 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const fetch = require("node-fetch")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
-const { prepareUpload } = require("../deploy/utils")
 const { processString } = require("@budibase/string-templates")
 const { budibaseTempDir } = require("../../../utilities/budibaseDir")
-const { getDeployedApps } = require("../../../utilities/builder/hosting")
+const { getDeployedApps } = require("../../../utilities/workerRequests")
 const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
@@ -17,6 +16,27 @@ const {
 } = require("../../../utilities/fileSystem")
 const env = require("../../../environment")
 const { objectStoreUrl, clientLibraryPath } = require("../../../utilities")
+const { upload } = require("../../../utilities/fileSystem")
+const { attachmentsRelativeURL } = require("../../../utilities")
+
+async function prepareUpload({ s3Key, bucket, metadata, file }) {
+  const response = await upload({
+    bucket,
+    metadata,
+    filename: s3Key,
+    path: file.path,
+    type: file.type,
+  })
+
+  // don't store a URL, work this out on the way out as the URL could change
+  return {
+    size: file.size,
+    name: file.name,
+    url: attachmentsRelativeURL(response.Key),
+    extension: [...file.name.split(".")].pop(),
+    key: response.Key,
+  }
+}
 
 async function checkForSelfHostedURL(ctx) {
   // the "appId" component of the URL may actually be a specific self hosted URL
@@ -95,10 +115,14 @@ exports.serveComponentLibrary = async function (ctx) {
   if (env.isDev() || env.isTest()) {
     const componentLibraryPath = join(
       budibaseTempDir(),
-      decodeURI(ctx.query.library),
+      appId,
+      "node_modules",
+      "@budibase",
+      "standard-components",
+      "package",
       "dist"
     )
-    return send(ctx, "/awsDeploy.js", { root: componentLibraryPath })
+    return send(ctx, "/index.js", { root: componentLibraryPath })
   }
   const db = new CouchDB(appId)
   const appInfo = await db.get(appId)

packages/server/src/api/controllers/user.js

@@ -5,7 +5,7 @@ const {
   getGlobalIDFromUserMetadataID,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { getRole } = require("../../utilities/security/roles")
+const { getRole, BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   getGlobalUsers,
   saveGlobalUser,
@@ -73,6 +73,9 @@ exports.createMetadata = async function (ctx) {
 exports.updateSelfMetadata = async function (ctx) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user._id
+  if (ctx.user.builder && ctx.user.builder.global) {
+    ctx.request.body.roleId = BUILTIN_ROLE_IDS.ADMIN
+  }
   // make sure no stale rev
   delete ctx.request.body._rev
   await exports.updateMetadata(ctx)

packages/server/src/api/index.js

@@ -37,6 +37,7 @@ router
     })
   )
   .use(currentApp)
+// .use(development)
 
 // error handling middleware
 router.use(async (ctx, next) => {

packages/server/src/api/routes/analytics.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
 const controller = require("../controllers/analytics")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/apikeys.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/apikeys")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/application.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/application")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/automation.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const Joi = require("joi")
 const { bodyResource, paramResource } = require("../../middleware/resourceId")
 

packages/server/src/api/routes/backup.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/backup")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/component.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/component")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/datasource.js

@@ -5,7 +5,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/deploy.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/deploy")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/dev.js

@@ -1,6 +1,8 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/dev")
 const env = require("../../environment")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
@@ -11,4 +13,6 @@ if (env.isDev() || env.isTest()) {
     .delete("/api/admin/:devPath(.*)", controller.redirectDelete)
 }
 
+router.delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
+
 module.exports = router

packages/server/src/api/routes/hosting.js

@@ -2,15 +2,12 @@ const Router = require("@koa/router")
 const controller = require("../controllers/hosting")
 const authorized = require("../../middleware/authorized")
 const selfhost = require("../../middleware/selfhost")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
 router
-  .get("/api/hosting/info", authorized(BUILDER), controller.fetchInfo)
   .get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
-  .get("/api/hosting", authorized(BUILDER), controller.fetch)
-  .post("/api/hosting", authorized(BUILDER), controller.save)
   // this isn't risky, doesn't return anything about apps other than names and URLs
   .get("/api/hosting/apps", selfhost, controller.getDeployedApps)
 

packages/server/src/api/routes/integration.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/integration")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/layout.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const controller = require("../controllers/layout")
 
 const router = Router()

packages/server/src/api/routes/permission.js

@@ -4,7 +4,7 @@ const authorized = require("../../middleware/authorized")
 const {
   BUILDER,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const Joi = require("joi")
 const joiValidator = require("../../middleware/joi-validator")
 

packages/server/src/api/routes/query.js

@@ -1,12 +1,12 @@
 const Router = require("@koa/router")
 const queryController = require("../controllers/query")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
 const Joi = require("joi")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+  BUILDER,
+} = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const {
   bodyResource,

packages/server/src/api/routes/role.js

@@ -1,15 +1,13 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/role")
 const authorized = require("../../middleware/authorized")
-const {
-  BUILDER,
-  PermissionLevels,
-} = require("../../utilities/security/permissions")
 const Joi = require("joi")
 const joiValidator = require("../../middleware/joi-validator")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("../../utilities/security/permissions")
+  BUILDER,
+  PermissionLevels,
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/routing.js

@@ -1,6 +1,6 @@
 const Router = require("@koa/router")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const controller = require("../controllers/routing")
 
 const router = Router()

packages/server/src/api/routes/row.js

@@ -9,7 +9,7 @@ const {
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/screen.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/screen")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

packages/server/src/api/routes/script.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
-const controller = require("../controllers/hosting")
+const controller = require("../controllers/script")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/search.js

@@ -3,7 +3,7 @@ const controller = require("../controllers/search")
 const {
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const authorized = require("../../middleware/authorized")
 const { paramResource } = require("../../middleware/resourceId")
 

packages/server/src/api/routes/static.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 const env = require("../../environment")
 

packages/server/src/api/routes/table.js

@@ -6,7 +6,7 @@ const {
   BUILDER,
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const joiValidator = require("../../middleware/joi-validator")
 const Joi = require("joi")
 

packages/server/src/api/routes/templates.js

@@ -1,7 +1,7 @@
 const Router = require("@koa/router")
 const controller = require("../controllers/templates")
 const authorized = require("../../middleware/authorized")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 

packages/server/src/api/routes/tests/application.spec.js

@@ -1,6 +1,14 @@
 const { clearAllApps, checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 
+jest.mock("../../../utilities/redis", () => ({
+  init: jest.fn(),
+  getAllLocks: () => {
+    return []
+  },
+  updateLock: jest.fn(),
+}))
+
 describe("/applications", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -40,7 +48,7 @@ describe("/applications", () => {
       await config.createApp(request, "app2")
 
       const res = await request
-        .get("/api/applications")
+        .get("/api/applications?status=dev")
         .set(config.defaultHeaders())
         .expect('Content-Type', /json/)
         .expect(200)

packages/server/src/api/routes/tests/hosting.spec.js

@@ -15,25 +15,6 @@ describe("/hosting", () => {
     app = await config.init()
   })
 
-  describe("fetchInfo", () => {
-    it("should be able to fetch hosting information", async () => {
-      const res = await request
-        .get(`/api/hosting/info`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body).toEqual({ types: ["cloud", "self"]})
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "GET",
-        url: `/api/hosting/info`,
-      })
-    })
-  })
-
   describe("fetchUrls", () => {
     it("should be able to fetch current app URLs", async () => {
       const res = await request
@@ -41,7 +22,7 @@ describe("/hosting", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(res.body.app).toEqual(`https://${config.getAppId()}.app.budi.live`)
+      expect(res.body.app).toEqual(`http://localhost:10000/app`)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -52,78 +33,4 @@ describe("/hosting", () => {
       })
     })
   })
-
-  describe("fetch", () => {
-    it("should be able to fetch the current hosting information", async () => {
-      const res = await request
-        .get(`/api/hosting`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
-      expect(res.body.hostingUrl).toBeDefined()
-      expect(res.body.type).toEqual("cloud")
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "GET",
-        url: `/api/hosting`,
-      })
-    })
-  })
-
-  describe("save", () => {
-    it("should be able to update the hosting information", async () => {
-      const res = await request
-        .post(`/api/hosting`)
-        .send({
-          type: "self",
-          selfHostKey: "budibase",
-          hostingUrl: "localhost:10000",
-          useHttps: false,
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.ok).toEqual(true)
-      // make sure URL updated
-      const urlRes = await request
-        .get(`/api/hosting/urls`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(urlRes.body.app).toEqual(`http://localhost:10000/app`)
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await checkBuilderEndpoint({
-        config,
-        method: "POST",
-        url: `/api/hosting`,
-      })
-    })
-  })
-
-  describe("getDeployedApps", () => {
-    it("should fail when not self hosted", async () => {
-      await request
-        .get(`/api/hosting/apps`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-    })
-
-    it("should get apps when in cloud", async () => {
-      await setup.switchToSelfHosted(async () => {
-        const res = await request
-          .get(`/api/hosting/apps`)
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
-        expect(res.body.app1).toEqual({url: "/app1"})
-      })
-    })
-  })
 })

packages/server/src/api/routes/tests/permissions.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
 

packages/server/src/api/routes/tests/role.spec.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("../../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const setup = require("./utilities")
 const { basicRole } = setup.structures
 

packages/server/src/api/routes/tests/routing.spec.js

@@ -1,7 +1,7 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const workerRequests = require("../../../utilities/workerRequests")
 
 const route = "/test"

packages/server/src/api/routes/tests/user.spec.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
 const { basicUser } = setup.structures

packages/server/src/api/routes/tests/utilities/TestFunctions.js

@@ -14,7 +14,7 @@ exports.getAllTableRows = async config => {
 }
 
 exports.clearAllApps = async () => {
-  const req = {}
+  const req = { query: { status: "dev"} }
   await appController.fetch(req)
   const apps = req.body
   if (!apps || apps.length <= 0) {

packages/server/src/api/routes/user.js

@@ -4,7 +4,7 @@ const authorized = require("../../middleware/authorized")
 const {
   PermissionLevels,
   PermissionTypes,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

packages/server/src/api/routes/view.js

@@ -7,7 +7,7 @@ const {
   BUILDER,
   PermissionTypes,
   PermissionLevels,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const usage = require("../../middleware/usageQuota")
 
 const router = Router()

packages/server/src/api/routes/webhook.js

@@ -2,7 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/webhook")
 const authorized = require("../../middleware/authorized")
 const joiValidator = require("../../middleware/joi-validator")
-const { BUILDER } = require("../../utilities/security/permissions")
+const { BUILDER } = require("@budibase/auth/permissions")
 const Joi = require("joi")
 
 const router = Router()

packages/server/src/app.js

@@ -13,6 +13,7 @@ const automations = require("./automations/index")
 const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
+const redis = require("./utilities/redis")
 
 const app = new Koa()
 
@@ -84,6 +85,7 @@ module.exports = server.listen(env.PORT || 0, async () => {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await automations.init()
+  await redis.init()
 })
 
 process.on("uncaughtException", err => {

packages/server/src/automations/steps/createUser.js

@@ -1,4 +1,4 @@
-const roles = require("../../utilities/security/roles")
+const roles = require("@budibase/auth/roles")
 const userController = require("../../api/controllers/user")
 const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")

packages/server/src/automations/tests/createUser.spec.js

@@ -1,6 +1,6 @@
 const usageQuota = require("../../utilities/usageQuota")
 const setup = require("./utilities")
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { InternalTables } = require("../../db/utils")
 
 jest.mock("../../utilities/usageQuota")

packages/server/src/automations/triggers.js

@@ -6,7 +6,7 @@ const Queue = env.isTest()
   : require("bull")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")
-const { utils } = require("@budibase/auth").redis
+const { utils } = require("@budibase/auth/redis")
 
 const { opts } = utils.getRedisOptions()
 let automationQueue = new Queue("automationQueue", { redis: opts })

packages/server/src/constants/index.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { UserStatus } = require("@budibase/auth").constants
 const { ObjectStoreBuckets } = require("@budibase/auth").objectStore
 

packages/server/src/constants/screens.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { BASE_LAYOUT_PROP_IDS } = require("./layouts")
 const { LOGO_URL } = require("../constants")
 

packages/server/src/db/utils.js

@@ -1,28 +1,36 @@
 const newid = require("./newid")
+const {
+  DocumentTypes: CoreDocTypes,
+  getRoleParams,
+  generateRoleID,
+  APP_DEV_PREFIX,
+  APP_PREFIX,
+  SEPARATOR,
+} = require("@budibase/auth/db")
 
 const UNICODE_MAX = "\ufff0"
-const SEPARATOR = "_"
 
 const StaticDatabases = {
   BUILDER: {
     name: "builder-db",
     baseDoc: "builder-doc",
   },
-  // TODO: needs removed
-  BUILDER_HOSTING: {
-    name: "builder-config-db",
-    baseDoc: "hosting-doc",
-  },
+}
+
+const AppStatus = {
+  DEV: "dev",
+  DEPLOYED: "PUBLISHED",
 }
 
 const DocumentTypes = {
+  APP: CoreDocTypes.APP,
+  APP_DEV: CoreDocTypes.APP_DEV,
+  ROLE: CoreDocTypes.ROLE,
   TABLE: "ta",
   ROW: "ro",
   USER: "us",
   AUTOMATION: "au",
   LINK: "li",
-  APP: "app",
-  ROLE: "role",
   WEBHOOK: "wh",
   INSTANCE: "inst",
   LAYOUT: "layout",
@@ -44,6 +52,8 @@ const SearchIndexes = {
   ROWS: "rows",
 }
 
+exports.APP_PREFIX = APP_PREFIX
+exports.APP_DEV_PREFIX = APP_DEV_PREFIX
 exports.StaticDatabases = StaticDatabases
 exports.ViewNames = ViewNames
 exports.InternalTables = InternalTables
@@ -51,6 +61,10 @@ exports.DocumentTypes = DocumentTypes
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
 exports.SearchIndexes = SearchIndexes
+exports.AppStatus = AppStatus
+
+exports.generateRoleID = generateRoleID
+exports.getRoleParams = getRoleParams
 
 exports.getQueryIndex = viewName => {
   return `database/${viewName}`
@@ -143,9 +157,11 @@ exports.generateUserMetadataID = globalId => {
  * Breaks up the ID to get the global ID.
  */
 exports.getGlobalIDFromUserMetadataID = id => {
-  return id.split(
-    `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-  )[1]
+  const prefix = `${DocumentTypes.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+  if (!id.includes(prefix)) {
+    return id
+  }
+  return id.split(prefix)[1]
 }
 
 /**
@@ -204,25 +220,13 @@ exports.generateAppID = () => {
 }
 
 /**
- * Gets parameters for retrieving apps, this is a utility function for the getDocParams function.
+ * Generates a development app ID from a real app ID.
+ * @returns {string} the dev app ID which can be used for dev database.
  */
-exports.getAppParams = (appId = null, otherProps = {}) => {
-  return getDocParams(DocumentTypes.APP, appId, otherProps)
-}
-
-/**
- * Generates a new role ID.
- * @returns {string} The new role ID which the role doc can be stored under.
- */
-exports.generateRoleID = id => {
-  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
-}
-
-/**
- * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
- */
-exports.getRoleParams = (roleId = null, otherProps = {}) => {
-  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
+exports.generateDevAppID = appId => {
+  const prefix = `${DocumentTypes.APP}${SEPARATOR}`
+  const uuid = appId.split(prefix)[1]
+  return `${DocumentTypes.APP_DEV}${SEPARATOR}${uuid}`
 }
 
 /**

packages/server/src/middleware/authorized.js

@@ -1,9 +1,11 @@
-const { getUserPermissions } = require("../utilities/security/roles")
+const { getUserPermissions } = require("@budibase/auth/roles")
 const {
   PermissionTypes,
   doesHaveResourcePermission,
   doesHaveBasePermission,
-} = require("../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
+const { APP_DEV_PREFIX } = require("../db/utils")
+const { doesUserHaveLock, updateLock } = require("../utilities/redis")
 
 function hasResource(ctx) {
   return ctx.resourceId != null
@@ -13,6 +15,21 @@ const WEBHOOK_ENDPOINTS = new RegExp(
   ["webhooks/trigger", "webhooks/schema"].join("|")
 )
 
+async function checkDevAppLocks(ctx) {
+  const appId = ctx.appId
+
+  // not a development app, don't need to do anything
+  if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
+    return
+  }
+  if (!(await doesUserHaveLock(appId, ctx.user))) {
+    ctx.throw(403, "User does not hold app lock.")
+  }
+
+  // they do have lock, update it
+  await updateLock(appId, ctx.user)
+}
+
 module.exports = (permType, permLevel = null) => async (ctx, next) => {
   // webhooks don't need authentication, each webhook unique
   if (WEBHOOK_ENDPOINTS.test(ctx.request.url)) {
@@ -23,8 +40,15 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
     return ctx.throw(403, "No user info found")
   }
 
-  const isAuthed = ctx.isAuthenticated
+  const builderCall = permType === PermissionTypes.BUILDER
+  const referer = ctx.headers["referer"]
+  const editingApp = referer ? referer.includes(ctx.appId) : false
+  // this makes sure that builder calls abide by dev locks
+  if (builderCall && editingApp) {
+    await checkDevAppLocks(ctx)
+  }
 
+  const isAuthed = ctx.isAuthenticated
   const { basePermissions, permissions } = await getUserPermissions(
     ctx.appId,
     ctx.roleId
@@ -35,7 +59,7 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => {
   let isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
   if (isBuilder) {
     return next()
-  } else if (permType === PermissionTypes.BUILDER && !isBuilder) {
+  } else if (builderCall && !isBuilder) {
     return ctx.throw(403, "Not Authorized")
   }
 

packages/server/src/middleware/currentapp.js

@@ -1,8 +1,8 @@
 const { getAppId, setCookie, getCookie } = require("@budibase/auth").utils
 const { Cookies } = require("@budibase/auth").constants
-const { getRole } = require("../utilities/security/roles")
+const { getRole } = require("@budibase/auth/roles")
 const { getGlobalUsers } = require("../utilities/workerRequests")
-const { BUILTIN_ROLE_IDS } = require("../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const { generateUserMetadataID } = require("../db/utils")
 
 module.exports = async (ctx, next) => {
@@ -31,9 +31,8 @@ module.exports = async (ctx, next) => {
     const globalUser = await getGlobalUsers(ctx, requestAppId, ctx.user._id)
     updateCookie = true
     appId = requestAppId
-    if (globalUser.roles && globalUser.roles[requestAppId]) {
-      roleId = globalUser.roles[requestAppId]
-    }
+    // retrieving global user gets the right role
+    roleId = globalUser.roleId
   } else if (appCookie != null) {
     appId = appCookie.appId
     roleId = appCookie.roleId || BUILTIN_ROLE_IDS.PUBLIC

packages/server/src/middleware/tests/authorized.spec.js

@@ -1,6 +1,6 @@
 const authorizedMiddleware = require("../authorized")
 const env = require("../../environment")
-const { PermissionTypes, PermissionLevels } = require("../../utilities/security/permissions")
+const { PermissionTypes, PermissionLevels } = require("@budibase/auth/permissions")
 jest.mock("../../environment", () => ({
     prod: false,
     isTest: () => true,

packages/server/src/middleware/tests/currentapp.spec.js

@@ -8,7 +8,8 @@ function mockWorker() {
         _id: "us_uuid1",
         roles: {
           "app_test": "BASIC",
-        }
+        },
+        roleId: "BASIC",
       }
     }
   }))

packages/server/src/middleware/tests/selfhost.spec.js

@@ -1,7 +1,6 @@
 const selfHostMiddleware = require("../selfhost")
 const env = require("../../environment")
-jest.mock("../../environment") 
-jest.mock("../../utilities/builder/hosting") 
+jest.mock("../../environment")
 
 class TestConfiguration {
   constructor() {

packages/server/src/tests/utilities/TestConfiguration.js

@@ -1,4 +1,4 @@
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const env = require("../../environment")
 const {
   basicTable,
@@ -16,7 +16,7 @@ const supertest = require("supertest")
 const { cleanup } = require("../../utilities/fileSystem")
 const { Cookies } = require("@budibase/auth").constants
 const { jwt } = require("@budibase/auth").auth
-const { StaticDatabases } = require("@budibase/auth").db
+const { StaticDatabases } = require("@budibase/auth/db")
 const CouchDB = require("../../db")
 
 const GLOBAL_USER_ID = "us_uuid1"

packages/server/src/tests/utilities/structures.js

@@ -1,7 +1,7 @@
-const { BUILTIN_ROLE_IDS } = require("../../utilities/security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 const {
   BUILTIN_PERMISSION_IDS,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const { createHomeScreen } = require("../../constants/screens")
 const { EMPTY_LAYOUT } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")

packages/server/src/utilities/builder/hosting.js

@@ -1,86 +0,0 @@
-const CouchDB = require("../../db")
-const { StaticDatabases } = require("../../db/utils")
-const { getDeployedApps } = require("../../utilities/workerRequests")
-
-const PROD_HOSTING_URL = "app.budi.live"
-
-function getProtocol(hostingInfo) {
-  return hostingInfo.useHttps ? "https://" : "http://"
-}
-
-async function getURLWithPath(pathIfSelfHosted) {
-  const hostingInfo = await exports.getHostingInfo()
-  const protocol = getProtocol(hostingInfo)
-  const path =
-    hostingInfo.type === exports.HostingTypes.SELF ? pathIfSelfHosted : ""
-  return `${protocol}${hostingInfo.hostingUrl}${path}`
-}
-
-exports.HostingTypes = {
-  CLOUD: "cloud",
-  SELF: "self",
-}
-
-exports.getHostingInfo = async () => {
-  const db = new CouchDB(StaticDatabases.BUILDER_HOSTING.name)
-  let doc
-  try {
-    doc = await db.get(StaticDatabases.BUILDER_HOSTING.baseDoc)
-  } catch (err) {
-    // don't write this doc, want to be able to update these default props
-    // for our servers with a new release without needing to worry about state of
-    // PouchDB in peoples installations
-    doc = {
-      _id: StaticDatabases.BUILDER_HOSTING.baseDoc,
-      type: exports.HostingTypes.CLOUD,
-      hostingUrl: PROD_HOSTING_URL,
-      selfHostKey: "",
-      templatesUrl: "prod-budi-templates.s3-eu-west-1.amazonaws.com",
-      useHttps: true,
-    }
-  }
-  return doc
-}
-
-exports.getAppUrl = async appId => {
-  const hostingInfo = await exports.getHostingInfo()
-  const protocol = getProtocol(hostingInfo)
-  let url
-  if (hostingInfo.type === exports.HostingTypes.CLOUD) {
-    url = `${protocol}${appId}.${hostingInfo.hostingUrl}`
-  } else {
-    url = `${protocol}${hostingInfo.hostingUrl}/app`
-  }
-  return url
-}
-
-exports.getWorkerUrl = async () => {
-  return getURLWithPath("/worker")
-}
-
-exports.getMinioUrl = async () => {
-  return getURLWithPath("/")
-}
-
-exports.getCouchUrl = async () => {
-  return getURLWithPath("/db")
-}
-
-exports.getSelfHostKey = async () => {
-  const hostingInfo = await exports.getHostingInfo()
-  return hostingInfo.selfHostKey
-}
-
-exports.getTemplatesUrl = async (appId, type, name) => {
-  const hostingInfo = await exports.getHostingInfo()
-  const protocol = getProtocol(hostingInfo)
-  let path
-  if (type && name) {
-    path = `templates/type/${name}.tar.gz`
-  } else {
-    path = "manifest.json"
-  }
-  return `${protocol}${hostingInfo.templatesUrl}/${path}`
-}
-
-exports.getDeployedApps = getDeployedApps

packages/server/src/utilities/index.js

@@ -1,35 +1,15 @@
 const env = require("../environment")
-const { DocumentTypes, SEPARATOR } = require("../db/utils")
+const { APP_PREFIX } = require("../db/utils")
 const CouchDB = require("../db")
 const { OBJ_STORE_DIRECTORY, ObjectStoreBuckets } = require("../constants")
+const { getAllApps } = require("@budibase/auth/db")
 
 const BB_CDN = "https://cdn.app.budi.live/assets"
-const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
 exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
 exports.isDev = env.isDev
-
-/**
- * Lots of different points in the app need to find the full list of apps, this will
- * enumerate the entire CouchDB cluster and get the list of databases (every app).
- * NOTE: this operation is fine in self hosting, but cannot be used when hosting many
- * different users/companies apps as there is no security around it - all apps are returned.
- * @return {Promise<object[]>} returns the app information document stored in each app database.
- */
-exports.getAllApps = async () => {
-  let allDbs = await CouchDB.allDbs()
-  const appDbNames = allDbs.filter(dbName => dbName.startsWith(APP_PREFIX))
-  const appPromises = appDbNames.map(db => new CouchDB(db).get(db))
-  if (appPromises.length === 0) {
-    return []
-  } else {
-    const response = await Promise.allSettled(appPromises)
-    return response
-      .filter(result => result.status === "fulfilled")
-      .map(({ value }) => value)
-  }
-}
+exports.getAllApps = getAllApps
 
 /**
  * Makes sure that a URL has the correct number of slashes, while maintaining the

packages/server/src/utilities/redis.js

@@ -0,0 +1,54 @@
+const { Client, utils } = require("@budibase/auth/redis")
+const { getGlobalIDFromUserMetadataID } = require("../db/utils")
+
+const APP_DEV_LOCK_SECONDS = 600
+const DB_NAME = utils.Databases.DEV_LOCKS
+let devAppClient
+
+// we init this as we want to keep the connection open all the time
+// reduces the performance hit
+exports.init = async () => {
+  devAppClient = await new Client(DB_NAME).init()
+}
+
+exports.doesUserHaveLock = async (devAppId, user) => {
+  const value = await devAppClient.get(devAppId)
+  if (!value) {
+    return true
+  }
+  // make sure both IDs are global
+  const expected = getGlobalIDFromUserMetadataID(value._id)
+  const userId = getGlobalIDFromUserMetadataID(user._id)
+  return expected === userId
+}
+
+exports.getAllLocks = async () => {
+  const locks = await devAppClient.scan()
+  return locks.map(lock => ({
+    appId: lock.key,
+    user: lock.value,
+  }))
+}
+
+exports.updateLock = async (devAppId, user) => {
+  // make sure always global user ID
+  const globalId = getGlobalIDFromUserMetadataID(user._id)
+  const inputUser = {
+    ...user,
+    userId: globalId,
+    _id: globalId,
+  }
+  await devAppClient.store(devAppId, inputUser, APP_DEV_LOCK_SECONDS)
+}
+
+exports.clearLock = async (devAppId, user) => {
+  const value = await devAppClient.get(devAppId)
+  if (!value) {
+    return
+  }
+  const userId = getGlobalIDFromUserMetadataID(user._id)
+  if (value._id !== userId) {
+    throw "User does not hold lock, cannot clear it."
+  }
+  await devAppClient.delete(devAppId)
+}

packages/server/src/utilities/security.js

@@ -3,12 +3,12 @@ const {
   PermissionTypes,
   getBuiltinPermissionByID,
   isPermissionLevelHigherThanRead,
-} = require("../../utilities/security/permissions")
+} = require("@budibase/auth/permissions")
 const {
   lowerBuiltinRoleID,
   getBuiltinRoles,
-} = require("../../utilities/security/roles")
-const { DocumentTypes } = require("../../db/utils")
+} = require("@budibase/auth/roles")
+const { DocumentTypes } = require("../db/utils")
 
 const CURRENTLY_SUPPORTED_LEVELS = [
   PermissionLevels.WRITE,

packages/server/src/utilities/workerRequests.js

@@ -1,7 +1,7 @@
 const fetch = require("node-fetch")
 const env = require("../environment")
 const { checkSlashesInUrl } = require("./index")
-const { BUILTIN_ROLE_IDS } = require("./security/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
 
 function getAppRole(appId, user) {
   if (!user.roles) {
@@ -62,9 +62,6 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => {
 }
 
 exports.getDeployedApps = async ctx => {
-  if (!env.SELF_HOSTED) {
-    throw "Can only check apps for self hosted environments"
-  }
   try {
     const response = await fetch(
       checkSlashesInUrl(env.WORKER_URL + `/api/apps`),

packages/worker/package.json

@@ -2,7 +2,7 @@
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
   "version": "0.8.16",
-  "description": "Budibase Deployment Server",
+  "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
     "type": "git",

packages/worker/src/api/controllers/admin/roles.js

@@ -0,0 +1,24 @@
+const { getAllRoles } = require("@budibase/auth/roles")
+const { getAllApps } = require("@budibase/auth/db")
+
+exports.fetch = async ctx => {
+  // always use the dev apps as they'll be most up to date (true)
+  const apps = await getAllApps(true)
+  const promises = []
+  for (let app of apps) {
+    promises.push(getAllRoles(app._id))
+  }
+  const roles = await Promise.all(promises)
+  const response = {}
+  for (let app of apps) {
+    response[app._id] = roles.shift()
+  }
+  ctx.body = response
+}
+
+exports.find = async ctx => {
+  const appId = ctx.params.appId
+  ctx.body = {
+    roles: await getAllRoles(appId)
+  }
+}